cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Storage detach/attach in lotus-miner, cli commands

Browse Source
This commit is contained in:
Łukasz Magiera 2022-07-15 12:44:05 +02:00
parent 943d2a72c6
commit 8cff52aef6
18 changed files with 372 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/api_storage.go
View File

@ -176,9 +176,9 @@ type StorageMiner interface {
StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
StorageAddLocal(ctx context.Context, path string) error //perm:admin
//StorageDetachLocal(ctx context.Context, path string) error //perm:admin
//StorageRedeclareLocal(ctx context.Context, id storiface.ID, dropMissing bool) error //perm:admin
StorageAddLocal(ctx context.Context, path string) error //perm:admin
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
MarketListDeals(ctx context.Context) ([]*MarketDeal, error) //perm:read

6
api/api_worker.go
View File

@ -59,9 +59,9 @@ type Worker interface {
// Storage / Other
Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
StorageAddLocal(ctx context.Context, path string) error //perm:admin
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
StorageRedeclareLocal(ctx context.Context, id storiface.ID, dropMissing bool) error //perm:admin
StorageAddLocal(ctx context.Context, path string) error //perm:admin
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
// SetEnabled marks the worker as enabled/disabled. Not that this setting
// may take a few seconds to propagate to task scheduler

2
api/docgen/docgen.go
View File

@ -272,6 +272,8 @@ func init() {
Read: [storiface.FileTypes]uint{2, 3, 0},
},
})
storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
addExample(&storifaceid)
// worker specific
addExample(storiface.AcquireMove)

34
api/proxy_gen.go
View File

@ -850,7 +850,7 @@ type StorageMinerStruct struct {
SectorsUpdate func(p0 context.Context, p1 abi.SectorNumber, p2 SectorState) error `perm:"admin"`
StorageAddLocal func(p0 context.Context, p1 string) error ``
StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"`
StorageAttach func(p0 context.Context, p1 storiface.StorageInfo, p2 fsutil.FsStat) error `perm:"admin"`
@ -862,6 +862,8 @@ type StorageMinerStruct struct {
StorageDetach func(p0 context.Context, p1 storiface.ID, p2 string) error `perm:"admin"`
StorageDetachLocal func(p0 context.Context, p1 string) error `perm:"admin"`
StorageDropSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error `perm:"admin"`
StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]storiface.SectorStorageInfo, error) `perm:"admin"`
@ -876,6 +878,8 @@ type StorageMinerStruct struct {
StorageLock func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 storiface.SectorFileType) error `perm:"admin"`
StorageRedeclareLocal func(p0 context.Context, p1 *storiface.ID, p2 bool) error `perm:"admin"`
StorageReportHealth func(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error `perm:"admin"`
StorageStat func(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) `perm:"admin"`
@ -971,7 +975,7 @@ type WorkerStruct struct {
StorageDetachLocal func(p0 context.Context, p1 string) error `perm:"admin"`
StorageRedeclareLocal func(p0 context.Context, p1 storiface.ID, p2 bool) error `perm:"admin"`
StorageRedeclareLocal func(p0 context.Context, p1 *storiface.ID, p2 bool) error `perm:"admin"`
TaskDisable func(p0 context.Context, p1 sealtasks.TaskType) error `perm:"admin"`
@ -5093,6 +5097,17 @@ func (s *StorageMinerStub) StorageDetach(p0 context.Context, p1 storiface.ID, p2
return ErrNotSupported
}
func (s *StorageMinerStruct) StorageDetachLocal(p0 context.Context, p1 string) error {
if s.Internal.StorageDetachLocal == nil {
return ErrNotSupported
}
return s.Internal.StorageDetachLocal(p0, p1)
}
func (s *StorageMinerStub) StorageDetachLocal(p0 context.Context, p1 string) error {
return ErrNotSupported
}
func (s *StorageMinerStruct) StorageDropSector(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error {
if s.Internal.StorageDropSector == nil {
return ErrNotSupported
@ -5170,6 +5185,17 @@ func (s *StorageMinerStub) StorageLock(p0 context.Context, p1 abi.SectorID, p2 s
return ErrNotSupported
}
func (s *StorageMinerStruct) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
if s.Internal.StorageRedeclareLocal == nil {
return ErrNotSupported
}
return s.Internal.StorageRedeclareLocal(p0, p1, p2)
}
func (s *StorageMinerStub) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
return ErrNotSupported
}
func (s *StorageMinerStruct) StorageReportHealth(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error {
if s.Internal.StorageReportHealth == nil {
return ErrNotSupported
@ -5599,14 +5625,14 @@ func (s *WorkerStub) StorageDetachLocal(p0 context.Context, p1 string) error {
return ErrNotSupported
}
func (s *WorkerStruct) StorageRedeclareLocal(p0 context.Context, p1 storiface.ID, p2 bool) error {
func (s *WorkerStruct) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
if s.Internal.StorageRedeclareLocal == nil {
return ErrNotSupported
}
return s.Internal.StorageRedeclareLocal(p0, p1, p2)
}
func (s *WorkerStub) StorageRedeclareLocal(p0 context.Context, p1 storiface.ID, p2 bool) error {
func (s *WorkerStub) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
return ErrNotSupported
}

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/gateway.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

82
cmd/lotus-miner/storage.go
View File

@ -46,6 +46,8 @@ long term for proving (references as 'store') as well as how sectors will be
stored while moving through the sealing pipeline (references as 'seal').`,
Subcommands: []*cli.Command{
storageAttachCmd,
storageDetachCmd,
storageRedeclareCmd,
storageListCmd,
storageFindCmd,
storageCleanupCmd,
@ -174,6 +176,86 @@ over time
},
}
var storageDetachCmd = &cli.Command{
Name: "detach",
Usage: "detach local storage path",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
},
},
ArgsUsage: "[path]",
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if !cctx.Args().Present() {
return xerrors.Errorf("must specify storage path")
}
p, err := homedir.Expand(cctx.Args().First())
if err != nil {
return xerrors.Errorf("expanding path: %w", err)
}
if !cctx.Bool("really-do-id") {
return xerrors.Errorf("pass --really-do-it to execute the action")
}
return nodeApi.StorageDetachLocal(ctx, p)
},
}
var storageRedeclareCmd = &cli.Command{
Name: "redeclare",
Usage: "redeclare sectors in a local storage path",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "id",
Usage: "storage path ID",
},
&cli.BoolFlag{
Name: "all",
Usage: "redeclare all storage paths",
},
&cli.BoolFlag{
Name: "drop-missing",
Usage: "Drop index entries with missing files",
},
},
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if !cctx.Args().Present() {
return xerrors.Errorf("must specify storage path")
}
if cctx.IsSet("id") && cctx.Bool("all") {
return xerrors.Errorf("--id and --all can't be passed at the same time")
}
if cctx.IsSet("id") {
id := storiface.ID(cctx.String("id"))
return nodeApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
}
if cctx.Bool("all") {
return nodeApi.StorageRedeclareLocal(ctx, nil, cctx.Bool("drop-missing"))
}
return xerrors.Errorf("either --all or --id must be specified")
},
}
var storageListCmd = &cli.Command{
Name: "list",
Usage: "list local storage paths",

2
cmd/lotus-worker/main.go
View File

@ -571,7 +571,7 @@ var runCmd = &cli.Command{
if redeclareStorage {
log.Info("Redeclaring local storage")
if err := localStore.Redeclare(ctx); err != nil {
if err := localStore.Redeclare(ctx, nil, false); err != nil {
log.Errorf("Redeclaring local storage failed: %+v", err)
select {

4
cmd/lotus-worker/sealworker/rpc.go
View File

@ -130,8 +130,8 @@ func (w *Worker) StorageDetachLocal(ctx context.Context, path string) error {
return w.LocalStore.ClosePath(ctx, localPath.ID)
}
func (w *Worker) StorageRedeclareLocal(ctx context.Context, id storiface.ID, dropMissing bool) error {
return w.LocalStore.Redeclare(ctx, &id, dropMissing)
func (w *Worker) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error {
return w.LocalStore.Redeclare(ctx, id, dropMissing)
}
func (w *Worker) SetEnabled(ctx context.Context, enabled bool) error {

82
cmd/lotus-worker/storage.go
View File

@ -24,6 +24,8 @@ var storageCmd = &cli.Command{
Usage: "manage sector storage",
Subcommands: []*cli.Command{
storageAttachCmd,
storageDetachCmd,
storageRedeclareCmd,
},
}
@ -128,3 +130,83 @@ var storageAttachCmd = &cli.Command{
return nodeApi.StorageAddLocal(ctx, p)
},
}
var storageDetachCmd = &cli.Command{
Name: "detach",
Usage: "detach local storage path",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
},
},
ArgsUsage: "[path]",
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if !cctx.Args().Present() {
return xerrors.Errorf("must specify storage path")
}
p, err := homedir.Expand(cctx.Args().First())
if err != nil {
return xerrors.Errorf("expanding path: %w", err)
}
if !cctx.Bool("really-do-id") {
return xerrors.Errorf("pass --really-do-it to execute the action")
}
return nodeApi.StorageDetachLocal(ctx, p)
},
}
var storageRedeclareCmd = &cli.Command{
Name: "redeclare",
Usage: "redeclare sectors in a local storage path",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "id",
Usage: "storage path ID",
},
&cli.BoolFlag{
Name: "all",
Usage: "redeclare all storage paths",
},
&cli.BoolFlag{
Name: "drop-missing",
Usage: "Drop index entries with missing files",
},
},
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if !cctx.Args().Present() {
return xerrors.Errorf("must specify storage path")
}
if cctx.IsSet("id") && cctx.Bool("all") {
return xerrors.Errorf("--id and --all can't be passed at the same time")
}
if cctx.IsSet("id") {
id := storiface.ID(cctx.String("id"))
return nodeApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
}
if cctx.Bool("all") {
return nodeApi.StorageRedeclareLocal(ctx, nil, cctx.Bool("drop-missing"))
}
return xerrors.Errorf("either --all or --id must be specified")
},
}

33
documentation/en/api-v0-methods-miner.md
View File

@ -161,6 +161,7 @@
* [StorageBestAlloc](#StorageBestAlloc)
* [StorageDeclareSector](#StorageDeclareSector)
* [StorageDetach](#StorageDetach)
* [StorageDetachLocal](#StorageDetachLocal)
* [StorageDropSector](#StorageDropSector)
* [StorageFindSector](#StorageFindSector)
* [StorageGetLocks](#StorageGetLocks)
@ -168,6 +169,7 @@
* [StorageList](#StorageList)
* [StorageLocal](#StorageLocal)
* [StorageLock](#StorageLock)
* [StorageRedeclareLocal](#StorageRedeclareLocal)
* [StorageReportHealth](#StorageReportHealth)
* [StorageStat](#StorageStat)
* [StorageTryLock](#StorageTryLock)
@ -3260,7 +3262,7 @@ Response: `{}`
### StorageAddLocal
Perms:
Perms: admin
Inputs:
```json
@ -3414,6 +3416,20 @@ Inputs:
Response: `{}`
### StorageDetachLocal
Perms: admin
Inputs:
```json
[
"string value"
]
```
Response: `{}`
### StorageDropSector
@ -3610,6 +3626,21 @@ Inputs:
Response: `{}`
### StorageRedeclareLocal
Perms: admin
Inputs:
```json
[
"1399aa04-2625-44b1-bad4-bd07b59b22c4",
true
]
```
Response: `{}`
### StorageReportHealth

2
documentation/en/api-v0-methods-worker.md
View File

@ -2131,7 +2131,7 @@ Perms: admin
Inputs:
```json
[
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8",
"1399aa04-2625-44b1-bad4-bd07b59b22c4",
true
]
```

42
documentation/en/cli-lotus-miner.md
View File

@ -2150,12 +2150,14 @@ DESCRIPTION:
stored while moving through the sealing pipeline (references as 'seal').
COMMANDS:
attach attach local storage path
list list local storage paths
find find sector in the storage system
cleanup trigger cleanup actions
locks show active sector locks
help, h Shows a list of commands or help for one command
attach attach local storage path
detach detach local storage path
redeclare redeclare sectors in a local storage path
list list local storage paths
find find sector in the storage system
cleanup trigger cleanup actions
locks show active sector locks
help, h Shows a list of commands or help for one command
OPTIONS:
--help, -h show help (default: false)
@ -2201,6 +2203,34 @@ OPTIONS:
```
### lotus-miner storage detach
```
NAME:
lotus-miner storage detach - detach local storage path
USAGE:
lotus-miner storage detach [command options] [path]
OPTIONS:
--really-do-it (default: false)
```
### lotus-miner storage redeclare
```
NAME:
lotus-miner storage redeclare - redeclare sectors in a local storage path
USAGE:
lotus-miner storage redeclare [command options] [arguments...]
OPTIONS:
--all redeclare all storage paths (default: false)
--drop-missing Drop index entries with missing files (default: false)
--id value storage path ID
```
### lotus-miner storage list
```
NAME:

34
documentation/en/cli-lotus-worker.md
View File

@ -80,8 +80,10 @@ USAGE:
lotus-worker storage command [command options] [arguments...]
COMMANDS:
attach attach local storage path
help, h Shows a list of commands or help for one command
attach attach local storage path
detach detach local storage path
redeclare redeclare sectors in a local storage path
help, h Shows a list of commands or help for one command
OPTIONS:
--help, -h show help (default: false)
@ -107,6 +109,34 @@ OPTIONS:
```
### lotus-worker storage detach
```
NAME:
lotus-worker storage detach - detach local storage path
USAGE:
lotus-worker storage detach [command options] [path]
OPTIONS:
--really-do-it (default: false)
```
### lotus-worker storage redeclare
```
NAME:
lotus-worker storage redeclare - redeclare sectors in a local storage path
USAGE:
lotus-worker storage redeclare [command options] [arguments...]
OPTIONS:
--all redeclare all storage paths (default: false)
--drop-missing Drop index entries with missing files (default: false)
--id value storage path ID
```
## lotus-worker set
```
NAME:

16
node/impl/storminer.go
View File

@ -1206,6 +1206,22 @@ func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) err
return sm.StorageMgr.AddLocalStorage(ctx, path)
}
func (sm *StorageMinerAPI) StorageDetachLocal(ctx context.Context, path string) error {
if sm.StorageMgr == nil {
return xerrors.Errorf("no storage manager")
}
return sm.StorageMgr.DetachLocalStorage(ctx, path)
}
func (sm *StorageMinerAPI) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error {
if sm.StorageMgr == nil {
return xerrors.Errorf("no storage manager")
}
return sm.StorageMgr.RedeclareLocalStorage(ctx, id, false)
}
func (sm *StorageMinerAPI) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
return sm.PieceStore.ListPieceInfoKeys()
}

50
storage/sealer/manager.go
View File

@ -235,6 +235,56 @@ func (m *Manager) AddLocalStorage(ctx context.Context, path string) error {
return nil
}
func (m *Manager) DetachLocalStorage(ctx context.Context, path string) error {
path, err := homedir.Expand(path)
if err != nil {
return xerrors.Errorf("expanding local path: %w", err)
}
// check that we have the path opened
lps, err := m.localStore.Local(ctx)
if err != nil {
return xerrors.Errorf("getting local path list: %w", err)
}
var localPath *storiface.StoragePath
for _, lp := range lps {
if lp.LocalPath == path {
localPath = &lp
break
}
}
if localPath == nil {
return xerrors.Errorf("no local paths match '%s'", path)
}
// drop from the persisted storage.json
var found bool
if err := m.ls.SetStorage(func(sc *paths.StorageConfig) {
out := make([]paths.LocalPath, 0, len(sc.StoragePaths))
for _, storagePath := range sc.StoragePaths {
if storagePath.Path != path {
out = append(out, storagePath)
return
}
found = true
}
}); err != nil {
return xerrors.Errorf("set storage config: %w", err)
}
if !found {
// maybe this is fine?
return xerrors.Errorf("path not found in storage.json")
}
// unregister locally, drop from sector index
return m.localStore.ClosePath(ctx, localPath.ID)
}
func (m *Manager) RedeclareLocalStorage(ctx context.Context, id *storiface.ID, dropMissing bool) error {
return m.localStore.Redeclare(ctx, id, dropMissing)
}
func (m *Manager) AddWorker(ctx context.Context, w Worker) error {
sessID, err := w.Session(ctx)
if err != nil {