Storage detach/attach in lotus-miner, cli commands

api/api_storage.go

@@ -176,9 +176,9 @@ type StorageMiner interface {
 
 	StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
+	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
-	//StorageDetachLocal(ctx context.Context, path string) error    //perm:admin
+	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
-	//StorageRedeclareLocal(ctx context.Context, id storiface.ID, dropMissing bool) error //perm:admin
+	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
 
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
 	MarketListDeals(ctx context.Context) ([]*MarketDeal, error)                                                                                                                          //perm:read

api/api_worker.go

@@ -59,9 +59,9 @@ type Worker interface {
 	// Storage / Other
 	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error                             //perm:admin
+	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
-	StorageDetachLocal(ctx context.Context, path string) error                          //perm:admin
+	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
-	StorageRedeclareLocal(ctx context.Context, id storiface.ID, dropMissing bool) error //perm:admin
+	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
 
 	// SetEnabled marks the worker as enabled/disabled. Not that this setting
 	// may take a few seconds to propagate to task scheduler

api/docgen/docgen.go

@@ -272,6 +272,8 @@ func init() {
 			Read:   [storiface.FileTypes]uint{2, 3, 0},
 		},
 	})
+	storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
+	addExample(&storifaceid)
 
 	// worker specific
 	addExample(storiface.AcquireMove)

api/proxy_gen.go

@@ -850,7 +850,7 @@ type StorageMinerStruct struct {
 
 		SectorsUpdate func(p0 context.Context, p1 abi.SectorNumber, p2 SectorState) error `perm:"admin"`
 
-		StorageAddLocal func(p0 context.Context, p1 string) error ``
+		StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"`
 
 		StorageAttach func(p0 context.Context, p1 storiface.StorageInfo, p2 fsutil.FsStat) error `perm:"admin"`
 
@@ -862,6 +862,8 @@ type StorageMinerStruct struct {
 
 		StorageDetach func(p0 context.Context, p1 storiface.ID, p2 string) error `perm:"admin"`
 
+		StorageDetachLocal func(p0 context.Context, p1 string) error `perm:"admin"`
+
 		StorageDropSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error `perm:"admin"`
 
 		StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]storiface.SectorStorageInfo, error) `perm:"admin"`
@@ -876,6 +878,8 @@ type StorageMinerStruct struct {
 
 		StorageLock func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 storiface.SectorFileType) error `perm:"admin"`
 
+		StorageRedeclareLocal func(p0 context.Context, p1 *storiface.ID, p2 bool) error `perm:"admin"`
+
 		StorageReportHealth func(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error `perm:"admin"`
 
 		StorageStat func(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) `perm:"admin"`
@@ -971,7 +975,7 @@ type WorkerStruct struct {
 
 		StorageDetachLocal func(p0 context.Context, p1 string) error `perm:"admin"`
 
-		StorageRedeclareLocal func(p0 context.Context, p1 storiface.ID, p2 bool) error `perm:"admin"`
+		StorageRedeclareLocal func(p0 context.Context, p1 *storiface.ID, p2 bool) error `perm:"admin"`
 
 		TaskDisable func(p0 context.Context, p1 sealtasks.TaskType) error `perm:"admin"`
 
@@ -5093,6 +5097,17 @@ func (s *StorageMinerStub) StorageDetach(p0 context.Context, p1 storiface.ID, p2
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) StorageDetachLocal(p0 context.Context, p1 string) error {
+	if s.Internal.StorageDetachLocal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.StorageDetachLocal(p0, p1)
+}
+
+func (s *StorageMinerStub) StorageDetachLocal(p0 context.Context, p1 string) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) StorageDropSector(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error {
 	if s.Internal.StorageDropSector == nil {
 		return ErrNotSupported
@@ -5170,6 +5185,17 @@ func (s *StorageMinerStub) StorageLock(p0 context.Context, p1 abi.SectorID, p2 s
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
+	if s.Internal.StorageRedeclareLocal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.StorageRedeclareLocal(p0, p1, p2)
+}
+
+func (s *StorageMinerStub) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) StorageReportHealth(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error {
 	if s.Internal.StorageReportHealth == nil {
 		return ErrNotSupported
@@ -5599,14 +5625,14 @@ func (s *WorkerStub) StorageDetachLocal(p0 context.Context, p1 string) error {
 	return ErrNotSupported
 }
 
-func (s *WorkerStruct) StorageRedeclareLocal(p0 context.Context, p1 storiface.ID, p2 bool) error {
+func (s *WorkerStruct) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
 	if s.Internal.StorageRedeclareLocal == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.StorageRedeclareLocal(p0, p1, p2)
 }
 
-func (s *WorkerStub) StorageRedeclareLocal(p0 context.Context, p1 storiface.ID, p2 bool) error {
+func (s *WorkerStub) StorageRedeclareLocal(p0 context.Context, p1 *storiface.ID, p2 bool) error {
 	return ErrNotSupported
 }
 

cmd/lotus-miner/storage.go

@@ -46,6 +46,8 @@ long term for proving (references as 'store') as well as how sectors will be
 stored while moving through the sealing pipeline (references as 'seal').`,
 	Subcommands: []*cli.Command{
 		storageAttachCmd,
+		storageDetachCmd,
+		storageRedeclareCmd,
 		storageListCmd,
 		storageFindCmd,
 		storageCleanupCmd,
@@ -174,6 +176,86 @@ over time
 	},
 }
 
+var storageDetachCmd = &cli.Command{
+	Name:  "detach",
+	Usage: "detach local storage path",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name: "really-do-it",
+		},
+	},
+	ArgsUsage: "[path]",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return xerrors.Errorf("must specify storage path")
+		}
+
+		p, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return xerrors.Errorf("expanding path: %w", err)
+		}
+
+		if !cctx.Bool("really-do-id") {
+			return xerrors.Errorf("pass --really-do-it to execute the action")
+		}
+
+		return nodeApi.StorageDetachLocal(ctx, p)
+	},
+}
+
+var storageRedeclareCmd = &cli.Command{
+	Name:  "redeclare",
+	Usage: "redeclare sectors in a local storage path",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "id",
+			Usage: "storage path ID",
+		},
+		&cli.BoolFlag{
+			Name:  "all",
+			Usage: "redeclare all storage paths",
+		},
+		&cli.BoolFlag{
+			Name:  "drop-missing",
+			Usage: "Drop index entries with missing files",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return xerrors.Errorf("must specify storage path")
+		}
+
+		if cctx.IsSet("id") && cctx.Bool("all") {
+			return xerrors.Errorf("--id and --all can't be passed at the same time")
+		}
+
+		if cctx.IsSet("id") {
+			id := storiface.ID(cctx.String("id"))
+			return nodeApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
+		}
+
+		if cctx.Bool("all") {
+			return nodeApi.StorageRedeclareLocal(ctx, nil, cctx.Bool("drop-missing"))
+		}
+
+		return xerrors.Errorf("either --all or --id must be specified")
+	},
+}
+
 var storageListCmd = &cli.Command{
 	Name:  "list",
 	Usage: "list local storage paths",

cmd/lotus-worker/main.go

@@ -571,7 +571,7 @@ var runCmd = &cli.Command{
 				if redeclareStorage {
 					log.Info("Redeclaring local storage")
 
-					if err := localStore.Redeclare(ctx); err != nil {
+					if err := localStore.Redeclare(ctx, nil, false); err != nil {
 						log.Errorf("Redeclaring local storage failed: %+v", err)
 
 						select {

cmd/lotus-worker/sealworker/rpc.go

@@ -130,8 +130,8 @@ func (w *Worker) StorageDetachLocal(ctx context.Context, path string) error {
 	return w.LocalStore.ClosePath(ctx, localPath.ID)
 }
 
-func (w *Worker) StorageRedeclareLocal(ctx context.Context, id storiface.ID, dropMissing bool) error {
+func (w *Worker) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error {
-	return w.LocalStore.Redeclare(ctx, &id, dropMissing)
+	return w.LocalStore.Redeclare(ctx, id, dropMissing)
 }
 
 func (w *Worker) SetEnabled(ctx context.Context, enabled bool) error {

cmd/lotus-worker/storage.go

@@ -24,6 +24,8 @@ var storageCmd = &cli.Command{
 	Usage: "manage sector storage",
 	Subcommands: []*cli.Command{
 		storageAttachCmd,
+		storageDetachCmd,
+		storageRedeclareCmd,
 	},
 }
 
@@ -128,3 +130,83 @@ var storageAttachCmd = &cli.Command{
 		return nodeApi.StorageAddLocal(ctx, p)
 	},
 }
+
+var storageDetachCmd = &cli.Command{
+	Name:  "detach",
+	Usage: "detach local storage path",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name: "really-do-it",
+		},
+	},
+	ArgsUsage: "[path]",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return xerrors.Errorf("must specify storage path")
+		}
+
+		p, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return xerrors.Errorf("expanding path: %w", err)
+		}
+
+		if !cctx.Bool("really-do-id") {
+			return xerrors.Errorf("pass --really-do-it to execute the action")
+		}
+
+		return nodeApi.StorageDetachLocal(ctx, p)
+	},
+}
+
+var storageRedeclareCmd = &cli.Command{
+	Name:  "redeclare",
+	Usage: "redeclare sectors in a local storage path",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "id",
+			Usage: "storage path ID",
+		},
+		&cli.BoolFlag{
+			Name:  "all",
+			Usage: "redeclare all storage paths",
+		},
+		&cli.BoolFlag{
+			Name:  "drop-missing",
+			Usage: "Drop index entries with missing files",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return xerrors.Errorf("must specify storage path")
+		}
+
+		if cctx.IsSet("id") && cctx.Bool("all") {
+			return xerrors.Errorf("--id and --all can't be passed at the same time")
+		}
+
+		if cctx.IsSet("id") {
+			id := storiface.ID(cctx.String("id"))
+			return nodeApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
+		}
+
+		if cctx.Bool("all") {
+			return nodeApi.StorageRedeclareLocal(ctx, nil, cctx.Bool("drop-missing"))
+		}
+
+		return xerrors.Errorf("either --all or --id must be specified")
+	},
+}

documentation/en/api-v0-methods-miner.md

@@ -161,6 +161,7 @@
   * [StorageBestAlloc](#StorageBestAlloc)
   * [StorageDeclareSector](#StorageDeclareSector)
   * [StorageDetach](#StorageDetach)
+  * [StorageDetachLocal](#StorageDetachLocal)
   * [StorageDropSector](#StorageDropSector)
   * [StorageFindSector](#StorageFindSector)
   * [StorageGetLocks](#StorageGetLocks)
@@ -168,6 +169,7 @@
   * [StorageList](#StorageList)
   * [StorageLocal](#StorageLocal)
   * [StorageLock](#StorageLock)
+  * [StorageRedeclareLocal](#StorageRedeclareLocal)
   * [StorageReportHealth](#StorageReportHealth)
   * [StorageStat](#StorageStat)
   * [StorageTryLock](#StorageTryLock)
@@ -3260,7 +3262,7 @@ Response: `{}`
 ### StorageAddLocal
 
 
-Perms: 
+Perms: admin
 
 Inputs:
 ```json
@@ -3414,6 +3416,20 @@ Inputs:
 
 Response: `{}`
 
+### StorageDetachLocal
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  "string value"
+]
+```
+
+Response: `{}`
+
 ### StorageDropSector
 
 
@@ -3610,6 +3626,21 @@ Inputs:
 
 Response: `{}`
 
+### StorageRedeclareLocal
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  "1399aa04-2625-44b1-bad4-bd07b59b22c4",
+  true
+]
+```
+
+Response: `{}`
+
 ### StorageReportHealth
 
 

documentation/en/api-v0-methods-worker.md

@@ -2131,7 +2131,7 @@ Perms: admin
 Inputs:
 ```json
 [
-  "76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8",
+  "1399aa04-2625-44b1-bad4-bd07b59b22c4",
   true
 ]
 ```

documentation/en/cli-lotus-miner.md

@@ -2150,12 +2150,14 @@ DESCRIPTION:
    stored while moving through the sealing pipeline (references as 'seal').
 
 COMMANDS:
-   attach   attach local storage path
+   attach     attach local storage path
-   list     list local storage paths
+   detach     detach local storage path
-   find     find sector in the storage system
+   redeclare  redeclare sectors in a local storage path
-   cleanup  trigger cleanup actions
+   list       list local storage paths
-   locks    show active sector locks
+   find       find sector in the storage system
-   help, h  Shows a list of commands or help for one command
+   cleanup    trigger cleanup actions
+   locks      show active sector locks
+   help, h    Shows a list of commands or help for one command
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -2201,6 +2203,34 @@ OPTIONS:
    
 ```
 
+### lotus-miner storage detach
+```
+NAME:
+   lotus-miner storage detach - detach local storage path
+
+USAGE:
+   lotus-miner storage detach [command options] [path]
+
+OPTIONS:
+   --really-do-it  (default: false)
+   
+```
+
+### lotus-miner storage redeclare
+```
+NAME:
+   lotus-miner storage redeclare - redeclare sectors in a local storage path
+
+USAGE:
+   lotus-miner storage redeclare [command options] [arguments...]
+
+OPTIONS:
+   --all           redeclare all storage paths (default: false)
+   --drop-missing  Drop index entries with missing files (default: false)
+   --id value      storage path ID
+   
+```
+
 ### lotus-miner storage list
 ```
 NAME:

documentation/en/cli-lotus-worker.md

@@ -80,8 +80,10 @@ USAGE:
    lotus-worker storage command [command options] [arguments...]
 
 COMMANDS:
-   attach   attach local storage path
+   attach     attach local storage path
-   help, h  Shows a list of commands or help for one command
+   detach     detach local storage path
+   redeclare  redeclare sectors in a local storage path
+   help, h    Shows a list of commands or help for one command
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -107,6 +109,34 @@ OPTIONS:
    
 ```
 
+### lotus-worker storage detach
+```
+NAME:
+   lotus-worker storage detach - detach local storage path
+
+USAGE:
+   lotus-worker storage detach [command options] [path]
+
+OPTIONS:
+   --really-do-it  (default: false)
+   
+```
+
+### lotus-worker storage redeclare
+```
+NAME:
+   lotus-worker storage redeclare - redeclare sectors in a local storage path
+
+USAGE:
+   lotus-worker storage redeclare [command options] [arguments...]
+
+OPTIONS:
+   --all           redeclare all storage paths (default: false)
+   --drop-missing  Drop index entries with missing files (default: false)
+   --id value      storage path ID
+   
+```
+
 ## lotus-worker set
 ```
 NAME:

node/impl/storminer.go

@@ -1206,6 +1206,22 @@ func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) err
 	return sm.StorageMgr.AddLocalStorage(ctx, path)
 }
 
+func (sm *StorageMinerAPI) StorageDetachLocal(ctx context.Context, path string) error {
+	if sm.StorageMgr == nil {
+		return xerrors.Errorf("no storage manager")
+	}
+
+	return sm.StorageMgr.DetachLocalStorage(ctx, path)
+}
+
+func (sm *StorageMinerAPI) StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error {
+	if sm.StorageMgr == nil {
+		return xerrors.Errorf("no storage manager")
+	}
+
+	return sm.StorageMgr.RedeclareLocalStorage(ctx, id, false)
+}
+
 func (sm *StorageMinerAPI) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
 	return sm.PieceStore.ListPieceInfoKeys()
 }

storage/sealer/manager.go

@@ -235,6 +235,56 @@ func (m *Manager) AddLocalStorage(ctx context.Context, path string) error {
 	return nil
 }
 
+func (m *Manager) DetachLocalStorage(ctx context.Context, path string) error {
+	path, err := homedir.Expand(path)
+	if err != nil {
+		return xerrors.Errorf("expanding local path: %w", err)
+	}
+
+	// check that we have the path opened
+	lps, err := m.localStore.Local(ctx)
+	if err != nil {
+		return xerrors.Errorf("getting local path list: %w", err)
+	}
+
+	var localPath *storiface.StoragePath
+	for _, lp := range lps {
+		if lp.LocalPath == path {
+			localPath = &lp
+			break
+		}
+	}
+	if localPath == nil {
+		return xerrors.Errorf("no local paths match '%s'", path)
+	}
+
+	// drop from the persisted storage.json
+	var found bool
+	if err := m.ls.SetStorage(func(sc *paths.StorageConfig) {
+		out := make([]paths.LocalPath, 0, len(sc.StoragePaths))
+		for _, storagePath := range sc.StoragePaths {
+			if storagePath.Path != path {
+				out = append(out, storagePath)
+				return
+			}
+			found = true
+		}
+	}); err != nil {
+		return xerrors.Errorf("set storage config: %w", err)
+	}
+	if !found {
+		// maybe this is fine?
+		return xerrors.Errorf("path not found in storage.json")
+	}
+
+	// unregister locally, drop from sector index
+	return m.localStore.ClosePath(ctx, localPath.ID)
+}
+
+func (m *Manager) RedeclareLocalStorage(ctx context.Context, id *storiface.ID, dropMissing bool) error {
+	return m.localStore.Redeclare(ctx, id, dropMissing)
+}
+
 func (m *Manager) AddWorker(ctx context.Context, w Worker) error {
 	sessID, err := w.Session(ctx)
 	if err != nil {