Don't remove sector data when moving data into a shared path

extern/sector-storage/manager.go

@@ -588,13 +588,13 @@ func (m *Manager) Remove(ctx context.Context, sector storage.SectorRef) error {
 
 	var err error
 
-	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTSealed, true); rerr != nil {
+	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTSealed, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (sealed): %w", rerr))
 	}
-	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTCache, true); rerr != nil {
+	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTCache, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (cache): %w", rerr))
 	}
-	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true); rerr != nil {
+	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr))
 	}
 

extern/sector-storage/piece_provider_test.go

@@ -298,7 +298,7 @@ func (p *pieceProviderTestHarness) addRemoteWorker(t *testing.T, tasks []sealtas
 func (p *pieceProviderTestHarness) removeAllUnsealedSectorFiles(t *testing.T) {
 	for i := range p.localStores {
 		ls := p.localStores[i]
-		require.NoError(t, ls.Remove(p.ctx, p.sector.ID, storiface.FTUnsealed, false))
+		require.NoError(t, ls.Remove(p.ctx, p.sector.ID, storiface.FTUnsealed, false, nil))
 	}
 }
 

extern/sector-storage/stores/http_handler.go

@@ -21,9 +21,9 @@ import (
 
 var log = logging.Logger("stores")
 
-var _ partialFileHandler = &DefaultPartialFileHandler{}
+var _ PartialFileHandler = &DefaultPartialFileHandler{}
 
-// DefaultPartialFileHandler is the default implementation of the partialFileHandler interface.
+// DefaultPartialFileHandler is the default implementation of the PartialFileHandler interface.
 // This is probably the only implementation we'll ever use because the purpose of the
 // interface to is to mock out partial file related functionality during testing.
 type DefaultPartialFileHandler struct{}
@@ -46,7 +46,7 @@ func (d *DefaultPartialFileHandler) Close(pf *partialfile.PartialFile) error {
 
 type FetchHandler struct {
 	Local     Store
-	PfHandler partialFileHandler
+	PfHandler PartialFileHandler
 }
 
 func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /remote/
@@ -179,7 +179,7 @@ func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	if err := handler.Local.Remove(r.Context(), id, ft, false); err != nil {
+	if err := handler.Local.Remove(r.Context(), id, ft, false, []ID{ID(r.FormValue("keep"))}); err != nil {
 		log.Errorf("%+v", err)
 		w.WriteHeader(500)
 		return

extern/sector-storage/stores/http_handler_test.go

@@ -63,7 +63,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 	tcs := map[string]struct {
 		piFnc    func(pi *pieceInfo)
 		storeFnc func(s *mocks.MockStore)
-		pfFunc   func(s *mocks.MockpartialFileHandler)
+		pfFunc   func(s *mocks.MockPartialFileHandler)
 
 		// expectation
 		expectedStatusCode int
@@ -129,7 +129,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(&partialfile.PartialFile{},
 					xerrors.New("some error")).Times(1)
 			},
@@ -146,7 +146,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(emptyPartialFile,
 					nil).Times(1)
 
@@ -165,7 +165,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(emptyPartialFile,
 					nil).Times(1)
 
@@ -184,7 +184,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(emptyPartialFile,
 					nil).Times(1)
 
@@ -203,7 +203,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 			defer mockCtrl.Finish()
 
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)
 
 			handler := &stores.FetchHandler{
 				lstore,
@@ -371,7 +371,7 @@ func TestRemoteGetSector(t *testing.T) {
 			// when test is done, assert expectations on all mock objects.
 			defer mockCtrl.Finish()
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)
 
 			var path string
 

extern/sector-storage/stores/index.go

@@ -55,6 +55,8 @@ type SectorStorageInfo struct {
 	Primary bool
 }
 
+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/index.go -package=mocks . SectorIndex
+
 type SectorIndex interface { // part of storage-miner api
 	StorageAttach(context.Context, StorageInfo, fsutil.FsStat) error
 	StorageInfo(context.Context, ID) (StorageInfo, error)

extern/sector-storage/stores/interface.go

@@ -13,8 +13,10 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )
 
+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/pf.go -package=mocks . PartialFileHandler
+
 // PartialFileHandler helps mock out the partial file functionality during testing.
-type partialFileHandler interface {
+type PartialFileHandler interface {
 	// OpenPartialFile opens and returns a partial file at the given path and also verifies it has the given
 	// size
 	OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)
@@ -30,9 +32,11 @@ type partialFileHandler interface {
 	Close(pf *partialfile.PartialFile) error
 }
 
+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/store.go -package=mocks . Store
+
 type Store interface {
 	AcquireSector(ctx context.Context, s storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode) (paths storiface.SectorPaths, stores storiface.SectorPaths, err error)
-	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool) error
+	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool, keepIn []ID) error
 
 	// like remove, but doesn't remove the primary sector copy, nor the last
 	// non-primary copy if there no primary copies

extern/sector-storage/stores/local.go

@@ -544,7 +544,7 @@ func (st *Local) Local(ctx context.Context) ([]StoragePath, error) {
 	return out, nil
 }
 
-func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool) error {
+func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []ID) error {
 	if bits.OnesCount(uint(typ)) != 1 {
 		return xerrors.New("delete expects one file type")
 	}
@@ -558,7 +558,14 @@ func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.Sec
 		return xerrors.Errorf("can't delete sector %v(%d), not found", sid, typ)
 	}
 
+storeLoop:
 	for _, info := range si {
+		for _, id := range keepIn {
+			if id == info.ID {
+				continue storeLoop
+			}
+		}
+
 		if err := st.removeSector(ctx, sid, typ, info.ID); err != nil {
 			return err
 		}

extern/sector-storage/stores/mocks/index.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: index.go
+// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: SectorIndex)
 
 // Package mocks is a generated GoMock package.
 package mocks
@@ -53,61 +53,61 @@ func (mr *MockSectorIndexMockRecorder) StorageAttach(arg0, arg1, arg2 interface{
 }
 
 // StorageBestAlloc mocks base method.
-func (m *MockSectorIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error) {
+func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType) ([]stores.StorageInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageBestAlloc", ctx, allocate, ssize, pathType)
+	ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].([]stores.StorageInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // StorageBestAlloc indicates an expected call of StorageBestAlloc.
-func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(ctx, allocate, ssize, pathType interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), ctx, allocate, ssize, pathType)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3)
 }
 
 // StorageDeclareSector mocks base method.
-func (m *MockSectorIndex) StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error {
+func (m *MockSectorIndex) StorageDeclareSector(arg0 context.Context, arg1 stores.ID, arg2 abi.SectorID, arg3 storiface.SectorFileType, arg4 bool) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageDeclareSector", ctx, storageID, s, ft, primary)
+	ret := m.ctrl.Call(m, "StorageDeclareSector", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // StorageDeclareSector indicates an expected call of StorageDeclareSector.
-func (mr *MockSectorIndexMockRecorder) StorageDeclareSector(ctx, storageID, s, ft, primary interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageDeclareSector(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDeclareSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDeclareSector), ctx, storageID, s, ft, primary)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDeclareSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDeclareSector), arg0, arg1, arg2, arg3, arg4)
 }
 
 // StorageDropSector mocks base method.
-func (m *MockSectorIndex) StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error {
+func (m *MockSectorIndex) StorageDropSector(arg0 context.Context, arg1 stores.ID, arg2 abi.SectorID, arg3 storiface.SectorFileType) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageDropSector", ctx, storageID, s, ft)
+	ret := m.ctrl.Call(m, "StorageDropSector", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // StorageDropSector indicates an expected call of StorageDropSector.
-func (mr *MockSectorIndexMockRecorder) StorageDropSector(ctx, storageID, s, ft interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageDropSector(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDropSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDropSector), ctx, storageID, s, ft)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDropSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDropSector), arg0, arg1, arg2, arg3)
 }
 
 // StorageFindSector mocks base method.
-func (m *MockSectorIndex) StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) {
+func (m *MockSectorIndex) StorageFindSector(arg0 context.Context, arg1 abi.SectorID, arg2 storiface.SectorFileType, arg3 abi.SectorSize, arg4 bool) ([]stores.SectorStorageInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageFindSector", ctx, sector, ft, ssize, allowFetch)
+	ret := m.ctrl.Call(m, "StorageFindSector", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].([]stores.SectorStorageInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // StorageFindSector indicates an expected call of StorageFindSector.
-func (mr *MockSectorIndexMockRecorder) StorageFindSector(ctx, sector, ft, ssize, allowFetch interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageFindSector(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageFindSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageFindSector), ctx, sector, ft, ssize, allowFetch)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageFindSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageFindSector), arg0, arg1, arg2, arg3, arg4)
 }
 
 // StorageInfo mocks base method.
@@ -126,32 +126,32 @@ func (mr *MockSectorIndexMockRecorder) StorageInfo(arg0, arg1 interface{}) *gomo
 }
 
 // StorageList mocks base method.
-func (m *MockSectorIndex) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) {
+func (m *MockSectorIndex) StorageList(arg0 context.Context) (map[stores.ID][]stores.Decl, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageList", ctx)
+	ret := m.ctrl.Call(m, "StorageList", arg0)
 	ret0, _ := ret[0].(map[stores.ID][]stores.Decl)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // StorageList indicates an expected call of StorageList.
-func (mr *MockSectorIndexMockRecorder) StorageList(ctx interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageList(arg0 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageList", reflect.TypeOf((*MockSectorIndex)(nil).StorageList), ctx)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageList", reflect.TypeOf((*MockSectorIndex)(nil).StorageList), arg0)
 }
 
 // StorageLock mocks base method.
-func (m *MockSectorIndex) StorageLock(ctx context.Context, sector abi.SectorID, read, write storiface.SectorFileType) error {
+func (m *MockSectorIndex) StorageLock(arg0 context.Context, arg1 abi.SectorID, arg2, arg3 storiface.SectorFileType) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageLock", ctx, sector, read, write)
+	ret := m.ctrl.Call(m, "StorageLock", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // StorageLock indicates an expected call of StorageLock.
-func (mr *MockSectorIndexMockRecorder) StorageLock(ctx, sector, read, write interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageLock(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageLock), ctx, sector, read, write)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageLock), arg0, arg1, arg2, arg3)
 }
 
 // StorageReportHealth mocks base method.
@@ -169,16 +169,16 @@ func (mr *MockSectorIndexMockRecorder) StorageReportHealth(arg0, arg1, arg2 inte
 }
 
 // StorageTryLock mocks base method.
-func (m *MockSectorIndex) StorageTryLock(ctx context.Context, sector abi.SectorID, read, write storiface.SectorFileType) (bool, error) {
+func (m *MockSectorIndex) StorageTryLock(arg0 context.Context, arg1 abi.SectorID, arg2, arg3 storiface.SectorFileType) (bool, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageTryLock", ctx, sector, read, write)
+	ret := m.ctrl.Call(m, "StorageTryLock", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(bool)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // StorageTryLock indicates an expected call of StorageTryLock.
-func (mr *MockSectorIndexMockRecorder) StorageTryLock(ctx, sector, read, write interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageTryLock(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageTryLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageTryLock), ctx, sector, read, write)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageTryLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageTryLock), arg0, arg1, arg2, arg3)
 }

extern/sector-storage/stores/mocks/pf.go

@@ -0,0 +1,97 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: PartialFileHandler)
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	os "os"
+	reflect "reflect"
+
+	abi "github.com/filecoin-project/go-state-types/abi"
+	partialfile "github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
+	storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockPartialFileHandler is a mock of PartialFileHandler interface.
+type MockPartialFileHandler struct {
+	ctrl     *gomock.Controller
+	recorder *MockPartialFileHandlerMockRecorder
+}
+
+// MockPartialFileHandlerMockRecorder is the mock recorder for MockPartialFileHandler.
+type MockPartialFileHandlerMockRecorder struct {
+	mock *MockPartialFileHandler
+}
+
+// NewMockPartialFileHandler creates a new mock instance.
+func NewMockPartialFileHandler(ctrl *gomock.Controller) *MockPartialFileHandler {
+	mock := &MockPartialFileHandler{ctrl: ctrl}
+	mock.recorder = &MockPartialFileHandlerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockPartialFileHandler) EXPECT() *MockPartialFileHandlerMockRecorder {
+	return m.recorder
+}
+
+// Close mocks base method.
+func (m *MockPartialFileHandler) Close(arg0 *partialfile.PartialFile) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockPartialFileHandlerMockRecorder) Close(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPartialFileHandler)(nil).Close), arg0)
+}
+
+// HasAllocated mocks base method.
+func (m *MockPartialFileHandler) HasAllocated(arg0 *partialfile.PartialFile, arg1 storiface.UnpaddedByteIndex, arg2 abi.UnpaddedPieceSize) (bool, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "HasAllocated", arg0, arg1, arg2)
+	ret0, _ := ret[0].(bool)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// HasAllocated indicates an expected call of HasAllocated.
+func (mr *MockPartialFileHandlerMockRecorder) HasAllocated(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasAllocated", reflect.TypeOf((*MockPartialFileHandler)(nil).HasAllocated), arg0, arg1, arg2)
+}
+
+// OpenPartialFile mocks base method.
+func (m *MockPartialFileHandler) OpenPartialFile(arg0 abi.PaddedPieceSize, arg1 string) (*partialfile.PartialFile, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "OpenPartialFile", arg0, arg1)
+	ret0, _ := ret[0].(*partialfile.PartialFile)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// OpenPartialFile indicates an expected call of OpenPartialFile.
+func (mr *MockPartialFileHandlerMockRecorder) OpenPartialFile(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenPartialFile", reflect.TypeOf((*MockPartialFileHandler)(nil).OpenPartialFile), arg0, arg1)
+}
+
+// Reader mocks base method.
+func (m *MockPartialFileHandler) Reader(arg0 *partialfile.PartialFile, arg1 storiface.PaddedByteIndex, arg2 abi.PaddedPieceSize) (*os.File, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Reader", arg0, arg1, arg2)
+	ret0, _ := ret[0].(*os.File)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Reader indicates an expected call of Reader.
+func (mr *MockPartialFileHandlerMockRecorder) Reader(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockPartialFileHandler)(nil).Reader), arg0, arg1, arg2)
+}

extern/sector-storage/stores/mocks/store.go

@@ -0,0 +1,128 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: Store)
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	context "context"
+	reflect "reflect"
+
+	abi "github.com/filecoin-project/go-state-types/abi"
+	fsutil "github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
+	stores "github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	storage "github.com/filecoin-project/specs-storage/storage"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockStore is a mock of Store interface.
+type MockStore struct {
+	ctrl     *gomock.Controller
+	recorder *MockStoreMockRecorder
+}
+
+// MockStoreMockRecorder is the mock recorder for MockStore.
+type MockStoreMockRecorder struct {
+	mock *MockStore
+}
+
+// NewMockStore creates a new mock instance.
+func NewMockStore(ctrl *gomock.Controller) *MockStore {
+	mock := &MockStore{ctrl: ctrl}
+	mock.recorder = &MockStoreMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockStore) EXPECT() *MockStoreMockRecorder {
+	return m.recorder
+}
+
+// AcquireSector mocks base method.
+func (m *MockStore) AcquireSector(arg0 context.Context, arg1 storage.SectorRef, arg2, arg3 storiface.SectorFileType, arg4 storiface.PathType, arg5 storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AcquireSector", arg0, arg1, arg2, arg3, arg4, arg5)
+	ret0, _ := ret[0].(storiface.SectorPaths)
+	ret1, _ := ret[1].(storiface.SectorPaths)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// AcquireSector indicates an expected call of AcquireSector.
+func (mr *MockStoreMockRecorder) AcquireSector(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireSector", reflect.TypeOf((*MockStore)(nil).AcquireSector), arg0, arg1, arg2, arg3, arg4, arg5)
+}
+
+// FsStat mocks base method.
+func (m *MockStore) FsStat(arg0 context.Context, arg1 stores.ID) (fsutil.FsStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FsStat", arg0, arg1)
+	ret0, _ := ret[0].(fsutil.FsStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FsStat indicates an expected call of FsStat.
+func (mr *MockStoreMockRecorder) FsStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FsStat", reflect.TypeOf((*MockStore)(nil).FsStat), arg0, arg1)
+}
+
+// MoveStorage mocks base method.
+func (m *MockStore) MoveStorage(arg0 context.Context, arg1 storage.SectorRef, arg2 storiface.SectorFileType) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MoveStorage", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MoveStorage indicates an expected call of MoveStorage.
+func (mr *MockStoreMockRecorder) MoveStorage(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MoveStorage", reflect.TypeOf((*MockStore)(nil).MoveStorage), arg0, arg1, arg2)
+}
+
+// Remove mocks base method.
+func (m *MockStore) Remove(arg0 context.Context, arg1 abi.SectorID, arg2 storiface.SectorFileType, arg3 bool, arg4 []stores.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Remove", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Remove indicates an expected call of Remove.
+func (mr *MockStoreMockRecorder) Remove(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockStore)(nil).Remove), arg0, arg1, arg2, arg3, arg4)
+}
+
+// RemoveCopies mocks base method.
+func (m *MockStore) RemoveCopies(arg0 context.Context, arg1 abi.SectorID, arg2 storiface.SectorFileType) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveCopies", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RemoveCopies indicates an expected call of RemoveCopies.
+func (mr *MockStoreMockRecorder) RemoveCopies(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveCopies", reflect.TypeOf((*MockStore)(nil).RemoveCopies), arg0, arg1, arg2)
+}
+
+// Reserve mocks base method.
+func (m *MockStore) Reserve(arg0 context.Context, arg1 storage.SectorRef, arg2 storiface.SectorFileType, arg3 storiface.SectorPaths, arg4 map[storiface.SectorFileType]int) (func(), error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Reserve", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(func())
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Reserve indicates an expected call of Reserve.
+func (mr *MockStoreMockRecorder) Reserve(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reserve", reflect.TypeOf((*MockStore)(nil).Reserve), arg0, arg1, arg2, arg3, arg4)
+}

extern/sector-storage/stores/mocks/stores.go

@@ -1,212 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: interface.go
-
-// Package mocks is a generated GoMock package.
-package mocks
-
-import (
-	context "context"
-	os "os"
-	reflect "reflect"
-
-	abi "github.com/filecoin-project/go-state-types/abi"
-	fsutil "github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	partialfile "github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
-	stores "github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	storage "github.com/filecoin-project/specs-storage/storage"
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockpartialFileHandler is a mock of partialFileHandler interface.
-type MockpartialFileHandler struct {
-	ctrl     *gomock.Controller
-	recorder *MockpartialFileHandlerMockRecorder
-}
-
-// MockpartialFileHandlerMockRecorder is the mock recorder for MockpartialFileHandler.
-type MockpartialFileHandlerMockRecorder struct {
-	mock *MockpartialFileHandler
-}
-
-// NewMockpartialFileHandler creates a new mock instance.
-func NewMockpartialFileHandler(ctrl *gomock.Controller) *MockpartialFileHandler {
-	mock := &MockpartialFileHandler{ctrl: ctrl}
-	mock.recorder = &MockpartialFileHandlerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockpartialFileHandler) EXPECT() *MockpartialFileHandlerMockRecorder {
-	return m.recorder
-}
-
-// Close mocks base method.
-func (m *MockpartialFileHandler) Close(pf *partialfile.PartialFile) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Close", pf)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Close indicates an expected call of Close.
-func (mr *MockpartialFileHandlerMockRecorder) Close(pf interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockpartialFileHandler)(nil).Close), pf)
-}
-
-// HasAllocated mocks base method.
-func (m *MockpartialFileHandler) HasAllocated(pf *partialfile.PartialFile, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "HasAllocated", pf, offset, size)
-	ret0, _ := ret[0].(bool)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// HasAllocated indicates an expected call of HasAllocated.
-func (mr *MockpartialFileHandlerMockRecorder) HasAllocated(pf, offset, size interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasAllocated", reflect.TypeOf((*MockpartialFileHandler)(nil).HasAllocated), pf, offset, size)
-}
-
-// OpenPartialFile mocks base method.
-func (m *MockpartialFileHandler) OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "OpenPartialFile", maxPieceSize, path)
-	ret0, _ := ret[0].(*partialfile.PartialFile)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// OpenPartialFile indicates an expected call of OpenPartialFile.
-func (mr *MockpartialFileHandlerMockRecorder) OpenPartialFile(maxPieceSize, path interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenPartialFile", reflect.TypeOf((*MockpartialFileHandler)(nil).OpenPartialFile), maxPieceSize, path)
-}
-
-// Reader mocks base method.
-func (m *MockpartialFileHandler) Reader(pf *partialfile.PartialFile, offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Reader", pf, offset, size)
-	ret0, _ := ret[0].(*os.File)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Reader indicates an expected call of Reader.
-func (mr *MockpartialFileHandlerMockRecorder) Reader(pf, offset, size interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockpartialFileHandler)(nil).Reader), pf, offset, size)
-}
-
-// MockStore is a mock of Store interface.
-type MockStore struct {
-	ctrl     *gomock.Controller
-	recorder *MockStoreMockRecorder
-}
-
-// MockStoreMockRecorder is the mock recorder for MockStore.
-type MockStoreMockRecorder struct {
-	mock *MockStore
-}
-
-// NewMockStore creates a new mock instance.
-func NewMockStore(ctrl *gomock.Controller) *MockStore {
-	mock := &MockStore{ctrl: ctrl}
-	mock.recorder = &MockStoreMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockStore) EXPECT() *MockStoreMockRecorder {
-	return m.recorder
-}
-
-// AcquireSector mocks base method.
-func (m *MockStore) AcquireSector(ctx context.Context, s storage.SectorRef, existing, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "AcquireSector", ctx, s, existing, allocate, sealing, op)
-	ret0, _ := ret[0].(storiface.SectorPaths)
-	ret1, _ := ret[1].(storiface.SectorPaths)
-	ret2, _ := ret[2].(error)
-	return ret0, ret1, ret2
-}
-
-// AcquireSector indicates an expected call of AcquireSector.
-func (mr *MockStoreMockRecorder) AcquireSector(ctx, s, existing, allocate, sealing, op interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireSector", reflect.TypeOf((*MockStore)(nil).AcquireSector), ctx, s, existing, allocate, sealing, op)
-}
-
-// FsStat mocks base method.
-func (m *MockStore) FsStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "FsStat", ctx, id)
-	ret0, _ := ret[0].(fsutil.FsStat)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// FsStat indicates an expected call of FsStat.
-func (mr *MockStoreMockRecorder) FsStat(ctx, id interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FsStat", reflect.TypeOf((*MockStore)(nil).FsStat), ctx, id)
-}
-
-// MoveStorage mocks base method.
-func (m *MockStore) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "MoveStorage", ctx, s, types)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// MoveStorage indicates an expected call of MoveStorage.
-func (mr *MockStoreMockRecorder) MoveStorage(ctx, s, types interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MoveStorage", reflect.TypeOf((*MockStore)(nil).MoveStorage), ctx, s, types)
-}
-
-// Remove mocks base method.
-func (m *MockStore) Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Remove", ctx, s, types, force)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Remove indicates an expected call of Remove.
-func (mr *MockStoreMockRecorder) Remove(ctx, s, types, force interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockStore)(nil).Remove), ctx, s, types, force)
-}
-
-// RemoveCopies mocks base method.
-func (m *MockStore) RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "RemoveCopies", ctx, s, types)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// RemoveCopies indicates an expected call of RemoveCopies.
-func (mr *MockStoreMockRecorder) RemoveCopies(ctx, s, types interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveCopies", reflect.TypeOf((*MockStore)(nil).RemoveCopies), ctx, s, types)
-}
-
-// Reserve mocks base method.
-func (m *MockStore) Reserve(ctx context.Context, sid storage.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Reserve", ctx, sid, ft, storageIDs, overheadTab)
-	ret0, _ := ret[0].(func())
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Reserve indicates an expected call of Reserve.
-func (mr *MockStoreMockRecorder) Reserve(ctx, sid, ft, storageIDs, overheadTab interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reserve", reflect.TypeOf((*MockStore)(nil).Reserve), ctx, sid, ft, storageIDs, overheadTab)
-}

extern/sector-storage/stores/remote.go

@@ -41,7 +41,7 @@ type Remote struct {
 	fetchLk  sync.Mutex
 	fetching map[abi.SectorID]chan struct{}
 
-	pfHandler partialFileHandler
+	pfHandler PartialFileHandler
 }
 
 func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error {
@@ -52,7 +52,7 @@ func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, types storifa
 	return r.local.RemoveCopies(ctx, s, types)
 }
 
-func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler partialFileHandler) *Remote {
+func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler PartialFileHandler) *Remote {
 	return &Remote{
 		local: local,
 		index: index,
@@ -155,7 +155,8 @@ func (r *Remote) AcquireSector(ctx context.Context, s storage.SectorRef, existin
 		}
 
 		if op == storiface.AcquireMove {
-			if err := r.deleteFromRemote(ctx, url); err != nil {
+			id := ID(storageID)
+			if err := r.deleteFromRemote(ctx, url, &id); err != nil {
 				log.Warnf("deleting sector %v from %s (delete %s): %+v", s, storageID, url, err)
 			}
 		}
@@ -333,12 +334,12 @@ func (r *Remote) MoveStorage(ctx context.Context, s storage.SectorRef, types sto
 	return r.local.MoveStorage(ctx, s, types)
 }
 
-func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool) error {
+func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []ID) error {
 	if bits.OnesCount(uint(typ)) != 1 {
 		return xerrors.New("delete expects one file type")
 	}
 
-	if err := r.local.Remove(ctx, sid, typ, force); err != nil {
+	if err := r.local.Remove(ctx, sid, typ, force, keepIn); err != nil {
 		return xerrors.Errorf("remove from local: %w", err)
 	}
 
@@ -347,9 +348,15 @@ func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.Sec
 		return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err)
 	}
 
+storeLoop:
 	for _, info := range si {
+		for _, id := range keepIn {
+			if id == info.ID {
+				continue storeLoop
+			}
+		}
 		for _, url := range info.URLs {
-			if err := r.deleteFromRemote(ctx, url); err != nil {
+			if err := r.deleteFromRemote(ctx, url, nil); err != nil {
 				log.Warnf("remove %s: %+v", url, err)
 				continue
 			}
@@ -360,7 +367,11 @@ func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.Sec
 	return nil
 }
 
-func (r *Remote) deleteFromRemote(ctx context.Context, url string) error {
+func (r *Remote) deleteFromRemote(ctx context.Context, url string, keepIn *ID) error {
+	if keepIn != nil {
+		url = url + "?keep=" + string(*keepIn)
+	}
+
 	log.Infof("Delete %s", url)
 
 	req, err := http.NewRequest("DELETE", url, nil)

extern/sector-storage/stores/remote_test.go

@@ -2,26 +2,156 @@ package stores_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/golang/mock/gomock"
+	"github.com/google/uuid"
+	"github.com/gorilla/mux"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores/mocks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/specs-storage/storage"
-	"github.com/golang/mock/gomock"
-	"github.com/gorilla/mux"
-	logging "github.com/ipfs/go-log/v2"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
 )
 
+const metaFile = "sectorstore.json"
+
+func createTestStorage(t *testing.T, p string, seal bool, att ...*stores.Local) stores.ID {
+	if err := os.MkdirAll(p, 0755); err != nil {
+		if !os.IsExist(err) {
+			require.NoError(t, err)
+		}
+	}
+
+	cfg := &stores.LocalStorageMeta{
+		ID:       stores.ID(uuid.New().String()),
+		Weight:   10,
+		CanSeal:  seal,
+		CanStore: !seal,
+	}
+
+	b, err := json.MarshalIndent(cfg, "", "  ")
+	require.NoError(t, err)
+
+	require.NoError(t, ioutil.WriteFile(filepath.Join(p, metaFile), b, 0644))
+
+	for _, s := range att {
+		require.NoError(t, s.OpenPath(context.Background(), p))
+	}
+
+	return cfg.ID
+}
+
+func TestMoveShared(t *testing.T) {
+	logging.SetAllLoggers(logging.LevelDebug)
+
+	index := stores.NewIndex()
+
+	ctx := context.Background()
+
+	dir, err := ioutil.TempDir("", "stores-remote-test-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	openRepo := func(dir string) repo.LockedRepo {
+		r, err := repo.NewFS(dir)
+		require.NoError(t, err)
+		require.NoError(t, r.Init(repo.Worker))
+		lr, err := r.Lock(repo.Worker)
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			_ = lr.Close()
+		})
+
+		err = lr.SetStorage(func(config *stores.StorageConfig) {
+			*config = stores.StorageConfig{}
+		})
+		require.NoError(t, err)
+
+		return lr
+	}
+
+	// setup two repos with two storage paths:
+	// repo 1 with both paths
+	// repo 2 with one path (shared)
+
+	lr1 := openRepo(filepath.Join(dir, "l1"))
+	lr2 := openRepo(filepath.Join(dir, "l2"))
+
+	mux1 := mux.NewRouter()
+	mux2 := mux.NewRouter()
+	hs1 := httptest.NewServer(mux1)
+	hs2 := httptest.NewServer(mux2)
+
+	ls1, err := stores.NewLocal(ctx, lr1, index, []string{hs1.URL + "/remote"})
+	require.NoError(t, err)
+	ls2, err := stores.NewLocal(ctx, lr2, index, []string{hs2.URL + "/remote"})
+	require.NoError(t, err)
+
+	dirStor := filepath.Join(dir, "stor")
+	dirSeal := filepath.Join(dir, "seal")
+
+	id1 := createTestStorage(t, dirStor, false, ls1, ls2)
+	id2 := createTestStorage(t, dirSeal, true, ls1)
+
+	rs1 := stores.NewRemote(ls1, index, nil, 20, &stores.DefaultPartialFileHandler{})
+	rs2 := stores.NewRemote(ls2, index, nil, 20, &stores.DefaultPartialFileHandler{})
+	_ = rs2
+	mux1.PathPrefix("/").Handler(&stores.FetchHandler{Local: ls1, PfHandler: &stores.DefaultPartialFileHandler{}})
+	mux2.PathPrefix("/").Handler(&stores.FetchHandler{Local: ls2, PfHandler: &stores.DefaultPartialFileHandler{}})
+
+	// add a sealed replica file to the sealing (non-shared) path
+
+	s1ref := storage.SectorRef{
+		ID: abi.SectorID{
+			Miner:  12,
+			Number: 1,
+		},
+		ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1,
+	}
+
+	sp, sid, err := rs1.AcquireSector(ctx, s1ref, storiface.FTNone, storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove)
+	require.NoError(t, err)
+	require.Equal(t, id2, stores.ID(sid.Sealed))
+
+	data := make([]byte, 2032)
+	data[1] = 54
+	require.NoError(t, ioutil.WriteFile(sp.Sealed, data, 0666))
+	fmt.Println("write to ", sp.Sealed)
+
+	require.NoError(t, index.StorageDeclareSector(ctx, stores.ID(sid.Sealed), s1ref.ID, storiface.FTSealed, true))
+
+	// move to the shared path from the second node (remote move / delete)
+
+	require.NoError(t, rs2.MoveStorage(ctx, s1ref, storiface.FTSealed))
+
+	// check that the file still exists
+	sp, sid, err = rs2.AcquireSector(ctx, s1ref, storiface.FTSealed, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
+	require.NoError(t, err)
+	require.Equal(t, id1, stores.ID(sid.Sealed))
+	fmt.Println("read from ", sp.Sealed)
+
+	read, err := ioutil.ReadFile(sp.Sealed)
+	require.NoError(t, err)
+	require.EqualValues(t, data, read)
+}
+
 func TestReader(t *testing.T) {
 	logging.SetAllLoggers(logging.LevelDebug)
 	bz := []byte("Hello World")
@@ -46,7 +176,7 @@ func TestReader(t *testing.T) {
 
 	tcs := map[string]struct {
 		storeFnc func(s *mocks.MockStore)
-		pfFunc   func(s *mocks.MockpartialFileHandler)
+		pfFunc   func(s *mocks.MockPartialFileHandler)
 		indexFnc func(s *mocks.MockSectorIndex, serverURL string)
 
 		needHttpServer bool
@@ -76,7 +206,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, xerrors.New("pf open error"))
 			},
 			errStr: "pf open error",
@@ -87,7 +217,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, xerrors.New("piece check error"))
@@ -101,7 +231,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -115,7 +245,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, nil)
@@ -157,7 +287,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -223,7 +353,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, nil)
@@ -251,7 +381,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -308,7 +438,7 @@ func TestReader(t *testing.T) {
 
 			// create them mocks
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)
 			index := mocks.NewMockSectorIndex(mockCtrl)
 
 			if tc.storeFnc != nil {
@@ -393,7 +523,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 
 	tcs := map[string]struct {
 		storeFnc func(s *mocks.MockStore)
-		pfFunc   func(s *mocks.MockpartialFileHandler)
+		pfFunc   func(s *mocks.MockPartialFileHandler)
 		indexFnc func(s *mocks.MockSectorIndex, serverURL string)
 
 		needHttpServer bool
@@ -421,7 +551,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, xerrors.New("pf open error"))
 			},
 			errStr: "pf open error",
@@ -432,7 +562,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, xerrors.New("piece check error"))
@@ -446,7 +576,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
@@ -488,7 +618,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -533,7 +663,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, nil)
@@ -569,7 +699,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},
 
-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -602,7 +732,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 
 			// create them mocks
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)
 			index := mocks.NewMockSectorIndex(mockCtrl)
 
 			if tc.storeFnc != nil {
@@ -656,18 +786,18 @@ func mockSectorAcquire(l *mocks.MockStore, sectorRef storage.SectorRef, pfPath s
 		storiface.SectorPaths{}, err).Times(1)
 }
 
-func mockPartialFileOpen(pf *mocks.MockpartialFileHandler, sectorSize abi.SectorSize, pfPath string, err error) {
+func mockPartialFileOpen(pf *mocks.MockPartialFileHandler, sectorSize abi.SectorSize, pfPath string, err error) {
 	pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(&partialfile.PartialFile{},
 		err).Times(1)
 }
 
-func mockCheckAllocation(pf *mocks.MockpartialFileHandler, offset, size abi.PaddedPieceSize, file *partialfile.PartialFile,
+func mockCheckAllocation(pf *mocks.MockPartialFileHandler, offset, size abi.PaddedPieceSize, file *partialfile.PartialFile,
 	out bool, err error) {
 	pf.EXPECT().HasAllocated(file, storiface.UnpaddedByteIndex(offset.Unpadded()),
 		size.Unpadded()).Return(out, err).Times(1)
 }
 
-func mockPfReader(pf *mocks.MockpartialFileHandler, file *partialfile.PartialFile, offset, size abi.PaddedPieceSize,
+func mockPfReader(pf *mocks.MockPartialFileHandler, file *partialfile.PartialFile, offset, size abi.PaddedPieceSize,
 	outFile *os.File, err error) {
 	pf.EXPECT().Reader(file, storiface.PaddedByteIndex(offset), size).Return(outFile, err)
 }

extern/sector-storage/worker_local.go

@@ -331,11 +331,11 @@ func (l *LocalWorker) SealPreCommit1(ctx context.Context, sector storage.SectorR
 
 		{
 			// cleanup previous failed attempts if they exist
-			if err := l.storage.Remove(ctx, sector.ID, storiface.FTSealed, true); err != nil {
+			if err := l.storage.Remove(ctx, sector.ID, storiface.FTSealed, true, nil); err != nil {
 				return nil, xerrors.Errorf("cleaning up sealed data: %w", err)
 			}
 
-			if err := l.storage.Remove(ctx, sector.ID, storiface.FTCache, true); err != nil {
+			if err := l.storage.Remove(ctx, sector.ID, storiface.FTCache, true, nil); err != nil {
 				return nil, xerrors.Errorf("cleaning up cache data: %w", err)
 			}
 		}
@@ -394,7 +394,7 @@ func (l *LocalWorker) FinalizeSector(ctx context.Context, sector storage.SectorR
 		}
 
 		if len(keepUnsealed) == 0 {
-			if err := l.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true); err != nil {
+			if err := l.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true, nil); err != nil {
 				return nil, xerrors.Errorf("removing unsealed data: %w", err)
 			}
 		}
@@ -410,13 +410,13 @@ func (l *LocalWorker) ReleaseUnsealed(ctx context.Context, sector storage.Sector
 func (l *LocalWorker) Remove(ctx context.Context, sector abi.SectorID) error {
 	var err error
 
-	if rerr := l.storage.Remove(ctx, sector, storiface.FTSealed, true); rerr != nil {
+	if rerr := l.storage.Remove(ctx, sector, storiface.FTSealed, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (sealed): %w", rerr))
 	}
-	if rerr := l.storage.Remove(ctx, sector, storiface.FTCache, true); rerr != nil {
+	if rerr := l.storage.Remove(ctx, sector, storiface.FTCache, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (cache): %w", rerr))
 	}
-	if rerr := l.storage.Remove(ctx, sector, storiface.FTUnsealed, true); rerr != nil {
+	if rerr := l.storage.Remove(ctx, sector, storiface.FTUnsealed, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr))
 	}