fix: curio seal: Fix partial Finalize retry

curiosrc/ffi/sdr_funcs.go

@@ -374,12 +374,62 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector
 		// * we truncate the temp file to the sector size
 		// * we move the temp file to the unsealed location
 
-		// move tree-d to temp file
+		// temp path in cache where we'll move tree-d before truncating
+		// it is in the cache directory so that we can use os.Rename to move it
+		// to unsealed (which may be on a different filesystem)
 		tempUnsealed := filepath.Join(sectorPaths.Cache, storiface.SectorName(sector.ID))
+
+		_, terr := os.Stat(tempUnsealed)
+		tempUnsealedExists := terr == nil
+
+		// First handle an edge case where we have already gone through this step,
+		// but ClearCache or later steps failed. In that case we'll see tree-d missing and unsealed present
+
+		if _, err := os.Stat(filepath.Join(sectorPaths.Cache, proofpaths.TreeDName)); err != nil {
+			if os.IsNotExist(err) {
+				// check that unsealed exists and is the right size
+				st, err := os.Stat(sectorPaths.Unsealed)
+				if err != nil {
+					if os.IsNotExist(err) {
+						if tempUnsealedExists {
+							// unsealed file does not exist, but temp unsealed file does
+							// so we can just resume where the previous attempt left off
+							goto retryUnsealedMove
+						}
+						return xerrors.Errorf("neither unsealed file nor temp-unsealed file exists")
+					}
+					return xerrors.Errorf("stat unsealed file: %w", err)
+				}
+				if st.Size() != int64(ssize) {
+					if tempUnsealedExists {
+						// unsealed file exists but is the wrong size, and temp unsealed file exists
+						// so we can just resume where the previous attempt left off with some cleanup
+
+						if err := os.Remove(sectorPaths.Unsealed); err != nil {
+							return xerrors.Errorf("removing unsealed file from last attempt: %w", err)
+						}
+
+						goto retryUnsealedMove
+					}
+					return xerrors.Errorf("unsealed file is not the right size: %d != %d and temp unsealed is missing", st.Size(), ssize)
+				}
+
+				// all good, just log that this edge case happened
+				log.Warnw("unsealed file exists but tree-d is missing, skipping move", "sector", sector.ID, "unsealed", sectorPaths.Unsealed, "cache", sectorPaths.Cache)
+				goto afterUnsealedMove
+			}
+			return xerrors.Errorf("stat tree-d file: %w", err)
+		}
+
+		// If the state in clean do the move
+
+		// move tree-d to temp file
 		if err := os.Rename(filepath.Join(sectorPaths.Cache, proofpaths.TreeDName), tempUnsealed); err != nil {
 			return xerrors.Errorf("moving tree-d to temp file: %w", err)
 		}
 
+	retryUnsealedMove:
+
 		// truncate sealed file to sector size
 		if err := os.Truncate(tempUnsealed, int64(ssize)); err != nil {
 			return xerrors.Errorf("truncating unsealed file to sector size: %w", err)
@@ -391,6 +441,7 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector
 		}
 	}
 
+afterUnsealedMove:
 	if err := ffi.ClearCache(uint64(ssize), sectorPaths.Cache); err != nil {
 		return xerrors.Errorf("clearing cache: %w", err)
 	}

itests/sector_import_simple_test.go

@@ -42,7 +42,7 @@ func TestSectorImportAfterPC2(t *testing.T) {
 	// We use two miners so that in case the actively tested miner misses PoSt, we still have a blockchain
 	client, miner, _, ens := kit.EnsembleOneTwo(t, kit.ThroughRPC())
 
-	ens.InterconnectAll().BeginMiningMustPost(blockTime)
+	ens.InterconnectAll().BeginMining(blockTime)
 
 	ctx := context.Background()
 

storage/paths/local.go

@@ -476,14 +476,18 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
 	var out storiface.SectorPaths
 	var storageIDs storiface.SectorPaths
 
+	// First find existing files
 	for _, fileType := range storiface.PathTypes {
-		if fileType&existing == 0 {
+		// also try to find existing sectors if we're allocating
+		if fileType&(existing|allocate) == 0 {
 			continue
 		}
 
 		si, err := st.index.StorageFindSector(ctx, sid.ID, fileType, ssize, false)
 		if err != nil {
+			if fileType&existing != 0 {
 				log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err)
+			}
 			continue
 		}
 
@@ -501,11 +505,13 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
 			storiface.SetPathByType(&out, fileType, spath)
 			storiface.SetPathByType(&storageIDs, fileType, string(info.ID))
 
-			existing ^= fileType
+			existing = existing.Unset(fileType)
+			allocate = allocate.Unset(fileType)
 			break
 		}
 	}
 
+	// Then allocate for allocation requests
 	for _, fileType := range storiface.PathTypes {
 		if fileType&allocate == 0 {
 			continue

storage/sealer/storiface/filetype.go

@@ -174,6 +174,10 @@ func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) Sect
 	return t & denyMask
 }
 
+func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType {
+	return t &^ toUnset
+}
+
 func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool {
 	return t.SubAllowed(allowTypes, denyTypes) != t
 }