feat: curio: Cleanup files after failed TreeDRC (#11827)

curiosrc/ffi/sdr_funcs.go

@@ -161,7 +161,7 @@ func (sb *SealCalls) GenerateSDR(ctx context.Context, taskID harmonytask.TaskID,
 	return nil
 }
 
-func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, unsealed cid.Cid, size abi.PaddedPieceSize, data io.Reader, unpaddedData bool) (cid.Cid, cid.Cid, error) {
+func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, unsealed cid.Cid, size abi.PaddedPieceSize, data io.Reader, unpaddedData bool) (scid cid.Cid, ucid cid.Cid, err error) {
 	p1o, err := sb.makePhase1Out(unsealed, sector.ProofType)
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("make phase1 output: %w", err)
@@ -173,6 +173,15 @@ func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sect
 	}
 	defer releaseSector()
 
+	defer func() {
+		if err != nil {
+			clerr := removeDRCTrees(paths.Cache)
+			if clerr != nil {
+				log.Errorw("removing tree files after TreeDRC error", "error", clerr, "exec-error", err, "sector", sector, "cache", paths.Cache)
+			}
+		}
+	}()
+
 	treeDUnsealed, err := proof.BuildTreeD(data, unpaddedData, filepath.Join(paths.Cache, proofpaths.TreeDName), size)
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("building tree-d: %w", err)
@@ -237,6 +246,25 @@ func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sect
 	return sl, uns, nil
 }
 
+func removeDRCTrees(cache string) error {
+	// list files in cache
+	files, err := os.ReadDir(cache)
+	if err != nil {
+		return xerrors.Errorf("listing cache: %w", err)
+	}
+
+	for _, file := range files {
+		if proofpaths.IsTreeFile(file.Name()) {
+			err := os.Remove(filepath.Join(cache, file.Name()))
+			if err != nil {
+				return xerrors.Errorf("removing tree file: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (sb *SealCalls) GenerateSynthPoRep() {
 	panic("todo")
 }

curiosrc/proof/treed_build.go

@@ -58,6 +58,15 @@ func BuildTreeD(data io.Reader, unpaddedData bool, outPath string, size abi.Padd
 	}
 	defer func() {
 		cerr := out.Close()
+
+		if err != nil {
+			// remove the file, it's probably bad
+			rerr := os.Remove(outPath)
+			if rerr != nil {
+				err = multierror.Append(err, rerr)
+			}
+		}
+
 		if cerr != nil {
 			err = multierror.Append(err, cerr)
 		}

storage/sealer/proofpaths/cachefiles.go

@@ -2,6 +2,7 @@ package proofpaths
 
 import (
 	"fmt"
+	"regexp"
 
 	"github.com/filecoin-project/go-state-types/abi"
 )
@@ -9,6 +10,29 @@ import (
 const dataFilePrefix = "sc-02-data-"
 const TreeDName = dataFilePrefix + "tree-d.dat"
 
+const TreeRLastPrefix = dataFilePrefix + "tree-r-last-"
+const TreeCPrefix = dataFilePrefix + "tree-c-"
+
+func IsFileTreeD(baseName string) bool {
+	return baseName == TreeDName
+}
+
+func IsFileTreeRLast(baseName string) bool {
+	// TreeRLastPrefix<int>.dat
+	reg := fmt.Sprintf(`^%s\d+\.dat$`, TreeRLastPrefix)
+	return regexp.MustCompile(reg).MatchString(baseName)
+}
+
+func IsFileTreeC(baseName string) bool {
+	// TreeCPrefix<int>.dat
+	reg := fmt.Sprintf(`^%s\d+\.dat$`, TreeCPrefix)
+	return regexp.MustCompile(reg).MatchString(baseName)
+}
+
+func IsTreeFile(baseName string) bool {
+	return IsFileTreeD(baseName) || IsFileTreeRLast(baseName) || IsFileTreeC(baseName)
+}
+
 func LayerFileName(layer int) string {
 	return fmt.Sprintf("%slayer-%d.dat", dataFilePrefix, layer)
 }