feat: curio: Cleanup files after failed TreeDRC (#11827)

This commit is contained in:
Łukasz Magiera 2024-04-04 17:25:06 +02:00 committed by GitHub
parent 7bd21bc675
commit a99d8c8791
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 62 additions and 1 deletions

View File

@ -161,7 +161,7 @@ func (sb *SealCalls) GenerateSDR(ctx context.Context, taskID harmonytask.TaskID,
return nil
}
func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, unsealed cid.Cid, size abi.PaddedPieceSize, data io.Reader, unpaddedData bool) (cid.Cid, cid.Cid, error) {
func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, unsealed cid.Cid, size abi.PaddedPieceSize, data io.Reader, unpaddedData bool) (scid cid.Cid, ucid cid.Cid, err error) {
p1o, err := sb.makePhase1Out(unsealed, sector.ProofType)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("make phase1 output: %w", err)
@ -173,6 +173,15 @@ func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sect
}
defer releaseSector()
defer func() {
if err != nil {
clerr := removeDRCTrees(paths.Cache)
if clerr != nil {
log.Errorw("removing tree files after TreeDRC error", "error", clerr, "exec-error", err, "sector", sector, "cache", paths.Cache)
}
}
}()
treeDUnsealed, err := proof.BuildTreeD(data, unpaddedData, filepath.Join(paths.Cache, proofpaths.TreeDName), size)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("building tree-d: %w", err)
@ -237,6 +246,25 @@ func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sect
return sl, uns, nil
}
func removeDRCTrees(cache string) error {
// list files in cache
files, err := os.ReadDir(cache)
if err != nil {
return xerrors.Errorf("listing cache: %w", err)
}
for _, file := range files {
if proofpaths.IsTreeFile(file.Name()) {
err := os.Remove(filepath.Join(cache, file.Name()))
if err != nil {
return xerrors.Errorf("removing tree file: %w", err)
}
}
}
return nil
}
func (sb *SealCalls) GenerateSynthPoRep() {
panic("todo")
}

View File

@ -58,6 +58,15 @@ func BuildTreeD(data io.Reader, unpaddedData bool, outPath string, size abi.Padd
}
defer func() {
cerr := out.Close()
if err != nil {
// remove the file, it's probably bad
rerr := os.Remove(outPath)
if rerr != nil {
err = multierror.Append(err, rerr)
}
}
if cerr != nil {
err = multierror.Append(err, cerr)
}

View File

@ -2,6 +2,7 @@ package proofpaths
import (
"fmt"
"regexp"
"github.com/filecoin-project/go-state-types/abi"
)
@ -9,6 +10,29 @@ import (
const dataFilePrefix = "sc-02-data-"
const TreeDName = dataFilePrefix + "tree-d.dat"
const TreeRLastPrefix = dataFilePrefix + "tree-r-last-"
const TreeCPrefix = dataFilePrefix + "tree-c-"
func IsFileTreeD(baseName string) bool {
return baseName == TreeDName
}
func IsFileTreeRLast(baseName string) bool {
// TreeRLastPrefix<int>.dat
reg := fmt.Sprintf(`^%s\d+\.dat$`, TreeRLastPrefix)
return regexp.MustCompile(reg).MatchString(baseName)
}
func IsFileTreeC(baseName string) bool {
// TreeCPrefix<int>.dat
reg := fmt.Sprintf(`^%s\d+\.dat$`, TreeCPrefix)
return regexp.MustCompile(reg).MatchString(baseName)
}
func IsTreeFile(baseName string) bool {
return IsFileTreeD(baseName) || IsFileTreeRLast(baseName) || IsFileTreeC(baseName)
}
func LayerFileName(layer int) string {
return fmt.Sprintf("%slayer-%d.dat", dataFilePrefix, layer)
}