fix: curio seal: Fix partial Finalize retry

This commit is contained in:
Łukasz Magiera 2024-03-17 12:19:27 +01:00 committed by Łukasz Magiera
parent 8ec74fdb05
commit e86d81b26a
4 changed files with 66 additions and 5 deletions

View File

@ -374,12 +374,62 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector
// * we truncate the temp file to the sector size
// * we move the temp file to the unsealed location
// move tree-d to temp file
// temp path in cache where we'll move tree-d before truncating
// it is in the cache directory so that we can use os.Rename to move it
// to unsealed (which may be on a different filesystem)
tempUnsealed := filepath.Join(sectorPaths.Cache, storiface.SectorName(sector.ID))
_, terr := os.Stat(tempUnsealed)
tempUnsealedExists := terr == nil
// First handle an edge case where we have already gone through this step,
// but ClearCache or later steps failed. In that case we'll see tree-d missing and unsealed present
if _, err := os.Stat(filepath.Join(sectorPaths.Cache, proofpaths.TreeDName)); err != nil {
if os.IsNotExist(err) {
// check that unsealed exists and is the right size
st, err := os.Stat(sectorPaths.Unsealed)
if err != nil {
if os.IsNotExist(err) {
if tempUnsealedExists {
// unsealed file does not exist, but temp unsealed file does
// so we can just resume where the previous attempt left off
goto retryUnsealedMove
}
return xerrors.Errorf("neither unsealed file nor temp-unsealed file exists")
}
return xerrors.Errorf("stat unsealed file: %w", err)
}
if st.Size() != int64(ssize) {
if tempUnsealedExists {
// unsealed file exists but is the wrong size, and temp unsealed file exists
// so we can just resume where the previous attempt left off with some cleanup
if err := os.Remove(sectorPaths.Unsealed); err != nil {
return xerrors.Errorf("removing unsealed file from last attempt: %w", err)
}
goto retryUnsealedMove
}
return xerrors.Errorf("unsealed file is not the right size: %d != %d and temp unsealed is missing", st.Size(), ssize)
}
// all good, just log that this edge case happened
log.Warnw("unsealed file exists but tree-d is missing, skipping move", "sector", sector.ID, "unsealed", sectorPaths.Unsealed, "cache", sectorPaths.Cache)
goto afterUnsealedMove
}
return xerrors.Errorf("stat tree-d file: %w", err)
}
// If the state in clean do the move
// move tree-d to temp file
if err := os.Rename(filepath.Join(sectorPaths.Cache, proofpaths.TreeDName), tempUnsealed); err != nil {
return xerrors.Errorf("moving tree-d to temp file: %w", err)
}
retryUnsealedMove:
// truncate sealed file to sector size
if err := os.Truncate(tempUnsealed, int64(ssize)); err != nil {
return xerrors.Errorf("truncating unsealed file to sector size: %w", err)
@ -391,6 +441,7 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector
}
}
afterUnsealedMove:
if err := ffi.ClearCache(uint64(ssize), sectorPaths.Cache); err != nil {
return xerrors.Errorf("clearing cache: %w", err)
}

View File

@ -42,7 +42,7 @@ func TestSectorImportAfterPC2(t *testing.T) {
// We use two miners so that in case the actively tested miner misses PoSt, we still have a blockchain
client, miner, _, ens := kit.EnsembleOneTwo(t, kit.ThroughRPC())
ens.InterconnectAll().BeginMiningMustPost(blockTime)
ens.InterconnectAll().BeginMining(blockTime)
ctx := context.Background()

View File

@ -476,14 +476,18 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
var out storiface.SectorPaths
var storageIDs storiface.SectorPaths
// First find existing files
for _, fileType := range storiface.PathTypes {
if fileType&existing == 0 {
// also try to find existing sectors if we're allocating
if fileType&(existing|allocate) == 0 {
continue
}
si, err := st.index.StorageFindSector(ctx, sid.ID, fileType, ssize, false)
if err != nil {
log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err)
if fileType&existing != 0 {
log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err)
}
continue
}
@ -501,11 +505,13 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
storiface.SetPathByType(&out, fileType, spath)
storiface.SetPathByType(&storageIDs, fileType, string(info.ID))
existing ^= fileType
existing = existing.Unset(fileType)
allocate = allocate.Unset(fileType)
break
}
}
// Then allocate for allocation requests
for _, fileType := range storiface.PathTypes {
if fileType&allocate == 0 {
continue

View File

@ -174,6 +174,10 @@ func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) Sect
return t & denyMask
}
func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType {
return t &^ toUnset
}
func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) != t
}