lotus/storage/sealer/faults.go

134 lines
3.4 KiB
Go
Raw Normal View History

package sealer
2020-05-16 21:03:29 +00:00
import (
"context"
2020-12-01 23:32:01 +00:00
"crypto/rand"
"fmt"
2022-03-29 01:19:11 +00:00
"sync"
"time"
2022-01-18 11:11:59 +00:00
2020-05-16 21:03:29 +00:00
"golang.org/x/xerrors"
2020-12-01 23:32:01 +00:00
ffi "github.com/filecoin-project/filecoin-ffi"
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
2020-05-16 21:03:29 +00:00
)
var PostCheckTimeout = 160 * time.Second
2020-08-16 10:40:35 +00:00
// FaultTracker TODO: Track things more actively
2020-05-16 21:03:29 +00:00
type FaultTracker interface {
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error)
2020-05-16 21:03:29 +00:00
}
2020-08-16 10:40:35 +00:00
// CheckProvable returns unprovable sectors
func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
2022-03-29 01:19:11 +00:00
ctx, cancel := context.WithCancel(ctx)
defer cancel()
2022-01-14 13:11:04 +00:00
if rg == nil {
return nil, xerrors.Errorf("rg is nil")
}
2020-11-26 07:02:43 +00:00
var bad = make(map[abi.SectorID]string)
2022-03-29 01:19:11 +00:00
var badLk sync.Mutex
var postRand abi.PoStRandomness = make([]byte, abi.RandomnessLength)
_, _ = rand.Read(postRand)
postRand[31] &= 0x3f
limit := m.parallelCheckLimit
if limit <= 0 {
limit = len(sectors)
}
throttle := make(chan struct{}, limit)
addBad := func(s abi.SectorID, reason string) {
badLk.Lock()
bad[s] = reason
badLk.Unlock()
}
var wg sync.WaitGroup
wg.Add(len(sectors))
2020-05-16 21:03:29 +00:00
for _, sector := range sectors {
2022-03-29 01:19:11 +00:00
select {
case throttle <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
}
go func(sector storiface.SectorRef) {
2022-03-29 01:19:11 +00:00
defer wg.Done()
defer func() {
<-throttle
}()
2020-06-08 16:47:59 +00:00
ctx, cancel := context.WithCancel(ctx)
defer cancel()
commr, update, err := rg(ctx, sector.ID)
if err != nil {
log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", "err", err)
2022-03-29 01:19:11 +00:00
addBad(sector.ID, fmt.Sprintf("getting commR: %s", err))
return
}
toLock := storiface.FTSealed | storiface.FTCache
if update {
toLock = storiface.FTUpdate | storiface.FTUpdateCache
}
locked, err := m.index.StorageTryLock(ctx, sector.ID, toLock, storiface.FTNone)
2020-06-08 16:47:59 +00:00
if err != nil {
2022-03-29 01:19:11 +00:00
addBad(sector.ID, fmt.Sprintf("tryLock error: %s", err))
return
2020-06-08 16:47:59 +00:00
}
if !locked {
2020-11-19 15:48:28 +00:00
log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
2022-03-29 01:19:11 +00:00
addBad(sector.ID, fmt.Sprint("can't acquire read lock"))
return
2020-06-08 16:47:59 +00:00
}
2022-01-14 13:11:04 +00:00
wpp, err := sector.ProofType.RegisteredWindowPoStProof()
if err != nil {
2022-03-29 01:19:11 +00:00
addBad(sector.ID, fmt.Sprint("can't get proof type"))
return
2022-01-14 13:11:04 +00:00
}
2022-03-29 01:19:11 +00:00
ch, err := ffi.GeneratePoStFallbackSectorChallenges(wpp, sector.ID.Miner, postRand, []abi.SectorNumber{
2022-01-14 13:11:04 +00:00
sector.ID.Number,
})
2020-05-16 21:03:29 +00:00
if err != nil {
2022-01-14 13:11:04 +00:00
log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "err", err)
2022-03-29 01:19:11 +00:00
addBad(sector.ID, fmt.Sprintf("generating fallback challenges: %s", err))
return
2020-05-16 21:03:29 +00:00
}
vctx, cancel2 := context.WithTimeout(ctx, PostCheckTimeout)
defer cancel2()
_, err = m.storage.GenerateSingleVanillaProof(vctx, sector.ID.Miner, storiface.PostSectorChallenge{
2022-01-14 13:11:04 +00:00
SealProof: sector.ProofType,
SectorNumber: sector.ID.Number,
SealedCID: commr,
Challenge: ch.Challenges[sector.ID.Number],
2022-01-18 10:25:04 +00:00
Update: update,
2022-01-14 13:11:04 +00:00
}, wpp)
if err != nil {
log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "err", err)
2022-03-29 01:19:11 +00:00
addBad(sector.ID, fmt.Sprintf("generating vanilla proof: %s", err))
return
2020-12-01 23:32:01 +00:00
}
2022-03-29 01:19:11 +00:00
}(sector)
2020-05-16 21:03:29 +00:00
}
2022-03-29 19:28:37 +00:00
wg.Wait()
2020-05-16 21:03:29 +00:00
return bad, nil
}
var _ FaultTracker = &Manager{}