Merge pull request #5086 from filecoin-project/feat/better-sector-check

miner: Add slow mode to proving check

api/api_storage.go

@@ -118,7 +118,7 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error)
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error)
 }
 
 type SealRes struct {

api/apistruct/struct.go

@@ -364,7 +364,7 @@ type StorageMinerStruct struct {
 
 		CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"`
 
-		CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) `perm:"admin"`
+		CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
 	}
 }
 
@@ -1516,8 +1516,8 @@ func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) err
 	return c.Internal.CreateBackup(ctx, fpath)
 }
 
-func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) {
+func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) {
-	return c.Internal.CheckProvable(ctx, pp, sectors)
+	return c.Internal.CheckProvable(ctx, pp, sectors, expensive)
 }
 
 // WorkerStruct

cmd/lotus-storage-miner/proving.go

@@ -386,6 +386,10 @@ var provingCheckProvableCmd = &cli.Command{
 			Usage: "print only bad sectors",
 			Value: false,
 		},
+		&cli.BoolFlag{
+			Name:  "slow",
+			Usage: "run slower checks",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 1 {
@@ -459,7 +463,7 @@ var provingCheckProvableCmd = &cli.Command{
 				})
 			}
 
-			bad, err := sapi.CheckProvable(ctx, pf, tocheck)
+			bad, err := sapi.CheckProvable(ctx, pf, tocheck, cctx.Bool("slow"))
 			if err != nil {
 				return err
 			}

documentation/en/api-methods-miner.md

@@ -232,7 +232,8 @@ Inputs:
 ```json
 [
   8,
-  null
+  null,
+  true
 ]
 ```
 

extern/sector-storage/faults.go

@@ -2,13 +2,16 @@ package sectorstorage
 
 import (
 	"context"
+	"crypto/rand"
 	"fmt"
 	"os"
 	"path/filepath"
 
 	"golang.org/x/xerrors"
 
+	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
@@ -16,11 +19,11 @@ import (
 
 // FaultTracker TODO: Track things more actively
 type FaultTracker interface {
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error)
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error)
 }
 
 // CheckProvable returns unprovable sectors
-func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) {
+func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	var bad = make(map[abi.SectorID]string)
 
 	ssize, err := pp.SectorSize()
@@ -83,6 +86,49 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 				}
 			}
 
+			if rg != nil {
+				wpp, err := sector.ProofType.RegisteredWindowPoStProof()
+				if err != nil {
+					return err
+				}
+
+				var pr abi.PoStRandomness = make([]byte, abi.RandomnessLength)
+				_, _ = rand.Read(pr)
+				pr[31] &= 0x3f
+
+				ch, err := ffi.GeneratePoStFallbackSectorChallenges(wpp, sector.ID.Miner, pr, []abi.SectorNumber{
+					sector.ID.Number,
+				})
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					bad[sector.ID] = fmt.Sprintf("generating fallback challenges: %s", err)
+					return nil
+				}
+
+				commr, err := rg(ctx, sector.ID)
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					bad[sector.ID] = fmt.Sprintf("getting commR: %s", err)
+					return nil
+				}
+
+				_, err = ffi.GenerateSingleVanillaProof(ffi.PrivateSectorInfo{
+					SectorInfo: proof.SectorInfo{
+						SealProof:    sector.ProofType,
+						SectorNumber: sector.ID.Number,
+						SealedCID:    commr,
+					},
+					CacheDirPath:     lp.Cache,
+					PoStProofType:    wpp,
+					SealedSectorPath: lp.Sealed,
+				}, ch.Challenges[sector.ID.Number])
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					bad[sector.ID] = fmt.Sprintf("generating vanilla proof: %s", err)
+					return nil
+				}
+			}
+
 			return nil
 		}()
 		if err != nil {

extern/sector-storage/mock/mock.go

@@ -405,7 +405,7 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro
 	return nil
 }
 
-func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) (map[abi.SectorID]string, error) {
+func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	bad := map[abi.SectorID]string{}
 
 	for _, sid := range ids {

extern/sector-storage/storiface/ffi.go

@@ -1,8 +1,11 @@
 package storiface
 
 import (
+	"context"
 	"errors"
 
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-state-types/abi"
 )
 
@@ -15,3 +18,5 @@ func (i UnpaddedByteIndex) Padded() PaddedByteIndex {
 }
 
 type PaddedByteIndex uint64
+
+type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error)

node/impl/storminer.go

@@ -544,8 +544,23 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error
 	return backup(sm.DS, fpath)
 }
 
-func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef) (map[abi.SectorNumber]string, error) {
+func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) {
-	bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors)
+	var rg storiface.RGetter
+	if expensive {
+		rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) {
+			si, err := sm.Miner.GetSectorInfo(id.Number)
+			if err != nil {
+				return cid.Undef, err
+			}
+			if si.CommR == nil {
+				return cid.Undef, xerrors.Errorf("commr is nil")
+			}
+
+			return *si.CommR, nil
+		}
+	}
+
+	bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors, rg)
 	if err != nil {
 		return nil, err
 	}

storage/wdpost_run.go

@@ -214,7 +214,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B
 		})
 	}
 
-	bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck)
+	bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck, nil)
 	if err != nil {
 		return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err)
 	}

storage/wdpost_run_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/lotus/journal"
 )
 
@@ -125,7 +126,7 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si
 type mockFaultTracker struct {
 }
 
-func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) {
+func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	// Returns "bad" sectors so just return empty map meaning all sectors are good
 	return map[abi.SectorID]string{}, nil
 }