post: Handle Recovering sectors

api/api_full.go

@@ -147,7 +147,8 @@ type FullNode interface {
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
 	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)
 	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error)
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) ([]abi.SectorNumber, error)
+	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
+	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error)
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
 	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)

api/apistruct/struct.go

@@ -120,7 +120,8 @@ type FullNodeStruct struct {
 		StateMinerPower                   func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)                                    `perm:"read"`
 		StateMinerInfo                    func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)                                    `perm:"read"`
 		StateMinerDeadlines               func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error)                                   `perm:"read"`
-		StateMinerFaults                  func(context.Context, address.Address, types.TipSetKey) ([]abi.SectorNumber, error)                                 `perm:"read"`
+		StateMinerFaults                  func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateMinerRecoveries              func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
 		StateMinerInitialPledgeCollateral func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error)                     `perm:"read"`
 		StateMinerAvailableBalance        func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
 		StateSectorPreCommitInfo          func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
@@ -533,10 +534,14 @@ func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Addr
 	return c.Internal.StateMinerDeadlines(ctx, m, tsk)
 }
 
-func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) ([]abi.SectorNumber, error) {
+func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
 	return c.Internal.StateMinerFaults(ctx, actor, tsk)
 }
 
+func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
+	return c.Internal.StateMinerRecoveries(ctx, actor, tsk)
+}
+
 func (c *FullNodeStruct) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, snum abi.SectorNumber, tsk types.TipSetKey) (types.BigInt, error) {
 	return c.Internal.StateMinerInitialPledgeCollateral(ctx, maddr, snum, tsk)
 }

chain/stmgr/utils.go

@@ -248,24 +248,24 @@ func GetMinerDeadlines(ctx context.Context, sm *StateManager, ts *types.TipSet,
 	return mas.LoadDeadlines(sm.cs.Store(ctx))
 }
 
-func GetMinerFaults(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]abi.SectorNumber, error) {
+func GetMinerFaults(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*abi.BitField, error) {
 	var mas miner.State
 	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
 	if err != nil {
-		return nil, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err)
+		return nil, xerrors.Errorf("(get faults) failed to load miner actor state: %w", err)
 	}
 
-	faults, err := mas.Faults.All(miner.SectorsMax)
+	return mas.Faults, nil
+}
+
+func GetMinerRecoveries(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*abi.BitField, error) {
+	var mas miner.State
+	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
 	if err != nil {
-		return nil, xerrors.Errorf("reading fault bit set: %w", err)
+		return nil, xerrors.Errorf("(get recoveries) failed to load miner actor state: %w", err)
 	}
 
-	out := make([]abi.SectorNumber, len(faults))
+	return mas.Recoveries, nil
-	for i, fault := range faults {
-		out[i] = abi.SectorNumber(fault)
-	}
-
-	return out, nil
 }
 
 func GetStorageDeal(ctx context.Context, sm *StateManager, dealId abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) {

cmd/lotus-storage-miner/info.go

@@ -98,17 +98,22 @@ var infoCmd = &cli.Command{
 			return err
 		}
 
+		nfaults, err := faults.Count()
+		if err != nil {
+			return xerrors.Errorf("counting faults: %w", err)
+		}
+
 		fmt.Printf("\tCommitted: %s\n", types.SizeStr(types.BigMul(types.NewInt(secCounts.Sset), types.NewInt(uint64(mi.SectorSize)))))
-		if len(faults) == 0 {
+		if nfaults == 0 {
 			fmt.Printf("\tProving: %s\n", types.SizeStr(types.BigMul(types.NewInt(secCounts.Pset), types.NewInt(uint64(mi.SectorSize)))))
 		} else {
 			var faultyPercentage float64
 			if secCounts.Sset != 0 {
-				faultyPercentage = float64(10000*uint64(len(faults))/secCounts.Sset) / 100.
+				faultyPercentage = float64(10000*nfaults/secCounts.Sset) / 100.
 			}
 			fmt.Printf("\tProving: %s (%s Faulty, %.2f%%)\n",
 				types.SizeStr(types.BigMul(types.NewInt(secCounts.Pset), types.NewInt(uint64(mi.SectorSize)))),
-				types.SizeStr(types.BigMul(types.NewInt(uint64(len(faults))), types.NewInt(uint64(mi.SectorSize)))),
+				types.SizeStr(types.BigMul(types.NewInt(nfaults), types.NewInt(uint64(mi.SectorSize)))),
 				faultyPercentage)
 		}
 

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200505180321-973f8949ea8e
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/sector-storage v0.0.0-20200515123304-20817dc51db5
+	github.com/filecoin-project/sector-storage v0.0.0-20200516212231-e620df4c43a4
 	github.com/filecoin-project/specs-actors v0.5.3
 	github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102
 	github.com/filecoin-project/storage-fsm v0.0.0-20200427182014-01487d5ad3c8

go.sum

@@ -180,8 +180,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/
 github.com/filecoin-project/lotus v0.2.10/go.mod h1:om5PQA9ZT0lf16qI7Fz/ZGLn4LDCMqPC8ntZA9uncRE=
 github.com/filecoin-project/sector-storage v0.0.0-20200411000242-61616264b16d/go.mod h1:/yueJueMh0Yc+0G1adS0lhnedcSnjY86EjKsA20+DVY=
 github.com/filecoin-project/sector-storage v0.0.0-20200508203401-a74812ba12f3/go.mod h1:B+xzopr/oWZJz2hBL5Ekb7Obcum5ntmfbaAUlaaho28=
-github.com/filecoin-project/sector-storage v0.0.0-20200515123304-20817dc51db5 h1:eGHeK6+O9/GcQWwlRmXuCaBOUtS0AvEO/9hdXpJG5ZY=
+github.com/filecoin-project/sector-storage v0.0.0-20200516212231-e620df4c43a4 h1:am8Jl4jJ+y7y2fQHrsKaV9aa8uZmwR0/VTP1wlCy0jM=
-github.com/filecoin-project/sector-storage v0.0.0-20200515123304-20817dc51db5/go.mod h1:AeiT6Szz4XSnSJwHF1+flTRMspkwekbTP8zX8/wlhbY=
+github.com/filecoin-project/sector-storage v0.0.0-20200516212231-e620df4c43a4/go.mod h1:AeiT6Szz4XSnSJwHF1+flTRMspkwekbTP8zX8/wlhbY=
 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
 github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA=
 github.com/filecoin-project/specs-actors v0.2.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
@@ -1092,6 +1092,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
 go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
+go4.org v0.0.0-20190218023631-ce4c26f7be8e h1:m9LfARr2VIOW0vsV19kEKp/sWQvZnGobA8JHui/XJoY=
 go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw=
 go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=

node/impl/full/state.go

@@ -108,7 +108,7 @@ func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.A
 	return miner.ComputeProvingPeriodDeadline(mas.ProvingPeriodStart, ts.Height()), nil
 }
 
-func (a *StateAPI) StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]abi.SectorNumber, error) {
+func (a *StateAPI) StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
 	ts, err := a.Chain.GetTipSetFromKey(tsk)
 	if err != nil {
 		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
@@ -116,6 +116,14 @@ func (a *StateAPI) StateMinerFaults(ctx context.Context, addr address.Address, t
 	return stmgr.GetMinerFaults(ctx, a.StateManager, ts, addr)
 }
 
+func (a *StateAPI) StateMinerRecoveries(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
+	ts, err := a.Chain.GetTipSetFromKey(tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+	return stmgr.GetMinerRecoveries(ctx, a.StateManager, ts, addr)
+}
+
 func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.MinerPower, error) {
 	ts, err := a.Chain.GetTipSetFromKey(tsk)
 	if err != nil {

node/modules/storageminer.go

@@ -138,7 +138,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h
 		return nil, err
 	}
 
-	fps, err := storage.NewWindowedPoStScheduler(api, sealer, maddr, worker)
+	fps, err := storage.NewWindowedPoStScheduler(api, sealer, sealer, maddr, worker)
 	if err != nil {
 		return nil, err
 	}

storage/miner.go

@@ -57,7 +57,8 @@ type storageMinerApi interface {
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
 	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
 	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error)
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) ([]abi.SectorNumber, error)
+	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
+	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
 
 	MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error)
 

storage/wdpost_run.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"github.com/filecoin-project/go-bitfield"
 	"time"
 
 	"github.com/filecoin-project/go-address"
@@ -59,11 +60,128 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli
 	}()
 }
 
+func (s *WindowPoStScheduler) checkRecoveries(ctx context.Context, deadline uint64, ts *types.TipSet) error {
+	faults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key())
+	if err != nil {
+		return xerrors.Errorf("getting on-chain faults: %w", err)
+	}
+
+	fc, err := faults.Count()
+	if err != nil {
+		return xerrors.Errorf("counting faulty sectors: %w", err)
+	}
+
+	if fc == 0 {
+		return nil
+	}
+
+	recov, err := s.api.StateMinerRecoveries(ctx, s.actor, ts.Key())
+	if err != nil {
+		return xerrors.Errorf("getting on-chain recoveries: %w", err)
+	}
+
+	unrecovered, err := bitfield.SubtractBitField(faults, recov)
+	if err != nil {
+		return xerrors.Errorf("subtracting recovered set from fault set: %w", err)
+	}
+
+	uc, err := unrecovered.Count()
+	if err != nil {
+		return xerrors.Errorf("counting unrecovered sectors: %w", err)
+	}
+
+	if uc == 0 {
+		return nil
+	}
+
+	spt, err := s.proofType.RegisteredSealProof()
+	if err != nil {
+		return xerrors.Errorf("getting seal proof type: %w", err)
+	}
+
+	mid, err := address.IDFromAddress(s.actor)
+	if err != nil {
+		return err
+	}
+
+	var sectors map[abi.SectorID]struct{}
+	var tocheck []abi.SectorID
+	err = unrecovered.ForEach(func(snum uint64) error {
+		s := abi.SectorID{
+			Miner:  abi.ActorID(mid),
+			Number: abi.SectorNumber(snum),
+		}
+
+		tocheck = append(tocheck, s)
+		sectors[s] = struct{}{}
+		return nil
+	})
+	if err != nil {
+		return xerrors.Errorf("iterating over unrecovered bitfield: %w", err)
+	}
+
+	bad, err := s.faultTracker.CheckProvable(ctx, spt, tocheck)
+	if err != nil {
+		return xerrors.Errorf("checking provable sectors: %w", err)
+	}
+	for _, id := range bad {
+		delete(sectors, id)
+	}
+
+	log.Warnw("Recoverable sectors", "faulty", len(tocheck), "recoverable", len(sectors))
+
+	if len(sectors) == 0 { // nothing to recover
+		return nil
+	}
+
+	sbf := bitfield.New()
+	for s := range sectors {
+		(&sbf).Set(uint64(s.Number))
+	}
+
+	params := &miner.DeclareFaultsRecoveredParams{
+		Recoveries: []miner.RecoveryDeclaration{{Deadline: deadline, Sectors: &sbf}},
+	}
+
+	enc, aerr := actors.SerializeParams(params)
+	if aerr != nil {
+		return xerrors.Errorf("could not serialize declare recoveries parameters: %w", aerr)
+	}
+
+	msg := &types.Message{
+		To:       s.actor,
+		From:     s.worker,
+		Method:   builtin.MethodsMiner.DeclareFaultsRecovered,
+		Params:   enc,
+		Value:    types.NewInt(0),
+		GasLimit: 10000000, // i dont know help
+		GasPrice: types.NewInt(2),
+	}
+
+	sm, err := s.api.MpoolPushMessage(ctx, msg)
+	if err != nil {
+		return xerrors.Errorf("pushing message to mpool: %w", err)
+	}
+
+	log.Warnw("declare faults recovered Message CID", "cid", sm.Cid())
+
+	rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid())
+	if err != nil {
+		return xerrors.Errorf("declare faults recovered wait error: %w", err)
+	}
+
+	if rec.Receipt.ExitCode == 0 {
+		return xerrors.Errorf("declare faults recovered wait non-0 exit code: %d", rec.Receipt.ExitCode)
+	}
+
+	return nil
+}
+
 func (s *WindowPoStScheduler) checkFaults(ctx context.Context, ssi []abi.SectorNumber) ([]abi.SectorNumber, error) {
 	//faults := s.prover.Scrub(ssi)
 	log.Warnf("Stub checkFaults")
 
-	declaredFaults := map[abi.SectorNumber]struct{}{}
+	/*declaredFaults := map[abi.SectorNumber]struct{}{}
 
 	{
 		chainFaults, err := s.api.StateMinerFaults(ctx, s.actor, types.EmptyTSK)
@@ -74,7 +192,7 @@ func (s *WindowPoStScheduler) checkFaults(ctx context.Context, ssi []abi.SectorN
 		for _, fault := range chainFaults {
 			declaredFaults[fault] = struct{}{}
 		}
-	}
+	}*/
 
 	return nil, nil
 }
@@ -83,6 +201,11 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 	ctx, span := trace.StartSpan(ctx, "storage.runPost")
 	defer span.End()
 
+	if err := s.checkRecoveries(ctx, di.Index, ts); err != nil {
+		// TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse
+		log.Errorf("checking sector recoveries: %v")
+	}
+
 	buf := new(bytes.Buffer)
 	if err := s.actor.MarshalCBOR(buf); err != nil {
 		return nil, xerrors.Errorf("failed to marshal address to cbor: %w", err)

storage/wdpost_sched.go

@@ -8,6 +8,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	sectorstorage "github.com/filecoin-project/sector-storage"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-storage/storage"
@@ -22,6 +23,7 @@ const StartConfidence = 4 // TODO: config
 type WindowPoStScheduler struct {
 	api              storageMinerApi
 	prover           storage.Prover
+	faultTracker     sectorstorage.FaultTracker
 	proofType        abi.RegisteredProof
 	partitionSectors uint64
 
@@ -38,7 +40,7 @@ type WindowPoStScheduler struct {
 	//failLk sync.Mutex
 }
 
-func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) {
+func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) {
 	mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK)
 	if err != nil {
 		return nil, xerrors.Errorf("getting sector size: %w", err)
@@ -52,6 +54,7 @@ func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, actor addr
 	return &WindowPoStScheduler{
 		api:              api,
 		prover:           sb,
+		faultTracker:     ft,
 		proofType:        rt,
 		partitionSectors: mi.WindowPoStPartitionSectors,