storagefsm: Handle sectors with expired deals better

api/api_full.go

@@ -323,7 +323,7 @@ type FullNode interface {
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
 	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
 	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
-	// StateSectorGetInfo returns the on-chain info for the specified miner's sector
+	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)

chain/stmgr/utils.go

@@ -157,7 +157,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
 		return nil, err
 	}
 	if !ok {
-		return nil, xerrors.New("sector not found")
+		return nil, nil
 	}
 
 	return sectorInfo, nil

cmd/lotus-storage-miner/info.go

@@ -266,6 +266,7 @@ var stateList = []stateMeta{
 	{col: color.FgRed, state: sealing.FaultReported},
 	{col: color.FgRed, state: sealing.FaultedFinal},
 	{col: color.FgRed, state: sealing.RemoveFailed},
+	{col: color.FgRed, state: sealing.DealsExpired},
 }
 
 func init() {

extern/storage-sealing/checks.go

@@ -33,7 +33,7 @@ type ErrInvalidProof struct{ error }
 type ErrNoPrecommit struct{ error }
 type ErrCommitWaitFailed struct{ error }
 
-func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
+func checkPieces(ctx context.Context, maddr address.Address, si SectorInfo, api SealingAPI) error {
 	tok, height, err := api.ChainHead(ctx)
 	if err != nil {
 		return &ErrApi{xerrors.Errorf("getting chain head: %w", err)}
@@ -55,6 +55,10 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
 			return &ErrInvalidDeals{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)}
 		}
 
+		if proposal.Provider != maddr {
+			return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong provider: %s != %s", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, proposal.Provider, maddr)}
+		}
+
 		if proposal.PieceCID != p.Piece.PieceCID {
 			return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID, proposal.PieceCID)}
 		}
@@ -74,6 +78,10 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
 // checkPrecommit checks that data commitment generated in the sealing process
 //  matches pieces, and that the seal ticket isn't expired
 func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, tok TipSetToken, height abi.ChainEpoch, api SealingAPI) (err error) {
+	if err := checkPieces(ctx, maddr, si, api); err != nil {
+		return err
+	}
+
 	commD, err := api.StateComputeDataCommitment(ctx, maddr, si.SectorType, si.dealIDs(), tok)
 	if err != nil {
 		return &ErrApi{xerrors.Errorf("calling StateComputeDataCommitment: %w", err)}
@@ -176,5 +184,9 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte,
 		return &ErrInvalidProof{xerrors.New("invalid proof (compute error?)")}
 	}
 
+	if err := checkPieces(ctx, m.maddr, si, m.api); err != nil {
+		return err
+	}
+
 	return nil
 }

extern/storage-sealing/fsm.go

@@ -61,6 +61,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorPreCommitted{}, PreCommitWait),
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
 		on(SectorPreCommitLanded{}, WaitSeed),
+		on(SectorDealsExpired{}, DealsExpired),
 	),
 	PreCommitWait: planOne(
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
@@ -99,6 +100,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorRetryWaitSeed{}, WaitSeed),
 		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 		on(SectorPreCommitLanded{}, WaitSeed),
+		on(SectorDealsExpired{}, DealsExpired),
 	),
 	ComputeProofFailed: planOne(
 		on(SectorRetryComputeProof{}, Committing),
@@ -113,10 +115,14 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
 		on(SectorRetryPreCommit{}, PreCommitting),
 		on(SectorRetryCommitWait{}, CommitWait),
+		on(SectorDealsExpired{}, DealsExpired),
 	),
 	FinalizeFailed: planOne(
 		on(SectorRetryFinalize{}, FinalizeSector),
 	),
+	DealsExpired: planOne(
+	// SectorRemove (global)
+	),
 
 	// Post-seal
 
@@ -275,6 +281,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleCommitFailed, processed, nil
 	case FinalizeFailed:
 		return m.handleFinalizeFailed, processed, nil
+	case DealsExpired:
+		return m.handleDealsExpired, processed, nil
 
 	// Post-seal
 	case Proving:

extern/storage-sealing/fsm_events.go

@@ -191,6 +191,11 @@ type SectorCommitFailed struct{ error }
 func (evt SectorCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error }
 func (evt SectorCommitFailed) apply(*SectorInfo)                        {}
 
+type SectorDealsExpired struct{ error }
+
+func (evt SectorDealsExpired) FormatError(xerrors.Printer) (next error) { return evt.error }
+func (evt SectorDealsExpired) apply(*SectorInfo)                        {}
+
 type SectorCommitted struct {
 	Proof []byte
 }

extern/storage-sealing/sector_state.go

@@ -28,6 +28,7 @@ const (
 	CommitFailed         SectorState = "CommitFailed"
 	PackingFailed        SectorState = "PackingFailed"
 	FinalizeFailed       SectorState = "FinalizeFailed"
+	DealsExpired         SectorState = "DealsExpired"
 
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain

extern/storage-sealing/states_failed.go

@@ -81,6 +81,12 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
 		case *ErrBadTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+		case *ErrInvalidDeals:
+			// TODO: Deals got reorged, figure out what to do about this
+			//  (this will probably require tracking the deal submit message CID, and re-checking what's on chain)
+			return xerrors.Errorf("invalid deals in sector %d: %w", sector.SectorNumber, err)
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case *ErrNoPrecommit:
 			return ctx.Send(SectorRetryPreCommit{})
 		case *ErrPrecommitOnChain:
@@ -88,6 +94,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 		case *ErrSectorNumberAllocated:
 			log.Errorf("handlePreCommitFailed: sector number already allocated, not proceeding: %+v", err)
 			// TODO: check if the sector is committed (not sure how we'd end up here)
+			// TODO: check on-chain state, adjust local sector number counter to not give out allocated numbers
 			return nil
 		default:
 			return xerrors.Errorf("checkPrecommit sanity check error: %w", err)
@@ -157,7 +164,13 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 		case *ErrExpiredTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
 		case *ErrBadTicket:
-			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)})
+		case *ErrInvalidDeals:
+			// TODO: Deals got reorged, figure out what to do about this
+			//  (this will probably require tracking the deal submit message CID, and re-checking what's on chain)
+			return xerrors.Errorf("invalid deals in sector %d: %w", sector.SectorNumber, err)
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case nil:
 			return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("no precommit: %w", err)})
 		case *ErrPrecommitOnChain:
@@ -192,6 +205,12 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 			return ctx.Send(SectorRetryPreCommitWait{})
 		case *ErrNoPrecommit:
 			return ctx.Send(SectorRetryPreCommit{})
+		case *ErrInvalidDeals:
+			// TODO: Deals got reorged, figure out what to do about this
+			//  (this will probably require tracking the deal submit message CID, and re-checking what's on chain)
+			return xerrors.Errorf("invalid deals in sector %d: %w", sector.SectorNumber, err)
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case *ErrCommitWaitFailed:
 			if err := failedCooldown(ctx, sector); err != nil {
 				return err
@@ -221,3 +240,24 @@ func (m *Sealing) handleFinalizeFailed(ctx statemachine.Context, sector SectorIn
 
 	return ctx.Send(SectorRetryFinalize{})
 }
+
+func (m *Sealing) handleDealsExpired(ctx statemachine.Context, sector SectorInfo) error {
+	// First make vary sure the sector isn't committed
+	si, err := m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, nil)
+	if err != nil {
+		return xerrors.Errorf("getting sector info: %w", err)
+	}
+	if si != nil {
+		// TODO: this should never happen, but in case it does, try to go back to
+		//  the proving state after running some checks
+		return xerrors.Errorf("sector is committed on-chain, but we're in DealsExpired")
+	}
+
+	if sector.PreCommitInfo == nil {
+		// TODO: Create a separate state which will remove those pieces, and go back to PC1
+		return xerrors.Errorf("non-precommitted sector with expired deals, can't recover from this yet")
+	}
+
+	// Not much to do here, we can't go back in time to commit this sector
+	return ctx.Send(SectorRemove{})
+}

extern/storage-sealing/states_sealing.go

@@ -79,7 +79,7 @@ func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.Se
 }
 
 func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error {
-	if err := checkPieces(ctx.Context(), sector, m.api); err != nil { // Sanity check state
+	if err := checkPieces(ctx.Context(), m.maddr, sector, m.api); err != nil { // Sanity check state
 		switch err.(type) {
 		case *ErrApi:
 			log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err)
@@ -155,6 +155,12 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired: %w", err)})
 		case *ErrBadTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)})
+		case *ErrInvalidDeals:
+			// TODO: Deals got reorged, figure out what to do about this
+			//  (this will probably require tracking the deal submit message CID, and re-checking what's on chain)
+			return xerrors.Errorf("invalid deals in sector %d: %w", sector.SectorNumber, err)
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case *ErrPrecommitOnChain:
 			return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit
 		case *ErrSectorNumberAllocated:
@@ -402,9 +408,12 @@ func (m *Sealing) handleCommitWait(ctx statemachine.Context, sector SectorInfo)
 		return ctx.Send(SectorCommitFailed{xerrors.Errorf("submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.TicketValue, sector.SeedValue, sector.SeedEpoch, sector.Proof)})
 	}
 
-	_, err = m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, mw.TipSetTok)
+	si, err := m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, mw.TipSetTok)
 	if err != nil {
-		return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, sector not found in sector set after cron: %w", err)})
+		return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, calling StateSectorGetInfo: %w", err)})
+	}
+	if si == nil {
+		return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, sector not found in sector set after cron")})
 	}
 
 	return ctx.Send(SectorProving{})

extern/storage-sealing/upgrade_queue.go

@@ -72,6 +72,10 @@ func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreC
 			log.Errorf("error calling StateSectorGetInfo for replaced sector: %+v", err)
 			return big.Zero()
 		}
+		if ri == nil {
+			log.Errorf("couldn't find sector info for sector to replace: %+v", replace)
+			return big.Zero()
+		}
 
 		if params.Expiration < ri.Expiration {
 			// TODO: Some limit on this

node/impl/storminer.go

@@ -177,6 +177,9 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 
 	onChainInfo, err := sm.Full.StateSectorGetInfo(ctx, sm.Miner.Address(), sid, types.EmptyTSK)
 	if err != nil {
+		return sInfo, err
+	}
+	if onChainInfo == nil {
 		return sInfo, nil
 	}
 	sInfo.SealProof = onChainInfo.SealProof