Merge pull request #24 from filecoin-project/feat/improvements

A bunch of improvements for more reliable sealing

cbor_gen.go

@@ -382,7 +382,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{179}); err != nil {
+	if _, err := w.Write([]byte{180}); err != nil {
 		return err
 	}
 
@@ -425,22 +425,6 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Nonce (uint64) (uint64)
-	if len("Nonce") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"Nonce\" was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Nonce")))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte("Nonce")); err != nil {
-		return err
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil {
-		return err
-	}
-
 	// t.SectorType (abi.RegisteredProof) (int64)
 	if len("SectorType") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"SectorType\" was too long")
@@ -645,6 +629,45 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
+	// t.PreCommitTipSet (sealing.TipSetToken) (slice)
+	if len("PreCommitTipSet") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"PreCommitTipSet\" was too long")
+	}
+
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitTipSet")))); err != nil {
+		return err
+	}
+	if _, err := w.Write([]byte("PreCommitTipSet")); err != nil {
+		return err
+	}
+
+	if len(t.PreCommitTipSet) > cbg.ByteArrayMaxLen {
+		return xerrors.Errorf("Byte array in field t.PreCommitTipSet was too long")
+	}
+
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PreCommitTipSet)))); err != nil {
+		return err
+	}
+	if _, err := w.Write(t.PreCommitTipSet); err != nil {
+		return err
+	}
+
+	// t.PreCommit2Fails (uint64) (uint64)
+	if len("PreCommit2Fails") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"PreCommit2Fails\" was too long")
+	}
+
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommit2Fails")))); err != nil {
+		return err
+	}
+	if _, err := w.Write([]byte("PreCommit2Fails")); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PreCommit2Fails))); err != nil {
+		return err
+	}
+
 	// t.SeedValue (abi.InteractiveSealRandomness) (slice)
 	if len("SeedValue") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"SeedValue\" was too long")
@@ -855,21 +878,6 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 				}
 				t.SectorNumber = abi.SectorNumber(extra)
 
-			}
-			// t.Nonce (uint64) (uint64)
-		case "Nonce":
-
-			{
-
-				maj, extra, err = cbg.CborReadHeader(br)
-				if err != nil {
-					return err
-				}
-				if maj != cbg.MajUnsignedInt {
-					return fmt.Errorf("wrong type for uint64 field")
-				}
-				t.Nonce = uint64(extra)
-
 			}
 			// t.SectorType (abi.RegisteredProof) (int64)
 		case "SectorType":
@@ -912,9 +920,11 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajArray {
 				return fmt.Errorf("expected cbor array")
 			}
+
 			if extra > 0 {
 				t.Pieces = make([]Piece, extra)
 			}
+
 			for i := 0; i < int(extra); i++ {
 
 				var v Piece
@@ -1079,6 +1089,39 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 					t.PreCommitMessage = &c
 				}
 
+			}
+			// t.PreCommitTipSet (sealing.TipSetToken) (slice)
+		case "PreCommitTipSet":
+
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+
+			if extra > cbg.ByteArrayMaxLen {
+				return fmt.Errorf("t.PreCommitTipSet: byte array too large (%d)", extra)
+			}
+			if maj != cbg.MajByteString {
+				return fmt.Errorf("expected byte array")
+			}
+			t.PreCommitTipSet = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.PreCommitTipSet); err != nil {
+				return err
+			}
+			// t.PreCommit2Fails (uint64) (uint64)
+		case "PreCommit2Fails":
+
+			{
+
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.PreCommit2Fails = uint64(extra)
+
 			}
 			// t.SeedValue (abi.InteractiveSealRandomness) (slice)
 		case "SeedValue":
@@ -1215,9 +1258,11 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajArray {
 				return fmt.Errorf("expected cbor array")
 			}
+
 			if extra > 0 {
 				t.Log = make([]Log, extra)
 			}
+
 			for i := 0; i < int(extra); i++ {
 
 				var v Log

checks.go

@@ -24,6 +24,8 @@ type ErrExpiredDeals struct{ error }
 
 type ErrBadCommD struct{ error }
 type ErrExpiredTicket struct{ error }
+type ErrBadTicket struct{ error }
+type ErrPrecommitOnChain struct{ error }
 
 type ErrBadSeed struct{ error }
 type ErrInvalidProof struct{ error }
@@ -78,10 +80,22 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t
 		return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)}
 	}
 
-	if int64(height)-int64(si.TicketEpoch+SealRandomnessLookback) > SealRandomnessLookbackLimit {
+	if height-si.TicketEpoch+SealRandomnessLookback > SealRandomnessLookbackLimit(si.SectorType) {
 		return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+SealRandomnessLookback, height)}
 	}
 
+	pci, err := api.StateSectorPreCommitInfo(ctx, maddr, si.SectorNumber, tok)
+	if err != nil {
+		return &ErrApi{xerrors.Errorf("getting precommit info: %w", err)}
+	}
+
+	if pci != nil {
+		if pci.Info.SealRandEpoch != si.TicketEpoch {
+			return &ErrBadTicket{}
+		}
+		return &ErrPrecommitOnChain{}
+	}
+
 	return nil
 }
 

constants.go

@@ -1,13 +1,17 @@
 package sealing
 
-// Epochs
+import (
-const Finality = 500
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+)
 
 // Epochs
-const SealRandomnessLookback = Finality
+const SealRandomnessLookback = miner.ChainFinalityish
 
 // Epochs
-const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
+func SealRandomnessLookbackLimit(spt abi.RegisteredProof) abi.ChainEpoch {
+	return miner.MaxSealDuration[spt]
+}
 
 // Epochs
 const InteractivePoRepConfidence = 6

fsm.go

@@ -36,18 +36,19 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	Packing:              planOne(on(SectorPacked{}, PreCommit1)),
 	PreCommit1: planOne(
 		on(SectorPreCommit1{}, PreCommit2),
-		on(SectorSealPreCommitFailed{}, SealFailed),
+		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 		on(SectorPackingFailed{}, PackingFailed),
 	),
 	PreCommit2: planOne(
 		on(SectorPreCommit2{}, PreCommitting),
-		on(SectorSealPreCommitFailed{}, SealFailed),
+		on(SectorSealPreCommit2Failed{}, SealPreCommit2Failed),
 		on(SectorPackingFailed{}, PackingFailed),
 	),
 	PreCommitting: planOne(
-		on(SectorSealPreCommitFailed{}, SealFailed),
+		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 		on(SectorPreCommitted{}, PreCommitWait),
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
+		on(SectorPreCommitLanded{}, WaitSeed),
 	),
 	PreCommitWait: planOne(
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
@@ -65,6 +66,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 
 	FinalizeSector: planOne(
 		on(SectorFinalized{}, Proving),
+		on(SectorFinalizeFailed{}, FinalizeFailed),
 	),
 
 	Proving: planOne(
@@ -72,23 +74,32 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorFaulty{}, Faulty),
 	),
 
-	SealFailed: planOne(
+	SealPreCommit1Failed: planOne(
-		on(SectorRetrySeal{}, PreCommit1),
+		on(SectorRetrySealPreCommit1{}, PreCommit1),
+	),
+	SealPreCommit2Failed: planOne(
+		on(SectorRetrySealPreCommit1{}, PreCommit1),
+		on(SectorRetrySealPreCommit2{}, PreCommit2),
 	),
 	PreCommitFailed: planOne(
 		on(SectorRetryPreCommit{}, PreCommitting),
 		on(SectorRetryWaitSeed{}, WaitSeed),
-		on(SectorSealPreCommitFailed{}, SealFailed),
+		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
+		on(SectorPreCommitLanded{}, WaitSeed),
 	),
 	ComputeProofFailed: planOne(
 		on(SectorRetryComputeProof{}, Committing),
+		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 	),
 	CommitFailed: planOne(
-		on(SectorSealPreCommitFailed{}, SealFailed),
+		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 		on(SectorRetryWaitSeed{}, WaitSeed),
 		on(SectorRetryComputeProof{}, Committing),
 		on(SectorRetryInvalidProof{}, Committing),
 	),
+	FinalizeFailed: planOne(
+		on(SectorRetryFinalize{}, FinalizeSector),
+	),
 
 	Faulty: planOne(
 		on(SectorFaultReported{}, FaultReported),
@@ -140,10 +151,11 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		*<- Packing <- incoming
 		|   |
 		|   v
-		*<- PreCommit1 <--> SealFailed
+		*<- PreCommit1 <--> SealPreCommit1Failed
-		|   |                 ^^^
+		|   |       ^          ^^
-		|   v                 |||
+		|   |       *----------++----\
-		*<- PreCommit2 -------/||
+		|   v       v          ||    |
+		*<- PreCommit2 --------++--> SealPreCommit2Failed
 		|   |                  ||
 		|   v          /-------/|
 		*   PreCommitting <-----+---> PreCommitFailed
@@ -160,6 +172,9 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		*<- CommitWait ---/
 		|   |
 		|   v
+		|   FinalizeSector <--> FinalizeFailed
+		|   |
+		|   v
 		*<- Proving
 		|
 		v
@@ -196,14 +211,18 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		log.Infof("Proving sector %d", state.SectorNumber)
 
 	// Handled failure modes
-	case SealFailed:
+	case SealPreCommit1Failed:
-		return m.handleSealFailed, nil
+		return m.handleSealPrecommit1Failed, nil
+	case SealPreCommit2Failed:
+		return m.handleSealPrecommit2Failed, nil
 	case PreCommitFailed:
 		return m.handlePreCommitFailed, nil
 	case ComputeProofFailed:
 		return m.handleComputeProofFailed, nil
 	case CommitFailed:
 		return m.handleCommitFailed, nil
+	case FinalizeFailed:
+		return m.handleFinalizeFailed, nil
 
 		// Faults
 	case Faulty:
@@ -244,7 +263,7 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
 			return nil
 		case SectorComputeProofFailed:
 			state.State = ComputeProofFailed
-		case SectorSealPreCommitFailed:
+		case SectorSealPreCommit1Failed:
 			state.State = CommitFailed
 		case SectorCommitFailed:
 			state.State = CommitFailed

fsm_events.go

@@ -84,6 +84,7 @@ func (evt SectorPreCommit1) apply(state *SectorInfo) {
 	state.PreCommit1Out = evt.PreCommit1Out
 	state.TicketEpoch = evt.TicketEpoch
 	state.TicketValue = evt.TicketValue
+	state.PreCommit2Fails = 0
 }
 
 type SectorPreCommit2 struct {
@@ -106,11 +107,20 @@ func (evt SectorPreCommitLanded) apply(si *SectorInfo) {
 	si.PreCommitTipSet = evt.TipSet
 }
 
-type SectorSealPreCommitFailed struct{ error }
+type SectorSealPreCommit1Failed struct{ error }
 
-func (evt SectorSealPreCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error }
+func (evt SectorSealPreCommit1Failed) FormatError(xerrors.Printer) (next error) { return evt.error }
-func (evt SectorSealPreCommitFailed) apply(si *SectorInfo) {
+func (evt SectorSealPreCommit1Failed) apply(si *SectorInfo) {
 	si.InvalidProofs = 0 // reset counter
+	si.PreCommit2Fails = 0
+}
+
+type SectorSealPreCommit2Failed struct{ error }
+
+func (evt SectorSealPreCommit2Failed) FormatError(xerrors.Printer) (next error) { return evt.error }
+func (evt SectorSealPreCommit2Failed) apply(si *SectorInfo) {
+	si.InvalidProofs = 0 // reset counter
+	si.PreCommit2Fails++
 }
 
 type SectorChainPreCommitFailed struct{ error }
@@ -164,6 +174,10 @@ type SectorFinalized struct{}
 
 func (evt SectorFinalized) apply(*SectorInfo) {}
 
+type SectorRetryFinalize struct{}
+
+func (evt SectorRetryFinalize) apply(*SectorInfo) {}
+
 type SectorFinalizeFailed struct{ error }
 
 func (evt SectorFinalizeFailed) FormatError(xerrors.Printer) (next error) { return evt.error }
@@ -171,9 +185,13 @@ func (evt SectorFinalizeFailed) apply(*SectorInfo)                        {}
 
 // Failed state recovery
 
-type SectorRetrySeal struct{}
+type SectorRetrySealPreCommit1 struct{}
 
-func (evt SectorRetrySeal) apply(state *SectorInfo) {}
+func (evt SectorRetrySealPreCommit1) apply(state *SectorInfo) {}
+
+type SectorRetrySealPreCommit2 struct{}
+
+func (evt SectorRetrySealPreCommit2) apply(state *SectorInfo) {}
 
 type SectorRetryPreCommit struct{}
 
@@ -185,7 +203,9 @@ func (evt SectorRetryWaitSeed) apply(state *SectorInfo) {}
 
 type SectorRetryComputeProof struct{}
 
-func (evt SectorRetryComputeProof) apply(state *SectorInfo) {}
+func (evt SectorRetryComputeProof) apply(state *SectorInfo) {
+	state.InvalidProofs++
+}
 
 type SectorRetryInvalidProof struct{}
 

sector_state.go

@@ -19,11 +19,14 @@ const (
 	Proving        SectorState = "Proving"
 	// error modes
 	FailedUnrecoverable  SectorState = "FailedUnrecoverable"
-	SealFailed          SectorState = "SealFailed"
+	SealPreCommit1Failed SectorState = "SealPreCommit1Failed"
+	SealPreCommit2Failed SectorState = "SealPreCommit2Failed"
 	PreCommitFailed      SectorState = "PreCommitFailed"
 	ComputeProofFailed   SectorState = "ComputeProofFailed"
 	CommitFailed         SectorState = "CommitFailed"
 	PackingFailed        SectorState = "PackingFailed"
+	FinalizeFailed       SectorState = "FinalizeFailed"
+
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain
 	FaultedFinal  SectorState = "FaultedFinal"  // fault declared on chain

states.go

@@ -46,13 +46,37 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err
 	return ctx.Send(SectorPacked{FillerPieces: fillerPieces})
 }
 
-func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error {
+func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.SealRandomness, abi.ChainEpoch, error) {
 	tok, epoch, err := m.api.ChainHead(ctx.Context())
 	if err != nil {
 		log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err)
-		return nil
+		return nil, 0, nil
 	}
 
+	ticketEpoch := epoch - SealRandomnessLookback
+	buf := new(bytes.Buffer)
+	if err := m.maddr.MarshalCBOR(buf); err != nil {
+		return nil, 0, err
+	}
+
+	pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
+	if err != nil {
+		return nil, 0, xerrors.Errorf("getting precommit info: %w", err)
+	}
+
+	if pci != nil {
+		ticketEpoch = pci.Info.SealRandEpoch
+	}
+
+	rand, err := m.api.ChainGetRandomness(ctx.Context(), tok, crypto.DomainSeparationTag_SealRandomness, ticketEpoch, buf.Bytes())
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return abi.SealRandomness(rand), ticketEpoch, nil
+}
+
+func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error {
 	if err := checkPieces(ctx.Context(), sector, m.api); err != nil { // Sanity check state
 		switch err.(type) {
 		case *ErrApi:
@@ -68,20 +92,14 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo)
 	}
 
 	log.Infow("performing sector replication...", "sector", sector.SectorNumber)
-	ticketEpoch := epoch - miner.ChainFinalityish
+	ticketValue, ticketEpoch, err := m.getTicket(ctx, sector)
-	buf := new(bytes.Buffer)
-	if err := m.maddr.MarshalCBOR(buf); err != nil {
-		return err
-	}
-	rand, err := m.api.ChainGetRandomness(ctx.Context(), tok, crypto.DomainSeparationTag_SealRandomness, ticketEpoch, buf.Bytes())
 	if err != nil {
-		return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("getting ticket failed: %w", err)})
+		return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("getting ticket failed: %w", err)})
 	}
-	ticketValue := abi.SealRandomness(rand)
 
 	pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), m.minerSector(sector.SectorNumber), ticketValue, sector.pieceInfos())
 	if err != nil {
-		return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("seal pre commit(1) failed: %w", err)})
+		return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("seal pre commit(1) failed: %w", err)})
 	}
 
 	return ctx.Send(SectorPreCommit1{
@@ -94,7 +112,7 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo)
 func (m *Sealing) handlePreCommit2(ctx statemachine.Context, sector SectorInfo) error {
 	cids, err := m.sealer.SealPreCommit2(ctx.Context(), m.minerSector(sector.SectorNumber), sector.PreCommit1Out)
 	if err != nil {
-		return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("seal pre commit(2) failed: %w", err)})
+		return ctx.Send(SectorSealPreCommit2Failed{xerrors.Errorf("seal pre commit(2) failed: %w", err)})
 	}
 
 	return ctx.Send(SectorPreCommit2{
@@ -117,14 +135,18 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 	}
 
 	if err := checkPrecommit(ctx.Context(), m.Address(), sector, tok, height, m.api); err != nil {
-		switch err.(type) {
+		switch err := err.(type) {
 		case *ErrApi:
 			log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err)
 			return nil
 		case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too)
-			return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
 		case *ErrExpiredTicket:
-			return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired: %w", err)})
+		case *ErrBadTicket:
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)})
+		case *ErrPrecommitOnChain:
+			return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit
 		default:
 			return xerrors.Errorf("checkPrecommit sanity check error: %w", err)
 		}
@@ -132,7 +154,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 
 	expiration, err := m.pcp.Expiration(ctx.Context(), sector.Pieces...)
 	if err != nil {
-		return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)})
+		return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)})
 	}
 
 	params := &miner.SectorPreCommitInfo{
@@ -224,12 +246,12 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo)
 	}
 	c2in, err := m.sealer.SealCommit1(ctx.Context(), m.minerSector(sector.SectorNumber), sector.TicketValue, sector.SeedValue, sector.pieceInfos(), cids)
 	if err != nil {
-		return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)})
+		return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(1): %w", err)})
 	}
 
 	proof, err := m.sealer.SealCommit2(ctx.Context(), m.minerSector(sector.SectorNumber), c2in)
 	if err != nil {
-		return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)})
+		return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(2): %w", err)})
 	}
 
 	tok, _, err := m.api.ChainHead(ctx.Context())

states_failed.go

@@ -30,30 +30,37 @@ func failedCooldown(ctx statemachine.Context, sector SectorInfo) error {
 func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) (*miner.SectorPreCommitOnChainInfo, bool) {
 	tok, _, err := m.api.ChainHead(ctx.Context())
 	if err != nil {
-		log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorNumber, err)
+		log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
 		return nil, true
 	}
 
 	info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
 	if err != nil {
-		log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorNumber, err)
+		log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
 		return nil, true
 	}
 
 	return info, false
 }
 
-func (m *Sealing) handleSealFailed(ctx statemachine.Context, sector SectorInfo) error {
+func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error {
-	if _, is := m.checkPreCommitted(ctx, sector); is {
-		// TODO: Remove this after we can re-precommit
-		return nil // noop, for now
-	}
-
 	if err := failedCooldown(ctx, sector); err != nil {
 		return err
 	}
 
-	return ctx.Send(SectorRetrySeal{})
+	return ctx.Send(SectorRetrySealPreCommit1{})
+}
+
+func (m *Sealing) handleSealPrecommit2Failed(ctx statemachine.Context, sector SectorInfo) error {
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	if sector.PreCommit2Fails > 1 {
+		return ctx.Send(SectorRetrySealPreCommit1{})
+	}
+
+	return ctx.Send(SectorRetrySealPreCommit2{})
 }
 
 func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorInfo) error {
@@ -69,9 +76,13 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 			log.Errorf("handlePreCommitFailed: api error, not proceeding: %+v", err)
 			return nil
 		case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too)
-			return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
 		case *ErrExpiredTicket:
-			return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
+		case *ErrBadTicket:
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+		case *ErrPrecommitOnChain:
+			// noop
 		default:
 			return xerrors.Errorf("checkPrecommit sanity check error: %w", err)
 		}
@@ -80,7 +91,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 	if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil {
 		if sector.PreCommitMessage != nil {
 			log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber)
-			return nil // TODO: SeedWait needs this currently
+			return ctx.Send(SectorPreCommitLanded{TipSet: tok})
 		}
 
 		if pci.Info.SealedCID != *sector.CommR {
@@ -116,6 +127,10 @@ func (m *Sealing) handleComputeProofFailed(ctx statemachine.Context, sector Sect
 		return err
 	}
 
+	if sector.InvalidProofs > 1 {
+		return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive compute fails")})
+	}
+
 	return ctx.Send(SectorRetryComputeProof{})
 }
 
@@ -132,9 +147,13 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 			log.Errorf("handleCommitFailed: api error, not proceeding: %+v", err)
 			return nil
 		case *ErrBadCommD:
-			return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
 		case *ErrExpiredTicket:
-			return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
+		case *ErrBadTicket:
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+		case *ErrPrecommitOnChain:
+			// noop, this is expected
 		default:
 			return xerrors.Errorf("checkPrecommit sanity check error: %w", err)
 		}
@@ -154,7 +173,7 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 			}
 
 			if sector.InvalidProofs > 0 {
-				return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("consecutive invalid proofs")})
+				return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive invalid proofs")})
 			}
 
 			return ctx.Send(SectorRetryInvalidProof{})
@@ -171,3 +190,13 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 
 	return ctx.Send(SectorRetryComputeProof{})
 }
+
+func (m *Sealing) handleFinalizeFailed(ctx statemachine.Context, sector SectorInfo) error {
+	// TODO: Check sector files
+
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	return ctx.Send(SectorRetryFinalize{})
+}

types.go

@@ -48,8 +48,7 @@ type Log struct {
 
 type SectorInfo struct {
 	State        SectorState
-	SectorNumber abi.SectorNumber // TODO: this field's name should be changed to SectorNumber
+	SectorNumber abi.SectorNumber
-	Nonce        uint64           // TODO: remove
 
 	SectorType abi.RegisteredProof
 
@@ -69,13 +68,15 @@ type SectorInfo struct {
 	PreCommitMessage *cid.Cid
 	PreCommitTipSet  TipSetToken
 
+	PreCommit2Fails uint64
+
 	// WaitSeed
 	SeedValue abi.InteractiveSealRandomness
 	SeedEpoch abi.ChainEpoch
 
 	// Committing
 	CommitMessage *cid.Cid
-	InvalidProofs uint64 // failed proof computations (doesn't validate with proof inputs)
+	InvalidProofs uint64 // failed proof computations (doesn't validate with proof inputs; can't compute)
 
 	// Faults
 	FaultReportMsg *cid.Cid

types_test.go

@@ -27,7 +27,6 @@ func TestSectorInfoSelialization(t *testing.T) {
 	si := &SectorInfo{
 		State:        "stateful",
 		SectorNumber: 234,
-		Nonce:        345,
 		Pieces: []Piece{{
 			Piece: abi.PieceInfo{
 				Size:     5,
@@ -59,7 +58,6 @@ func TestSectorInfoSelialization(t *testing.T) {
 	}
 
 	assert.Equal(t, si.State, si2.State)
-	assert.Equal(t, si.Nonce, si2.Nonce)
 	assert.Equal(t, si.SectorNumber, si2.SectorNumber)
 
 	assert.Equal(t, si.Pieces, si2.Pieces)