Merge pull request #63 from filecoin-project/fix/bugfixes

Fix a bunch sealing issues

cbor_gen.go

@@ -1350,7 +1350,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 				return err
 			}
 
-			if extra > cbg.MaxLength {
+			if extra > cbg.MaxLength+1 { // +1 placed here to recover broken state machines in calibration net; feel free to drop
 				return fmt.Errorf("t.Log: array too large (%d)", extra)
 			}
 

checks.go

@@ -50,7 +50,7 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
 
 		proposal, err := api.StateMarketStorageDeal(ctx, p.DealInfo.DealID, tok)
 		if err != nil {
-			return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)}
+			return &ErrInvalidDeals{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)}
 		}
 
 		if proposal.PieceCID != p.Piece.PieceCID {
@@ -92,9 +92,9 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t
 
 	if pci != nil {
 		if pci.Info.SealRandEpoch != si.TicketEpoch {
-			return &ErrBadTicket{}
+			return &ErrBadTicket{xerrors.Errorf("bad ticket epoch: %d != %d", pci.Info.SealRandEpoch, si.TicketEpoch)}
 		}
-		return &ErrPrecommitOnChain{}
+		return &ErrPrecommitOnChain{xerrors.Errorf("precommit already on chain")}
 	}
 
 	return nil

fsm.go

@@ -104,6 +104,8 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorRetryComputeProof{}, Committing),
 		on(SectorRetryInvalidProof{}, Committing),
 		on(SectorRetryPreCommitWait{}, PreCommitWait),
+		on(SectorChainPreCommitFailed{}, PreCommitFailed),
+		on(SectorRetryPreCommit{}, PreCommitting),
 	),
 	FinalizeFailed: planOne(
 		on(SectorRetryFinalize{}, FinalizeSector),
@@ -149,6 +151,17 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 			l.Trace = fmt.Sprintf("%+v", err)
 		}
 
+		if len(state.Log) > 8000 {
+			log.Warnw("truncating sector log", "sector", state.SectorNumber)
+			state.Log[2000] = Log{
+				Timestamp: uint64(time.Now().Unix()),
+				Message:   "truncating log (above 8000 entries)",
+				Kind:      fmt.Sprintf("truncate"),
+			}
+
+			state.Log = append(state.Log[:2000], state.Log[:6000]...)
+		}
+
 		state.Log = append(state.Log, l)
 	}
 

states_failed.go

@@ -81,6 +81,8 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
 		case *ErrBadTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+		case *ErrNoPrecommit:
+			return ctx.Send(SectorRetryPreCommit{})
 		case *ErrPrecommitOnChain:
 			// noop
 		default:
@@ -152,10 +154,12 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
 		case *ErrBadTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+		case nil:
+			return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("no precommit: %w", err)})
 		case *ErrPrecommitOnChain:
 			// noop, this is expected
 		default:
-			return xerrors.Errorf("checkPrecommit sanity check error: %w", err)
+			return xerrors.Errorf("checkPrecommit sanity check error (%T): %w", err, err)
 		}
 	}
 
@@ -180,8 +184,10 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 		case *ErrPrecommitOnChain:
 			log.Errorf("no precommit on chain, will retry: %+v", err)
 			return ctx.Send(SectorRetryPreCommitWait{})
+		case *ErrNoPrecommit:
+			return ctx.Send(SectorRetryPreCommit{})
 		default:
-			return xerrors.Errorf("checkCommit sanity check error: %w", err)
+			return xerrors.Errorf("checkCommit sanity check error (%T): %w", err, err)
 		}
 	}
 

states_sealing.go

@@ -183,7 +183,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 
 	deposit := big.Max(depositMinimum, collateral)
 
-	log.Infof("submitting precommit for sector (deposit: %s): ", sector.SectorNumber, deposit)
+	log.Infof("submitting precommit for sector %d (deposit: %s): ", sector.SectorNumber, deposit)
 	mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.PreCommitSector, deposit, big.NewInt(0), 0, enc.Bytes())
 	if err != nil {
 		return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)})
@@ -215,7 +215,13 @@ func (m *Sealing) handlePreCommitWait(ctx statemachine.Context, sector SectorInf
 }
 
 func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) error {
-	pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, sector.PreCommitTipSet)
+	tok, _, err := m.api.ChainHead(ctx.Context())
+	if err != nil {
+		log.Errorf("handleCommitting: api error, not proceeding: %+v", err)
+		return nil
+	}
+
+	pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
 	if err != nil {
 		return xerrors.Errorf("getting precommit info: %w", err)
 	}
@@ -225,7 +231,15 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er
 
 	randHeight := pci.PreCommitEpoch + miner.PreCommitChallengeDelay
 
-	err = m.events.ChainAt(func(ectx context.Context, tok TipSetToken, curH abi.ChainEpoch) error {
+	err = m.events.ChainAt(func(ectx context.Context, _ TipSetToken, curH abi.ChainEpoch) error {
+		// in case of null blocks the randomness can land after the tipset we
+		// get from the events API
+		tok, _, err := m.api.ChainHead(ctx.Context())
+		if err != nil {
+			log.Errorf("handleCommitting: api error, not proceeding: %+v", err)
+			return nil
+		}
+
 		buf := new(bytes.Buffer)
 		if err := m.maddr.MarshalCBOR(buf); err != nil {
 			return err
@@ -305,7 +319,7 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo)
 		return xerrors.Errorf("getting initial pledge collateral: %w", err)
 	}
 
-	pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, sector.PreCommitTipSet)
+	pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
 	if err != nil {
 		return xerrors.Errorf("getting precommit info: %w", err)
 	}