From 9d9ea4e6331f4793e81698aa29f596277498870a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 5 Aug 2020 03:30:58 +0200 Subject: [PATCH] Fix a bunch sealing issues --- cbor_gen.go | 2 +- checks.go | 6 +++--- fsm.go | 13 +++++++++++++ states_failed.go | 10 ++++++++-- states_sealing.go | 22 ++++++++++++++++++---- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/cbor_gen.go b/cbor_gen.go index 42a54c103..9410e6f28 100644 --- a/cbor_gen.go +++ b/cbor_gen.go @@ -1350,7 +1350,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return err } - if extra > cbg.MaxLength { + if extra > cbg.MaxLength+1 { // +1 placed here to recover broken state machines in calibration net; feel free to drop return fmt.Errorf("t.Log: array too large (%d)", extra) } diff --git a/checks.go b/checks.go index 3ce3e2796..bfc292202 100644 --- a/checks.go +++ b/checks.go @@ -50,7 +50,7 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { proposal, err := api.StateMarketStorageDeal(ctx, p.DealInfo.DealID, tok) if err != nil { - return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)} + return &ErrInvalidDeals{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)} } if proposal.PieceCID != p.Piece.PieceCID { @@ -92,9 +92,9 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t if pci != nil { if pci.Info.SealRandEpoch != si.TicketEpoch { - return &ErrBadTicket{} + return &ErrBadTicket{xerrors.Errorf("bad ticket epoch: %d != %d", pci.Info.SealRandEpoch, si.TicketEpoch)} } - return &ErrPrecommitOnChain{} + return &ErrPrecommitOnChain{xerrors.Errorf("precommit already on chain")} } return nil diff --git a/fsm.go b/fsm.go index 7c0091113..25b8f364a 100644 --- a/fsm.go +++ b/fsm.go @@ -104,6 +104,8 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorRetryComputeProof{}, Committing), on(SectorRetryInvalidProof{}, Committing), on(SectorRetryPreCommitWait{}, PreCommitWait), + on(SectorChainPreCommitFailed{}, PreCommitFailed), + on(SectorRetryPreCommit{}, PreCommitting), ), FinalizeFailed: planOne( on(SectorRetryFinalize{}, FinalizeSector), @@ -149,6 +151,17 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta l.Trace = fmt.Sprintf("%+v", err) } + if len(state.Log) > 8000 { + log.Warnw("truncating sector log", "sector", state.SectorNumber) + state.Log[2000] = Log{ + Timestamp: uint64(time.Now().Unix()), + Message: "truncating log (above 8000 entries)", + Kind: fmt.Sprintf("truncate"), + } + + state.Log = append(state.Log[:2000], state.Log[:6000]...) + } + state.Log = append(state.Log, l) } diff --git a/states_failed.go b/states_failed.go index 6c2eb11bc..e208a8cca 100644 --- a/states_failed.go +++ b/states_failed.go @@ -81,6 +81,8 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)}) case *ErrBadTicket: return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)}) + case *ErrNoPrecommit: + return ctx.Send(SectorRetryPreCommit{}) case *ErrPrecommitOnChain: // noop default: @@ -152,10 +154,12 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)}) case *ErrBadTicket: return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)}) + case nil: + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("no precommit: %w", err)}) case *ErrPrecommitOnChain: // noop, this is expected default: - return xerrors.Errorf("checkPrecommit sanity check error: %w", err) + return xerrors.Errorf("checkPrecommit sanity check error (%T): %w", err, err) } } @@ -180,8 +184,10 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo case *ErrPrecommitOnChain: log.Errorf("no precommit on chain, will retry: %+v", err) return ctx.Send(SectorRetryPreCommitWait{}) + case *ErrNoPrecommit: + return ctx.Send(SectorRetryPreCommit{}) default: - return xerrors.Errorf("checkCommit sanity check error: %w", err) + return xerrors.Errorf("checkCommit sanity check error (%T): %w", err, err) } } diff --git a/states_sealing.go b/states_sealing.go index fb9ac3b65..11e8a39a5 100644 --- a/states_sealing.go +++ b/states_sealing.go @@ -183,7 +183,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf deposit := big.Max(depositMinimum, collateral) - log.Infof("submitting precommit for sector (deposit: %s): ", sector.SectorNumber, deposit) + log.Infof("submitting precommit for sector %d (deposit: %s): ", sector.SectorNumber, deposit) mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.PreCommitSector, deposit, big.NewInt(0), 0, enc.Bytes()) if err != nil { return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)}) @@ -215,7 +215,13 @@ func (m *Sealing) handlePreCommitWait(ctx statemachine.Context, sector SectorInf } func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) error { - pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, sector.PreCommitTipSet) + tok, _, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + + pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) if err != nil { return xerrors.Errorf("getting precommit info: %w", err) } @@ -225,7 +231,15 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er randHeight := pci.PreCommitEpoch + miner.PreCommitChallengeDelay - err = m.events.ChainAt(func(ectx context.Context, tok TipSetToken, curH abi.ChainEpoch) error { + err = m.events.ChainAt(func(ectx context.Context, _ TipSetToken, curH abi.ChainEpoch) error { + // in case of null blocks the randomness can land after the tipset we + // get from the events API + tok, _, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + buf := new(bytes.Buffer) if err := m.maddr.MarshalCBOR(buf); err != nil { return err @@ -305,7 +319,7 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo) return xerrors.Errorf("getting initial pledge collateral: %w", err) } - pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, sector.PreCommitTipSet) + pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) if err != nil { return xerrors.Errorf("getting precommit info: %w", err) }