From 9b92e145e71709c82189356bbbbf659893cd1b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 2 Jun 2020 22:30:40 +0200 Subject: [PATCH 1/6] Try to re-precommit on consecutive failed commit proof compute --- cbor_gen.go | 76 ++++++++++++++++++++++++++++-------------------- fsm.go | 1 + fsm_events.go | 4 ++- states.go | 4 +-- states_failed.go | 4 +++ types.go | 5 ++-- 6 files changed, 57 insertions(+), 37 deletions(-) diff --git a/cbor_gen.go b/cbor_gen.go index b2bd03971..1b05f00e0 100644 --- a/cbor_gen.go +++ b/cbor_gen.go @@ -425,22 +425,6 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.Nonce (uint64) (uint64) - if len("Nonce") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Nonce\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Nonce")))); err != nil { - return err - } - if _, err := w.Write([]byte("Nonce")); err != nil { - return err - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { - return err - } - // t.SectorType (abi.RegisteredProof) (int64) if len("SectorType") > cbg.MaxLength { return xerrors.Errorf("Value in field \"SectorType\" was too long") @@ -645,6 +629,29 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } + // t.PreCommitTipSet (sealing.TipSetToken) (slice) + if len("PreCommitTipSet") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitTipSet\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitTipSet")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommitTipSet")); err != nil { + return err + } + + if len(t.PreCommitTipSet) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.PreCommitTipSet was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PreCommitTipSet)))); err != nil { + return err + } + if _, err := w.Write(t.PreCommitTipSet); err != nil { + return err + } + // t.SeedValue (abi.InteractiveSealRandomness) (slice) if len("SeedValue") > cbg.MaxLength { return xerrors.Errorf("Value in field \"SeedValue\" was too long") @@ -855,21 +862,6 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } t.SectorNumber = abi.SectorNumber(extra) - } - // t.Nonce (uint64) (uint64) - case "Nonce": - - { - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) - } // t.SectorType (abi.RegisteredProof) (int64) case "SectorType": @@ -912,9 +904,11 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajArray { return fmt.Errorf("expected cbor array") } + if extra > 0 { t.Pieces = make([]Piece, extra) } + for i := 0; i < int(extra); i++ { var v Piece @@ -1080,6 +1074,24 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } } + // t.PreCommitTipSet (sealing.TipSetToken) (slice) + case "PreCommitTipSet": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.PreCommitTipSet: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.PreCommitTipSet = make([]byte, extra) + if _, err := io.ReadFull(br, t.PreCommitTipSet); err != nil { + return err + } // t.SeedValue (abi.InteractiveSealRandomness) (slice) case "SeedValue": @@ -1215,9 +1227,11 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajArray { return fmt.Errorf("expected cbor array") } + if extra > 0 { t.Log = make([]Log, extra) } + for i := 0; i < int(extra); i++ { var v Log diff --git a/fsm.go b/fsm.go index e99f452ff..5f3f25c87 100644 --- a/fsm.go +++ b/fsm.go @@ -82,6 +82,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto ), ComputeProofFailed: planOne( on(SectorRetryComputeProof{}, Committing), + on(SectorSealPreCommitFailed{}, SealFailed), ), CommitFailed: planOne( on(SectorSealPreCommitFailed{}, SealFailed), diff --git a/fsm_events.go b/fsm_events.go index a9d2de87f..76106e0cd 100644 --- a/fsm_events.go +++ b/fsm_events.go @@ -185,7 +185,9 @@ func (evt SectorRetryWaitSeed) apply(state *SectorInfo) {} type SectorRetryComputeProof struct{} -func (evt SectorRetryComputeProof) apply(state *SectorInfo) {} +func (evt SectorRetryComputeProof) apply(state *SectorInfo) { + state.InvalidProofs++ +} type SectorRetryInvalidProof struct{} diff --git a/states.go b/states.go index 7ef6d32a1..04f4f1a40 100644 --- a/states.go +++ b/states.go @@ -224,12 +224,12 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo) } c2in, err := m.sealer.SealCommit1(ctx.Context(), m.minerSector(sector.SectorNumber), sector.TicketValue, sector.SeedValue, sector.pieceInfos(), cids) if err != nil { - return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)}) + return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(1): %w", err)}) } proof, err := m.sealer.SealCommit2(ctx.Context(), m.minerSector(sector.SectorNumber), c2in) if err != nil { - return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)}) + return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(2): %w", err)}) } tok, _, err := m.api.ChainHead(ctx.Context()) diff --git a/states_failed.go b/states_failed.go index 73212d1f5..f5a496661 100644 --- a/states_failed.go +++ b/states_failed.go @@ -116,6 +116,10 @@ func (m *Sealing) handleComputeProofFailed(ctx statemachine.Context, sector Sect return err } + if sector.InvalidProofs > 1 { + return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("consecutive compute fails")}) + } + return ctx.Send(SectorRetryComputeProof{}) } diff --git a/types.go b/types.go index 7f60d0a70..08f55f572 100644 --- a/types.go +++ b/types.go @@ -48,8 +48,7 @@ type Log struct { type SectorInfo struct { State SectorState - SectorNumber abi.SectorNumber // TODO: this field's name should be changed to SectorNumber - Nonce uint64 // TODO: remove + SectorNumber abi.SectorNumber SectorType abi.RegisteredProof @@ -75,7 +74,7 @@ type SectorInfo struct { // Committing CommitMessage *cid.Cid - InvalidProofs uint64 // failed proof computations (doesn't validate with proof inputs) + InvalidProofs uint64 // failed proof computations (doesn't validate with proof inputs; can't compute) // Faults FaultReportMsg *cid.Cid From 222f3d876ca1db9edd9dfff3c5b5d4ea930e71fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 2 Jun 2020 23:45:28 +0200 Subject: [PATCH 2/6] Fix handling of re-precommit --- checks.go | 14 ++++++++++++++ fsm.go | 2 ++ states.go | 42 ++++++++++++++++++++++++++++++++---------- states_failed.go | 10 +++++++++- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/checks.go b/checks.go index 4a6aa79fb..b967ea081 100644 --- a/checks.go +++ b/checks.go @@ -24,6 +24,8 @@ type ErrExpiredDeals struct{ error } type ErrBadCommD struct{ error } type ErrExpiredTicket struct{ error } +type ErrBadTicket struct{ error } +type ErrPrecommitOnChain struct{ error } type ErrBadSeed struct{ error } type ErrInvalidProof struct{ error } @@ -82,6 +84,18 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+SealRandomnessLookback, height)} } + pci, err := api.StateSectorPreCommitInfo(ctx, maddr, si.SectorNumber, tok) + if err != nil { + return &ErrApi{xerrors.Errorf("getting precommit info: %w", err)} + } + + if pci != nil { + if pci.Info.SealRandEpoch != si.TicketEpoch { + return &ErrBadTicket{} + } + return &ErrPrecommitOnChain{} + } + return nil } diff --git a/fsm.go b/fsm.go index 5f3f25c87..3f946832d 100644 --- a/fsm.go +++ b/fsm.go @@ -48,6 +48,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorSealPreCommitFailed{}, SealFailed), on(SectorPreCommitted{}, PreCommitWait), on(SectorChainPreCommitFailed{}, PreCommitFailed), + on(SectorPreCommitLanded{}, WaitSeed), ), PreCommitWait: planOne( on(SectorChainPreCommitFailed{}, PreCommitFailed), @@ -79,6 +80,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorRetryPreCommit{}, PreCommitting), on(SectorRetryWaitSeed{}, WaitSeed), on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorPreCommitLanded{}, WaitSeed), ), ComputeProofFailed: planOne( on(SectorRetryComputeProof{}, Committing), diff --git a/states.go b/states.go index 04f4f1a40..af27c8b31 100644 --- a/states.go +++ b/states.go @@ -46,13 +46,37 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err return ctx.Send(SectorPacked{FillerPieces: fillerPieces}) } -func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error { +func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.SealRandomness, abi.ChainEpoch, error) { tok, epoch, err := m.api.ChainHead(ctx.Context()) if err != nil { log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err) - return nil + return nil, 0, nil } + ticketEpoch := epoch - miner.ChainFinalityish + buf := new(bytes.Buffer) + if err := m.maddr.MarshalCBOR(buf); err != nil { + return nil, 0, err + } + + pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) + if err != nil { + return nil, 0, xerrors.Errorf("getting precommit info: %w", err) + } + + if pci != nil { + ticketEpoch = pci.Info.SealRandEpoch + } + + rand, err := m.api.ChainGetRandomness(ctx.Context(), tok, crypto.DomainSeparationTag_SealRandomness, ticketEpoch, buf.Bytes()) + if err != nil { + return nil, 0, err + } + + return abi.SealRandomness(rand), ticketEpoch, nil +} + +func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error { if err := checkPieces(ctx.Context(), sector, m.api); err != nil { // Sanity check state switch err.(type) { case *ErrApi: @@ -68,16 +92,10 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) } log.Infow("performing sector replication...", "sector", sector.SectorNumber) - ticketEpoch := epoch - miner.ChainFinalityish - buf := new(bytes.Buffer) - if err := m.maddr.MarshalCBOR(buf); err != nil { - return err - } - rand, err := m.api.ChainGetRandomness(ctx.Context(), tok, crypto.DomainSeparationTag_SealRandomness, ticketEpoch, buf.Bytes()) + ticketValue, ticketEpoch, err := m.getTicket(ctx, sector) if err != nil { return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("getting ticket failed: %w", err)}) } - ticketValue := abi.SealRandomness(rand) pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), m.minerSector(sector.SectorNumber), ticketValue, sector.pieceInfos()) if err != nil { @@ -117,7 +135,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf } if err := checkPrecommit(ctx.Context(), m.Address(), sector, tok, height, m.api); err != nil { - switch err.(type) { + switch err := err.(type) { case *ErrApi: log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err) return nil @@ -125,6 +143,10 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)}) case *ErrExpiredTicket: return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired: %w", err)}) + case *ErrBadTicket: + return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)}) + case *ErrPrecommitOnChain: + return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit default: return xerrors.Errorf("checkPrecommit sanity check error: %w", err) } diff --git a/states_failed.go b/states_failed.go index f5a496661..bf48f9898 100644 --- a/states_failed.go +++ b/states_failed.go @@ -72,6 +72,10 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)}) case *ErrExpiredTicket: return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)}) + case *ErrBadTicket: + return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)}) + case *ErrPrecommitOnChain: + // noop default: return xerrors.Errorf("checkPrecommit sanity check error: %w", err) } @@ -80,7 +84,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil { if sector.PreCommitMessage != nil { log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber) - return nil // TODO: SeedWait needs this currently + return ctx.Send(SectorPreCommitLanded{TipSet: tok}) } if pci.Info.SealedCID != *sector.CommR { @@ -139,6 +143,10 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)}) case *ErrExpiredTicket: return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)}) + case *ErrBadTicket: + return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)}) + case *ErrPrecommitOnChain: + // noop, this is expected default: return xerrors.Errorf("checkPrecommit sanity check error: %w", err) } From 1467fe42268580c41fcd094aa3415985f17d7898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Jun 2020 12:50:56 +0200 Subject: [PATCH 3/6] Fix tests --- types_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/types_test.go b/types_test.go index 9bb1df8cf..c11cc66b7 100644 --- a/types_test.go +++ b/types_test.go @@ -27,7 +27,6 @@ func TestSectorInfoSelialization(t *testing.T) { si := &SectorInfo{ State: "stateful", SectorNumber: 234, - Nonce: 345, Pieces: []Piece{{ Piece: abi.PieceInfo{ Size: 5, @@ -59,7 +58,6 @@ func TestSectorInfoSelialization(t *testing.T) { } assert.Equal(t, si.State, si2.State) - assert.Equal(t, si.Nonce, si2.Nonce) assert.Equal(t, si.SectorNumber, si2.SectorNumber) assert.Equal(t, si.Pieces, si2.Pieces) From 7fc72213483ab909d6b9f0530a70b87a3bd1f40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Jun 2020 14:14:06 +0200 Subject: [PATCH 4/6] Use correct SealRandomnessLookbackLimit --- checks.go | 2 +- constants.go | 12 ++++++++---- states.go | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/checks.go b/checks.go index b967ea081..3d63ecdee 100644 --- a/checks.go +++ b/checks.go @@ -80,7 +80,7 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)} } - if int64(height)-int64(si.TicketEpoch+SealRandomnessLookback) > SealRandomnessLookbackLimit { + if height-si.TicketEpoch+SealRandomnessLookback > SealRandomnessLookbackLimit(si.SectorType) { return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+SealRandomnessLookback, height)} } diff --git a/constants.go b/constants.go index b898443e6..56c72fde5 100644 --- a/constants.go +++ b/constants.go @@ -1,13 +1,17 @@ package sealing -// Epochs -const Finality = 500 +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) // Epochs -const SealRandomnessLookback = Finality +const SealRandomnessLookback = miner.ChainFinalityish // Epochs -const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 +func SealRandomnessLookbackLimit(spt abi.RegisteredProof) abi.ChainEpoch { + return miner.MaxSealDuration[spt] +} // Epochs const InteractivePoRepConfidence = 6 diff --git a/states.go b/states.go index af27c8b31..7f39de9b6 100644 --- a/states.go +++ b/states.go @@ -53,7 +53,7 @@ func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.Se return nil, 0, nil } - ticketEpoch := epoch - miner.ChainFinalityish + ticketEpoch := epoch - SealRandomnessLookback buf := new(bytes.Buffer) if err := m.maddr.MarshalCBOR(buf); err != nil { return nil, 0, err From 1f5aa7fb7d3339d28581b037e32c113b8f3c5c38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Jun 2020 23:42:13 +0200 Subject: [PATCH 5/6] Auto-retry failed finalize --- fsm.go | 6 ++++++ fsm_events.go | 4 ++++ sector_state.go | 8 +++++--- states_failed.go | 10 ++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/fsm.go b/fsm.go index 3f946832d..ca143208e 100644 --- a/fsm.go +++ b/fsm.go @@ -66,6 +66,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto FinalizeSector: planOne( on(SectorFinalized{}, Proving), + on(SectorFinalizeFailed{}, FinalizeFailed), ), Proving: planOne( @@ -92,6 +93,9 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorRetryComputeProof{}, Committing), on(SectorRetryInvalidProof{}, Committing), ), + FinalizeFailed: planOne( + on(SectorRetryFinalize{}, FinalizeSector), + ), Faulty: planOne( on(SectorFaultReported{}, FaultReported), @@ -207,6 +211,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta return m.handleComputeProofFailed, nil case CommitFailed: return m.handleCommitFailed, nil + case FinalizeFailed: + return m.handleFinalizeFailed, nil // Faults case Faulty: diff --git a/fsm_events.go b/fsm_events.go index 76106e0cd..9522e9986 100644 --- a/fsm_events.go +++ b/fsm_events.go @@ -164,6 +164,10 @@ type SectorFinalized struct{} func (evt SectorFinalized) apply(*SectorInfo) {} +type SectorRetryFinalize struct{} + +func (evt SectorRetryFinalize) apply(*SectorInfo) {} + type SectorFinalizeFailed struct{ error } func (evt SectorFinalizeFailed) FormatError(xerrors.Printer) (next error) { return evt.error } diff --git a/sector_state.go b/sector_state.go index b584ef24a..d9fc085f6 100644 --- a/sector_state.go +++ b/sector_state.go @@ -24,7 +24,9 @@ const ( ComputeProofFailed SectorState = "ComputeProofFailed" CommitFailed SectorState = "CommitFailed" PackingFailed SectorState = "PackingFailed" - Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason - FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain - FaultedFinal SectorState = "FaultedFinal" // fault declared on chain + FinalizeFailed SectorState = "FinalizeFailed" + + Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason + FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain + FaultedFinal SectorState = "FaultedFinal" // fault declared on chain ) diff --git a/states_failed.go b/states_failed.go index bf48f9898..772db0f62 100644 --- a/states_failed.go +++ b/states_failed.go @@ -183,3 +183,13 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo return ctx.Send(SectorRetryComputeProof{}) } + +func (m *Sealing) handleFinalizeFailed(ctx statemachine.Context, sector SectorInfo) error { + // TODO: Check sector files + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorRetryFinalize{}) +} From 8ad1af106d628b119ae2595ffa1a88ad74eccafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 4 Jun 2020 17:29:31 +0200 Subject: [PATCH 6/6] Retry precommit2 once before redoing precommit1 --- cbor_gen.go | 33 ++++++++++++++++++++++++++++++++- fsm.go | 44 +++++++++++++++++++++++++++----------------- fsm_events.go | 24 +++++++++++++++++++----- sector_state.go | 15 ++++++++------- states.go | 14 +++++++------- states_failed.go | 41 ++++++++++++++++++++++++----------------- types.go | 2 ++ 7 files changed, 119 insertions(+), 54 deletions(-) diff --git a/cbor_gen.go b/cbor_gen.go index 1b05f00e0..1948e39aa 100644 --- a/cbor_gen.go +++ b/cbor_gen.go @@ -382,7 +382,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{179}); err != nil { + if _, err := w.Write([]byte{180}); err != nil { return err } @@ -652,6 +652,22 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + // t.PreCommit2Fails (uint64) (uint64) + if len("PreCommit2Fails") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommit2Fails\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommit2Fails")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommit2Fails")); err != nil { + return err + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PreCommit2Fails))); err != nil { + return err + } + // t.SeedValue (abi.InteractiveSealRandomness) (slice) if len("SeedValue") > cbg.MaxLength { return xerrors.Errorf("Value in field \"SeedValue\" was too long") @@ -1092,6 +1108,21 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.PreCommitTipSet); err != nil { return err } + // t.PreCommit2Fails (uint64) (uint64) + case "PreCommit2Fails": + + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PreCommit2Fails = uint64(extra) + + } // t.SeedValue (abi.InteractiveSealRandomness) (slice) case "SeedValue": diff --git a/fsm.go b/fsm.go index ca143208e..40b017a1f 100644 --- a/fsm.go +++ b/fsm.go @@ -36,16 +36,16 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto Packing: planOne(on(SectorPacked{}, PreCommit1)), PreCommit1: planOne( on(SectorPreCommit1{}, PreCommit2), - on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), on(SectorPackingFailed{}, PackingFailed), ), PreCommit2: planOne( on(SectorPreCommit2{}, PreCommitting), - on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorSealPreCommit2Failed{}, SealPreCommit2Failed), on(SectorPackingFailed{}, PackingFailed), ), PreCommitting: planOne( - on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), on(SectorPreCommitted{}, PreCommitWait), on(SectorChainPreCommitFailed{}, PreCommitFailed), on(SectorPreCommitLanded{}, WaitSeed), @@ -74,21 +74,25 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorFaulty{}, Faulty), ), - SealFailed: planOne( - on(SectorRetrySeal{}, PreCommit1), + SealPreCommit1Failed: planOne( + on(SectorRetrySealPreCommit1{}, PreCommit1), + ), + SealPreCommit2Failed: planOne( + on(SectorRetrySealPreCommit1{}, PreCommit1), + on(SectorRetrySealPreCommit2{}, PreCommit2), ), PreCommitFailed: planOne( on(SectorRetryPreCommit{}, PreCommitting), on(SectorRetryWaitSeed{}, WaitSeed), - on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), on(SectorPreCommitLanded{}, WaitSeed), ), ComputeProofFailed: planOne( on(SectorRetryComputeProof{}, Committing), - on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), ), CommitFailed: planOne( - on(SectorSealPreCommitFailed{}, SealFailed), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), on(SectorRetryWaitSeed{}, WaitSeed), on(SectorRetryComputeProof{}, Committing), on(SectorRetryInvalidProof{}, Committing), @@ -147,10 +151,11 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta *<- Packing <- incoming | | | v - *<- PreCommit1 <--> SealFailed - | | ^^^ - | v ||| - *<- PreCommit2 -------/|| + *<- PreCommit1 <--> SealPreCommit1Failed + | | ^ ^^ + | | *----------++----\ + | v v || | + *<- PreCommit2 --------++--> SealPreCommit2Failed | | || | v /-------/| * PreCommitting <-----+---> PreCommitFailed @@ -167,14 +172,17 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta *<- CommitWait ---/ | | | v + | FinalizeSector <--> FinalizeFailed + | | + | v *<- Proving | v FailedUnrecoverable UndefinedSectorState <- ¯\_(ツ)_/¯ - | ^ - *---------------------/ + | ^ + *---------------------/ */ @@ -203,8 +211,10 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta log.Infof("Proving sector %d", state.SectorNumber) // Handled failure modes - case SealFailed: - return m.handleSealFailed, nil + case SealPreCommit1Failed: + return m.handleSealPrecommit1Failed, nil + case SealPreCommit2Failed: + return m.handleSealPrecommit2Failed, nil case PreCommitFailed: return m.handlePreCommitFailed, nil case ComputeProofFailed: @@ -253,7 +263,7 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error { return nil case SectorComputeProofFailed: state.State = ComputeProofFailed - case SectorSealPreCommitFailed: + case SectorSealPreCommit1Failed: state.State = CommitFailed case SectorCommitFailed: state.State = CommitFailed diff --git a/fsm_events.go b/fsm_events.go index 9522e9986..7466daec0 100644 --- a/fsm_events.go +++ b/fsm_events.go @@ -84,6 +84,7 @@ func (evt SectorPreCommit1) apply(state *SectorInfo) { state.PreCommit1Out = evt.PreCommit1Out state.TicketEpoch = evt.TicketEpoch state.TicketValue = evt.TicketValue + state.PreCommit2Fails = 0 } type SectorPreCommit2 struct { @@ -106,11 +107,20 @@ func (evt SectorPreCommitLanded) apply(si *SectorInfo) { si.PreCommitTipSet = evt.TipSet } -type SectorSealPreCommitFailed struct{ error } +type SectorSealPreCommit1Failed struct{ error } -func (evt SectorSealPreCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error } -func (evt SectorSealPreCommitFailed) apply(si *SectorInfo) { +func (evt SectorSealPreCommit1Failed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorSealPreCommit1Failed) apply(si *SectorInfo) { si.InvalidProofs = 0 // reset counter + si.PreCommit2Fails = 0 +} + +type SectorSealPreCommit2Failed struct{ error } + +func (evt SectorSealPreCommit2Failed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorSealPreCommit2Failed) apply(si *SectorInfo) { + si.InvalidProofs = 0 // reset counter + si.PreCommit2Fails++ } type SectorChainPreCommitFailed struct{ error } @@ -175,9 +185,13 @@ func (evt SectorFinalizeFailed) apply(*SectorInfo) {} // Failed state recovery -type SectorRetrySeal struct{} +type SectorRetrySealPreCommit1 struct{} -func (evt SectorRetrySeal) apply(state *SectorInfo) {} +func (evt SectorRetrySealPreCommit1) apply(state *SectorInfo) {} + +type SectorRetrySealPreCommit2 struct{} + +func (evt SectorRetrySealPreCommit2) apply(state *SectorInfo) {} type SectorRetryPreCommit struct{} diff --git a/sector_state.go b/sector_state.go index d9fc085f6..27eebcdad 100644 --- a/sector_state.go +++ b/sector_state.go @@ -18,13 +18,14 @@ const ( FinalizeSector SectorState = "FinalizeSector" Proving SectorState = "Proving" // error modes - FailedUnrecoverable SectorState = "FailedUnrecoverable" - SealFailed SectorState = "SealFailed" - PreCommitFailed SectorState = "PreCommitFailed" - ComputeProofFailed SectorState = "ComputeProofFailed" - CommitFailed SectorState = "CommitFailed" - PackingFailed SectorState = "PackingFailed" - FinalizeFailed SectorState = "FinalizeFailed" + FailedUnrecoverable SectorState = "FailedUnrecoverable" + SealPreCommit1Failed SectorState = "SealPreCommit1Failed" + SealPreCommit2Failed SectorState = "SealPreCommit2Failed" + PreCommitFailed SectorState = "PreCommitFailed" + ComputeProofFailed SectorState = "ComputeProofFailed" + CommitFailed SectorState = "CommitFailed" + PackingFailed SectorState = "PackingFailed" + FinalizeFailed SectorState = "FinalizeFailed" Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain diff --git a/states.go b/states.go index 7f39de9b6..e5a113513 100644 --- a/states.go +++ b/states.go @@ -94,12 +94,12 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) log.Infow("performing sector replication...", "sector", sector.SectorNumber) ticketValue, ticketEpoch, err := m.getTicket(ctx, sector) if err != nil { - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("getting ticket failed: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("getting ticket failed: %w", err)}) } pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), m.minerSector(sector.SectorNumber), ticketValue, sector.pieceInfos()) if err != nil { - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("seal pre commit(1) failed: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("seal pre commit(1) failed: %w", err)}) } return ctx.Send(SectorPreCommit1{ @@ -112,7 +112,7 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) func (m *Sealing) handlePreCommit2(ctx statemachine.Context, sector SectorInfo) error { cids, err := m.sealer.SealPreCommit2(ctx.Context(), m.minerSector(sector.SectorNumber), sector.PreCommit1Out) if err != nil { - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("seal pre commit(2) failed: %w", err)}) + return ctx.Send(SectorSealPreCommit2Failed{xerrors.Errorf("seal pre commit(2) failed: %w", err)}) } return ctx.Send(SectorPreCommit2{ @@ -140,11 +140,11 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err) return nil case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too) - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)}) case *ErrExpiredTicket: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired: %w", err)}) case *ErrBadTicket: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)}) case *ErrPrecommitOnChain: return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit default: @@ -154,7 +154,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf expiration, err := m.pcp.Expiration(ctx.Context(), sector.Pieces...) if err != nil { - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)}) } params := &miner.SectorPreCommitInfo{ diff --git a/states_failed.go b/states_failed.go index 772db0f62..160c9e74f 100644 --- a/states_failed.go +++ b/states_failed.go @@ -30,30 +30,37 @@ func failedCooldown(ctx statemachine.Context, sector SectorInfo) error { func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) (*miner.SectorPreCommitOnChainInfo, bool) { tok, _, err := m.api.ChainHead(ctx.Context()) if err != nil { - log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorNumber, err) + log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) return nil, true } info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) if err != nil { - log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorNumber, err) + log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) return nil, true } return info, false } -func (m *Sealing) handleSealFailed(ctx statemachine.Context, sector SectorInfo) error { - if _, is := m.checkPreCommitted(ctx, sector); is { - // TODO: Remove this after we can re-precommit - return nil // noop, for now - } - +func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error { if err := failedCooldown(ctx, sector); err != nil { return err } - return ctx.Send(SectorRetrySeal{}) + return ctx.Send(SectorRetrySealPreCommit1{}) +} + +func (m *Sealing) handleSealPrecommit2Failed(ctx statemachine.Context, sector SectorInfo) error { + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + if sector.PreCommit2Fails > 1 { + return ctx.Send(SectorRetrySealPreCommit1{}) + } + + return ctx.Send(SectorRetrySealPreCommit2{}) } func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorInfo) error { @@ -69,11 +76,11 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI log.Errorf("handlePreCommitFailed: api error, not proceeding: %+v", err) return nil case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too) - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)}) case *ErrExpiredTicket: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)}) case *ErrBadTicket: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)}) case *ErrPrecommitOnChain: // noop default: @@ -121,7 +128,7 @@ func (m *Sealing) handleComputeProofFailed(ctx statemachine.Context, sector Sect } if sector.InvalidProofs > 1 { - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("consecutive compute fails")}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive compute fails")}) } return ctx.Send(SectorRetryComputeProof{}) @@ -140,11 +147,11 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo log.Errorf("handleCommitFailed: api error, not proceeding: %+v", err) return nil case *ErrBadCommD: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)}) case *ErrExpiredTicket: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)}) case *ErrBadTicket: - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)}) case *ErrPrecommitOnChain: // noop, this is expected default: @@ -166,7 +173,7 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo } if sector.InvalidProofs > 0 { - return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("consecutive invalid proofs")}) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive invalid proofs")}) } return ctx.Send(SectorRetryInvalidProof{}) diff --git a/types.go b/types.go index 08f55f572..1cfe37ef4 100644 --- a/types.go +++ b/types.go @@ -68,6 +68,8 @@ type SectorInfo struct { PreCommitMessage *cid.Cid PreCommitTipSet TipSetToken + PreCommit2Fails uint64 + // WaitSeed SeedValue abi.InteractiveSealRandomness SeedEpoch abi.ChainEpoch