cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Retry precommit2 once before redoing precommit1

Browse Source
This commit is contained in:
Łukasz Magiera 2020-06-04 17:29:31 +02:00
parent 1f5aa7fb7d
commit 8ad1af106d
7 changed files with 119 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
cbor_gen.go
View File

@ -382,7 +382,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{179}); err != nil {
if _, err := w.Write([]byte{180}); err != nil {
return err
}
@ -652,6 +652,22 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
return err
}
// t.PreCommit2Fails (uint64) (uint64)
if len("PreCommit2Fails") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"PreCommit2Fails\" was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommit2Fails")))); err != nil {
return err
}
if _, err := w.Write([]byte("PreCommit2Fails")); err != nil {
return err
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PreCommit2Fails))); err != nil {
return err
}
// t.SeedValue (abi.InteractiveSealRandomness) (slice)
if len("SeedValue") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"SeedValue\" was too long")
@ -1092,6 +1108,21 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
if _, err := io.ReadFull(br, t.PreCommitTipSet); err != nil {
return err
}
// t.PreCommit2Fails (uint64) (uint64)
case "PreCommit2Fails":
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.PreCommit2Fails = uint64(extra)
}
// t.SeedValue (abi.InteractiveSealRandomness) (slice)
case "SeedValue":

44
fsm.go
View File

@ -36,16 +36,16 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
Packing: planOne(on(SectorPacked{}, PreCommit1)),
PreCommit1: planOne(
on(SectorPreCommit1{}, PreCommit2),
on(SectorSealPreCommitFailed{}, SealFailed),
on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
on(SectorPackingFailed{}, PackingFailed),
),
PreCommit2: planOne(
on(SectorPreCommit2{}, PreCommitting),
on(SectorSealPreCommitFailed{}, SealFailed),
on(SectorSealPreCommit2Failed{}, SealPreCommit2Failed),
on(SectorPackingFailed{}, PackingFailed),
),
PreCommitting: planOne(
on(SectorSealPreCommitFailed{}, SealFailed),
on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
on(SectorPreCommitted{}, PreCommitWait),
on(SectorChainPreCommitFailed{}, PreCommitFailed),
on(SectorPreCommitLanded{}, WaitSeed),
@ -74,21 +74,25 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
on(SectorFaulty{}, Faulty),
),
SealFailed: planOne(
on(SectorRetrySeal{}, PreCommit1),
SealPreCommit1Failed: planOne(
on(SectorRetrySealPreCommit1{}, PreCommit1),
),
SealPreCommit2Failed: planOne(
on(SectorRetrySealPreCommit1{}, PreCommit1),
on(SectorRetrySealPreCommit2{}, PreCommit2),
),
PreCommitFailed: planOne(
on(SectorRetryPreCommit{}, PreCommitting),
on(SectorRetryWaitSeed{}, WaitSeed),
on(SectorSealPreCommitFailed{}, SealFailed),
on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
on(SectorPreCommitLanded{}, WaitSeed),
),
ComputeProofFailed: planOne(
on(SectorRetryComputeProof{}, Committing),
on(SectorSealPreCommitFailed{}, SealFailed),
on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
),
CommitFailed: planOne(
on(SectorSealPreCommitFailed{}, SealFailed),
on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
on(SectorRetryWaitSeed{}, WaitSeed),
on(SectorRetryComputeProof{}, Committing),
on(SectorRetryInvalidProof{}, Committing),
@ -147,10 +151,11 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
*<- Packing <- incoming
| |
| v
*<- PreCommit1 <--> SealFailed
| | ^^^
| v |||
*<- PreCommit2 -------/||
*<- PreCommit1 <--> SealPreCommit1Failed
| | ^ ^^
| | *----------++----\
| v v || |
*<- PreCommit2 --------++--> SealPreCommit2Failed
| | ||
| v /-------/|
* PreCommitting <-----+---> PreCommitFailed
@ -167,14 +172,17 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
*<- CommitWait ---/
| |
| v
| FinalizeSector <--> FinalizeFailed
| |
| v
*<- Proving
|
v
FailedUnrecoverable
UndefinedSectorState <- ¯\_(ツ)_/¯
| ^
*---------------------/
| ^
*---------------------/
*/
@ -203,8 +211,10 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
log.Infof("Proving sector %d", state.SectorNumber)
// Handled failure modes
case SealFailed:
return m.handleSealFailed, nil
case SealPreCommit1Failed:
return m.handleSealPrecommit1Failed, nil
case SealPreCommit2Failed:
return m.handleSealPrecommit2Failed, nil
case PreCommitFailed:
return m.handlePreCommitFailed, nil
case ComputeProofFailed:
@ -253,7 +263,7 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
return nil
case SectorComputeProofFailed:
state.State = ComputeProofFailed
case SectorSealPreCommitFailed:
case SectorSealPreCommit1Failed:
state.State = CommitFailed
case SectorCommitFailed:
state.State = CommitFailed

24
fsm_events.go
View File

@ -84,6 +84,7 @@ func (evt SectorPreCommit1) apply(state *SectorInfo) {
state.PreCommit1Out = evt.PreCommit1Out
state.TicketEpoch = evt.TicketEpoch
state.TicketValue = evt.TicketValue
state.PreCommit2Fails = 0
}
type SectorPreCommit2 struct {
@ -106,11 +107,20 @@ func (evt SectorPreCommitLanded) apply(si *SectorInfo) {
si.PreCommitTipSet = evt.TipSet
}
type SectorSealPreCommitFailed struct{ error }
type SectorSealPreCommit1Failed struct{ error }
func (evt SectorSealPreCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error }
func (evt SectorSealPreCommitFailed) apply(si *SectorInfo) {
func (evt SectorSealPreCommit1Failed) FormatError(xerrors.Printer) (next error) { return evt.error }
func (evt SectorSealPreCommit1Failed) apply(si *SectorInfo) {
si.InvalidProofs = 0 // reset counter
si.PreCommit2Fails = 0
}
type SectorSealPreCommit2Failed struct{ error }
func (evt SectorSealPreCommit2Failed) FormatError(xerrors.Printer) (next error) { return evt.error }
func (evt SectorSealPreCommit2Failed) apply(si *SectorInfo) {
si.InvalidProofs = 0 // reset counter
si.PreCommit2Fails++
}
type SectorChainPreCommitFailed struct{ error }
@ -175,9 +185,13 @@ func (evt SectorFinalizeFailed) apply(*SectorInfo) {}
// Failed state recovery
type SectorRetrySeal struct{}
type SectorRetrySealPreCommit1 struct{}
func (evt SectorRetrySeal) apply(state *SectorInfo) {}
func (evt SectorRetrySealPreCommit1) apply(state *SectorInfo) {}
type SectorRetrySealPreCommit2 struct{}
func (evt SectorRetrySealPreCommit2) apply(state *SectorInfo) {}
type SectorRetryPreCommit struct{}

15
sector_state.go
View File

@ -18,13 +18,14 @@ const (
FinalizeSector SectorState = "FinalizeSector"
Proving SectorState = "Proving"
// error modes
FailedUnrecoverable SectorState = "FailedUnrecoverable"
SealFailed SectorState = "SealFailed"
PreCommitFailed SectorState = "PreCommitFailed"
ComputeProofFailed SectorState = "ComputeProofFailed"
CommitFailed SectorState = "CommitFailed"
PackingFailed SectorState = "PackingFailed"
FinalizeFailed SectorState = "FinalizeFailed"
FailedUnrecoverable SectorState = "FailedUnrecoverable"
SealPreCommit1Failed SectorState = "SealPreCommit1Failed"
SealPreCommit2Failed SectorState = "SealPreCommit2Failed"
PreCommitFailed SectorState = "PreCommitFailed"
ComputeProofFailed SectorState = "ComputeProofFailed"
CommitFailed SectorState = "CommitFailed"
PackingFailed SectorState = "PackingFailed"
FinalizeFailed SectorState = "FinalizeFailed"
Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason
FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain

14
states.go
View File

@ -94,12 +94,12 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo)
log.Infow("performing sector replication...", "sector", sector.SectorNumber)
ticketValue, ticketEpoch, err := m.getTicket(ctx, sector)
if err != nil {
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("getting ticket failed: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("getting ticket failed: %w", err)})
}
pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), m.minerSector(sector.SectorNumber), ticketValue, sector.pieceInfos())
if err != nil {
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("seal pre commit(1) failed: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("seal pre commit(1) failed: %w", err)})
}
return ctx.Send(SectorPreCommit1{
@ -112,7 +112,7 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo)
func (m *Sealing) handlePreCommit2(ctx statemachine.Context, sector SectorInfo) error {
cids, err := m.sealer.SealPreCommit2(ctx.Context(), m.minerSector(sector.SectorNumber), sector.PreCommit1Out)
if err != nil {
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("seal pre commit(2) failed: %w", err)})
return ctx.Send(SectorSealPreCommit2Failed{xerrors.Errorf("seal pre commit(2) failed: %w", err)})
}
return ctx.Send(SectorPreCommit2{
@ -140,11 +140,11 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err)
return nil
case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too)
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
case *ErrExpiredTicket:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired: %w", err)})
case *ErrBadTicket:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)})
case *ErrPrecommitOnChain:
return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit
default:
@ -154,7 +154,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
expiration, err := m.pcp.Expiration(ctx.Context(), sector.Pieces...)
if err != nil {
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)})
}
params := &miner.SectorPreCommitInfo{

41
states_failed.go
View File

@ -30,30 +30,37 @@ func failedCooldown(ctx statemachine.Context, sector SectorInfo) error {
func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) (*miner.SectorPreCommitOnChainInfo, bool) {
tok, _, err := m.api.ChainHead(ctx.Context())
if err != nil {
log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorNumber, err)
log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
return nil, true
}
info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
if err != nil {
log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorNumber, err)
log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
return nil, true
}
return info, false
}
func (m *Sealing) handleSealFailed(ctx statemachine.Context, sector SectorInfo) error {
if _, is := m.checkPreCommitted(ctx, sector); is {
// TODO: Remove this after we can re-precommit
return nil // noop, for now
}
func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error {
if err := failedCooldown(ctx, sector); err != nil {
return err
}
return ctx.Send(SectorRetrySeal{})
return ctx.Send(SectorRetrySealPreCommit1{})
}
func (m *Sealing) handleSealPrecommit2Failed(ctx statemachine.Context, sector SectorInfo) error {
if err := failedCooldown(ctx, sector); err != nil {
return err
}
if sector.PreCommit2Fails > 1 {
return ctx.Send(SectorRetrySealPreCommit1{})
}
return ctx.Send(SectorRetrySealPreCommit2{})
}
func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorInfo) error {
@ -69,11 +76,11 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
log.Errorf("handlePreCommitFailed: api error, not proceeding: %+v", err)
return nil
case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too)
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
case *ErrExpiredTicket:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
case *ErrBadTicket:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
case *ErrPrecommitOnChain:
// noop
default:
@ -121,7 +128,7 @@ func (m *Sealing) handleComputeProofFailed(ctx statemachine.Context, sector Sect
}
if sector.InvalidProofs > 1 {
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("consecutive compute fails")})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive compute fails")})
}
return ctx.Send(SectorRetryComputeProof{})
@ -140,11 +147,11 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
log.Errorf("handleCommitFailed: api error, not proceeding: %+v", err)
return nil
case *ErrBadCommD:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad CommD error: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
case *ErrExpiredTicket:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("ticket expired error: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
case *ErrBadTicket:
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("bad expired: %w", err)})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
case *ErrPrecommitOnChain:
// noop, this is expected
default:
@ -166,7 +173,7 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
}
if sector.InvalidProofs > 0 {
return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("consecutive invalid proofs")})
return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive invalid proofs")})
}
return ctx.Send(SectorRetryInvalidProof{})

2
types.go
View File

@ -68,6 +68,8 @@ type SectorInfo struct {
PreCommitMessage *cid.Cid
PreCommitTipSet TipSetToken
PreCommit2Fails uint64
// WaitSeed
SeedValue abi.InteractiveSealRandomness
SeedEpoch abi.ChainEpoch