diff --git a/cmd/lotus-miner/info.go b/cmd/lotus-miner/info.go index 39de942aa..1133908ca 100644 --- a/cmd/lotus-miner/info.go +++ b/cmd/lotus-miner/info.go @@ -466,6 +466,7 @@ var stateOrder = map[sealing.SectorState]stateMeta{} var stateList = []stateMeta{ {col: 39, state: "Total"}, {col: color.FgGreen, state: sealing.Proving}, + {col: color.FgGreen, state: sealing.UpdateActivating}, {col: color.FgBlue, state: sealing.Empty}, {col: color.FgBlue, state: sealing.WaitDeals}, @@ -496,6 +497,7 @@ var stateList = []stateMeta{ {col: color.FgYellow, state: sealing.SubmitReplicaUpdate}, {col: color.FgYellow, state: sealing.ReplicaUpdateWait}, {col: color.FgYellow, state: sealing.FinalizeReplicaUpdate}, + {col: color.FgYellow, state: sealing.ReleaseSectorKey}, {col: color.FgCyan, state: sealing.Terminating}, {col: color.FgCyan, state: sealing.TerminateWait}, @@ -524,6 +526,7 @@ var stateList = []stateMeta{ {col: color.FgRed, state: sealing.SnapDealsAddPieceFailed}, {col: color.FgRed, state: sealing.SnapDealsDealsExpired}, {col: color.FgRed, state: sealing.ReplicaUpdateFailed}, + {col: color.FgRed, state: sealing.ReleaseSectorKeyFailed}, } func init() { diff --git a/extern/storage-sealing/fsm.go b/extern/storage-sealing/fsm.go index 1b18efcc7..251d3a74a 100644 --- a/extern/storage-sealing/fsm.go +++ b/extern/storage-sealing/fsm.go @@ -174,7 +174,14 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorAbortUpgrade{}, AbortUpgrade), ), FinalizeReplicaUpdate: planOne( - on(SectorFinalized{}, Proving), + on(SectorFinalized{}, UpdateActivating), + ), + UpdateActivating: planOne( + on(SectorUpdateActive{}, ReleaseSectorKey), + ), + ReleaseSectorKey: planOne( + on(SectorKeyReleased{}, Proving), + on(SectorReleaseKeyFailed{}, ReleaseSectorKeyFailed), ), // Sealing errors @@ -257,6 +264,9 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorDealsExpired{}, SnapDealsDealsExpired), on(SectorAbortUpgrade{}, AbortUpgrade), ), + ReleaseSectorKeyFailed: planOne( + on(SectorUpdateActive{}, ReleaseSectorKey), + ), // Post-seal @@ -484,6 +494,10 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta return m.handleReplicaUpdateWait, processed, nil case FinalizeReplicaUpdate: return m.handleFinalizeReplicaUpdate, processed, nil + case UpdateActivating: + return m.handleUpdateActivating, processed, nil + case ReleaseSectorKey: + return m.handleReleaseSectorKey, processed, nil // Handled failure modes case AddPieceFailed: @@ -520,6 +534,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta return m.handleSnapDealsRecoverDealIDs, processed, nil case ReplicaUpdateFailed: return m.handleSubmitReplicaUpdateFailed, processed, nil + case ReleaseSectorKeyFailed: + return m.handleReleaseSectorKeyFailed, 0, err case AbortUpgrade: return m.handleAbortUpgrade, processed, nil diff --git a/extern/storage-sealing/fsm_events.go b/extern/storage-sealing/fsm_events.go index 395c4b94a..fc3b774f9 100644 --- a/extern/storage-sealing/fsm_events.go +++ b/extern/storage-sealing/fsm_events.go @@ -335,6 +335,14 @@ type SectorReplicaUpdateLanded struct{} func (evt SectorReplicaUpdateLanded) apply(state *SectorInfo) {} +type SectorUpdateActive struct{} + +func (evt SectorUpdateActive) apply(state *SectorInfo) {} + +type SectorKeyReleased struct{} + +func (evt SectorKeyReleased) apply(state *SectorInfo) {} + // Failed state recovery type SectorRetrySealPreCommit1 struct{} @@ -445,6 +453,13 @@ type SectorSubmitReplicaUpdateFailed struct{} func (evt SectorSubmitReplicaUpdateFailed) apply(state *SectorInfo) {} +type SectorReleaseKeyFailed struct{ error } + +func (evt SectorReleaseKeyFailed) FormatError(xerrors.Printer) (next error) { + return evt.error +} +func (evt SectorReleaseKeyFailed) apply(state *SectorInfo) {} + // Faults type SectorFaulty struct{} diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index ba6df7ff4..5c2c56171 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -52,11 +52,14 @@ var ExistSectorStateList = map[SectorState]struct{}{ ProveReplicaUpdate: {}, SubmitReplicaUpdate: {}, ReplicaUpdateWait: {}, + UpdateActivating: {}, + ReleaseSectorKey: {}, FinalizeReplicaUpdate: {}, SnapDealsAddPieceFailed: {}, SnapDealsDealsExpired: {}, SnapDealsRecoverDealIDs: {}, ReplicaUpdateFailed: {}, + ReleaseSectorKeyFailed: {}, AbortUpgrade: {}, } @@ -104,6 +107,8 @@ const ( SubmitReplicaUpdate SectorState = "SubmitReplicaUpdate" ReplicaUpdateWait SectorState = "ReplicaUpdateWait" FinalizeReplicaUpdate SectorState = "FinalizeReplicaUpdate" + UpdateActivating SectorState = "UpdateActivating" + ReleaseSectorKey SectorState = "ReleaseSectorKey" // error modes FailedUnrecoverable SectorState = "FailedUnrecoverable" @@ -124,6 +129,7 @@ const ( SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs" AbortUpgrade SectorState = "AbortUpgrade" ReplicaUpdateFailed SectorState = "ReplicaUpdateFailed" + ReleaseSectorKeyFailed SectorState = "ReleaseSectorKeyFailed" Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain @@ -153,7 +159,7 @@ func toStatState(st SectorState, finEarly bool) statSectorState { return sstProving } return sstSealing - case Proving, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed: + case Proving, UpdateActivating, ReleaseSectorKey, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed: return sstProving } diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index 6d40bf097..a1c3be460 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -255,6 +255,16 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect return ctx.Send(SectorRetrySubmitReplicaUpdate{}) } +func (m *Sealing) handleReleaseSectorKeyFailed(ctx statemachine.Context, sector SectorInfo) error { + // not much we can do, wait for a bit and try again + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorUpdateActive{}) +} + func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo) error { tok, _, err := m.Api.ChainHead(ctx.Context()) if err != nil { diff --git a/extern/storage-sealing/states_replica_update.go b/extern/storage-sealing/states_replica_update.go index 6222d49db..4c3faed82 100644 --- a/extern/storage-sealing/states_replica_update.go +++ b/extern/storage-sealing/states_replica_update.go @@ -2,12 +2,15 @@ package sealing import ( "bytes" + "context" + "time" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" statemachine "github.com/filecoin-project/go-statemachine" api "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" "golang.org/x/xerrors" ) @@ -225,6 +228,75 @@ func (m *Sealing) handleFinalizeReplicaUpdate(ctx statemachine.Context, sector S return ctx.Send(SectorFinalized{}) } +func (m *Sealing) handleUpdateActivating(ctx statemachine.Context, sector SectorInfo) error { + try := func() error { + mw, err := m.Api.StateWaitMsg(ctx.Context(), *sector.ReplicaUpdateMessage) + if err != nil { + return err + } + + tok, height, err := m.Api.ChainHead(ctx.Context()) + if err != nil { + return err + } + + nv, err := m.Api.StateNetworkVersion(ctx.Context(), tok) + if err != nil { + return err + } + + lb := policy.GetWinningPoStSectorSetLookback(nv) + + targetHeight := mw.Height + lb + InteractivePoRepConfidence + delay := 50 * time.Millisecond + + for { + if height >= targetHeight { + break + } + + wctx, cancel := context.WithTimeout(ctx.Context(), delay) + <-wctx.Done() + cancel() + + // increasing backoff; can't just calculate the correct time because integration tests do funny things with time + delay = delay * 10 / 3 + if delay > 5*time.Minute { + delay = 5 * time.Minute + } + + _, height, err = m.Api.ChainHead(ctx.Context()) + if err != nil { + return err + } + } + + return nil + } + + for { + err := try() + if err == nil { + break + } + + log.Errorw("error in handleUpdateActivating", "error", err) + + // likely an API issue, sleep for a bit and retry + time.Sleep(time.Minute) + } + + return ctx.Send(SectorUpdateActive{}) +} + +func (m *Sealing) handleReleaseSectorKey(ctx statemachine.Context, sector SectorInfo) error { + if err := m.sealer.ReleaseSectorKey(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber)); err != nil { + return ctx.Send(SectorReleaseKeyFailed{err}) + } + + return ctx.Send(SectorKeyReleased{}) +} + func handleErrors(ctx statemachine.Context, err error, sector SectorInfo) error { switch err.(type) { case *ErrApi: