fsm: Call ReleaseSectorKey after WinningPoSt lookback period

cmd/lotus-miner/info.go

@@ -466,6 +466,7 @@ var stateOrder = map[sealing.SectorState]stateMeta{}
 var stateList = []stateMeta{
 	{col: 39, state: "Total"},
 	{col: color.FgGreen, state: sealing.Proving},
+	{col: color.FgGreen, state: sealing.UpdateActivating},
 
 	{col: color.FgBlue, state: sealing.Empty},
 	{col: color.FgBlue, state: sealing.WaitDeals},
@@ -496,6 +497,7 @@ var stateList = []stateMeta{
 	{col: color.FgYellow, state: sealing.SubmitReplicaUpdate},
 	{col: color.FgYellow, state: sealing.ReplicaUpdateWait},
 	{col: color.FgYellow, state: sealing.FinalizeReplicaUpdate},
+	{col: color.FgYellow, state: sealing.ReleaseSectorKey},
 
 	{col: color.FgCyan, state: sealing.Terminating},
 	{col: color.FgCyan, state: sealing.TerminateWait},
@@ -524,6 +526,7 @@ var stateList = []stateMeta{
 	{col: color.FgRed, state: sealing.SnapDealsAddPieceFailed},
 	{col: color.FgRed, state: sealing.SnapDealsDealsExpired},
 	{col: color.FgRed, state: sealing.ReplicaUpdateFailed},
+	{col: color.FgRed, state: sealing.ReleaseSectorKeyFailed},
 }
 
 func init() {

extern/storage-sealing/fsm.go

@@ -174,7 +174,14 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	FinalizeReplicaUpdate: planOne(
-		on(SectorFinalized{}, Proving),
+		on(SectorFinalized{}, UpdateActivating),
+	),
+	UpdateActivating: planOne(
+		on(SectorUpdateActive{}, ReleaseSectorKey),
+	),
+	ReleaseSectorKey: planOne(
+		on(SectorKeyReleased{}, Proving),
+		on(SectorReleaseKeyFailed{}, ReleaseSectorKeyFailed),
 	),
 	// Sealing errors
 
@@ -257,6 +264,9 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorDealsExpired{}, SnapDealsDealsExpired),
 		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
+	ReleaseSectorKeyFailed: planOne(
+		on(SectorUpdateActive{}, ReleaseSectorKey),
+	),
 
 	// Post-seal
 
@@ -484,6 +494,10 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleReplicaUpdateWait, processed, nil
 	case FinalizeReplicaUpdate:
 		return m.handleFinalizeReplicaUpdate, processed, nil
+	case UpdateActivating:
+		return m.handleUpdateActivating, processed, nil
+	case ReleaseSectorKey:
+		return m.handleReleaseSectorKey, processed, nil
 
 	// Handled failure modes
 	case AddPieceFailed:
@@ -520,6 +534,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleSnapDealsRecoverDealIDs, processed, nil
 	case ReplicaUpdateFailed:
 		return m.handleSubmitReplicaUpdateFailed, processed, nil
+	case ReleaseSectorKeyFailed:
+		return m.handleReleaseSectorKeyFailed, 0, err
 	case AbortUpgrade:
 		return m.handleAbortUpgrade, processed, nil
 

extern/storage-sealing/fsm_events.go

@@ -335,6 +335,14 @@ type SectorReplicaUpdateLanded struct{}
 
 func (evt SectorReplicaUpdateLanded) apply(state *SectorInfo) {}
 
+type SectorUpdateActive struct{}
+
+func (evt SectorUpdateActive) apply(state *SectorInfo) {}
+
+type SectorKeyReleased struct{}
+
+func (evt SectorKeyReleased) apply(state *SectorInfo) {}
+
 // Failed state recovery
 
 type SectorRetrySealPreCommit1 struct{}
@@ -445,6 +453,13 @@ type SectorSubmitReplicaUpdateFailed struct{}
 
 func (evt SectorSubmitReplicaUpdateFailed) apply(state *SectorInfo) {}
 
+type SectorReleaseKeyFailed struct{ error }
+
+func (evt SectorReleaseKeyFailed) FormatError(xerrors.Printer) (next error) {
+	return evt.error
+}
+func (evt SectorReleaseKeyFailed) apply(state *SectorInfo) {}
+
 // Faults
 
 type SectorFaulty struct{}

extern/storage-sealing/sector_state.go

@@ -52,11 +52,14 @@ var ExistSectorStateList = map[SectorState]struct{}{
 	ProveReplicaUpdate:      {},
 	SubmitReplicaUpdate:     {},
 	ReplicaUpdateWait:       {},
+	UpdateActivating:        {},
+	ReleaseSectorKey:        {},
 	FinalizeReplicaUpdate:   {},
 	SnapDealsAddPieceFailed: {},
 	SnapDealsDealsExpired:   {},
 	SnapDealsRecoverDealIDs: {},
 	ReplicaUpdateFailed:     {},
+	ReleaseSectorKeyFailed:  {},
 	AbortUpgrade:            {},
 }
 
@@ -104,6 +107,8 @@ const (
 	SubmitReplicaUpdate   SectorState = "SubmitReplicaUpdate"
 	ReplicaUpdateWait     SectorState = "ReplicaUpdateWait"
 	FinalizeReplicaUpdate SectorState = "FinalizeReplicaUpdate"
+	UpdateActivating      SectorState = "UpdateActivating"
+	ReleaseSectorKey      SectorState = "ReleaseSectorKey"
 
 	// error modes
 	FailedUnrecoverable  SectorState = "FailedUnrecoverable"
@@ -124,6 +129,7 @@ const (
 	SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs"
 	AbortUpgrade            SectorState = "AbortUpgrade"
 	ReplicaUpdateFailed     SectorState = "ReplicaUpdateFailed"
+	ReleaseSectorKeyFailed  SectorState = "ReleaseSectorKeyFailed"
 
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain
@@ -153,7 +159,7 @@ func toStatState(st SectorState, finEarly bool) statSectorState {
 			return sstProving
 		}
 		return sstSealing
-	case Proving, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed:
+	case Proving, UpdateActivating, ReleaseSectorKey, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed:
 		return sstProving
 	}
 

extern/storage-sealing/states_failed.go

@@ -255,6 +255,16 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 	return ctx.Send(SectorRetrySubmitReplicaUpdate{})
 }
 
+func (m *Sealing) handleReleaseSectorKeyFailed(ctx statemachine.Context, sector SectorInfo) error {
+	// not much we can do, wait for a bit and try again
+
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	return ctx.Send(SectorUpdateActive{})
+}
+
 func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo) error {
 	tok, _, err := m.Api.ChainHead(ctx.Context())
 	if err != nil {

extern/storage-sealing/states_replica_update.go

@@ -2,12 +2,15 @@ package sealing
 
 import (
 	"bytes"
+	"context"
+	"time"
 
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/exitcode"
 	statemachine "github.com/filecoin-project/go-statemachine"
 	api "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"golang.org/x/xerrors"
 )
 
@@ -225,6 +228,75 @@ func (m *Sealing) handleFinalizeReplicaUpdate(ctx statemachine.Context, sector S
 	return ctx.Send(SectorFinalized{})
 }
 
+func (m *Sealing) handleUpdateActivating(ctx statemachine.Context, sector SectorInfo) error {
+	try := func() error {
+		mw, err := m.Api.StateWaitMsg(ctx.Context(), *sector.ReplicaUpdateMessage)
+		if err != nil {
+			return err
+		}
+
+		tok, height, err := m.Api.ChainHead(ctx.Context())
+		if err != nil {
+			return err
+		}
+
+		nv, err := m.Api.StateNetworkVersion(ctx.Context(), tok)
+		if err != nil {
+			return err
+		}
+
+		lb := policy.GetWinningPoStSectorSetLookback(nv)
+
+		targetHeight := mw.Height + lb + InteractivePoRepConfidence
+		delay := 50 * time.Millisecond
+
+		for {
+			if height >= targetHeight {
+				break
+			}
+
+			wctx, cancel := context.WithTimeout(ctx.Context(), delay)
+			<-wctx.Done()
+			cancel()
+
+			// increasing backoff; can't just calculate the correct time because integration tests do funny things with time
+			delay = delay * 10 / 3
+			if delay > 5*time.Minute {
+				delay = 5 * time.Minute
+			}
+
+			_, height, err = m.Api.ChainHead(ctx.Context())
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	for {
+		err := try()
+		if err == nil {
+			break
+		}
+
+		log.Errorw("error in handleUpdateActivating", "error", err)
+
+		// likely an API issue, sleep for a bit and retry
+		time.Sleep(time.Minute)
+	}
+
+	return ctx.Send(SectorUpdateActive{})
+}
+
+func (m *Sealing) handleReleaseSectorKey(ctx statemachine.Context, sector SectorInfo) error {
+	if err := m.sealer.ReleaseSectorKey(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber)); err != nil {
+		return ctx.Send(SectorReleaseKeyFailed{err})
+	}
+
+	return ctx.Send(SectorKeyReleased{})
+}
+
 func handleErrors(ctx statemachine.Context, err error, sector SectorInfo) error {
 	switch err.(type) {
 	case *ErrApi: