Merge pull request #8243 from filecoin-project/release/v1.14.4

chore: build: v1.14.4

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Lotus changelog
 
+# 1.14.4 / 2022-03-03
+
+This is a *highly recommended* optional release for storage providers that are doing snap deals. This fix the bug 
+that causes some snap deal sectors are stuck in `FinalizeReplicaUpdate`. In addition, SPs should be able to force 
+update sectors status without getting blocked by `normal shutdown of state machine`. 
+
+# v1.14.3 / 2022-02-28
+
+This is an **optional** release, that includes a fix to properly register the `--really-do-it` flag for abort-upgrade.
+
 # 1.14.2 / 2022-02-24
 
 This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550). 

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.14.2"
+const BuildVersion = "1.14.4"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

cmd/lotus-miner/info.go

@@ -527,6 +527,7 @@ var stateList = []stateMeta{
 	{col: color.FgRed, state: sealing.SnapDealsDealsExpired},
 	{col: color.FgRed, state: sealing.ReplicaUpdateFailed},
 	{col: color.FgRed, state: sealing.ReleaseSectorKeyFailed},
+	{col: color.FgRed, state: sealing.FinalizeReplicaUpdateFailed},
 }
 
 func init() {

cmd/lotus-miner/sectors.go

@@ -1525,6 +1525,12 @@ var sectorsSnapAbortCmd = &cli.Command{
 	Name:      "abort-upgrade",
 	Usage:     "Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before",
 	ArgsUsage: "<sectorNum>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "pass this flag if you know what you are doing",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 1 {
 			return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number"))

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.14.2
+   1.14.4
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -1757,7 +1757,8 @@ USAGE:
    lotus-miner sectors abort-upgrade [command options] <sectorNum>
 
 OPTIONS:
-   --help, -h  show help (default: false)
+   --really-do-it  pass this flag if you know what you are doing (default: false)
+   --help, -h      show help (default: false)
    
 ```
 

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.14.2
+   1.14.4
 
 COMMANDS:
    run         Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.14.2
+   1.14.4
 
 COMMANDS:
    daemon   Start a lotus daemon process

extern/storage-sealing/fsm.go

@@ -19,7 +19,13 @@ import (
 func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) {
 	next, processed, err := m.plan(events, user.(*SectorInfo))
 	if err != nil || next == nil {
-		return nil, processed, err
+		l := Log{
+			Timestamp: uint64(time.Now().Unix()),
+			Message:   fmt.Sprintf("state machine error: %s", err),
+			Kind:      fmt.Sprintf("error;%T", err),
+		}
+		user.(*SectorInfo).logAppend(l)
+		return nil, processed, nil
 	}
 
 	return func(ctx statemachine.Context, si SectorInfo) error {
@@ -175,6 +181,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	),
 	FinalizeReplicaUpdate: planOne(
 		on(SectorFinalized{}, UpdateActivating),
+		on(SectorFinalizeFailed{}, FinalizeReplicaUpdateFailed),
 	),
 	UpdateActivating: planOne(
 		on(SectorUpdateActive{}, ReleaseSectorKey),
@@ -267,6 +274,9 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	ReleaseSectorKeyFailed: planOne(
 		on(SectorUpdateActive{}, ReleaseSectorKey),
 	),
+	FinalizeReplicaUpdateFailed: planOne(
+		on(SectorRetryFinalize{}, FinalizeReplicaUpdate),
+	),
 
 	// Post-seal
 
@@ -309,6 +319,21 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	FailedUnrecoverable: final,
 }
 
+func (state *SectorInfo) logAppend(l Log) {
+	if len(state.Log) > 8000 {
+		log.Warnw("truncating sector log", "sector", state.SectorNumber)
+		state.Log[2000] = Log{
+			Timestamp: uint64(time.Now().Unix()),
+			Message:   "truncating log (above 8000 entries)",
+			Kind:      fmt.Sprintf("truncate"),
+		}
+
+		state.Log = append(state.Log[:2000], state.Log[6000:]...)
+	}
+
+	state.Log = append(state.Log, l)
+}
+
 func (m *Sealing) logEvents(events []statemachine.Event, state *SectorInfo) {
 	for _, event := range events {
 		log.Debugw("sector event", "sector", state.SectorNumber, "type", fmt.Sprintf("%T", event.User), "event", event.User)
@@ -337,18 +362,7 @@ func (m *Sealing) logEvents(events []statemachine.Event, state *SectorInfo) {
 			l.Trace = fmt.Sprintf("%+v", err)
 		}
 
-		if len(state.Log) > 8000 {
-			log.Warnw("truncating sector log", "sector", state.SectorNumber)
-			state.Log[2000] = Log{
-				Timestamp: uint64(time.Now().Unix()),
-				Message:   "truncating log (above 8000 entries)",
-				Kind:      fmt.Sprintf("truncate"),
-			}
-
-			state.Log = append(state.Log[:2000], state.Log[6000:]...)
-		}
-
-		state.Log = append(state.Log, l)
+		state.logAppend(l)
 	}
 }
 
@@ -536,6 +550,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleSubmitReplicaUpdateFailed, processed, nil
 	case ReleaseSectorKeyFailed:
 		return m.handleReleaseSectorKeyFailed, 0, err
+	case FinalizeReplicaUpdateFailed:
+		return m.handleFinalizeFailed, processed, nil
 	case AbortUpgrade:
 		return m.handleAbortUpgrade, processed, nil
 

extern/storage-sealing/sector_state.go

@@ -3,64 +3,65 @@ package sealing
 type SectorState string
 
 var ExistSectorStateList = map[SectorState]struct{}{
-	Empty:                   {},
-	WaitDeals:               {},
-	Packing:                 {},
-	AddPiece:                {},
-	AddPieceFailed:          {},
-	GetTicket:               {},
-	PreCommit1:              {},
-	PreCommit2:              {},
-	PreCommitting:           {},
-	PreCommitWait:           {},
-	SubmitPreCommitBatch:    {},
-	PreCommitBatchWait:      {},
-	WaitSeed:                {},
-	Committing:              {},
-	CommitFinalize:          {},
-	CommitFinalizeFailed:    {},
-	SubmitCommit:            {},
-	CommitWait:              {},
-	SubmitCommitAggregate:   {},
-	CommitAggregateWait:     {},
-	FinalizeSector:          {},
-	Proving:                 {},
-	FailedUnrecoverable:     {},
-	SealPreCommit1Failed:    {},
-	SealPreCommit2Failed:    {},
-	PreCommitFailed:         {},
-	ComputeProofFailed:      {},
-	CommitFailed:            {},
-	PackingFailed:           {},
-	FinalizeFailed:          {},
-	DealsExpired:            {},
-	RecoverDealIDs:          {},
-	Faulty:                  {},
-	FaultReported:           {},
-	FaultedFinal:            {},
-	Terminating:             {},
-	TerminateWait:           {},
-	TerminateFinality:       {},
-	TerminateFailed:         {},
-	Removing:                {},
-	RemoveFailed:            {},
-	Removed:                 {},
-	SnapDealsWaitDeals:      {},
-	SnapDealsAddPiece:       {},
-	SnapDealsPacking:        {},
-	UpdateReplica:           {},
-	ProveReplicaUpdate:      {},
-	SubmitReplicaUpdate:     {},
-	ReplicaUpdateWait:       {},
-	UpdateActivating:        {},
-	ReleaseSectorKey:        {},
-	FinalizeReplicaUpdate:   {},
-	SnapDealsAddPieceFailed: {},
-	SnapDealsDealsExpired:   {},
-	SnapDealsRecoverDealIDs: {},
-	ReplicaUpdateFailed:     {},
-	ReleaseSectorKeyFailed:  {},
-	AbortUpgrade:            {},
+	Empty:                       {},
+	WaitDeals:                   {},
+	Packing:                     {},
+	AddPiece:                    {},
+	AddPieceFailed:              {},
+	GetTicket:                   {},
+	PreCommit1:                  {},
+	PreCommit2:                  {},
+	PreCommitting:               {},
+	PreCommitWait:               {},
+	SubmitPreCommitBatch:        {},
+	PreCommitBatchWait:          {},
+	WaitSeed:                    {},
+	Committing:                  {},
+	CommitFinalize:              {},
+	CommitFinalizeFailed:        {},
+	SubmitCommit:                {},
+	CommitWait:                  {},
+	SubmitCommitAggregate:       {},
+	CommitAggregateWait:         {},
+	FinalizeSector:              {},
+	Proving:                     {},
+	FailedUnrecoverable:         {},
+	SealPreCommit1Failed:        {},
+	SealPreCommit2Failed:        {},
+	PreCommitFailed:             {},
+	ComputeProofFailed:          {},
+	CommitFailed:                {},
+	PackingFailed:               {},
+	FinalizeFailed:              {},
+	DealsExpired:                {},
+	RecoverDealIDs:              {},
+	Faulty:                      {},
+	FaultReported:               {},
+	FaultedFinal:                {},
+	Terminating:                 {},
+	TerminateWait:               {},
+	TerminateFinality:           {},
+	TerminateFailed:             {},
+	Removing:                    {},
+	RemoveFailed:                {},
+	Removed:                     {},
+	SnapDealsWaitDeals:          {},
+	SnapDealsAddPiece:           {},
+	SnapDealsPacking:            {},
+	UpdateReplica:               {},
+	ProveReplicaUpdate:          {},
+	SubmitReplicaUpdate:         {},
+	ReplicaUpdateWait:           {},
+	UpdateActivating:            {},
+	ReleaseSectorKey:            {},
+	FinalizeReplicaUpdate:       {},
+	SnapDealsAddPieceFailed:     {},
+	SnapDealsDealsExpired:       {},
+	SnapDealsRecoverDealIDs:     {},
+	ReplicaUpdateFailed:         {},
+	ReleaseSectorKeyFailed:      {},
+	FinalizeReplicaUpdateFailed: {},
+	AbortUpgrade:                {},
 }
 
 // cmd/lotus-miner/info.go defines CLI colors corresponding to these states
@@ -124,12 +125,13 @@ const (
 	RecoverDealIDs       SectorState = "RecoverDealIDs"
 
 	// snap deals error modes
-	SnapDealsAddPieceFailed SectorState = "SnapDealsAddPieceFailed"
-	SnapDealsDealsExpired   SectorState = "SnapDealsDealsExpired"
-	SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs"
-	AbortUpgrade            SectorState = "AbortUpgrade"
-	ReplicaUpdateFailed     SectorState = "ReplicaUpdateFailed"
-	ReleaseSectorKeyFailed  SectorState = "ReleaseSectorKeyFailed"
+	SnapDealsAddPieceFailed     SectorState = "SnapDealsAddPieceFailed"
+	SnapDealsDealsExpired       SectorState = "SnapDealsDealsExpired"
+	SnapDealsRecoverDealIDs     SectorState = "SnapDealsRecoverDealIDs"
+	AbortUpgrade                SectorState = "AbortUpgrade"
+	ReplicaUpdateFailed         SectorState = "ReplicaUpdateFailed"
+	ReleaseSectorKeyFailed      SectorState = "ReleaseSectorKeyFailed"
+	FinalizeReplicaUpdateFailed SectorState = "FinalizeReplicaUpdateFailed"
 
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain