Stop recovery attempts after fault

extern/storage-sealing/states_failed.go

@@ -237,6 +237,17 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 		}
 	}
 
+	// Abort upgrade for sectors that went faulty since being marked for upgrade
+	active, err := sectorActive(ctx.Context(), m.Api, m.maddr, tok, sector.SectorNumber)
+	if err != nil {
+		log.Errorf("sector active check: api error, not proceeding: %+v", err)
+		return nil
+	}
+	if !active {
+		log.Errorf("sector marked for upgrade %d no longer active, aborting upgrade", sector.SectorNumber)
+		return ctx.Send(SectorAbortUpgrade{})
+	}
+
 	if err := failedCooldown(ctx, sector); err != nil {
 		return err
 	}

extern/storage-sealing/upgrade_queue.go

@@ -3,6 +3,7 @@ package sealing
 import (
 	"context"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	market7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/market"
 
@@ -86,19 +87,11 @@ func (m *Sealing) MarkForSnapUpgrade(ctx context.Context, id abi.SectorNumber) e
 		return xerrors.Errorf("failed to read sector on chain info: %w", err)
 	}
 
-	active, err := m.Api.StateMinerActiveSectors(ctx, m.maddr, tok)
+	active, err := sectorActive(ctx, m.Api, m.maddr, tok, id)
 	if err != nil {
-		return xerrors.Errorf("failed to check active sectors: %w", err)
+		return xerrors.Errorf("failed to check if sector is active")
 	}
-	// Ensure the upgraded sector is active
-	var found bool
-	for _, si := range active {
-		if si.SectorNumber == id {
-			found = true
-			break
-		}
-	}
-	if !found {
+	if !active {
 		return xerrors.Errorf("cannot mark inactive sector for upgrade")
 	}
 
@@ -110,6 +103,22 @@ func (m *Sealing) MarkForSnapUpgrade(ctx context.Context, id abi.SectorNumber) e
 	return m.sectors.Send(uint64(id), SectorStartCCUpdate{})
 }
 
+func sectorActive(ctx context.Context, api SealingAPI, maddr address.Address, tok TipSetToken, sector abi.SectorNumber) (bool, error) {
+	active, err := api.StateMinerActiveSectors(ctx, maddr, tok)
+	if err != nil {
+		return false, xerrors.Errorf("failed to check active sectors: %w", err)
+	}
+	// Ensure the upgraded sector is active
+	var found bool
+	for _, si := range active {
+		if si.SectorNumber == sector {
+			found = true
+			break
+		}
+	}
+	return found, nil
+}
+
 func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreCommitInfo) big.Int {
 	if len(params.DealIDs) == 0 {
 		return big.Zero()