Merge pull request #7140 from filecoin-project/feat/miner-expired-cmd

miner: Command to list/remove expired sectors

chain/actors/builtin/miner/actor.go.template

@@ -141,10 +141,21 @@ type Deadline interface {
 }
 
 type Partition interface {
+	// AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors
 	AllSectors() (bitfield.BitField, error)
+
+	// Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt).
+	// Faults ∩ Terminated = ∅
 	FaultySectors() (bitfield.BitField, error)
+
+	// Subset of faulty sectors expected to recover on next PoSt
+	// Recoveries ∩ Terminated = ∅
 	RecoveringSectors() (bitfield.BitField, error)
+
+	// Live sectors are those that are not terminated (but may be faulty).
 	LiveSectors() (bitfield.BitField, error)
+
+	// Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power.
 	ActiveSectors() (bitfield.BitField, error)
 }
 

chain/actors/builtin/miner/miner.go

@@ -200,10 +200,21 @@ type Deadline interface {
 }
 
 type Partition interface {
+	// AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors
 	AllSectors() (bitfield.BitField, error)
+
+	// Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt).
+	// Faults ∩ Terminated = ∅
 	FaultySectors() (bitfield.BitField, error)
+
+	// Subset of faulty sectors expected to recover on next PoSt
+	// Recoveries ∩ Terminated = ∅
 	RecoveringSectors() (bitfield.BitField, error)
+
+	// Live sectors are those that are not terminated (but may be faulty).
 	LiveSectors() (bitfield.BitField, error)
+
+	// Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power.
 	ActiveSectors() (bitfield.BitField, error)
 }
 

cmd/lotus-miner/sectors.go

@@ -44,6 +44,7 @@ var sectorsCmd = &cli.Command{
 		sectorsUpdateCmd,
 		sectorsPledgeCmd,
 		sectorsCheckExpireCmd,
+		sectorsExpiredCmd,
 		sectorsRenewCmd,
 		sectorsExtendCmd,
 		sectorsTerminateCmd,
@@ -1515,6 +1516,192 @@ var sectorsUpdateCmd = &cli.Command{
 	},
 }
 
+var sectorsExpiredCmd = &cli.Command{
+	Name:  "expired",
+	Usage: "Get or cleanup expired sectors",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "show-removed",
+			Usage: "show removed sectors",
+		},
+		&cli.BoolFlag{
+			Name:  "remove-expired",
+			Usage: "remove expired sectors",
+		},
+
+		&cli.Int64Flag{
+			Name:   "confirm-remove-count",
+			Hidden: true,
+		},
+		&cli.Int64Flag{
+			Name:        "expired-epoch",
+			Usage:       "epoch at which to check sector expirations",
+			DefaultText: "WinningPoSt lookback epoch",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return xerrors.Errorf("getting fullnode api: %w", err)
+		}
+		defer nCloser()
+		ctx := lcli.ReqContext(cctx)
+
+		head, err := fullApi.ChainHead(ctx)
+		if err != nil {
+			return xerrors.Errorf("getting chain head: %w", err)
+		}
+
+		lbEpoch := abi.ChainEpoch(cctx.Int64("expired-epoch"))
+		if !cctx.IsSet("expired-epoch") {
+			nv, err := fullApi.StateNetworkVersion(ctx, head.Key())
+			if err != nil {
+				return xerrors.Errorf("getting network version: %w", err)
+			}
+
+			lbEpoch = head.Height() - policy.GetWinningPoStSectorSetLookback(nv)
+			if lbEpoch < 0 {
+				return xerrors.Errorf("too early to terminate sectors")
+			}
+		}
+
+		if cctx.IsSet("confirm-remove-count") && !cctx.IsSet("expired-epoch") {
+			return xerrors.Errorf("--expired-epoch must be specified with --confirm-remove-count")
+		}
+
+		lbts, err := fullApi.ChainGetTipSetByHeight(ctx, lbEpoch, head.Key())
+		if err != nil {
+			return xerrors.Errorf("getting lookback tipset: %w", err)
+		}
+
+		maddr, err := nodeApi.ActorAddress(ctx)
+		if err != nil {
+			return xerrors.Errorf("getting actor address: %w", err)
+		}
+
+		// toCheck is a working bitfield which will only contain terminated sectors
+		toCheck := bitfield.New()
+		{
+			sectors, err := nodeApi.SectorsList(ctx)
+			if err != nil {
+				return xerrors.Errorf("getting sector list: %w", err)
+			}
+
+			for _, sector := range sectors {
+				toCheck.Set(uint64(sector))
+			}
+		}
+
+		mact, err := fullApi.StateGetActor(ctx, maddr, lbts.Key())
+		if err != nil {
+			return err
+		}
+
+		tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory())
+		mas, err := miner.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact)
+		if err != nil {
+			return err
+		}
+
+		alloc, err := mas.GetAllocatedSectors()
+		if err != nil {
+			return xerrors.Errorf("getting allocated sectors: %w", err)
+		}
+
+		// only allocated sectors can be expired,
+		toCheck, err = bitfield.IntersectBitField(toCheck, *alloc)
+		if err != nil {
+			return xerrors.Errorf("intersecting bitfields: %w", err)
+		}
+
+		if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error {
+			return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error {
+				live, err := part.LiveSectors()
+				if err != nil {
+					return err
+				}
+
+				toCheck, err = bitfield.SubtractBitField(toCheck, live)
+				return err
+			})
+		}); err != nil {
+			return err
+		}
+
+		if cctx.Bool("remove-expired") {
+			color.Red("Removing sectors:\n")
+		}
+
+		// toCheck now only contains sectors which either failed to precommit or are expired/terminated
+		fmt.Printf("Sector\tState\tExpiration\n")
+
+		var toRemove []abi.SectorNumber
+
+		err = toCheck.ForEach(func(u uint64) error {
+			s := abi.SectorNumber(u)
+
+			st, err := nodeApi.SectorsStatus(ctx, s, true)
+			if err != nil {
+				fmt.Printf("%d:\tError getting status: %s\n", u, err)
+				return nil
+			}
+
+			rmMsg := ""
+
+			if st.State == api.SectorState(sealing.Removed) {
+				if cctx.IsSet("confirm-remove-count") || !cctx.Bool("show-removed") {
+					return nil
+				}
+			} else { // not removed
+				toRemove = append(toRemove, s)
+			}
+
+			fmt.Printf("%d%s\t%s\t%s\n", s, rmMsg, st.State, lcli.EpochTime(head.Height(), st.Expiration))
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		if cctx.Bool("remove-expired") {
+			if !cctx.IsSet("confirm-remove-count") {
+				fmt.Println()
+				fmt.Println(color.YellowString("All"), color.GreenString("%d", len(toRemove)), color.YellowString("sectors listed above will be removed\n"))
+				fmt.Println(color.YellowString("To confirm removal of the above sectors, including\n all related sealed and unsealed data, run:\n"))
+				fmt.Println(color.RedString("lotus-miner sectors expired --remove-expired --confirm-remove-count=%d --expired-epoch=%d\n", len(toRemove), lbts.Height()))
+				fmt.Println(color.YellowString("WARNING: This operation is irreversible"))
+				return nil
+			}
+
+			fmt.Println()
+
+			if int64(len(toRemove)) != cctx.Int64("confirm-remove-count") {
+				return xerrors.Errorf("value of confirm-remove-count doesn't match the number of sectors which can be removed (%d)", len(toRemove))
+			}
+
+			for _, number := range toRemove {
+				fmt.Printf("Removing sector\t%s:\t", color.YellowString("%d", number))
+
+				err := nodeApi.SectorRemove(ctx, number)
+				if err != nil {
+					color.Red("ERROR: %s\n", err.Error())
+				} else {
+					color.Green("OK\n")
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
 var sectorsBatching = &cli.Command{
 	Name:  "batching",
 	Usage: "manage batch sector operations",

documentation/en/cli-lotus-miner.md

@@ -1472,6 +1472,7 @@ COMMANDS:
    update-state       ADVANCED: manually update the state of a sector, this may aid in error recovery
    pledge             store random data in a sector
    check-expire       Inspect expiring sectors
+   expired            Get or cleanup expired sectors
    renew              Renew expiring sectors while not exceeding each sector's max life
    extend             Extend sector expiration
    terminate          Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector)
@@ -1577,6 +1578,22 @@ OPTIONS:
    
 ```
 
+### lotus-miner sectors expired
+```
+NAME:
+   lotus-miner sectors expired - Get or cleanup expired sectors
+
+USAGE:
+   lotus-miner sectors expired [command options] [arguments...]
+
+OPTIONS:
+   --show-removed         show removed sectors (default: false)
+   --remove-expired       remove expired sectors (default: false)
+   --expired-epoch value  epoch at which to check sector expirations (default: WinningPoSt lookback epoch)
+   --help, -h             show help (default: false)
+   
+```
+
 ### lotus-miner sectors renew
 ```
 NAME: