From 927ef041f8d86c8dd86798f35988278b3cfa1efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Aug 2021 15:53:59 +0200 Subject: [PATCH 1/5] miner: Command to list expired sectors --- chain/actors/builtin/miner/actor.go.template | 11 ++ chain/actors/builtin/miner/miner.go | 11 ++ cmd/lotus-miner/sectors.go | 104 +++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/chain/actors/builtin/miner/actor.go.template b/chain/actors/builtin/miner/actor.go.template index edc298ca9..7ffe9f146 100644 --- a/chain/actors/builtin/miner/actor.go.template +++ b/chain/actors/builtin/miner/actor.go.template @@ -141,10 +141,21 @@ type Deadline interface { } type Partition interface { + // AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors AllSectors() (bitfield.BitField, error) + + // Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt). + // Faults ∩ Terminated = ∅ FaultySectors() (bitfield.BitField, error) + + // Subset of faulty sectors expected to recover on next PoSt + // Recoveries ∩ Terminated = ∅ RecoveringSectors() (bitfield.BitField, error) + + // Live sectors are those that are not terminated (but may be faulty). LiveSectors() (bitfield.BitField, error) + + // Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power. ActiveSectors() (bitfield.BitField, error) } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 43e3a0828..4621fa48b 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -200,10 +200,21 @@ type Deadline interface { } type Partition interface { + // AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors AllSectors() (bitfield.BitField, error) + + // Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt). + // Faults ∩ Terminated = ∅ FaultySectors() (bitfield.BitField, error) + + // Subset of faulty sectors expected to recover on next PoSt + // Recoveries ∩ Terminated = ∅ RecoveringSectors() (bitfield.BitField, error) + + // Live sectors are those that are not terminated (but may be faulty). LiveSectors() (bitfield.BitField, error) + + // Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power. ActiveSectors() (bitfield.BitField, error) } diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 39f9f53b9..aaae3d73f 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -44,6 +44,7 @@ var sectorsCmd = &cli.Command{ sectorsUpdateCmd, sectorsPledgeCmd, sectorsCheckExpireCmd, + sectorsExpiredCmd, sectorsRenewCmd, sectorsExtendCmd, sectorsTerminateCmd, @@ -1515,6 +1516,109 @@ var sectorsUpdateCmd = &cli.Command{ }, } +var sectorsExpiredCmd = &cli.Command{ + Name: "expired", + Usage: "Get or cleanup expired sectors", + ArgsUsage: "", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return xerrors.Errorf("getting fullnode api: %w", err) + } + defer nCloser() + ctx := lcli.ReqContext(cctx) + + maddr, err := api.ActorAddress(ctx) + if err != nil { + return xerrors.Errorf("getting actor address: %w", err) + } + + // toCheck is a working bitfield which will only contain terminated sectors + toCheck := bitfield.New() + { + sectors, err := api.SectorsList(ctx) + if err != nil { + return xerrors.Errorf("getting sector list: %w", err) + } + + for _, sector := range sectors { + toCheck.Set(uint64(sector)) + } + } + + head, err := fullApi.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting chain head: %w", err) + } + + nv, err := fullApi.StateNetworkVersion(ctx, head.Key()) + if err != nil { + return xerrors.Errorf("getting network version: %w", err) + } + + lbEpoch := head.Height() - policy.GetWinningPoStSectorSetLookback(nv) + if lbEpoch < 0 { + return xerrors.Errorf("too early to terminate sectors") + } + + lbts, err := fullApi.ChainGetTipSetByHeight(ctx, lbEpoch, head.Key()) + if err != nil { + return xerrors.Errorf("getting lookback tipset: %w", err) + } + + mact, err := fullApi.StateGetActor(ctx, maddr, lbts.Key()) + if err != nil { + return err + } + + tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory()) + mas, err := miner.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact) + if err != nil { + return err + } + + alloc, err := mas.GetAllocatedSectors() + if err != nil { + return xerrors.Errorf("getting allocated sectors: %w", err) + } + + // only allocated sectors can be expired, + toCheck, err = bitfield.IntersectBitField(toCheck, *alloc) + + if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { + return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { + live, err := part.LiveSectors() + if err != nil { + return err + } + + toCheck, err = bitfield.SubtractBitField(toCheck, live) + return err + }) + }); err != nil { + return err + } + + // toCheck now only contains sectors which either failed to precommit or are expired/terminated + err = toCheck.ForEach(func(u uint64) error { + fmt.Println(u) + return nil + }) + if err != nil { + return err + } + + return nil + }, +} + var sectorsBatching = &cli.Command{ Name: "batching", Usage: "manage batch sector operations", From d1759a43353d04955de341ef99eb7db937638864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Aug 2021 16:22:29 +0200 Subject: [PATCH 2/5] Show more info in sectors expired cmd --- cmd/lotus-miner/sectors.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index aaae3d73f..16d022997 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -1520,9 +1520,14 @@ var sectorsExpiredCmd = &cli.Command{ Name: "expired", Usage: "Get or cleanup expired sectors", ArgsUsage: "", - Flags: []cli.Flag{}, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-removed", + Usage: "show removed sectors", + }, + }, Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } @@ -1535,7 +1540,7 @@ var sectorsExpiredCmd = &cli.Command{ defer nCloser() ctx := lcli.ReqContext(cctx) - maddr, err := api.ActorAddress(ctx) + maddr, err := nodeApi.ActorAddress(ctx) if err != nil { return xerrors.Errorf("getting actor address: %w", err) } @@ -1543,7 +1548,7 @@ var sectorsExpiredCmd = &cli.Command{ // toCheck is a working bitfield which will only contain terminated sectors toCheck := bitfield.New() { - sectors, err := api.SectorsList(ctx) + sectors, err := nodeApi.SectorsList(ctx) if err != nil { return xerrors.Errorf("getting sector list: %w", err) } @@ -1607,8 +1612,23 @@ var sectorsExpiredCmd = &cli.Command{ } // toCheck now only contains sectors which either failed to precommit or are expired/terminated + fmt.Printf("Sector\tState\tExpiration\n") + err = toCheck.ForEach(func(u uint64) error { - fmt.Println(u) + s := abi.SectorNumber(u) + + st, err := nodeApi.SectorsStatus(ctx, s, true) + if err != nil { + fmt.Printf("%d:\tError getting status: %s\n", u, err) + return nil + } + + if !cctx.Bool("show-removed") && st.State == api.SectorState(sealing.Removed) { + return nil + } + + fmt.Printf("%d:\t%s\t%s\n", s, st.State, lcli.EpochTime(head.Height(), st.Expiration)) + return nil }) if err != nil { From ccf884468980697773712181045b8e25c5399fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Aug 2021 17:05:34 +0200 Subject: [PATCH 3/5] lotus-miner sectors expired --remove-expired --- cmd/lotus-miner/sectors.go | 86 ++++++++++++++++++++++++----- documentation/en/cli-lotus-miner.md | 17 ++++++ 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 16d022997..215995b9a 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -1517,14 +1517,27 @@ var sectorsUpdateCmd = &cli.Command{ } var sectorsExpiredCmd = &cli.Command{ - Name: "expired", - Usage: "Get or cleanup expired sectors", - ArgsUsage: "", - Flags: []cli.Flag{ + Name: "expired", + Usage: "Get or cleanup expired sectors", + Flags: []cli.Flag{ &cli.BoolFlag{ Name: "show-removed", Usage: "show removed sectors", }, + &cli.BoolFlag{ + Name: "remove-expired", + Usage: "remove expired sectors", + }, + + &cli.Int64Flag{ + Name: "confirm-remove-count", + Hidden: true, + }, + &cli.Int64Flag{ + Name: "expired-epoch", + Usage: "epoch at which to check sector expirations", + DefaultText: "WinningPoSt lookback epoch", + }, }, Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) @@ -1563,14 +1576,21 @@ var sectorsExpiredCmd = &cli.Command{ return xerrors.Errorf("getting chain head: %w", err) } - nv, err := fullApi.StateNetworkVersion(ctx, head.Key()) - if err != nil { - return xerrors.Errorf("getting network version: %w", err) + lbEpoch := abi.ChainEpoch(cctx.Int64("expired-epoch")) + if !cctx.IsSet("expired-epoch") { + nv, err := fullApi.StateNetworkVersion(ctx, head.Key()) + if err != nil { + return xerrors.Errorf("getting network version: %w", err) + } + + lbEpoch = head.Height() - policy.GetWinningPoStSectorSetLookback(nv) + if lbEpoch < 0 { + return xerrors.Errorf("too early to terminate sectors") + } } - lbEpoch := head.Height() - policy.GetWinningPoStSectorSetLookback(nv) - if lbEpoch < 0 { - return xerrors.Errorf("too early to terminate sectors") + if cctx.IsSet("confirm-remove-count") && !cctx.IsSet("expired-epoch") { + return xerrors.Errorf("--expired-epoch must be specified with --confirm-remove-count") } lbts, err := fullApi.ChainGetTipSetByHeight(ctx, lbEpoch, head.Key()) @@ -1611,9 +1631,15 @@ var sectorsExpiredCmd = &cli.Command{ return err } + if cctx.Bool("remove-expired") { + color.Red("Removing sectors:\n") + } + // toCheck now only contains sectors which either failed to precommit or are expired/terminated fmt.Printf("Sector\tState\tExpiration\n") + var toRemove []abi.SectorNumber + err = toCheck.ForEach(func(u uint64) error { s := abi.SectorNumber(u) @@ -1623,11 +1649,17 @@ var sectorsExpiredCmd = &cli.Command{ return nil } - if !cctx.Bool("show-removed") && st.State == api.SectorState(sealing.Removed) { - return nil + rmMsg := "" + + if st.State == api.SectorState(sealing.Removed) { + if cctx.IsSet("confirm-remove-count") || !cctx.Bool("show-removed") { + return nil + } + } else { // not removed + toRemove = append(toRemove, s) } - fmt.Printf("%d:\t%s\t%s\n", s, st.State, lcli.EpochTime(head.Height(), st.Expiration)) + fmt.Printf("%d%s\t%s\t%s\n", s, rmMsg, st.State, lcli.EpochTime(head.Height(), st.Expiration)) return nil }) @@ -1635,6 +1667,34 @@ var sectorsExpiredCmd = &cli.Command{ return err } + if cctx.Bool("remove-expired") { + if !cctx.IsSet("confirm-remove-count") { + fmt.Println() + fmt.Println(color.YellowString("All"), color.GreenString("%d", len(toRemove)), color.YellowString("sectors listed above will be removed\n")) + fmt.Println(color.YellowString("To confirm removal of the above sectors, including\n all related sealed and unsealed data, run:\n")) + fmt.Println(color.RedString("lotus-miner sectors expired --remove-expired --confirm-remove-count=%d --expired-epoch=%d\n", len(toRemove), lbts.Height())) + fmt.Println(color.YellowString("WARNING: This operation is irreversible")) + return nil + } + + fmt.Println() + + if int64(len(toRemove)) != cctx.Int64("confirm-remove-count") { + return xerrors.Errorf("value of confirm-remove-count doesn't match the number of sectors which can be removed (%d)", len(toRemove)) + } + + for _, number := range toRemove { + fmt.Printf("Removing sector\t%s:\t", color.YellowString("%d", number)) + + err := nodeApi.SectorRemove(ctx, number) + if err != nil { + color.Red("ERROR: %s\n", err.Error()) + } else { + color.Green("OK\n") + } + } + } + return nil }, } diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 80178c8a7..372f9deac 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1472,6 +1472,7 @@ COMMANDS: update-state ADVANCED: manually update the state of a sector, this may aid in error recovery pledge store random data in a sector check-expire Inspect expiring sectors + expired Get or cleanup expired sectors renew Renew expiring sectors while not exceeding each sector's max life extend Extend sector expiration terminate Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) @@ -1577,6 +1578,22 @@ OPTIONS: ``` +### lotus-miner sectors expired +``` +NAME: + lotus-miner sectors expired - Get or cleanup expired sectors + +USAGE: + lotus-miner sectors expired [command options] [arguments...] + +OPTIONS: + --show-removed show removed sectors (default: false) + --remove-expired remove expired sectors (default: false) + --expired-epoch value epoch at which to check sector expirations (default: WinningPoSt lookback epoch) + --help, -h show help (default: false) + +``` + ### lotus-miner sectors renew ``` NAME: From a9bf24695d6ac5a1496614bd95e8c33502209d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Aug 2021 17:10:34 +0200 Subject: [PATCH 4/5] Fix lint --- cmd/lotus-miner/sectors.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 215995b9a..a64a7632d 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -1616,6 +1616,9 @@ var sectorsExpiredCmd = &cli.Command{ // only allocated sectors can be expired, toCheck, err = bitfield.IntersectBitField(toCheck, *alloc) + if err != nil { + return xerrors.Errorf("intersecting bitfields: %w", err) + } if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { From 5a23c2bb90173e81425bf0d0a52e444e224bceb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 24 Aug 2021 11:28:58 +0200 Subject: [PATCH 5/5] sectors expired: Address review --- cmd/lotus-miner/sectors.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index a64a7632d..194b1e554 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -1553,24 +1553,6 @@ var sectorsExpiredCmd = &cli.Command{ defer nCloser() ctx := lcli.ReqContext(cctx) - maddr, err := nodeApi.ActorAddress(ctx) - if err != nil { - return xerrors.Errorf("getting actor address: %w", err) - } - - // toCheck is a working bitfield which will only contain terminated sectors - toCheck := bitfield.New() - { - sectors, err := nodeApi.SectorsList(ctx) - if err != nil { - return xerrors.Errorf("getting sector list: %w", err) - } - - for _, sector := range sectors { - toCheck.Set(uint64(sector)) - } - } - head, err := fullApi.ChainHead(ctx) if err != nil { return xerrors.Errorf("getting chain head: %w", err) @@ -1598,6 +1580,24 @@ var sectorsExpiredCmd = &cli.Command{ return xerrors.Errorf("getting lookback tipset: %w", err) } + maddr, err := nodeApi.ActorAddress(ctx) + if err != nil { + return xerrors.Errorf("getting actor address: %w", err) + } + + // toCheck is a working bitfield which will only contain terminated sectors + toCheck := bitfield.New() + { + sectors, err := nodeApi.SectorsList(ctx) + if err != nil { + return xerrors.Errorf("getting sector list: %w", err) + } + + for _, sector := range sectors { + toCheck.Set(uint64(sector)) + } + } + mact, err := fullApi.StateGetActor(ctx, maddr, lbts.Key()) if err != nil { return err