From 0ebdbed30b9c31539445485d9b5e201b55c7f27a Mon Sep 17 00:00:00 2001 From: Geoff Stuart Date: Tue, 20 Dec 2022 12:01:22 -0500 Subject: [PATCH] Refactor renew and remove extend --- cmd/lotus-miner/sectors.go | 348 +++++----------------------- documentation/en/cli-lotus-miner.md | 29 +-- 2 files changed, 63 insertions(+), 314 deletions(-) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index fc5fdcef6..455b5976b 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -51,7 +51,6 @@ var sectorsCmd = &cli.Command{ sectorPreCommitsCmd, sectorsCheckExpireCmd, sectorsExpiredCmd, - sectorsRenewCmd, sectorsExtendCmd, sectorsTerminateCmd, sectorsRemoveCmd, @@ -743,14 +742,14 @@ func ArrayToString(array []uint64) string { return strings.Join(sarray, ",") } -func getSectorsFromFile(filePath string) ([]uint64, error) { +func getSectorsFromFile(filePath string) ([]abi.SectorNumber, error) { file, err := os.Open(filePath) if err != nil { return nil, err } scanner := bufio.NewScanner(file) - sectors := make([]uint64, 0) + sectors := make([]abi.SectorNumber, 0) for scanner.Scan() { line := scanner.Text() @@ -760,7 +759,7 @@ func getSectorsFromFile(filePath string) ([]uint64, error) { return nil, xerrors.Errorf("could not parse %s as sector id: %s", line, err) } - sectors = append(sectors, id) + sectors = append(sectors, abi.SectorNumber(id)) } if err = file.Close(); err != nil { @@ -770,9 +769,19 @@ func getSectorsFromFile(filePath string) ([]uint64, error) { return sectors, nil } -var sectorsRenewCmd = &cli.Command{ - Name: "renew", - Usage: "Renew expiring sectors while not exceeding each sector's max life", +func SectorNumsToBitfield(sectors []abi.SectorNumber) bitfield.BitField { + var numbers []uint64 + for sector := range sectors { + numbers = append(numbers, uint64(sector)) + } + + return bitfield.NewFromSet(numbers) +} + +var sectorsExtendCmd = &cli.Command{ + Name: "extend", + Usage: "Extend expiring sectors while not exceeding each sector's max life", + ArgsUsage: "", Flags: []cli.Flag{ &cli.Int64Flag{ Name: "from", @@ -819,7 +828,7 @@ var sectorsRenewCmd = &cli.Command{ }, &cli.BoolFlag{ Name: "really-do-it", - Usage: "pass this flag to really renew sectors, otherwise will only print out json representation of parameters", + Usage: "pass this flag to really extend sectors, otherwise will only print out json representation of parameters", }, }, Action: func(cctx *cli.Context) error { @@ -896,8 +905,7 @@ var sectorsRenewCmd = &cli.Command{ return err } - excludeSet := make(map[uint64]struct{}) - + excludeSet := make(map[abi.SectorNumber]struct{}) if cctx.IsSet("exclude") { excludeSectors, err := getSectorsFromFile(cctx.String("exclude")) if err != nil { @@ -909,28 +917,24 @@ var sectorsRenewCmd = &cli.Command{ } } - var sis []*miner.SectorOnChainInfo - - if cctx.IsSet("sector-file") { - sectors, err := getSectorsFromFile(cctx.String("sector-file")) - if err != nil { - return err + var sectors []abi.SectorNumber + if cctx.Args().Present() { + if cctx.IsSet("sector-file") { + return xerrors.Errorf("sector-file specified along with command line params") } - for _, id := range sectors { - if _, exclude := excludeSet[id]; exclude { - continue + for i, s := range cctx.Args().Slice() { + id, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return xerrors.Errorf("could not parse sector %d: %w", i, err) } - si, found := activeSectorsInfo[abi.SectorNumber(id)] - if !found { - return xerrors.Errorf("sector %d is not active", id) - } - if len(si.DealIDs) > 0 && cctx.Bool("only-cc") { - continue - } - - sis = append(sis, si) + sectors = append(sectors, abi.SectorNumber(id)) + } + } else if cctx.IsSet("sector-file") { + sectors, err = getSectorsFromFile(cctx.String("sector-file")) + if err != nil { + return err } } else { from := currEpoch + 120 @@ -945,19 +949,28 @@ var sectorsRenewCmd = &cli.Command{ } for _, si := range activeSet { - if len(si.DealIDs) > 0 && cctx.Bool("only-cc") { - continue - } - if si.Expiration >= from && si.Expiration <= to { - if _, exclude := excludeSet[uint64(si.SectorNumber)]; !exclude { - sis = append(sis, si) - } + sectors = append(sectors, si.SectorNumber) } } } - extensions := map[lminer.SectorLocation]map[abi.ChainEpoch][]uint64{} + var sis []*miner.SectorOnChainInfo + for _, id := range sectors { + if _, exclude := excludeSet[id]; exclude { + continue + } + + si, found := activeSectorsInfo[id] + if !found { + return xerrors.Errorf("sector %d is not active", id) + } + if len(si.DealIDs) > 0 && cctx.Bool("only-cc") { + continue + } + + sis = append(sis, si) + } withinTolerance := func(a, b abi.ChainEpoch) bool { diff := a - b @@ -968,6 +981,7 @@ var sectorsRenewCmd = &cli.Command{ return diff <= abi.ChainEpoch(cctx.Int64("tolerance")) } + extensions := map[lminer.SectorLocation]map[abi.ChainEpoch][]abi.SectorNumber{} for _, si := range sis { extension := abi.ChainEpoch(cctx.Int64("extension")) newExp := si.Expiration + extension @@ -997,21 +1011,21 @@ var sectorsRenewCmd = &cli.Command{ es, found := extensions[*l] if !found { - ne := make(map[abi.ChainEpoch][]uint64) - ne[newExp] = []uint64{uint64(si.SectorNumber)} + ne := make(map[abi.ChainEpoch][]abi.SectorNumber) + ne[newExp] = []abi.SectorNumber{si.SectorNumber} extensions[*l] = ne } else { added := false for exp := range es { if withinTolerance(newExp, exp) { - es[exp] = append(es[exp], uint64(si.SectorNumber)) + es[exp] = append(es[exp], si.SectorNumber) added = true break } } if !added { - es[newExp] = []uint64{uint64(si.SectorNumber)} + es[newExp] = []abi.SectorNumber{si.SectorNumber} } } } @@ -1051,7 +1065,7 @@ var sectorsRenewCmd = &cli.Command{ p.Extensions = append(p.Extensions, miner.ExpirationExtension{ Deadline: l.Deadline, Partition: l.Partition, - Sectors: bitfield.NewFromSet(numbers), + Sectors: SectorNumsToBitfield(numbers), NewExpiration: newExp, }) } @@ -1083,7 +1097,7 @@ var sectorsRenewCmd = &cli.Command{ } scount += int(count) } - fmt.Printf("Renewing %d sectors: ", scount) + fmt.Printf("Extending %d sectors: ", scount) stotal += scount if !cctx.Bool("really-do-it") { @@ -1097,8 +1111,7 @@ var sectorsRenewCmd = &cli.Command{ return err } - fmt.Println() - fmt.Println(string(data)) + fmt.Println("\n", string(data)) continue } @@ -1121,252 +1134,7 @@ var sectorsRenewCmd = &cli.Command{ fmt.Println(smsg.Cid()) } - fmt.Printf("%d sectors renewed\n", stotal) - - return nil - }, -} - -var sectorsExtendCmd = &cli.Command{ - Name: "extend", - Usage: "Extend sector expiration", - ArgsUsage: "", - Flags: []cli.Flag{ - &cli.Int64Flag{ - Name: "new-expiration", - Usage: "new expiration epoch", - Required: false, - }, - &cli.BoolFlag{ - Name: "v1-sectors", - Usage: "renews all v1 sectors up to the maximum possible lifetime", - Required: false, - }, - &cli.Int64Flag{ - Name: "tolerance", - Value: 20160, - Usage: "when extending v1 sectors, don't try to extend sectors by fewer than this number of epochs", - Required: false, - }, - &cli.Int64Flag{ - Name: "expiration-ignore", - Value: 120, - Usage: "when extending v1 sectors, skip sectors whose current expiration is less than epochs from now", - Required: false, - }, - &cli.Int64Flag{ - Name: "expiration-cutoff", - Usage: "when extending v1 sectors, skip sectors whose current expiration is more than epochs from now (infinity if unspecified)", - Required: false, - }, - &cli.StringFlag{}, - }, - Action: func(cctx *cli.Context) error { - - api, nCloser, err := lcli.GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer nCloser() - - ctx := lcli.ReqContext(cctx) - - maddr, err := getActorAddress(ctx, cctx) - if err != nil { - return err - } - - var params []miner.ExtendSectorExpirationParams - - if cctx.Bool("v1-sectors") { - - head, err := api.ChainHead(ctx) - if err != nil { - return err - } - - nv, err := api.StateNetworkVersion(ctx, types.EmptyTSK) - if err != nil { - return err - } - - extensions := map[lminer.SectorLocation]map[abi.ChainEpoch][]uint64{} - - // are given durations within tolerance epochs - withinTolerance := func(a, b abi.ChainEpoch) bool { - diff := a - b - if diff < 0 { - diff = b - a - } - - return diff <= abi.ChainEpoch(cctx.Int64("tolerance")) - } - - sis, err := api.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting miner sector infos: %w", err) - } - - for _, si := range sis { - if si.SealProof >= abi.RegisteredSealProof_StackedDrg2KiBV1_1 { - continue - } - - if si.Expiration < (head.Height() + abi.ChainEpoch(cctx.Int64("expiration-ignore"))) { - continue - } - - if cctx.IsSet("expiration-cutoff") { - if si.Expiration > (head.Height() + abi.ChainEpoch(cctx.Int64("expiration-cutoff"))) { - continue - } - } - - ml := policy.GetSectorMaxLifetime(si.SealProof, nv) - // if the sector's missing less than "tolerance" of its maximum possible lifetime, don't bother extending it - if withinTolerance(si.Expiration-si.Activation, ml) { - continue - } - - // Set the new expiration to 48 hours less than the theoretical maximum lifetime - newExp := ml - (miner.WPoStProvingPeriod * 2) + si.Activation - if withinTolerance(si.Expiration, newExp) || si.Expiration >= newExp { - continue - } - - p, err := api.StateSectorPartition(ctx, maddr, si.SectorNumber, types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting sector location for sector %d: %w", si.SectorNumber, err) - } - - if p == nil { - return xerrors.Errorf("sector %d not found in any partition", si.SectorNumber) - } - - es, found := extensions[*p] - if !found { - ne := make(map[abi.ChainEpoch][]uint64) - ne[newExp] = []uint64{uint64(si.SectorNumber)} - extensions[*p] = ne - } else { - added := false - for exp := range es { - if withinTolerance(exp, newExp) && newExp >= exp && exp > si.Expiration { - es[exp] = append(es[exp], uint64(si.SectorNumber)) - added = true - break - } - } - - if !added { - es[newExp] = []uint64{uint64(si.SectorNumber)} - } - } - } - - p := miner.ExtendSectorExpirationParams{} - scount := 0 - - for l, exts := range extensions { - for newExp, numbers := range exts { - scount += len(numbers) - addressedMax, err := policy.GetAddressedSectorsMax(nv) - if err != nil { - return xerrors.Errorf("failed to get addressed sectors max") - } - declMax, err := policy.GetDeclarationsMax(nv) - if err != nil { - return xerrors.Errorf("failed to get declarations max") - } - if scount > addressedMax || len(p.Extensions) == declMax { - params = append(params, p) - p = miner.ExtendSectorExpirationParams{} - scount = len(numbers) - } - - p.Extensions = append(p.Extensions, miner.ExpirationExtension{ - Deadline: l.Deadline, - Partition: l.Partition, - Sectors: bitfield.NewFromSet(numbers), - NewExpiration: newExp, - }) - } - } - - // if we have any sectors, then one last append is needed here - if scount != 0 { - params = append(params, p) - } - - } else { - if !cctx.Args().Present() || !cctx.IsSet("new-expiration") { - return xerrors.Errorf("must pass at least one sector number and new expiration") - } - sectors := map[lminer.SectorLocation][]uint64{} - - for i, s := range cctx.Args().Slice() { - id, err := strconv.ParseUint(s, 10, 64) - if err != nil { - return xerrors.Errorf("could not parse sector %d: %w", i, err) - } - - p, err := api.StateSectorPartition(ctx, maddr, abi.SectorNumber(id), types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting sector location for sector %d: %w", id, err) - } - - if p == nil { - return xerrors.Errorf("sector %d not found in any partition", id) - } - - sectors[*p] = append(sectors[*p], id) - } - - p := miner.ExtendSectorExpirationParams{} - for l, numbers := range sectors { - - // TODO: Dedup with above loop - p.Extensions = append(p.Extensions, miner.ExpirationExtension{ - Deadline: l.Deadline, - Partition: l.Partition, - Sectors: bitfield.NewFromSet(numbers), - NewExpiration: abi.ChainEpoch(cctx.Int64("new-expiration")), - }) - } - - params = append(params, p) - } - - if len(params) == 0 { - fmt.Println("nothing to extend") - return nil - } - - mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting miner info: %w", err) - } - - for i := range params { - sp, aerr := actors.SerializeParams(¶ms[i]) - if aerr != nil { - return xerrors.Errorf("serializing params: %w", err) - } - - smsg, err := api.MpoolPushMessage(ctx, &types.Message{ - From: mi.Worker, - To: maddr, - Method: builtin.MethodsMiner.ExtendSectorExpiration, - - Value: big.Zero(), - Params: sp, - }, nil) - if err != nil { - return xerrors.Errorf("mpool push message: %w", err) - } - - fmt.Println(smsg.Cid()) - } + fmt.Printf("%d sectors extended\n", stotal) return nil }, diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 4b705adad..9bc3bac11 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1677,8 +1677,7 @@ COMMANDS: precommits Print on-chain precommit info check-expire Inspect expiring sectors expired Get or cleanup expired sectors - renew Renew expiring sectors while not exceeding each sector's max life - extend Extend sector expiration + extend Extend expiring sectors while not exceeding each sector's max life terminate Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) remove Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty)) snap-up Mark a committed capacity sector to be filled with deals @@ -1884,13 +1883,13 @@ OPTIONS: ``` -### lotus-miner sectors renew +### lotus-miner sectors extend ``` NAME: - lotus-miner sectors renew - Renew expiring sectors while not exceeding each sector's max life + lotus-miner sectors extend - Extend expiring sectors while not exceeding each sector's max life USAGE: - lotus-miner sectors renew [command options] [arguments...] + lotus-miner sectors extend [command options] OPTIONS: --exclude value optionally provide a file containing excluding sectors @@ -1900,31 +1899,13 @@ OPTIONS: --max-sectors value the maximum number of sectors contained in each message message (default: 0) --new-expiration value try to extend selected sectors to this epoch, ignoring extension (default: 0) --only-cc only extend CC sectors (useful for making sector ready for snap upgrade) (default: false) - --really-do-it pass this flag to really renew sectors, otherwise will only print out json representation of parameters (default: false) + --really-do-it pass this flag to really extend sectors, otherwise will only print out json representation of parameters (default: false) --sector-file value provide a file containing one sector number in each line, ignoring above selecting criteria --to value only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 92160 (32 days) (default: 0) --tolerance value don't try to extend sectors by fewer than this number of epochs, defaults to 7 days (default: 20160) ``` -### lotus-miner sectors extend -``` -NAME: - lotus-miner sectors extend - Extend sector expiration - -USAGE: - lotus-miner sectors extend [command options] - -OPTIONS: - - --expiration-cutoff value when extending v1 sectors, skip sectors whose current expiration is more than epochs from now (infinity if unspecified) (default: 0) - --expiration-ignore value when extending v1 sectors, skip sectors whose current expiration is less than epochs from now (default: 120) - --new-expiration value new expiration epoch (default: 0) - --tolerance value when extending v1 sectors, don't try to extend sectors by fewer than this number of epochs (default: 20160) - --v1-sectors renews all v1 sectors up to the maximum possible lifetime (default: false) - -``` - ### lotus-miner sectors terminate ``` NAME: