package main import ( "bufio" "encoding/csv" "encoding/json" "errors" "fmt" "math" "os" "sort" "strconv" "strings" "sync" "time" "github.com/docker/go-units" "github.com/fatih/color" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/lib/result" "github.com/filecoin-project/lotus/lib/strle" "github.com/filecoin-project/lotus/lib/tablewriter" sealing "github.com/filecoin-project/lotus/storage/pipeline" ) const parallelSectorChecks = 300 var sectorsCmd = &cli.Command{ Name: "sectors", Usage: "interact with sector store", Subcommands: []*cli.Command{ sectorsStatusCmd, sectorsListCmd, sectorsRefsCmd, sectorsUpdateCmd, sectorsPledgeCmd, sectorsNumbersCmd, sectorPreCommitsCmd, sectorsCheckExpireCmd, sectorsExpiredCmd, sectorsExtendCmd, sectorsTerminateCmd, sectorsRemoveCmd, sectorsSnapUpCmd, sectorsSnapAbortCmd, sectorsStartSealCmd, sectorsSealDelayCmd, sectorsCapacityCollateralCmd, sectorsBatching, sectorsRefreshPieceMatchingCmd, sectorsCompactPartitionsCmd, sectorsUnsealCmd, }, } var sectorsPledgeCmd = &cli.Command{ Name: "pledge", Usage: "store random data in a sector", Action: func(cctx *cli.Context) error { minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) id, err := minerApi.PledgeSector(ctx) if err != nil { return err } fmt.Println("Created CC sector: ", id.Number) return nil }, } var sectorsStatusCmd = &cli.Command{ Name: "status", Usage: "Get the seal status of a sector by its number", ArgsUsage: "", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "log", Usage: "display event log", Aliases: []string{"l"}, }, &cli.BoolFlag{ Name: "on-chain-info", Usage: "show sector on chain info", Aliases: []string{"c"}, }, &cli.BoolFlag{ Name: "partition-info", Usage: "show partition related info", Aliases: []string{"p"}, }, &cli.BoolFlag{ Name: "proof", Usage: "print snark proof bytes as hex", }, }, Action: func(cctx *cli.Context) error { minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) if cctx.NArg() != 1 { return lcli.IncorrectNumArgs(cctx) } id, err := strconv.ParseUint(cctx.Args().First(), 10, 64) if err != nil { return err } onChainInfo := cctx.Bool("on-chain-info") status, err := minerApi.SectorsStatus(ctx, abi.SectorNumber(id), onChainInfo) if err != nil { return err } fmt.Printf("SectorID:\t%d\n", status.SectorID) fmt.Printf("Status:\t\t%s\n", status.State) fmt.Printf("CIDcommD:\t%s\n", status.CommD) fmt.Printf("CIDcommR:\t%s\n", status.CommR) fmt.Printf("Ticket:\t\t%x\n", status.Ticket.Value) fmt.Printf("TicketH:\t%d\n", status.Ticket.Epoch) fmt.Printf("Seed:\t\t%x\n", status.Seed.Value) fmt.Printf("SeedH:\t\t%d\n", status.Seed.Epoch) fmt.Printf("Precommit:\t%s\n", status.PreCommitMsg) fmt.Printf("Commit:\t\t%s\n", status.CommitMsg) if cctx.Bool("proof") { fmt.Printf("Proof:\t\t%x\n", status.Proof) } fmt.Printf("Deals:\t\t%v\n", status.Deals) fmt.Printf("Retries:\t%d\n", status.Retries) if status.LastErr != "" { fmt.Printf("Last Error:\t\t%s\n", status.LastErr) } if onChainInfo { fmt.Printf("\nSector On Chain Info\n") fmt.Printf("SealProof:\t\t%x\n", status.SealProof) fmt.Printf("Activation:\t\t%v\n", status.Activation) fmt.Printf("Expiration:\t\t%v\n", status.Expiration) fmt.Printf("DealWeight:\t\t%v\n", status.DealWeight) fmt.Printf("VerifiedDealWeight:\t\t%v\n", status.VerifiedDealWeight) fmt.Printf("InitialPledge:\t\t%v\n", types.FIL(status.InitialPledge)) fmt.Printf("\nExpiration Info\n") fmt.Printf("OnTime:\t\t%v\n", status.OnTime) fmt.Printf("Early:\t\t%v\n", status.Early) } if cctx.Bool("partition-info") { fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer nCloser() maddr, err := getActorAddress(ctx, cctx) if err != nil { return err } mact, err := fullApi.StateGetActor(ctx, maddr, types.EmptyTSK) if err != nil { return err } tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory()) mas, err := miner.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact) if err != nil { return err } errFound := errors.New("found") if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { pas, err := part.AllSectors() if err != nil { return err } set, err := pas.IsSet(id) if err != nil { return err } if set { fmt.Printf("\nDeadline:\t%d\n", dlIdx) fmt.Printf("Partition:\t%d\n", partIdx) checkIn := func(name string, bg func() (bitfield.BitField, error)) error { bf, err := bg() if err != nil { return err } set, err := bf.IsSet(id) if err != nil { return err } setstr := "no" if set { setstr = "yes" } fmt.Printf("%s: \t%s\n", name, setstr) return nil } if err := checkIn("Unproven", part.UnprovenSectors); err != nil { return err } if err := checkIn("Live", part.LiveSectors); err != nil { return err } if err := checkIn("Active", part.ActiveSectors); err != nil { return err } if err := checkIn("Faulty", part.FaultySectors); err != nil { return err } if err := checkIn("Recovering", part.RecoveringSectors); err != nil { return err } return errFound } return nil }) }); err != errFound { if err != nil { return err } fmt.Println("\nNot found in any partition") } } if cctx.Bool("log") { fmt.Printf("--------\nEvent Log:\n") for i, l := range status.Log { fmt.Printf("%d.\t%s:\t[%s]\t%s\n", i, time.Unix(int64(l.Timestamp), 0), l.Kind, l.Message) if l.Trace != "" { fmt.Printf("\t%s\n", l.Trace) } } } return nil }, } var sectorsListCmd = &cli.Command{ Name: "list", Usage: "List sectors", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "show-removed", Usage: "show removed sectors", Aliases: []string{"r"}, }, &cli.BoolFlag{ Name: "fast", Usage: "don't show on-chain info for better performance", Aliases: []string{"f"}, }, &cli.BoolFlag{ Name: "events", Usage: "display number of events the sector has received", Aliases: []string{"e"}, }, &cli.BoolFlag{ Name: "initial-pledge", Usage: "display initial pledge", Aliases: []string{"p"}, }, &cli.BoolFlag{ Name: "seal-time", Usage: "display how long it took for the sector to be sealed", Aliases: []string{"t"}, }, &cli.StringFlag{ Name: "states", Usage: "filter sectors by a comma-separated list of states", }, &cli.BoolFlag{ Name: "unproven", Usage: "only show sectors which aren't in the 'Proving' state", Aliases: []string{"u"}, }, &cli.Int64Flag{ Name: "check-parallelism", Usage: "number of parallel requests to make for checking sector states", Value: parallelSectorChecks, }, }, Subcommands: []*cli.Command{ sectorsListUpgradeBoundsCmd, }, Action: func(cctx *cli.Context) error { // http mode allows for parallel json decoding/encoding, which was a bottleneck here minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) if err != nil { return err } defer closer() fullApi, closer2, err := lcli.GetFullNodeAPI(cctx) // TODO: consider storing full node address in config if err != nil { return err } defer closer2() ctx := lcli.ReqContext(cctx) var list []abi.SectorNumber showRemoved := cctx.Bool("show-removed") var states []api.SectorState if cctx.IsSet("states") && cctx.IsSet("unproven") { return xerrors.Errorf("only one of --states or --unproven can be specified at once") } if cctx.IsSet("states") { showRemoved = true sList := strings.Split(cctx.String("states"), ",") states = make([]api.SectorState, len(sList)) for i := range sList { states[i] = api.SectorState(sList[i]) } } if cctx.Bool("unproven") { for state := range sealing.ExistSectorStateList { if state == sealing.Proving || state == sealing.Available { continue } states = append(states, api.SectorState(state)) } } if len(states) == 0 { list, err = minerApi.SectorsList(ctx) } else { list, err = minerApi.SectorsListInStates(ctx, states) } if err != nil { return err } maddr, err := minerApi.ActorAddress(ctx) if err != nil { return err } head, err := fullApi.ChainHead(ctx) if err != nil { return err } activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, head.Key()) if err != nil { return err } activeIDs := make(map[abi.SectorNumber]struct{}, len(activeSet)) for _, info := range activeSet { activeIDs[info.SectorNumber] = struct{}{} } sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, head.Key()) if err != nil { return err } commitedIDs := make(map[abi.SectorNumber]struct{}, len(sset)) for _, info := range sset { commitedIDs[info.SectorNumber] = struct{}{} } sort.Slice(list, func(i, j int) bool { return list[i] < list[j] }) tw := tablewriter.New( tablewriter.Col("ID"), tablewriter.Col("State"), tablewriter.Col("OnChain"), tablewriter.Col("Active"), tablewriter.Col("Expiration"), tablewriter.Col("SealTime"), tablewriter.Col("Events"), tablewriter.Col("Deals"), tablewriter.Col("DealWeight"), tablewriter.Col("VerifiedPower"), tablewriter.Col("Pledge"), tablewriter.NewLineCol("Error"), tablewriter.NewLineCol("RecoveryTimeout")) fast := cctx.Bool("fast") throttle := make(chan struct{}, cctx.Int64("check-parallelism")) slist := make([]result.Result[api.SectorInfo], len(list)) var wg sync.WaitGroup for i, s := range list { throttle <- struct{}{} wg.Add(1) go func(i int, s abi.SectorNumber) { defer wg.Done() defer func() { <-throttle }() r := result.Wrap(minerApi.SectorsStatus(ctx, s, !fast)) if r.Error != nil { r.Value.SectorID = s } slist[i] = r }(i, s) } wg.Wait() for _, rsn := range slist { if rsn.Error != nil { tw.Write(map[string]interface{}{ "ID": rsn.Value.SectorID, "Error": err, }) continue } st := rsn.Value s := st.SectorID if !showRemoved && st.State == api.SectorState(sealing.Removed) { continue } _, inSSet := commitedIDs[s] _, inASet := activeIDs[s] const verifiedPowerGainMul = 9 dw, vp := .0, .0 estimate := (st.Expiration-st.Activation <= 0) || sealing.IsUpgradeState(sealing.SectorState(st.State)) if !estimate { rdw := big.Add(st.DealWeight, st.VerifiedDealWeight) dw = float64(big.Div(rdw, big.NewInt(int64(st.Expiration-st.Activation))).Uint64()) vp = float64(big.Div(big.Mul(st.VerifiedDealWeight, big.NewInt(verifiedPowerGainMul)), big.NewInt(int64(st.Expiration-st.Activation))).Uint64()) } else { for _, piece := range st.Pieces { if piece.DealInfo != nil { dw += float64(piece.Piece.Size) if piece.DealInfo.DealProposal != nil && piece.DealInfo.DealProposal.VerifiedDeal { vp += float64(piece.Piece.Size) * verifiedPowerGainMul } } } } var deals int for _, deal := range st.Deals { if deal != 0 { deals++ } } exp := st.Expiration if st.OnTime > 0 && st.OnTime < exp { exp = st.OnTime // Can be different when the sector was CC upgraded } m := map[string]interface{}{ "ID": s, "State": color.New(stateOrder[sealing.SectorState(st.State)].col).Sprint(st.State), "OnChain": yesno(inSSet), "Active": yesno(inASet), } if deals > 0 { m["Deals"] = color.GreenString("%d", deals) } else { m["Deals"] = color.BlueString("CC") if st.ToUpgrade { m["Deals"] = color.CyanString("CC(upgrade)") } } if !fast { if !inSSet { m["Expiration"] = "n/a" } else { m["Expiration"] = cliutil.EpochTime(head.Height(), exp) if st.Early > 0 { m["RecoveryTimeout"] = color.YellowString(cliutil.EpochTime(head.Height(), st.Early)) } } if inSSet && cctx.Bool("initial-pledge") { m["Pledge"] = types.FIL(st.InitialPledge).Short() } } if !fast && deals > 0 { estWrap := func(s string) string { if !estimate { return s } return fmt.Sprintf("[%s]", s) } m["DealWeight"] = estWrap(units.BytesSize(dw)) if vp > 0 { m["VerifiedPower"] = estWrap(color.GreenString(units.BytesSize(vp))) } } if cctx.Bool("events") { var events int for _, sectorLog := range st.Log { if !strings.HasPrefix(sectorLog.Kind, "event") { continue } if sectorLog.Kind == "event;sealing.SectorRestart" { continue } events++ } pieces := len(st.Deals) switch { case events < 12+pieces: m["Events"] = color.GreenString("%d", events) case events < 20+pieces: m["Events"] = color.YellowString("%d", events) default: m["Events"] = color.RedString("%d", events) } } if cctx.Bool("seal-time") && len(st.Log) > 1 { start := time.Unix(int64(st.Log[0].Timestamp), 0) for _, sectorLog := range st.Log { if sectorLog.Kind == "event;sealing.SectorProving" { // todo: figure out a good way to not hardcode end := time.Unix(int64(sectorLog.Timestamp), 0) dur := end.Sub(start) switch { case dur < 12*time.Hour: m["SealTime"] = color.GreenString("%s", dur) case dur < 24*time.Hour: m["SealTime"] = color.YellowString("%s", dur) default: m["SealTime"] = color.RedString("%s", dur) } break } } } tw.Write(m) } return tw.Flush(os.Stdout) }, } var sectorsListUpgradeBoundsCmd = &cli.Command{ Name: "upgrade-bounds", Usage: "Output upgrade bounds for available sectors", Flags: []cli.Flag{ &cli.IntFlag{ Name: "buckets", Value: 25, }, &cli.BoolFlag{ Name: "csv", Usage: "output machine-readable values", }, &cli.BoolFlag{ Name: "deal-terms", Usage: "bucket by how many deal-sectors can start at a given expiration", }, }, Action: func(cctx *cli.Context) error { minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) if err != nil { return err } defer closer() fullApi, closer2, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer closer2() ctx := lcli.ReqContext(cctx) list, err := minerApi.SectorsListInStates(ctx, []api.SectorState{ api.SectorState(sealing.Available), }) if err != nil { return xerrors.Errorf("getting sector list: %w", err) } head, err := fullApi.ChainHead(ctx) if err != nil { return xerrors.Errorf("getting chain head: %w", err) } filter := bitfield.New() for _, s := range list { filter.Set(uint64(s)) } maddr, err := minerApi.ActorAddress(ctx) if err != nil { return err } sset, err := fullApi.StateMinerSectors(ctx, maddr, &filter, head.Key()) if err != nil { return err } if len(sset) == 0 { return nil } var minExpiration, maxExpiration abi.ChainEpoch for _, s := range sset { if s.Expiration < minExpiration || minExpiration == 0 { minExpiration = s.Expiration } if s.Expiration > maxExpiration { maxExpiration = s.Expiration } } buckets := cctx.Int("buckets") bucketSize := (maxExpiration - minExpiration) / abi.ChainEpoch(buckets) bucketCounts := make([]int, buckets+1) for b := range bucketCounts { bucketMin := minExpiration + abi.ChainEpoch(b)*bucketSize bucketMax := minExpiration + abi.ChainEpoch(b+1)*bucketSize if cctx.Bool("deal-terms") { bucketMax = bucketMax + policy.MarketDefaultAllocationTermBuffer } for _, s := range sset { isInBucket := s.Expiration >= bucketMin && s.Expiration < bucketMax if isInBucket { bucketCounts[b]++ } } } // Creating CSV writer writer := csv.NewWriter(os.Stdout) // Writing CSV headers err = writer.Write([]string{"Max Expiration in Bucket", "Sector Count"}) if err != nil { return xerrors.Errorf("writing csv headers: %w", err) } // Writing bucket details if cctx.Bool("csv") { for i := 0; i < buckets; i++ { maxExp := minExpiration + abi.ChainEpoch(i+1)*bucketSize timeStr := strconv.FormatInt(int64(maxExp), 10) err = writer.Write([]string{ timeStr, strconv.Itoa(bucketCounts[i]), }) if err != nil { return xerrors.Errorf("writing csv row: %w", err) } } // Flush to make sure all data is written to the underlying writer writer.Flush() if err := writer.Error(); err != nil { return xerrors.Errorf("flushing csv writer: %w", err) } return nil } tw := tablewriter.New( tablewriter.Col("Bucket Expiration"), tablewriter.Col("Sector Count"), tablewriter.Col("Bar"), ) var barCols = 40 var maxCount int for _, c := range bucketCounts { if c > maxCount { maxCount = c } } for i := 0; i < buckets; i++ { maxExp := minExpiration + abi.ChainEpoch(i+1)*bucketSize timeStr := cliutil.EpochTime(head.Height(), maxExp) tw.Write(map[string]interface{}{ "Bucket Expiration": timeStr, "Sector Count": color.YellowString("%d", bucketCounts[i]), "Bar": "[" + color.GreenString(strings.Repeat("|", bucketCounts[i]*barCols/maxCount)) + strings.Repeat(" ", barCols-bucketCounts[i]*barCols/maxCount) + "]", }) } return tw.Flush(os.Stdout) }, } var sectorsRefsCmd = &cli.Command{ Name: "refs", Usage: "List References to sectors", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetMarketsAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) refs, err := nodeApi.SectorsRefs(ctx) if err != nil { return err } for name, refs := range refs { fmt.Printf("Block %s:\n", name) for _, ref := range refs { fmt.Printf("\t%d+%d %d bytes\n", ref.SectorID, ref.Offset, ref.Size) } } return nil }, } var sectorsCheckExpireCmd = &cli.Command{ Name: "check-expire", Usage: "Inspect expiring sectors", Flags: []cli.Flag{ &cli.Int64Flag{ Name: "cutoff", Usage: "skip sectors whose current expiration is more than epochs from now, defaults to 60 days", Value: 172800, }, }, Action: func(cctx *cli.Context) error { fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer nCloser() ctx := lcli.ReqContext(cctx) maddr, err := getActorAddress(ctx, cctx) if err != nil { return err } head, err := fullApi.ChainHead(ctx) if err != nil { return err } currEpoch := head.Height() nv, err := fullApi.StateNetworkVersion(ctx, types.EmptyTSK) if err != nil { return err } sectors, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) if err != nil { return err } n := 0 for _, s := range sectors { if s.Expiration-currEpoch <= abi.ChainEpoch(cctx.Int64("cutoff")) { sectors[n] = s n++ } } sectors = sectors[:n] sort.Slice(sectors, func(i, j int) bool { if sectors[i].Expiration == sectors[j].Expiration { return sectors[i].SectorNumber < sectors[j].SectorNumber } return sectors[i].Expiration < sectors[j].Expiration }) tw := tablewriter.New( tablewriter.Col("ID"), tablewriter.Col("SealProof"), tablewriter.Col("InitialPledge"), tablewriter.Col("Activation"), tablewriter.Col("Expiration"), tablewriter.Col("MaxExpiration"), tablewriter.Col("MaxExtendNow")) for _, sector := range sectors { MaxExpiration := sector.Activation + policy.GetSectorMaxLifetime(sector.SealProof, nv) MaxExtendNow := currEpoch + policy.GetMaxSectorExpirationExtension() if MaxExtendNow > MaxExpiration { MaxExtendNow = MaxExpiration } tw.Write(map[string]interface{}{ "ID": sector.SectorNumber, "SealProof": sector.SealProof, "InitialPledge": types.FIL(sector.InitialPledge).Short(), "Activation": cliutil.EpochTime(currEpoch, sector.Activation), "Expiration": cliutil.EpochTime(currEpoch, sector.Expiration), "MaxExpiration": cliutil.EpochTime(currEpoch, MaxExpiration), "MaxExtendNow": cliutil.EpochTime(currEpoch, MaxExtendNow), }) } return tw.Flush(os.Stdout) }, } type PseudoExpirationExtension struct { Deadline uint64 Partition uint64 Sectors string NewExpiration abi.ChainEpoch } type PseudoExtendSectorExpirationParams struct { Extensions []PseudoExpirationExtension } func NewPseudoExtendParams(p *miner.ExtendSectorExpiration2Params) (*PseudoExtendSectorExpirationParams, error) { res := PseudoExtendSectorExpirationParams{} for _, ext := range p.Extensions { scount, err := ext.Sectors.Count() if err != nil { return nil, err } sectors, err := ext.Sectors.All(scount) if err != nil { return nil, err } res.Extensions = append(res.Extensions, PseudoExpirationExtension{ Deadline: ext.Deadline, Partition: ext.Partition, Sectors: ArrayToString(sectors), NewExpiration: ext.NewExpiration, }) } return &res, nil } // ArrayToString Example: {1,3,4,5,8,9} -> "1,3-5,8-9" func ArrayToString(array []uint64) string { sort.Slice(array, func(i, j int) bool { return array[i] < array[j] }) var sarray []string s := "" for i, elm := range array { if i == 0 { s = strconv.FormatUint(elm, 10) continue } if elm == array[i-1] { continue // filter out duplicates } else if elm == array[i-1]+1 { s = strings.Split(s, "-")[0] + "-" + strconv.FormatUint(elm, 10) } else { sarray = append(sarray, s) s = strconv.FormatUint(elm, 10) } } if s != "" { sarray = append(sarray, s) } return strings.Join(sarray, ",") } func getSectorsFromFile(filePath string) ([]abi.SectorNumber, error) { file, err := os.Open(filePath) if err != nil { return nil, err } scanner := bufio.NewScanner(file) sectors := make([]abi.SectorNumber, 0) for scanner.Scan() { line := scanner.Text() id, err := strconv.ParseUint(line, 10, 64) if err != nil { return nil, xerrors.Errorf("could not parse %s as sector id: %s", line, err) } sectors = append(sectors, abi.SectorNumber(id)) } if err = file.Close(); err != nil { return nil, err } return sectors, nil } func SectorNumsToBitfield(sectors []abi.SectorNumber) bitfield.BitField { var numbers []uint64 for _, sector := range sectors { numbers = append(numbers, uint64(sector)) } return bitfield.NewFromSet(numbers) } var sectorsExtendCmd = &cli.Command{ Name: "extend", Usage: "Extend expiring sectors while not exceeding each sector's max life", ArgsUsage: "", Flags: []cli.Flag{ &cli.Int64Flag{ Name: "from", Usage: "only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 120 (1 hour)", }, &cli.Int64Flag{ Name: "to", Usage: "only consider sectors whose current expiration epoch is in the range of [from, to], defaults to: now + 92160 (32 days)", }, &cli.StringFlag{ Name: "sector-file", Usage: "provide a file containing one sector number in each line, ignoring above selecting criteria", }, &cli.StringFlag{ Name: "exclude", Usage: "optionally provide a file containing excluding sectors", }, &cli.Int64Flag{ Name: "extension", Usage: "try to extend selected sectors by this number of epochs, defaults to 540 days", Value: 1555200, }, &cli.Int64Flag{ Name: "new-expiration", Usage: "try to extend selected sectors to this epoch, ignoring extension", }, &cli.BoolFlag{ Name: "only-cc", Usage: "only extend CC sectors (useful for making sector ready for snap upgrade)", }, &cli.BoolFlag{ Name: "drop-claims", Usage: "drop claims for sectors that can be extended, but only by dropping some of their verified power claims", }, &cli.Int64Flag{ Name: "tolerance", Usage: "don't try to extend sectors by fewer than this number of epochs, defaults to 7 days", Value: 20160, }, &cli.StringFlag{ Name: "max-fee", Usage: "use up to this amount of FIL for one message. pass this flag to avoid message congestion.", Value: "0", }, &cli.Int64Flag{ Name: "max-sectors", Usage: "the maximum number of sectors contained in each message", }, &cli.BoolFlag{ Name: "really-do-it", Usage: "pass this flag to really extend sectors, otherwise will only print out json representation of parameters", }, }, Action: func(cctx *cli.Context) error { mf, err := types.ParseFIL(cctx.String("max-fee")) if err != nil { return err } spec := &api.MessageSendSpec{MaxFee: abi.TokenAmount(mf)} fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer nCloser() ctx := lcli.ReqContext(cctx) maddr, err := getActorAddress(ctx, cctx) if err != nil { return err } head, err := fullApi.ChainHead(ctx) if err != nil { return err } currEpoch := head.Height() nv, err := fullApi.StateNetworkVersion(ctx, types.EmptyTSK) if err != nil { return err } activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) if err != nil { return err } activeSectorsInfo := make(map[abi.SectorNumber]*miner.SectorOnChainInfo, len(activeSet)) for _, info := range activeSet { activeSectorsInfo[info.SectorNumber] = info } mact, err := fullApi.StateGetActor(ctx, maddr, types.EmptyTSK) if err != nil { return err } tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory()) adtStore := adt.WrapStore(ctx, cbor.NewCborStore(tbs)) mas, err := miner.Load(adtStore, mact) if err != nil { return err } activeSectorsLocation := make(map[abi.SectorNumber]*miner.SectorLocation, len(activeSet)) if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { pas, err := part.ActiveSectors() if err != nil { return err } return pas.ForEach(func(i uint64) error { activeSectorsLocation[abi.SectorNumber(i)] = &miner.SectorLocation{ Deadline: dlIdx, Partition: partIdx, } return nil }) }) }); err != nil { return err } excludeSet := make(map[abi.SectorNumber]struct{}) if cctx.IsSet("exclude") { excludeSectors, err := getSectorsFromFile(cctx.String("exclude")) if err != nil { return err } for _, id := range excludeSectors { excludeSet[id] = struct{}{} } } var sectors []abi.SectorNumber if cctx.Args().Present() { if cctx.IsSet("sector-file") { return xerrors.Errorf("sector-file specified along with command line params") } for i, s := range cctx.Args().Slice() { id, err := strconv.ParseUint(s, 10, 64) if err != nil { return xerrors.Errorf("could not parse sector %d: %w", i, err) } sectors = append(sectors, abi.SectorNumber(id)) } } else if cctx.IsSet("sector-file") { sectors, err = getSectorsFromFile(cctx.String("sector-file")) if err != nil { return err } } else { from := currEpoch + 120 to := currEpoch + 92160 if cctx.IsSet("from") { from = abi.ChainEpoch(cctx.Int64("from")) } if cctx.IsSet("to") { to = abi.ChainEpoch(cctx.Int64("to")) } for _, si := range activeSet { if si.Expiration >= from && si.Expiration <= to { sectors = append(sectors, si.SectorNumber) } } } var sis []*miner.SectorOnChainInfo for _, id := range sectors { if _, exclude := excludeSet[id]; exclude { continue } si, found := activeSectorsInfo[id] if !found { return xerrors.Errorf("sector %d is not active", id) } if len(si.DealIDs) > 0 && cctx.Bool("only-cc") { continue } sis = append(sis, si) } withinTolerance := func(a, b abi.ChainEpoch) bool { diff := a - b if diff < 0 { diff = -diff } return diff <= abi.ChainEpoch(cctx.Int64("tolerance")) } extensions := map[miner.SectorLocation]map[abi.ChainEpoch][]abi.SectorNumber{} for _, si := range sis { extension := abi.ChainEpoch(cctx.Int64("extension")) newExp := si.Expiration + extension if cctx.IsSet("new-expiration") { newExp = abi.ChainEpoch(cctx.Int64("new-expiration")) } maxExtendNow := currEpoch + policy.GetMaxSectorExpirationExtension() if newExp > maxExtendNow { newExp = maxExtendNow } maxExp := si.Activation + policy.GetSectorMaxLifetime(si.SealProof, nv) if newExp > maxExp { newExp = maxExp } if newExp <= si.Expiration || withinTolerance(newExp, si.Expiration) { continue } l, found := activeSectorsLocation[si.SectorNumber] if !found { return xerrors.Errorf("location for sector %d not found", si.SectorNumber) } es, found := extensions[*l] if !found { ne := make(map[abi.ChainEpoch][]abi.SectorNumber) ne[newExp] = []abi.SectorNumber{si.SectorNumber} extensions[*l] = ne } else { added := false for exp := range es { if withinTolerance(newExp, exp) { es[exp] = append(es[exp], si.SectorNumber) added = true break } } if !added { es[newExp] = []abi.SectorNumber{si.SectorNumber} } } } verifregAct, err := fullApi.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, types.EmptyTSK) if err != nil { return xerrors.Errorf("failed to lookup verifreg actor: %w", err) } verifregSt, err := verifreg.Load(adtStore, verifregAct) if err != nil { return xerrors.Errorf("failed to load verifreg state: %w", err) } claimsMap, err := verifregSt.GetClaims(maddr) if err != nil { return xerrors.Errorf("failed to lookup claims for miner: %w", err) } claimIdsBySector, err := verifregSt.GetClaimIdsBySector(maddr) if err != nil { return xerrors.Errorf("failed to lookup claim IDs by sector: %w", err) } sectorsMax, err := policy.GetAddressedSectorsMax(nv) if err != nil { return err } declMax, err := policy.GetDeclarationsMax(nv) if err != nil { return err } addrSectors := sectorsMax if cctx.Int("max-sectors") != 0 { addrSectors = cctx.Int("max-sectors") if addrSectors > sectorsMax { return xerrors.Errorf("the specified max-sectors exceeds the maximum limit") } } var params []miner.ExtendSectorExpiration2Params p := miner.ExtendSectorExpiration2Params{} scount := 0 for l, exts := range extensions { for newExp, numbers := range exts { sectorsWithoutClaimsToExtend := bitfield.New() var sectorsWithClaims []miner.SectorClaim for _, sectorNumber := range numbers { claimIdsToMaintain := make([]verifreg.ClaimId, 0) claimIdsToDrop := make([]verifreg.ClaimId, 0) cannotExtendSector := false claimIds, ok := claimIdsBySector[sectorNumber] // Nothing to check, add to ccSectors if !ok { sectorsWithoutClaimsToExtend.Set(uint64(sectorNumber)) } else { for _, claimId := range claimIds { claim, ok := claimsMap[claimId] if !ok { return xerrors.Errorf("failed to find claim for claimId %d", claimId) } claimExpiration := claim.TermStart + claim.TermMax // can be maintained in the extended sector if claimExpiration > newExp { claimIdsToMaintain = append(claimIdsToMaintain, claimId) } else { sectorInfo, ok := activeSectorsInfo[sectorNumber] if !ok { return xerrors.Errorf("failed to find sector in active sector set: %w", err) } if !cctx.Bool("drop-claims") || // FIP-0045 requires the claim minimum duration to have passed currEpoch <= (claim.TermStart+claim.TermMin) || // FIP-0045 requires the sector to be in its last 30 days of life (currEpoch <= sectorInfo.Expiration-builtin.EndOfLifeClaimDropPeriod) { fmt.Printf("skipping sector %d because claim %d does not live long enough \n", sectorNumber, claimId) cannotExtendSector = true break } claimIdsToDrop = append(claimIdsToDrop, claimId) } } if cannotExtendSector { continue } if len(claimIdsToMaintain)+len(claimIdsToDrop) != 0 { sectorsWithClaims = append(sectorsWithClaims, miner.SectorClaim{ SectorNumber: sectorNumber, MaintainClaims: claimIdsToMaintain, DropClaims: claimIdsToDrop, }) } } } sectorsWithoutClaimsCount, err := sectorsWithoutClaimsToExtend.Count() if err != nil { return xerrors.Errorf("failed to count cc sectors: %w", err) } sectorsInDecl := int(sectorsWithoutClaimsCount) + len(sectorsWithClaims) scount += sectorsInDecl if scount > addrSectors || len(p.Extensions) >= declMax { params = append(params, p) p = miner.ExtendSectorExpiration2Params{} scount = sectorsInDecl } p.Extensions = append(p.Extensions, miner.ExpirationExtension2{ Deadline: l.Deadline, Partition: l.Partition, Sectors: SectorNumsToBitfield(numbers), SectorsWithClaims: sectorsWithClaims, NewExpiration: newExp, }) } } // if we have any sectors, then one last append is needed here if scount != 0 { params = append(params, p) } if len(params) == 0 { fmt.Println("nothing to extend") return nil } mi, err := fullApi.StateMinerInfo(ctx, maddr, types.EmptyTSK) if err != nil { return xerrors.Errorf("getting miner info: %w", err) } stotal := 0 for i := range params { scount := 0 for _, ext := range params[i].Extensions { count, err := ext.Sectors.Count() if err != nil { return err } scount += int(count) } fmt.Printf("Extending %d sectors: ", scount) stotal += scount if !cctx.Bool("really-do-it") { pp, err := NewPseudoExtendParams(¶ms[i]) if err != nil { return err } data, err := json.MarshalIndent(pp, "", " ") if err != nil { return err } fmt.Println("\n", string(data)) continue } sp, aerr := actors.SerializeParams(¶ms[i]) if aerr != nil { return xerrors.Errorf("serializing params: %w", err) } smsg, err := fullApi.MpoolPushMessage(ctx, &types.Message{ From: mi.Worker, To: maddr, Method: builtin.MethodsMiner.ExtendSectorExpiration2, Value: big.Zero(), Params: sp, }, spec) if err != nil { return xerrors.Errorf("mpool push message: %w", err) } fmt.Println(smsg.Cid()) } fmt.Printf("%d sectors extended\n", stotal) return nil }, } var sectorsTerminateCmd = &cli.Command{ Name: "terminate", Usage: "Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector)", ArgsUsage: "", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "really-do-it", Usage: "pass this flag if you know what you are doing", }, }, Subcommands: []*cli.Command{ sectorsTerminateFlushCmd, sectorsTerminatePendingCmd, }, Action: func(cctx *cli.Context) error { minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) if cctx.NArg() != 1 { return lcli.IncorrectNumArgs(cctx) } if !cctx.Bool("really-do-it") { return xerrors.Errorf("pass --really-do-it to confirm this action") } id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) if err != nil { return xerrors.Errorf("could not parse sector number: %w", err) } return minerApi.SectorTerminate(ctx, abi.SectorNumber(id)) }, } var sectorsTerminateFlushCmd = &cli.Command{ Name: "flush", Usage: "Send a terminate message if there are sectors queued for termination", Action: func(cctx *cli.Context) error { minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) mcid, err := minerApi.SectorTerminateFlush(ctx) if err != nil { return err } if mcid == nil { return xerrors.New("no sectors were queued for termination") } fmt.Println(mcid) return nil }, } var sectorsTerminatePendingCmd = &cli.Command{ Name: "pending", Usage: "List sector numbers of sectors pending termination", Action: func(cctx *cli.Context) error { minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() api, nCloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer nCloser() ctx := lcli.ReqContext(cctx) pending, err := minerAPI.SectorTerminatePending(ctx) if err != nil { return err } maddr, err := minerAPI.ActorAddress(ctx) if err != nil { return err } dl, err := api.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) if err != nil { return xerrors.Errorf("getting proving deadline info failed: %w", err) } for _, id := range pending { loc, err := api.StateSectorPartition(ctx, maddr, id.Number, types.EmptyTSK) if err != nil { return xerrors.Errorf("finding sector partition: %w", err) } fmt.Print(id.Number) if loc.Deadline == (dl.Index+1)%miner.WPoStPeriodDeadlines || // not in next (in case the terminate message takes a while to get on chain) loc.Deadline == dl.Index || // not in current (loc.Deadline+1)%miner.WPoStPeriodDeadlines == dl.Index { // not in previous fmt.Print(" (in proving window)") } fmt.Println() } return nil }, } var sectorsRemoveCmd = &cli.Command{ Name: "remove", Usage: "Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty))", ArgsUsage: "", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "really-do-it", Usage: "pass this flag if you know what you are doing", }, }, Action: func(cctx *cli.Context) error { if !cctx.Bool("really-do-it") { return xerrors.Errorf("this is a command for advanced users, only use it if you are sure of what you are doing") } minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) if cctx.NArg() != 1 { return lcli.IncorrectNumArgs(cctx) } id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) if err != nil { return xerrors.Errorf("could not parse sector number: %w", err) } // Check if the sector exists _, err = minerAPI.SectorsStatus(ctx, abi.SectorNumber(id), false) if err != nil { return xerrors.Errorf("sectorID %d has not been created yet: %w", id, err) } return minerAPI.SectorRemove(ctx, abi.SectorNumber(id)) }, } var sectorsSnapUpCmd = &cli.Command{ Name: "snap-up", Usage: "Mark a committed capacity sector to be filled with deals", ArgsUsage: "", Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { return lcli.IncorrectNumArgs(cctx) } minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() api, nCloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } defer nCloser() ctx := lcli.ReqContext(cctx) nv, err := api.StateNetworkVersion(ctx, types.EmptyTSK) if err != nil { return xerrors.Errorf("failed to get network version: %w", err) } if nv < network.Version15 { return xerrors.Errorf("snap deals upgrades enabled in network v15") } id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) if err != nil { return xerrors.Errorf("could not parse sector number: %w", err) } return minerAPI.SectorMarkForUpgrade(ctx, abi.SectorNumber(id), true) }, } var sectorsSnapAbortCmd = &cli.Command{ Name: "abort-upgrade", Usage: "Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before", ArgsUsage: "", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "really-do-it", Usage: "pass this flag if you know what you are doing", }, }, Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number")) } really := cctx.Bool("really-do-it") if !really { //nolint:golint return fmt.Errorf("--really-do-it must be specified for this action to have an effect; you have been warned") } minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) if err != nil { return xerrors.Errorf("could not parse sector number: %w", err) } return minerAPI.SectorAbortUpgrade(ctx, abi.SectorNumber(id)) }, } var sectorsStartSealCmd = &cli.Command{ Name: "seal", Usage: "Manually start sealing a sector (filling any unused space with junk)", ArgsUsage: "", Action: func(cctx *cli.Context) error { minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } defer closer() ctx := lcli.ReqContext(cctx) if cctx.NArg() != 1 { return lcli.IncorrectNumArgs(cctx) } id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) if err != nil { return xerrors.Errorf("could not parse sector number: %w", err) } return minerAPI.SectorStartSealing(ctx, abi.SectorNumber(id)) }, } var sectorsSealDelayCmd = &cli.Command{ Name: "set-seal-delay", Usage: "Set the time (in minutes) that a new sector waits for deals before sealing starts", ArgsUsage: "