lotus/cmd/lotus-miner/sectors.go

2558 lines
64 KiB
Go
Raw Normal View History

package main
import (
"bufio"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"sort"
"strconv"
"strings"
2023-02-07 10:51:04 +00:00
"sync"
"time"
2020-09-30 11:33:42 +00:00
"github.com/docker/go-units"
"github.com/fatih/color"
2021-04-23 08:30:23 +00:00
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/urfave/cli/v2"
2020-06-05 22:59:01 +00:00
"golang.org/x/xerrors"
2021-02-23 08:22:30 +00:00
"github.com/filecoin-project/go-bitfield"
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
2020-09-30 11:33:42 +00:00
"github.com/filecoin-project/go-state-types/big"
2022-04-20 21:34:28 +00:00
"github.com/filecoin-project/go-state-types/builtin"
2022-09-06 15:49:29 +00:00
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
"github.com/filecoin-project/go-state-types/network"
2020-04-06 20:27:14 +00:00
"github.com/filecoin-project/lotus/api"
2021-04-23 08:30:23 +00:00
"github.com/filecoin-project/lotus/blockstore"
2021-02-23 08:22:30 +00:00
"github.com/filecoin-project/lotus/chain/actors"
2021-04-23 08:30:23 +00:00
"github.com/filecoin-project/lotus/chain/actors/adt"
2022-04-20 21:34:28 +00:00
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
2020-10-08 01:09:33 +00:00
"github.com/filecoin-project/lotus/chain/actors/policy"
2020-04-06 20:27:14 +00:00
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
2022-10-31 12:32:00 +00:00
cliutil "github.com/filecoin-project/lotus/cli/util"
2023-02-07 10:51:04 +00:00
"github.com/filecoin-project/lotus/lib/result"
2022-08-24 15:25:37 +00:00
"github.com/filecoin-project/lotus/lib/strle"
2022-06-14 15:00:51 +00:00
"github.com/filecoin-project/lotus/lib/tablewriter"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
)
2023-02-07 10:51:04 +00:00
const parallelSectorChecks = 300
2020-03-23 12:29:24 +00:00
var sectorsCmd = &cli.Command{
Name: "sectors",
Usage: "interact with sector store",
Subcommands: []*cli.Command{
sectorsStatusCmd,
sectorsListCmd,
sectorsRefsCmd,
sectorsUpdateCmd,
sectorsPledgeCmd,
2022-08-18 14:37:22 +00:00
sectorsNumbersCmd,
2022-05-23 04:54:00 +00:00
sectorPreCommitsCmd,
2021-04-23 08:16:56 +00:00
sectorsCheckExpireCmd,
2021-08-19 13:53:59 +00:00
sectorsExpiredCmd,
2021-02-23 08:22:30 +00:00
sectorsExtendCmd,
2021-01-12 23:42:01 +00:00
sectorsTerminateCmd,
2020-06-22 17:35:14 +00:00
sectorsRemoveCmd,
sectorsSnapUpCmd,
sectorsSnapAbortCmd,
sectorsStartSealCmd,
sectorsSealDelayCmd,
sectorsCapacityCollateralCmd,
2021-05-18 17:47:30 +00:00
sectorsBatching,
sectorsRefreshPieceMatchingCmd,
sectorsCompactPartitionsCmd,
sectorsUnsealCmd,
2020-03-23 12:29:24 +00:00
},
}
var sectorsPledgeCmd = &cli.Command{
Name: "pledge",
Usage: "store random data in a sector",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
2019-10-03 18:12:30 +00:00
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:51:18 +00:00
id, err := minerApi.PledgeSector(ctx)
2021-02-16 18:16:35 +00:00
if err != nil {
return err
}
fmt.Println("Created CC sector: ", id.Number)
return nil
},
}
var sectorsStatusCmd = &cli.Command{
2020-06-23 12:44:34 +00:00
Name: "status",
Usage: "Get the seal status of a sector by its number",
ArgsUsage: "<sectorNum>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "log",
Usage: "display event log",
Aliases: []string{"l"},
},
&cli.BoolFlag{
Name: "on-chain-info",
Usage: "show sector on chain info",
Aliases: []string{"c"},
},
&cli.BoolFlag{
Name: "partition-info",
Usage: "show partition related info",
Aliases: []string{"p"},
},
&cli.BoolFlag{
Name: "proof",
Usage: "print snark proof bytes as hex",
},
},
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
2019-10-03 18:12:30 +00:00
defer closer()
ctx := lcli.ReqContext(cctx)
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
}
id, err := strconv.ParseUint(cctx.Args().First(), 10, 64)
if err != nil {
return err
}
onChainInfo := cctx.Bool("on-chain-info")
2022-09-14 18:51:18 +00:00
status, err := minerApi.SectorsStatus(ctx, abi.SectorNumber(id), onChainInfo)
if err != nil {
return err
}
fmt.Printf("SectorID:\t%d\n", status.SectorID)
fmt.Printf("Status:\t\t%s\n", status.State)
fmt.Printf("CIDcommD:\t%s\n", status.CommD)
fmt.Printf("CIDcommR:\t%s\n", status.CommR)
2020-02-27 21:45:31 +00:00
fmt.Printf("Ticket:\t\t%x\n", status.Ticket.Value)
fmt.Printf("TicketH:\t%d\n", status.Ticket.Epoch)
2020-02-27 21:45:31 +00:00
fmt.Printf("Seed:\t\t%x\n", status.Seed.Value)
fmt.Printf("SeedH:\t\t%d\n", status.Seed.Epoch)
fmt.Printf("Precommit:\t%s\n", status.PreCommitMsg)
fmt.Printf("Commit:\t\t%s\n", status.CommitMsg)
if cctx.Bool("proof") {
fmt.Printf("Proof:\t\t%x\n", status.Proof)
}
2019-11-08 18:15:13 +00:00
fmt.Printf("Deals:\t\t%v\n", status.Deals)
fmt.Printf("Retries:\t%d\n", status.Retries)
2019-12-04 00:44:29 +00:00
if status.LastErr != "" {
fmt.Printf("Last Error:\t\t%s\n", status.LastErr)
}
if onChainInfo {
fmt.Printf("\nSector On Chain Info\n")
fmt.Printf("SealProof:\t\t%x\n", status.SealProof)
fmt.Printf("Activation:\t\t%v\n", status.Activation)
fmt.Printf("Expiration:\t\t%v\n", status.Expiration)
fmt.Printf("DealWeight:\t\t%v\n", status.DealWeight)
fmt.Printf("VerifiedDealWeight:\t\t%v\n", status.VerifiedDealWeight)
fmt.Printf("InitialPledge:\t\t%v\n", types.FIL(status.InitialPledge))
fmt.Printf("\nExpiration Info\n")
fmt.Printf("OnTime:\t\t%v\n", status.OnTime)
fmt.Printf("Early:\t\t%v\n", status.Early)
}
if cctx.Bool("partition-info") {
fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer nCloser()
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
mact, err := fullApi.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory())
2022-04-20 21:34:28 +00:00
mas, err := lminer.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact)
if err != nil {
return err
}
errFound := errors.New("found")
2022-04-20 21:34:28 +00:00
if err := mas.ForEachDeadline(func(dlIdx uint64, dl lminer.Deadline) error {
return dl.ForEachPartition(func(partIdx uint64, part lminer.Partition) error {
pas, err := part.AllSectors()
if err != nil {
return err
}
set, err := pas.IsSet(id)
if err != nil {
return err
}
if set {
fmt.Printf("\nDeadline:\t%d\n", dlIdx)
fmt.Printf("Partition:\t%d\n", partIdx)
checkIn := func(name string, bg func() (bitfield.BitField, error)) error {
bf, err := bg()
if err != nil {
return err
}
set, err := bf.IsSet(id)
if err != nil {
return err
}
setstr := "no"
if set {
setstr = "yes"
}
fmt.Printf("%s: \t%s\n", name, setstr)
return nil
}
if err := checkIn("Unproven", part.UnprovenSectors); err != nil {
return err
}
if err := checkIn("Live", part.LiveSectors); err != nil {
return err
}
if err := checkIn("Active", part.ActiveSectors); err != nil {
return err
}
if err := checkIn("Faulty", part.FaultySectors); err != nil {
return err
}
if err := checkIn("Recovering", part.RecoveringSectors); err != nil {
return err
}
return errFound
}
return nil
})
}); err != errFound {
if err != nil {
return err
}
fmt.Println("\nNot found in any partition")
}
}
if cctx.Bool("log") {
fmt.Printf("--------\nEvent Log:\n")
for i, l := range status.Log {
fmt.Printf("%d.\t%s:\t[%s]\t%s\n", i, time.Unix(int64(l.Timestamp), 0), l.Kind, l.Message)
if l.Trace != "" {
fmt.Printf("\t%s\n", l.Trace)
}
}
}
return nil
},
}
var sectorsListCmd = &cli.Command{
Name: "list",
Usage: "List sectors",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "show-removed",
Usage: "show removed sectors",
Aliases: []string{"r"},
},
2020-09-30 11:33:42 +00:00
&cli.BoolFlag{
Name: "fast",
Usage: "don't show on-chain info for better performance",
Aliases: []string{"f"},
2020-09-30 11:33:42 +00:00
},
&cli.BoolFlag{
Name: "events",
Usage: "display number of events the sector has received",
Aliases: []string{"e"},
},
&cli.BoolFlag{
Name: "initial-pledge",
Usage: "display initial pledge",
Aliases: []string{"p"},
},
&cli.BoolFlag{
Name: "seal-time",
Usage: "display how long it took for the sector to be sealed",
Aliases: []string{"t"},
},
2020-12-06 00:51:48 +00:00
&cli.StringFlag{
Name: "states",
Usage: "filter sectors by a comma-separated list of states",
},
&cli.BoolFlag{
Name: "unproven",
Usage: "only show sectors which aren't in the 'Proving' state",
Aliases: []string{"u"},
},
2023-02-07 10:51:04 +00:00
&cli.Int64Flag{
Name: "check-parallelism",
Usage: "number of parallel requests to make for checking sector states",
Value: parallelSectorChecks,
},
},
Subcommands: []*cli.Command{
sectorsListUpgradeBoundsCmd,
},
Action: func(cctx *cli.Context) error {
2023-02-15 18:41:46 +00:00
// http mode allows for parallel json decoding/encoding, which was a bottleneck here
2023-02-07 10:51:04 +00:00
minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
if err != nil {
return err
}
2019-10-03 18:12:30 +00:00
defer closer()
fullApi, closer2, err := lcli.GetFullNodeAPI(cctx) // TODO: consider storing full node address in config
if err != nil {
return err
}
defer closer2()
ctx := lcli.ReqContext(cctx)
2020-12-06 00:51:48 +00:00
var list []abi.SectorNumber
showRemoved := cctx.Bool("show-removed")
var states []api.SectorState
if cctx.IsSet("states") && cctx.IsSet("unproven") {
return xerrors.Errorf("only one of --states or --unproven can be specified at once")
}
if cctx.IsSet("states") {
2020-12-06 00:51:48 +00:00
showRemoved = true
sList := strings.Split(cctx.String("states"), ",")
states = make([]api.SectorState, len(sList))
2020-12-06 00:51:48 +00:00
for i := range sList {
states[i] = api.SectorState(sList[i])
}
}
if cctx.Bool("unproven") {
for state := range sealing.ExistSectorStateList {
if state == sealing.Proving || state == sealing.Available {
continue
}
states = append(states, api.SectorState(state))
2020-12-06 00:51:48 +00:00
}
}
if len(states) == 0 {
2022-09-14 18:51:18 +00:00
list, err = minerApi.SectorsList(ctx)
} else {
2022-09-14 18:51:18 +00:00
list, err = minerApi.SectorsListInStates(ctx, states)
2020-12-06 00:51:48 +00:00
}
if err != nil {
return err
}
2022-09-14 18:51:18 +00:00
maddr, err := minerApi.ActorAddress(ctx)
if err != nil {
return err
}
2020-09-30 11:33:42 +00:00
head, err := fullApi.ChainHead(ctx)
if err != nil {
return err
}
activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, head.Key())
if err != nil {
return err
}
2020-07-17 14:26:48 +00:00
activeIDs := make(map[abi.SectorNumber]struct{}, len(activeSet))
for _, info := range activeSet {
activeIDs[info.SectorNumber] = struct{}{}
2020-07-17 14:21:00 +00:00
}
2020-09-30 11:33:42 +00:00
sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, head.Key())
if err != nil {
return err
}
2020-11-10 09:37:36 +00:00
commitedIDs := make(map[abi.SectorNumber]struct{}, len(sset))
for _, info := range sset {
commitedIDs[info.SectorNumber] = struct{}{}
}
sort.Slice(list, func(i, j int) bool {
return list[i] < list[j]
})
2020-09-30 11:33:42 +00:00
tw := tablewriter.New(
tablewriter.Col("ID"),
tablewriter.Col("State"),
tablewriter.Col("OnChain"),
tablewriter.Col("Active"),
tablewriter.Col("Expiration"),
tablewriter.Col("SealTime"),
tablewriter.Col("Events"),
2020-09-30 11:33:42 +00:00
tablewriter.Col("Deals"),
tablewriter.Col("DealWeight"),
tablewriter.Col("VerifiedPower"),
tablewriter.Col("Pledge"),
2020-09-30 11:33:42 +00:00
tablewriter.NewLineCol("Error"),
tablewriter.NewLineCol("RecoveryTimeout"))
2020-09-30 11:33:42 +00:00
fast := cctx.Bool("fast")
2023-02-07 10:51:04 +00:00
throttle := make(chan struct{}, cctx.Int64("check-parallelism"))
slist := make([]result.Result[api.SectorInfo], len(list))
2023-02-07 10:51:04 +00:00
var wg sync.WaitGroup
for i, s := range list {
2023-02-07 10:51:04 +00:00
throttle <- struct{}{}
wg.Add(1)
go func(i int, s abi.SectorNumber) {
2023-02-07 10:51:04 +00:00
defer wg.Done()
defer func() { <-throttle }()
r := result.Wrap(minerApi.SectorsStatus(ctx, s, !fast))
if r.Error != nil {
r.Value.SectorID = s
}
slist[i] = r
}(i, s)
2023-02-07 10:51:04 +00:00
}
wg.Wait()
for _, rsn := range slist {
2023-02-07 10:51:04 +00:00
if rsn.Error != nil {
2020-09-30 11:33:42 +00:00
tw.Write(map[string]interface{}{
2023-02-07 10:51:04 +00:00
"ID": rsn.Value.SectorID,
2020-09-30 11:33:42 +00:00
"Error": err,
})
continue
}
2023-02-07 10:51:04 +00:00
st := rsn.Value
s := st.SectorID
2021-09-02 19:45:55 +00:00
if !showRemoved && st.State == api.SectorState(sealing.Removed) {
continue
}
_, inSSet := commitedIDs[s]
_, inASet := activeIDs[s]
2021-09-07 17:42:52 +00:00
const verifiedPowerGainMul = 9
2021-09-02 19:45:55 +00:00
dw, vp := .0, .0
estimate := (st.Expiration-st.Activation <= 0) || sealing.IsUpgradeState(sealing.SectorState(st.State))
2021-09-02 19:45:55 +00:00
if !estimate {
rdw := big.Add(st.DealWeight, st.VerifiedDealWeight)
dw = float64(big.Div(rdw, big.NewInt(int64(st.Expiration-st.Activation))).Uint64())
2021-09-07 17:42:52 +00:00
vp = float64(big.Div(big.Mul(st.VerifiedDealWeight, big.NewInt(verifiedPowerGainMul)), big.NewInt(int64(st.Expiration-st.Activation))).Uint64())
2021-09-02 19:45:55 +00:00
} else {
for _, piece := range st.Pieces {
if piece.DealInfo != nil {
dw += float64(piece.Piece.Size)
if piece.DealInfo.DealProposal != nil && piece.DealInfo.DealProposal.VerifiedDeal {
2021-09-07 17:42:52 +00:00
vp += float64(piece.Piece.Size) * verifiedPowerGainMul
2021-09-02 19:44:26 +00:00
}
}
2020-09-30 11:33:42 +00:00
}
2021-09-02 19:45:55 +00:00
}
2020-09-30 11:33:42 +00:00
2021-09-02 19:45:55 +00:00
var deals int
for _, deal := range st.Deals {
if deal != 0 {
deals++
2020-09-30 11:33:42 +00:00
}
2021-09-02 19:45:55 +00:00
}
2020-09-30 11:33:42 +00:00
2021-09-02 19:45:55 +00:00
exp := st.Expiration
if st.OnTime > 0 && st.OnTime < exp {
exp = st.OnTime // Can be different when the sector was CC upgraded
}
m := map[string]interface{}{
"ID": s,
"State": color.New(stateOrder[sealing.SectorState(st.State)].col).Sprint(st.State),
"OnChain": yesno(inSSet),
"Active": yesno(inASet),
}
2020-09-30 11:33:42 +00:00
2021-09-02 19:45:55 +00:00
if deals > 0 {
m["Deals"] = color.GreenString("%d", deals)
} else {
m["Deals"] = color.BlueString("CC")
if st.ToUpgrade {
m["Deals"] = color.CyanString("CC(upgrade)")
2020-09-30 11:33:42 +00:00
}
2021-09-02 19:45:55 +00:00
}
2020-09-30 11:33:42 +00:00
2021-09-02 19:45:55 +00:00
if !fast {
if !inSSet {
m["Expiration"] = "n/a"
2020-09-30 11:33:42 +00:00
} else {
2022-10-31 12:32:00 +00:00
m["Expiration"] = cliutil.EpochTime(head.Height(), exp)
2021-09-02 19:45:55 +00:00
if st.Early > 0 {
2022-10-31 12:32:00 +00:00
m["RecoveryTimeout"] = color.YellowString(cliutil.EpochTime(head.Height(), st.Early))
2020-09-30 11:33:42 +00:00
}
}
if inSSet && cctx.Bool("initial-pledge") {
m["Pledge"] = types.FIL(st.InitialPledge).Short()
}
2021-09-02 19:45:55 +00:00
}
2020-09-30 11:33:42 +00:00
2021-09-02 19:45:55 +00:00
if !fast && deals > 0 {
estWrap := func(s string) string {
if !estimate {
return s
2020-09-30 11:33:42 +00:00
}
2021-09-02 19:45:55 +00:00
return fmt.Sprintf("[%s]", s)
}
2021-09-02 19:45:55 +00:00
m["DealWeight"] = estWrap(units.BytesSize(dw))
if vp > 0 {
m["VerifiedPower"] = estWrap(color.GreenString(units.BytesSize(vp)))
2021-09-02 19:44:26 +00:00
}
2021-09-02 19:45:55 +00:00
}
2021-09-02 19:44:26 +00:00
2021-09-02 19:45:55 +00:00
if cctx.Bool("events") {
var events int
for _, sectorLog := range st.Log {
if !strings.HasPrefix(sectorLog.Kind, "event") {
continue
}
2021-09-02 19:45:55 +00:00
if sectorLog.Kind == "event;sealing.SectorRestart" {
continue
}
2021-09-02 19:45:55 +00:00
events++
}
2021-09-02 19:45:55 +00:00
pieces := len(st.Deals)
switch {
case events < 12+pieces:
m["Events"] = color.GreenString("%d", events)
case events < 20+pieces:
m["Events"] = color.YellowString("%d", events)
default:
m["Events"] = color.RedString("%d", events)
}
}
2021-09-02 19:45:55 +00:00
if cctx.Bool("seal-time") && len(st.Log) > 1 {
start := time.Unix(int64(st.Log[0].Timestamp), 0)
for _, sectorLog := range st.Log {
2021-09-07 17:42:52 +00:00
if sectorLog.Kind == "event;sealing.SectorProving" { // todo: figure out a good way to not hardcode
2021-09-02 19:45:55 +00:00
end := time.Unix(int64(sectorLog.Timestamp), 0)
dur := end.Sub(start)
switch {
case dur < 12*time.Hour:
m["SealTime"] = color.GreenString("%s", dur)
case dur < 24*time.Hour:
m["SealTime"] = color.YellowString("%s", dur)
default:
m["SealTime"] = color.RedString("%s", dur)
}
2021-09-02 19:45:55 +00:00
break
}
2020-09-30 11:33:42 +00:00
}
}
2021-09-02 19:45:55 +00:00
tw.Write(m)
}
2020-09-30 11:33:42 +00:00
return tw.Flush(os.Stdout)
},
}
var sectorsListUpgradeBoundsCmd = &cli.Command{
Name: "upgrade-bounds",
Usage: "Output upgrade bounds for available sectors",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "buckets",
Value: 25,
},
&cli.BoolFlag{
Name: "csv",
Usage: "output machine-readable values",
},
&cli.BoolFlag{
Name: "deal-terms",
Usage: "bucket by how many deal-sectors can start at a given expiration",
},
},
Action: func(cctx *cli.Context) error {
minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
if err != nil {
return err
}
defer closer()
fullApi, closer2, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer2()
ctx := lcli.ReqContext(cctx)
list, err := minerApi.SectorsListInStates(ctx, []api.SectorState{
api.SectorState(sealing.Available),
})
if err != nil {
return xerrors.Errorf("getting sector list: %w", err)
}
head, err := fullApi.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("getting chain head: %w", err)
}
filter := bitfield.New()
for _, s := range list {
filter.Set(uint64(s))
}
maddr, err := minerApi.ActorAddress(ctx)
if err != nil {
return err
}
sset, err := fullApi.StateMinerSectors(ctx, maddr, &filter, head.Key())
if err != nil {
return err
}
if len(sset) == 0 {
return nil
}
var minExpiration, maxExpiration abi.ChainEpoch
for _, s := range sset {
if s.Expiration < minExpiration || minExpiration == 0 {
minExpiration = s.Expiration
}
if s.Expiration > maxExpiration {
maxExpiration = s.Expiration
}
}
buckets := cctx.Int("buckets")
bucketSize := (maxExpiration - minExpiration) / abi.ChainEpoch(buckets)
bucketCounts := make([]int, buckets+1)
for b := range bucketCounts {
bucketMin := minExpiration + abi.ChainEpoch(b)*bucketSize
bucketMax := minExpiration + abi.ChainEpoch(b+1)*bucketSize
if cctx.Bool("deal-terms") {
bucketMax = bucketMax + policy.MarketDefaultAllocationTermBuffer
}
for _, s := range sset {
isInBucket := s.Expiration >= bucketMin && s.Expiration < bucketMax
if isInBucket {
bucketCounts[b]++
}
}
}
// Creating CSV writer
writer := csv.NewWriter(os.Stdout)
// Writing CSV headers
err = writer.Write([]string{"Max Expiration in Bucket", "Sector Count"})
if err != nil {
return xerrors.Errorf("writing csv headers: %w", err)
}
// Writing bucket details
if cctx.Bool("csv") {
for i := 0; i < buckets; i++ {
maxExp := minExpiration + abi.ChainEpoch(i+1)*bucketSize
timeStr := strconv.FormatInt(int64(maxExp), 10)
err = writer.Write([]string{
timeStr,
strconv.Itoa(bucketCounts[i]),
})
if err != nil {
return xerrors.Errorf("writing csv row: %w", err)
}
}
// Flush to make sure all data is written to the underlying writer
writer.Flush()
if err := writer.Error(); err != nil {
return xerrors.Errorf("flushing csv writer: %w", err)
}
return nil
}
tw := tablewriter.New(
tablewriter.Col("Bucket Expiration"),
tablewriter.Col("Sector Count"),
tablewriter.Col("Bar"),
)
var barCols = 40
var maxCount int
for _, c := range bucketCounts {
if c > maxCount {
maxCount = c
}
}
for i := 0; i < buckets; i++ {
maxExp := minExpiration + abi.ChainEpoch(i+1)*bucketSize
timeStr := cliutil.EpochTime(head.Height(), maxExp)
tw.Write(map[string]interface{}{
"Bucket Expiration": timeStr,
"Sector Count": color.YellowString("%d", bucketCounts[i]),
"Bar": "[" + color.GreenString(strings.Repeat("|", bucketCounts[i]*barCols/maxCount)) + strings.Repeat(" ", barCols-bucketCounts[i]*barCols/maxCount) + "]",
})
}
return tw.Flush(os.Stdout)
},
}
2019-08-26 10:04:57 +00:00
var sectorsRefsCmd = &cli.Command{
Name: "refs",
Usage: "List References to sectors",
Action: func(cctx *cli.Context) error {
integrate DAG store and CARv2 in deal-making (#6671) This commit removes badger from the deal-making processes, and moves to a new architecture with the dagstore as the cental component on the miner-side, and CARv2s on the client-side. Every deal that has been handed off to the sealing subsystem becomes a shard in the dagstore. Shards are mounted via the LotusMount, which teaches the dagstore how to load the related piece when serving retrievals. When the miner starts the Lotus for the first time with this patch, we will perform a one-time migration of all active deals into the dagstore. This is a lightweight process, and it consists simply of registering the shards in the dagstore. Shards are backed by the unsealed copy of the piece. This is currently a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so when it's time to acquire a shard to serve a retrieval, the unsealed CARv1 is joined with its index (safeguarded by the dagstore), to form a read-only blockstore, thus taking the place of the monolithic badger. Data transfers have been adjusted to interface directly with CARv2 files. On inbound transfers (client retrievals, miner storage deals), we stream the received data into a CARv2 ReadWrite blockstore. On outbound transfers (client storage deals, miner retrievals), we serve the data off a CARv2 ReadOnly blockstore. Client-side imports are managed by the refactored *imports.Manager component (when not using IPFS integration). Just like it before, we use the go-filestore library to avoid duplicating the data from the original file in the resulting UnixFS DAG (concretely the leaves). However, the target of those imports are what we call "ref-CARv2s": CARv2 files placed under the `$LOTUS_PATH/imports` directory, containing the intermediate nodes in full, and the leaves as positional references to the original file on disk. Client-side retrievals are placed into CARv2 files in the location: `$LOTUS_PATH/retrievals`. A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore` subcommands have been introduced on the miner-side to inspect and manage the dagstore. Despite moving to a CARv2-backed system, the IPFS integration has been respected, and it continues to be possible to make storage deals with data held in an IPFS node, and to perform retrievals directly into an IPFS node. NOTE: because the "staging" and "client" Badger blockstores are no longer used, existing imports on the client will be rendered useless. On startup, Lotus will enumerate all imports and print WARN statements on the log for each import that needs to be reimported. These log lines contain these messages: - import lacks carv2 path; import will not work; please reimport - import has missing/broken carv2; please reimport At the end, we will print a "sanity check completed" message indicating the count of imports found, and how many were deemed broken. Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com> Co-authored-by: Dirk McCormick <dirkmdev@gmail.com> Co-authored-by: Raúl Kripalani <raul@protocol.ai> Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
nodeApi, closer, err := lcli.GetMarketsAPI(cctx)
2019-08-26 10:04:57 +00:00
if err != nil {
return err
}
2019-10-03 18:12:30 +00:00
defer closer()
2019-08-26 10:04:57 +00:00
ctx := lcli.ReqContext(cctx)
refs, err := nodeApi.SectorsRefs(ctx)
if err != nil {
return err
}
for name, refs := range refs {
fmt.Printf("Block %s:\n", name)
for _, ref := range refs {
2019-12-01 17:58:31 +00:00
fmt.Printf("\t%d+%d %d bytes\n", ref.SectorID, ref.Offset, ref.Size)
2019-08-26 10:04:57 +00:00
}
}
return nil
},
}
2021-04-23 08:16:56 +00:00
var sectorsCheckExpireCmd = &cli.Command{
Name: "check-expire",
Usage: "Inspect expiring sectors",
Flags: []cli.Flag{
&cli.Int64Flag{
Name: "cutoff",
2021-08-13 05:47:48 +00:00
Usage: "skip sectors whose current expiration is more than <cutoff> epochs from now, defaults to 60 days",
Value: 172800,
2021-04-23 08:16:56 +00:00
},
},
Action: func(cctx *cli.Context) error {
2021-04-25 07:28:08 +00:00
fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
2021-04-23 08:16:56 +00:00
if err != nil {
return err
}
defer nCloser()
ctx := lcli.ReqContext(cctx)
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
2021-04-25 07:28:08 +00:00
head, err := fullApi.ChainHead(ctx)
2021-04-23 08:16:56 +00:00
if err != nil {
return err
}
currEpoch := head.Height()
2021-04-25 07:28:08 +00:00
nv, err := fullApi.StateNetworkVersion(ctx, types.EmptyTSK)
2021-04-23 08:16:56 +00:00
if err != nil {
return err
}
2021-04-25 07:28:08 +00:00
sectors, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK)
2021-08-14 02:41:01 +00:00
if err != nil {
return err
}
2021-04-23 08:16:56 +00:00
n := 0
for _, s := range sectors {
if s.Expiration-currEpoch <= abi.ChainEpoch(cctx.Int64("cutoff")) {
sectors[n] = s
n++
}
}
sectors = sectors[:n]
sort.Slice(sectors, func(i, j int) bool {
if sectors[i].Expiration == sectors[j].Expiration {
return sectors[i].SectorNumber < sectors[j].SectorNumber
}
return sectors[i].Expiration < sectors[j].Expiration
})
tw := tablewriter.New(
tablewriter.Col("ID"),
tablewriter.Col("SealProof"),
tablewriter.Col("InitialPledge"),
tablewriter.Col("Activation"),
tablewriter.Col("Expiration"),
tablewriter.Col("MaxExpiration"),
tablewriter.Col("MaxExtendNow"))
for _, sector := range sectors {
2021-08-02 08:26:29 +00:00
MaxExpiration := sector.Activation + policy.GetSectorMaxLifetime(sector.SealProof, nv)
MaxExtendNow := currEpoch + policy.GetMaxSectorExpirationExtension()
2021-04-23 08:16:56 +00:00
if MaxExtendNow > MaxExpiration {
MaxExtendNow = MaxExpiration
}
tw.Write(map[string]interface{}{
"ID": sector.SectorNumber,
"SealProof": sector.SealProof,
"InitialPledge": types.FIL(sector.InitialPledge).Short(),
2022-10-31 12:32:00 +00:00
"Activation": cliutil.EpochTime(currEpoch, sector.Activation),
"Expiration": cliutil.EpochTime(currEpoch, sector.Expiration),
"MaxExpiration": cliutil.EpochTime(currEpoch, MaxExpiration),
"MaxExtendNow": cliutil.EpochTime(currEpoch, MaxExtendNow),
2021-04-23 08:16:56 +00:00
})
}
return tw.Flush(os.Stdout)
},
}
type PseudoExpirationExtension struct {
Deadline uint64
Partition uint64
Sectors string
NewExpiration abi.ChainEpoch
}
type PseudoExtendSectorExpirationParams struct {
Extensions []PseudoExpirationExtension
}
func NewPseudoExtendParams(p *miner.ExtendSectorExpiration2Params) (*PseudoExtendSectorExpirationParams, error) {
res := PseudoExtendSectorExpirationParams{}
for _, ext := range p.Extensions {
scount, err := ext.Sectors.Count()
if err != nil {
return nil, err
}
sectors, err := ext.Sectors.All(scount)
if err != nil {
return nil, err
}
res.Extensions = append(res.Extensions, PseudoExpirationExtension{
Deadline: ext.Deadline,
Partition: ext.Partition,
Sectors: ArrayToString(sectors),
NewExpiration: ext.NewExpiration,
})
}
return &res, nil
}
2021-08-14 02:41:01 +00:00
// ArrayToString Example: {1,3,4,5,8,9} -> "1,3-5,8-9"
func ArrayToString(array []uint64) string {
2021-08-14 02:41:01 +00:00
sort.Slice(array, func(i, j int) bool {
return array[i] < array[j]
})
var sarray []string
s := ""
for i, elm := range array {
if i == 0 {
s = strconv.FormatUint(elm, 10)
continue
}
2021-08-14 02:41:01 +00:00
if elm == array[i-1] {
continue // filter out duplicates
} else if elm == array[i-1]+1 {
s = strings.Split(s, "-")[0] + "-" + strconv.FormatUint(elm, 10)
} else {
sarray = append(sarray, s)
s = strconv.FormatUint(elm, 10)
}
}
if s != "" {
sarray = append(sarray, s)
}
return strings.Join(sarray, ",")
}
2022-12-20 17:01:22 +00:00
func getSectorsFromFile(filePath string) ([]abi.SectorNumber, error) {
2021-08-14 02:41:01 +00:00
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
scanner := bufio.NewScanner(file)
2022-12-20 17:01:22 +00:00
sectors := make([]abi.SectorNumber, 0)
2021-08-14 02:41:01 +00:00
for scanner.Scan() {
line := scanner.Text()
id, err := strconv.ParseUint(line, 10, 64)
if err != nil {
return nil, xerrors.Errorf("could not parse %s as sector id: %s", line, err)
}
2022-12-20 17:01:22 +00:00
sectors = append(sectors, abi.SectorNumber(id))
2021-08-14 02:41:01 +00:00
}
if err = file.Close(); err != nil {
return nil, err
}
return sectors, nil
}
2022-12-20 17:01:22 +00:00
func SectorNumsToBitfield(sectors []abi.SectorNumber) bitfield.BitField {
var numbers []uint64
for _, sector := range sectors {
2022-12-20 17:01:22 +00:00
numbers = append(numbers, uint64(sector))
}
return bitfield.NewFromSet(numbers)
}
var sectorsExtendCmd = &cli.Command{
Name: "extend",
Usage: "Extend expiring sectors while not exceeding each sector's max life",
ArgsUsage: "<sectorNumbers...(optional)>",
Flags: []cli.Flag{
&cli.Int64Flag{
2021-08-02 08:44:22 +00:00
Name: "from",
2021-08-13 05:47:48 +00:00
Usage: "only consider sectors whose current expiration epoch is in the range of [from, to], <from> defaults to: now + 120 (1 hour)",
},
&cli.Int64Flag{
2021-08-02 08:44:22 +00:00
Name: "to",
2021-08-13 05:47:48 +00:00
Usage: "only consider sectors whose current expiration epoch is in the range of [from, to], <to> defaults to: now + 92160 (32 days)",
},
&cli.StringFlag{
Name: "sector-file",
Usage: "provide a file containing one sector number in each line, ignoring above selecting criteria",
},
2021-08-02 08:39:16 +00:00
&cli.StringFlag{
Name: "exclude",
Usage: "optionally provide a file containing excluding sectors",
},
&cli.Int64Flag{
Name: "extension",
2021-08-13 05:47:48 +00:00
Usage: "try to extend selected sectors by this number of epochs, defaults to 540 days",
Value: 1555200,
},
&cli.Int64Flag{
Name: "new-expiration",
Usage: "try to extend selected sectors to this epoch, ignoring extension",
},
2022-08-18 15:03:26 +00:00
&cli.BoolFlag{
Name: "only-cc",
Usage: "only extend CC sectors (useful for making sector ready for snap upgrade)",
},
&cli.BoolFlag{
Name: "drop-claims",
Usage: "drop claims for sectors that can be extended, but only by dropping some of their verified power claims",
},
&cli.Int64Flag{
Name: "tolerance",
2021-08-13 05:47:48 +00:00
Usage: "don't try to extend sectors by fewer than this number of epochs, defaults to 7 days",
Value: 20160,
},
&cli.StringFlag{
2021-04-25 07:28:08 +00:00
Name: "max-fee",
2021-08-14 02:41:01 +00:00
Usage: "use up to this amount of FIL for one message. pass this flag to avoid message congestion.",
Value: "0",
},
2022-12-25 13:05:44 +00:00
&cli.Int64Flag{
Name: "max-sectors",
Usage: "the maximum number of sectors contained in each message",
2022-12-25 13:05:44 +00:00
},
2021-08-13 05:47:48 +00:00
&cli.BoolFlag{
Name: "really-do-it",
2022-12-20 17:01:22 +00:00
Usage: "pass this flag to really extend sectors, otherwise will only print out json representation of parameters",
2021-08-13 05:47:48 +00:00
},
},
Action: func(cctx *cli.Context) error {
2021-08-14 02:41:01 +00:00
mf, err := types.ParseFIL(cctx.String("max-fee"))
if err != nil {
return err
}
2021-08-14 02:41:01 +00:00
spec := &api.MessageSendSpec{MaxFee: abi.TokenAmount(mf)}
2021-04-25 07:28:08 +00:00
fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer nCloser()
ctx := lcli.ReqContext(cctx)
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
2021-04-25 07:28:08 +00:00
head, err := fullApi.ChainHead(ctx)
if err != nil {
return err
}
currEpoch := head.Height()
2021-04-25 07:28:08 +00:00
nv, err := fullApi.StateNetworkVersion(ctx, types.EmptyTSK)
if err != nil {
return err
}
2021-04-25 07:28:08 +00:00
activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
activeSectorsInfo := make(map[abi.SectorNumber]*miner.SectorOnChainInfo, len(activeSet))
for _, info := range activeSet {
activeSectorsInfo[info.SectorNumber] = info
}
2021-04-25 07:28:08 +00:00
mact, err := fullApi.StateGetActor(ctx, maddr, types.EmptyTSK)
2021-04-23 08:30:23 +00:00
if err != nil {
return err
}
2021-04-25 07:28:08 +00:00
tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory())
adtStore := adt.WrapStore(ctx, cbor.NewCborStore(tbs))
mas, err := lminer.Load(adtStore, mact)
2021-04-23 08:30:23 +00:00
if err != nil {
return err
}
2022-04-20 21:34:28 +00:00
activeSectorsLocation := make(map[abi.SectorNumber]*lminer.SectorLocation, len(activeSet))
2021-04-23 08:30:23 +00:00
2022-04-20 21:34:28 +00:00
if err := mas.ForEachDeadline(func(dlIdx uint64, dl lminer.Deadline) error {
return dl.ForEachPartition(func(partIdx uint64, part lminer.Partition) error {
2021-04-23 08:30:23 +00:00
pas, err := part.ActiveSectors()
if err != nil {
return err
}
return pas.ForEach(func(i uint64) error {
2022-04-20 21:34:28 +00:00
activeSectorsLocation[abi.SectorNumber(i)] = &lminer.SectorLocation{
2021-04-23 08:30:23 +00:00
Deadline: dlIdx,
Partition: partIdx,
}
return nil
})
})
}); err != nil {
return err
}
2022-12-20 17:01:22 +00:00
excludeSet := make(map[abi.SectorNumber]struct{})
2021-08-02 08:39:16 +00:00
if cctx.IsSet("exclude") {
2021-08-14 02:41:01 +00:00
excludeSectors, err := getSectorsFromFile(cctx.String("exclude"))
2021-08-02 08:39:16 +00:00
if err != nil {
return err
}
2021-08-14 02:41:01 +00:00
for _, id := range excludeSectors {
2021-08-02 08:39:16 +00:00
excludeSet[id] = struct{}{}
}
}
2022-12-20 17:01:22 +00:00
var sectors []abi.SectorNumber
if cctx.Args().Present() {
if cctx.IsSet("sector-file") {
return xerrors.Errorf("sector-file specified along with command line params")
}
2022-12-20 17:01:22 +00:00
for i, s := range cctx.Args().Slice() {
id, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector %d: %w", i, err)
}
2022-12-20 17:01:22 +00:00
sectors = append(sectors, abi.SectorNumber(id))
}
} else if cctx.IsSet("sector-file") {
sectors, err = getSectorsFromFile(cctx.String("sector-file"))
if err != nil {
return err
}
} else {
2021-08-02 08:44:22 +00:00
from := currEpoch + 120
2021-08-13 05:47:48 +00:00
to := currEpoch + 92160
2021-08-02 08:44:22 +00:00
if cctx.IsSet("from") {
from = abi.ChainEpoch(cctx.Int64("from"))
}
if cctx.IsSet("to") {
to = abi.ChainEpoch(cctx.Int64("to"))
}
for _, si := range activeSet {
2021-08-02 08:44:22 +00:00
if si.Expiration >= from && si.Expiration <= to {
2022-12-20 17:01:22 +00:00
sectors = append(sectors, si.SectorNumber)
}
}
}
2022-12-20 17:01:22 +00:00
var sis []*miner.SectorOnChainInfo
for _, id := range sectors {
if _, exclude := excludeSet[id]; exclude {
continue
}
si, found := activeSectorsInfo[id]
if !found {
return xerrors.Errorf("sector %d is not active", id)
}
if len(si.DealIDs) > 0 && cctx.Bool("only-cc") {
continue
}
sis = append(sis, si)
}
withinTolerance := func(a, b abi.ChainEpoch) bool {
diff := a - b
if diff < 0 {
diff = -diff
}
return diff <= abi.ChainEpoch(cctx.Int64("tolerance"))
}
2022-12-20 17:01:22 +00:00
extensions := map[lminer.SectorLocation]map[abi.ChainEpoch][]abi.SectorNumber{}
for _, si := range sis {
extension := abi.ChainEpoch(cctx.Int64("extension"))
newExp := si.Expiration + extension
if cctx.IsSet("new-expiration") {
newExp = abi.ChainEpoch(cctx.Int64("new-expiration"))
}
2021-08-02 08:26:29 +00:00
maxExtendNow := currEpoch + policy.GetMaxSectorExpirationExtension()
if newExp > maxExtendNow {
newExp = maxExtendNow
}
maxExp := si.Activation + policy.GetSectorMaxLifetime(si.SealProof, nv)
if newExp > maxExp {
newExp = maxExp
}
if newExp <= si.Expiration || withinTolerance(newExp, si.Expiration) {
continue
}
2021-04-23 08:30:23 +00:00
l, found := activeSectorsLocation[si.SectorNumber]
if !found {
return xerrors.Errorf("location for sector %d not found", si.SectorNumber)
}
es, found := extensions[*l]
if !found {
2022-12-20 17:01:22 +00:00
ne := make(map[abi.ChainEpoch][]abi.SectorNumber)
ne[newExp] = []abi.SectorNumber{si.SectorNumber}
extensions[*l] = ne
} else {
added := false
for exp := range es {
if withinTolerance(newExp, exp) {
2022-12-20 17:01:22 +00:00
es[exp] = append(es[exp], si.SectorNumber)
added = true
break
}
}
if !added {
2022-12-20 17:01:22 +00:00
es[newExp] = []abi.SectorNumber{si.SectorNumber}
}
}
}
verifregAct, err := fullApi.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("failed to lookup verifreg actor: %w", err)
}
verifregSt, err := verifreg.Load(adtStore, verifregAct)
if err != nil {
return xerrors.Errorf("failed to load verifreg state: %w", err)
}
claimsMap, err := verifregSt.GetClaims(maddr)
if err != nil {
return xerrors.Errorf("failed to lookup claims for miner: %w", err)
}
claimIdsBySector, err := verifregSt.GetClaimIdsBySector(maddr)
if err != nil {
return xerrors.Errorf("failed to lookup claim IDs by sector: %w", err)
}
sectorsMax, err := policy.GetAddressedSectorsMax(nv)
if err != nil {
return err
}
declMax, err := policy.GetDeclarationsMax(nv)
if err != nil {
return err
}
addrSectors := sectorsMax
if cctx.Int("max-sectors") != 0 {
addrSectors = cctx.Int("max-sectors")
if addrSectors > sectorsMax {
return xerrors.Errorf("the specified max-sectors exceeds the maximum limit")
}
}
var params []miner.ExtendSectorExpiration2Params
p := miner.ExtendSectorExpiration2Params{}
scount := 0
for l, exts := range extensions {
for newExp, numbers := range exts {
sectorsWithoutClaimsToExtend := bitfield.New()
var sectorsWithClaims []miner.SectorClaim
for _, sectorNumber := range numbers {
claimIdsToMaintain := make([]verifreg.ClaimId, 0)
claimIdsToDrop := make([]verifreg.ClaimId, 0)
cannotExtendSector := false
claimIds, ok := claimIdsBySector[sectorNumber]
// Nothing to check, add to ccSectors
if !ok {
sectorsWithoutClaimsToExtend.Set(uint64(sectorNumber))
} else {
for _, claimId := range claimIds {
claim, ok := claimsMap[claimId]
if !ok {
return xerrors.Errorf("failed to find claim for claimId %d", claimId)
}
claimExpiration := claim.TermStart + claim.TermMax
// can be maintained in the extended sector
if claimExpiration > newExp {
claimIdsToMaintain = append(claimIdsToMaintain, claimId)
} else {
sectorInfo, ok := activeSectorsInfo[sectorNumber]
if !ok {
return xerrors.Errorf("failed to find sector in active sector set: %w", err)
}
if !cctx.Bool("drop-claims") ||
// FIP-0045 requires the claim minimum duration to have passed
currEpoch <= (claim.TermStart+claim.TermMin) ||
// FIP-0045 requires the sector to be in its last 30 days of life
(currEpoch <= sectorInfo.Expiration-builtin.EndOfLifeClaimDropPeriod) {
fmt.Printf("skipping sector %d because claim %d does not live long enough \n", sectorNumber, claimId)
cannotExtendSector = true
break
}
claimIdsToDrop = append(claimIdsToDrop, claimId)
}
}
if cannotExtendSector {
continue
}
if len(claimIdsToMaintain)+len(claimIdsToDrop) != 0 {
sectorsWithClaims = append(sectorsWithClaims, miner.SectorClaim{
SectorNumber: sectorNumber,
MaintainClaims: claimIdsToMaintain,
DropClaims: claimIdsToDrop,
})
}
2023-01-04 11:39:40 +00:00
}
}
2022-12-25 13:05:44 +00:00
sectorsWithoutClaimsCount, err := sectorsWithoutClaimsToExtend.Count()
if err != nil {
return xerrors.Errorf("failed to count cc sectors: %w", err)
}
sectorsInDecl := int(sectorsWithoutClaimsCount) + len(sectorsWithClaims)
scount += sectorsInDecl
if scount > addrSectors || len(p.Extensions) >= declMax {
params = append(params, p)
p = miner.ExtendSectorExpiration2Params{}
scount = sectorsInDecl
}
p.Extensions = append(p.Extensions, miner.ExpirationExtension2{
Deadline: l.Deadline,
Partition: l.Partition,
Sectors: SectorNumsToBitfield(numbers),
SectorsWithClaims: sectorsWithClaims,
NewExpiration: newExp,
})
}
}
// if we have any sectors, then one last append is needed here
if scount != 0 {
params = append(params, p)
}
if len(params) == 0 {
fmt.Println("nothing to extend")
return nil
}
2021-04-25 07:28:08 +00:00
mi, err := fullApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("getting miner info: %w", err)
}
stotal := 0
2021-08-14 02:41:01 +00:00
for i := range params {
scount := 0
2021-08-14 02:41:01 +00:00
for _, ext := range params[i].Extensions {
count, err := ext.Sectors.Count()
if err != nil {
return err
}
scount += int(count)
}
2022-12-20 17:01:22 +00:00
fmt.Printf("Extending %d sectors: ", scount)
stotal += scount
if !cctx.Bool("really-do-it") {
2021-08-14 02:41:01 +00:00
pp, err := NewPseudoExtendParams(&params[i])
if err != nil {
return err
}
data, err := json.MarshalIndent(pp, "", " ")
if err != nil {
return err
}
2022-12-20 17:01:22 +00:00
fmt.Println("\n", string(data))
continue
}
2021-08-14 02:41:01 +00:00
sp, aerr := actors.SerializeParams(&params[i])
if aerr != nil {
return xerrors.Errorf("serializing params: %w", err)
}
2021-04-25 07:28:08 +00:00
smsg, err := fullApi.MpoolPushMessage(ctx, &types.Message{
2021-08-02 08:26:29 +00:00
From: mi.Worker,
To: maddr,
Method: builtin.MethodsMiner.ExtendSectorExpiration2,
2021-08-02 08:26:29 +00:00
Value: big.Zero(),
Params: sp,
2021-04-25 07:28:08 +00:00
}, spec)
if err != nil {
return xerrors.Errorf("mpool push message: %w", err)
}
fmt.Println(smsg.Cid())
}
2022-12-20 17:01:22 +00:00
fmt.Printf("%d sectors extended\n", stotal)
2021-02-23 08:22:30 +00:00
return nil
},
}
2021-01-12 23:42:01 +00:00
var sectorsTerminateCmd = &cli.Command{
Name: "terminate",
Usage: "Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector)",
ArgsUsage: "<sectorNum>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
Usage: "pass this flag if you know what you are doing",
},
},
2021-01-14 11:37:31 +00:00
Subcommands: []*cli.Command{
sectorsTerminateFlushCmd,
sectorsTerminatePendingCmd,
},
2021-01-12 23:42:01 +00:00
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
2021-01-12 23:42:01 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
2021-01-12 23:42:01 +00:00
}
if !cctx.Bool("really-do-it") {
return xerrors.Errorf("pass --really-do-it to confirm this action")
}
2021-01-12 23:42:01 +00:00
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
2022-09-14 18:51:18 +00:00
return minerApi.SectorTerminate(ctx, abi.SectorNumber(id))
2021-01-12 23:42:01 +00:00
},
}
2021-01-14 11:37:31 +00:00
var sectorsTerminateFlushCmd = &cli.Command{
Name: "flush",
Usage: "Send a terminate message if there are sectors queued for termination",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
2021-01-14 11:37:31 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:51:18 +00:00
mcid, err := minerApi.SectorTerminateFlush(ctx)
2021-01-14 11:37:31 +00:00
if err != nil {
return err
}
if mcid == nil {
return xerrors.New("no sectors were queued for termination")
}
2021-01-14 15:47:30 +00:00
fmt.Println(mcid)
2021-01-14 11:37:31 +00:00
return nil
},
}
var sectorsTerminatePendingCmd = &cli.Command{
Name: "pending",
Usage: "List sector numbers of sectors pending termination",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2021-01-14 11:37:31 +00:00
if err != nil {
return err
}
defer closer()
api, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer nCloser()
2021-01-14 11:37:31 +00:00
ctx := lcli.ReqContext(cctx)
2022-09-14 18:51:18 +00:00
pending, err := minerAPI.SectorTerminatePending(ctx)
2021-01-14 11:37:31 +00:00
if err != nil {
return err
}
2022-09-14 18:51:18 +00:00
maddr, err := minerAPI.ActorAddress(ctx)
if err != nil {
return err
}
dl, err := api.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("getting proving deadline info failed: %w", err)
}
2021-01-14 11:37:31 +00:00
for _, id := range pending {
loc, err := api.StateSectorPartition(ctx, maddr, id.Number, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("finding sector partition: %w", err)
}
fmt.Print(id.Number)
if loc.Deadline == (dl.Index+1)%miner.WPoStPeriodDeadlines || // not in next (in case the terminate message takes a while to get on chain)
loc.Deadline == dl.Index || // not in current
(loc.Deadline+1)%miner.WPoStPeriodDeadlines == dl.Index { // not in previous
fmt.Print(" (in proving window)")
}
fmt.Println()
2021-01-14 11:37:31 +00:00
}
return nil
},
}
2020-06-22 17:35:14 +00:00
var sectorsRemoveCmd = &cli.Command{
2020-06-23 12:44:34 +00:00
Name: "remove",
2021-01-12 23:42:01 +00:00
Usage: "Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty))",
2020-06-23 12:44:34 +00:00
ArgsUsage: "<sectorNum>",
2020-06-22 17:35:14 +00:00
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
Usage: "pass this flag if you know what you are doing",
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Bool("really-do-it") {
return xerrors.Errorf("this is a command for advanced users, only use it if you are sure of what you are doing")
}
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2020-06-22 17:35:14 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
2020-06-22 17:35:14 +00:00
}
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
2020-06-23 12:44:34 +00:00
return xerrors.Errorf("could not parse sector number: %w", err)
2020-06-22 17:35:14 +00:00
}
// Check if the sector exists
_, err = minerAPI.SectorsStatus(ctx, abi.SectorNumber(id), false)
if err != nil {
return xerrors.Errorf("sectorID %d has not been created yet: %w", id, err)
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorRemove(ctx, abi.SectorNumber(id))
2020-06-22 17:35:14 +00:00
},
}
var sectorsSnapUpCmd = &cli.Command{
Name: "snap-up",
Usage: "Mark a committed capacity sector to be filled with deals",
ArgsUsage: "<sectorNum>",
Action: func(cctx *cli.Context) error {
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
}
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
api, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer nCloser()
ctx := lcli.ReqContext(cctx)
nv, err := api.StateNetworkVersion(ctx, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("failed to get network version: %w", err)
}
if nv < network.Version15 {
return xerrors.Errorf("snap deals upgrades enabled in network v15")
}
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorMarkForUpgrade(ctx, abi.SectorNumber(id), true)
},
}
var sectorsSnapAbortCmd = &cli.Command{
Name: "abort-upgrade",
Usage: "Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before",
ArgsUsage: "<sectorNum>",
2022-02-25 15:28:47 +00:00
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
Usage: "pass this flag if you know what you are doing",
},
},
Action: func(cctx *cli.Context) error {
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number"))
}
2022-02-24 22:20:21 +00:00
really := cctx.Bool("really-do-it")
if !really {
//nolint:golint
return fmt.Errorf("--really-do-it must be specified for this action to have an effect; you have been warned")
}
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorAbortUpgrade(ctx, abi.SectorNumber(id))
},
}
var sectorsStartSealCmd = &cli.Command{
Name: "seal",
Usage: "Manually start sealing a sector (filling any unused space with junk)",
ArgsUsage: "<sectorNum>",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
}
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorStartSealing(ctx, abi.SectorNumber(id))
},
}
var sectorsSealDelayCmd = &cli.Command{
Name: "set-seal-delay",
Usage: "Set the time (in minutes) that a new sector waits for deals before sealing starts",
ArgsUsage: "<time>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "seconds",
Usage: "Specifies that the time argument should be in seconds",
},
},
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
}
hs, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
var delay uint64
if cctx.Bool("seconds") {
delay = hs * uint64(time.Second)
} else {
delay = hs * uint64(time.Minute)
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorSetSealDelay(ctx, time.Duration(delay))
},
}
var sectorsCapacityCollateralCmd = &cli.Command{
Name: "get-cc-collateral",
Usage: "Get the collateral required to pledge a committed capacity sector",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "expiration",
Usage: "the epoch when the sector will expire",
},
},
Action: func(cctx *cli.Context) error {
nApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer nCloser()
ctx := lcli.ReqContext(cctx)
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
2021-02-28 07:10:22 +00:00
mi, err := nApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
nv, err := nApi.StateNetworkVersion(ctx, types.EmptyTSK)
if err != nil {
return err
}
2022-04-20 21:34:28 +00:00
spt, err := lminer.PreferredSealProofTypeFromWindowPoStType(nv, mi.WindowPoStProofType)
2021-02-28 07:10:22 +00:00
if err != nil {
return err
}
pci := miner.SectorPreCommitInfo{
2021-02-28 07:10:22 +00:00
SealProof: spt,
Expiration: abi.ChainEpoch(cctx.Uint64("expiration")),
}
if pci.Expiration == 0 {
2021-02-28 07:10:22 +00:00
h, err := nApi.ChainHead(ctx)
if err != nil {
return err
}
pci.Expiration = policy.GetMaxSectorExpirationExtension() + h.Height()
}
2021-02-28 07:10:22 +00:00
pc, err := nApi.StateMinerInitialPledgeCollateral(ctx, maddr, pci, types.EmptyTSK)
if err != nil {
return err
}
2021-02-28 07:10:22 +00:00
pcd, err := nApi.StateMinerPreCommitDepositForPower(ctx, maddr, pci, types.EmptyTSK)
if err != nil {
return err
}
fmt.Printf("Estimated collateral: %s\n", types.FIL(big.Max(pc, pcd)))
return nil
},
}
var sectorsUpdateCmd = &cli.Command{
Name: "update-state",
Usage: "ADVANCED: manually update the state of a sector, this may aid in error recovery",
ArgsUsage: "<sectorNum> <newState>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
Usage: "pass this flag if you know what you are doing",
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Bool("really-do-it") {
2019-12-05 11:15:39 +00:00
return xerrors.Errorf("this is a command for advanced users, only use it if you are sure of what you are doing")
}
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if cctx.NArg() != 2 {
return lcli.IncorrectNumArgs(cctx)
}
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
2020-06-23 12:44:34 +00:00
return xerrors.Errorf("could not parse sector number: %w", err)
}
2022-09-14 18:51:18 +00:00
_, err = minerAPI.SectorsStatus(ctx, abi.SectorNumber(id), false)
if err != nil {
return xerrors.Errorf("sector %d not found, could not change state", id)
}
newState := cctx.Args().Get(1)
if _, ok := sealing.ExistSectorStateList[sealing.SectorState(newState)]; !ok {
fmt.Printf(" \"%s\" is not a valid state. Possible states for sectors are: \n", newState)
for state := range sealing.ExistSectorStateList {
fmt.Printf("%s\n", string(state))
}
return nil
2020-09-16 11:49:45 +00:00
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorsUpdate(ctx, abi.SectorNumber(id), api.SectorState(cctx.Args().Get(1)))
},
}
2021-08-19 13:53:59 +00:00
var sectorsExpiredCmd = &cli.Command{
Name: "expired",
Usage: "Get or cleanup expired sectors",
Flags: []cli.Flag{
2021-08-19 14:22:29 +00:00
&cli.BoolFlag{
Name: "show-removed",
Usage: "show removed sectors",
},
&cli.BoolFlag{
Name: "remove-expired",
Usage: "remove expired sectors",
},
&cli.Int64Flag{
Name: "confirm-remove-count",
Hidden: true,
},
&cli.Int64Flag{
Name: "expired-epoch",
Usage: "epoch at which to check sector expirations",
DefaultText: "WinningPoSt lookback epoch",
},
2021-08-19 14:22:29 +00:00
},
2021-08-19 13:53:59 +00:00
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2021-08-19 13:53:59 +00:00
if err != nil {
return err
}
defer closer()
fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return xerrors.Errorf("getting fullnode api: %w", err)
}
defer nCloser()
ctx := lcli.ReqContext(cctx)
head, err := fullApi.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("getting chain head: %w", err)
}
lbEpoch := abi.ChainEpoch(cctx.Int64("expired-epoch"))
if !cctx.IsSet("expired-epoch") {
nv, err := fullApi.StateNetworkVersion(ctx, head.Key())
if err != nil {
return xerrors.Errorf("getting network version: %w", err)
}
lbEpoch = head.Height() - policy.GetWinningPoStSectorSetLookback(nv)
if lbEpoch < 0 {
return xerrors.Errorf("too early to terminate sectors")
}
2021-08-19 13:53:59 +00:00
}
if cctx.IsSet("confirm-remove-count") && !cctx.IsSet("expired-epoch") {
return xerrors.Errorf("--expired-epoch must be specified with --confirm-remove-count")
2021-08-19 13:53:59 +00:00
}
lbts, err := fullApi.ChainGetTipSetByHeight(ctx, lbEpoch, head.Key())
if err != nil {
return xerrors.Errorf("getting lookback tipset: %w", err)
}
2022-09-14 18:51:18 +00:00
maddr, err := minerAPI.ActorAddress(ctx)
2021-08-24 09:28:58 +00:00
if err != nil {
return xerrors.Errorf("getting actor address: %w", err)
}
// toCheck is a working bitfield which will only contain terminated sectors
toCheck := bitfield.New()
{
2022-09-14 18:51:18 +00:00
sectors, err := minerAPI.SectorsList(ctx)
2021-08-24 09:28:58 +00:00
if err != nil {
return xerrors.Errorf("getting sector list: %w", err)
}
for _, sector := range sectors {
toCheck.Set(uint64(sector))
}
}
2021-08-19 13:53:59 +00:00
mact, err := fullApi.StateGetActor(ctx, maddr, lbts.Key())
if err != nil {
return err
}
tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory())
2022-04-20 21:34:28 +00:00
mas, err := lminer.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact)
2021-08-19 13:53:59 +00:00
if err != nil {
return err
}
alloc, err := mas.GetAllocatedSectors()
if err != nil {
return xerrors.Errorf("getting allocated sectors: %w", err)
}
// only allocated sectors can be expired,
toCheck, err = bitfield.IntersectBitField(toCheck, *alloc)
2021-08-19 15:10:34 +00:00
if err != nil {
return xerrors.Errorf("intersecting bitfields: %w", err)
}
2021-08-19 13:53:59 +00:00
2022-04-20 21:34:28 +00:00
if err := mas.ForEachDeadline(func(dlIdx uint64, dl lminer.Deadline) error {
return dl.ForEachPartition(func(partIdx uint64, part lminer.Partition) error {
2021-08-19 13:53:59 +00:00
live, err := part.LiveSectors()
if err != nil {
return err
}
toCheck, err = bitfield.SubtractBitField(toCheck, live)
if err != nil {
return err
}
unproven, err := part.UnprovenSectors()
if err != nil {
return err
}
toCheck, err = bitfield.SubtractBitField(toCheck, unproven)
2021-08-19 13:53:59 +00:00
return err
})
}); err != nil {
return err
}
err = mas.ForEachPrecommittedSector(func(pci miner.SectorPreCommitOnChainInfo) error {
toCheck.Unset(uint64(pci.Info.SectorNumber))
return nil
})
if err != nil {
return err
}
if cctx.Bool("remove-expired") {
color.Red("Removing sectors:\n")
}
2021-08-19 13:53:59 +00:00
// toCheck now only contains sectors which either failed to precommit or are expired/terminated
2021-08-19 14:22:29 +00:00
fmt.Printf("Sector\tState\tExpiration\n")
var toRemove []abi.SectorNumber
2021-08-19 13:53:59 +00:00
err = toCheck.ForEach(func(u uint64) error {
2021-08-19 14:22:29 +00:00
s := abi.SectorNumber(u)
2022-09-14 18:51:18 +00:00
st, err := minerAPI.SectorsStatus(ctx, s, true)
2021-08-19 14:22:29 +00:00
if err != nil {
fmt.Printf("%d:\tError getting status: %s\n", u, err)
return nil
}
rmMsg := ""
if st.State == api.SectorState(sealing.Removed) {
if cctx.IsSet("confirm-remove-count") || !cctx.Bool("show-removed") {
return nil
}
} else { // not removed
toRemove = append(toRemove, s)
2021-08-19 14:22:29 +00:00
}
2022-10-31 12:32:00 +00:00
fmt.Printf("%d%s\t%s\t%s\n", s, rmMsg, st.State, cliutil.EpochTime(head.Height(), st.Expiration))
2021-08-19 14:22:29 +00:00
2021-08-19 13:53:59 +00:00
return nil
})
if err != nil {
return err
}
if cctx.Bool("remove-expired") {
if !cctx.IsSet("confirm-remove-count") {
fmt.Println()
fmt.Println(color.YellowString("All"), color.GreenString("%d", len(toRemove)), color.YellowString("sectors listed above will be removed\n"))
fmt.Println(color.YellowString("To confirm removal of the above sectors, including\n all related sealed and unsealed data, run:\n"))
fmt.Println(color.RedString("lotus-miner sectors expired --remove-expired --confirm-remove-count=%d --expired-epoch=%d\n", len(toRemove), lbts.Height()))
fmt.Println(color.YellowString("WARNING: This operation is irreversible"))
return nil
}
fmt.Println()
if int64(len(toRemove)) != cctx.Int64("confirm-remove-count") {
return xerrors.Errorf("value of confirm-remove-count doesn't match the number of sectors which can be removed (%d)", len(toRemove))
}
for _, number := range toRemove {
fmt.Printf("Removing sector\t%s:\t", color.YellowString("%d", number))
2022-09-14 18:51:18 +00:00
err := minerAPI.SectorRemove(ctx, number)
if err != nil {
color.Red("ERROR: %s\n", err.Error())
} else {
color.Green("OK\n")
}
}
}
2021-08-19 13:53:59 +00:00
return nil
},
}
2021-05-18 17:47:30 +00:00
var sectorsBatching = &cli.Command{
Name: "batching",
Usage: "manage batch sector operations",
Subcommands: []*cli.Command{
sectorsBatchingPendingCommit,
sectorsBatchingPendingPreCommit,
},
}
var sectorsBatchingPendingCommit = &cli.Command{
2021-06-01 10:33:24 +00:00
Name: "commit",
2021-05-18 17:47:30 +00:00
Usage: "list sectors waiting in commit batch queue",
2021-03-10 15:16:44 +00:00
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "publish-now",
Usage: "send a batch now",
},
},
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2021-03-10 15:16:44 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if cctx.Bool("publish-now") {
2022-09-14 18:51:18 +00:00
res, err := minerAPI.SectorCommitFlush(ctx)
2021-03-10 15:16:44 +00:00
if err != nil {
return xerrors.Errorf("flush: %w", err)
}
if res == nil {
2021-03-10 15:16:44 +00:00
return xerrors.Errorf("no sectors to publish")
}
for i, re := range res {
fmt.Printf("Batch %d:\n", i)
if re.Error != "" {
fmt.Printf("\tError: %s\n", re.Error)
} else {
fmt.Printf("\tMessage: %s\n", re.Msg)
}
fmt.Printf("\tSectors:\n")
for _, sector := range re.Sectors {
if e, found := re.FailedSectors[sector]; found {
fmt.Printf("\t\t%d\tERROR %s\n", sector, e)
} else {
fmt.Printf("\t\t%d\tOK\n", sector)
}
}
}
2021-03-10 15:16:44 +00:00
return nil
}
2022-09-14 18:51:18 +00:00
pending, err := minerAPI.SectorCommitPending(ctx)
2021-03-10 15:16:44 +00:00
if err != nil {
return xerrors.Errorf("getting pending deals: %w", err)
}
if len(pending) > 0 {
for _, sector := range pending {
fmt.Println(sector.Number)
}
reader := bufio.NewReader(os.Stdin)
fmt.Print("Do you want to publish these sectors now? (yes/no): ")
userInput, err := reader.ReadString('\n')
if err != nil {
return xerrors.Errorf("reading user input: %w", err)
}
userInput = strings.ToLower(strings.TrimSpace(userInput))
if userInput == "yes" {
2023-05-05 12:59:25 +00:00
err := cctx.Set("publish-now", "true")
if err != nil {
return xerrors.Errorf("setting publish-now flag: %w", err)
}
return cctx.Command.Action(cctx)
} else if userInput == "no" {
return nil
} else {
fmt.Println("Invalid input. Please answer with 'yes' or 'no'.")
return nil
}
2021-03-10 15:16:44 +00:00
} else {
fmt.Println("No sectors queued to be committed")
}
2021-03-10 15:16:44 +00:00
return nil
},
}
2021-05-18 17:47:30 +00:00
var sectorsBatchingPendingPreCommit = &cli.Command{
2021-06-01 10:33:24 +00:00
Name: "precommit",
2021-05-18 17:47:30 +00:00
Usage: "list sectors waiting in precommit batch queue",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "publish-now",
Usage: "send a batch now",
},
},
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2021-05-18 17:47:30 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if cctx.Bool("publish-now") {
2022-09-14 18:51:18 +00:00
res, err := minerAPI.SectorPreCommitFlush(ctx)
2021-05-18 17:47:30 +00:00
if err != nil {
return xerrors.Errorf("flush: %w", err)
}
if res == nil {
2021-05-18 17:47:30 +00:00
return xerrors.Errorf("no sectors to publish")
}
for i, re := range res {
fmt.Printf("Batch %d:\n", i)
if re.Error != "" {
fmt.Printf("\tError: %s\n", re.Error)
} else {
fmt.Printf("\tMessage: %s\n", re.Msg)
}
fmt.Printf("\tSectors:\n")
for _, sector := range re.Sectors {
fmt.Printf("\t\t%d\tOK\n", sector)
}
}
2021-05-18 17:47:30 +00:00
return nil
}
2022-09-14 18:51:18 +00:00
pending, err := minerAPI.SectorPreCommitPending(ctx)
2021-05-18 17:47:30 +00:00
if err != nil {
return xerrors.Errorf("getting pending deals: %w", err)
}
if len(pending) > 0 {
for _, sector := range pending {
fmt.Println(sector.Number)
}
reader := bufio.NewReader(os.Stdin)
fmt.Print("Do you want to publish these sectors now? (yes/no): ")
userInput, err := reader.ReadString('\n')
if err != nil {
return xerrors.Errorf("reading user input: %w", err)
}
userInput = strings.ToLower(strings.TrimSpace(userInput))
if userInput == "yes" {
2023-05-05 12:59:25 +00:00
err := cctx.Set("publish-now", "true")
if err != nil {
return xerrors.Errorf("setting publish-now flag: %w", err)
}
return cctx.Command.Action(cctx)
} else if userInput == "no" {
return nil
} else {
fmt.Println("Invalid input. Please answer with 'yes' or 'no'.")
return nil
}
2021-05-18 17:47:30 +00:00
} else {
fmt.Println("No sectors queued to be committed")
}
2021-05-18 17:47:30 +00:00
return nil
},
}
var sectorsRefreshPieceMatchingCmd = &cli.Command{
Name: "match-pending-pieces",
Usage: "force a refreshed match of pending pieces to open sectors without manually waiting for more deals",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:51:18 +00:00
if err := minerAPI.SectorMatchPendingPiecesToOpenSectors(ctx); err != nil {
return err
}
return nil
},
}
func yesno(b bool) string {
if b {
2020-09-30 11:33:42 +00:00
return color.GreenString("YES")
}
2020-09-30 11:33:42 +00:00
return color.RedString("NO")
}
var sectorsCompactPartitionsCmd = &cli.Command{
Name: "compact-partitions",
Usage: "removes dead sectors from partitions and reduces the number of partitions used if possible",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "deadline",
Usage: "the deadline to compact the partitions in",
Required: true,
},
&cli.Int64SliceFlag{
Name: "partitions",
Usage: "list of partitions to compact sectors in",
Required: true,
},
&cli.BoolFlag{
Name: "really-do-it",
Usage: "Actually send transaction performing the action",
Value: false,
},
2022-05-10 21:35:59 +00:00
&cli.StringFlag{
Name: "actor",
2022-05-25 18:37:33 +00:00
Usage: "Specify the address of the miner to run this command",
2022-05-10 21:35:59 +00:00
},
},
Action: func(cctx *cli.Context) error {
fullNodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer acloser()
ctx := lcli.ReqContext(cctx)
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
minfo, err := fullNodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
deadline := cctx.Uint64("deadline")
2022-05-25 18:37:33 +00:00
if deadline > miner.WPoStPeriodDeadlines {
return fmt.Errorf("deadline %d out of range", deadline)
}
parts := cctx.Int64Slice("partitions")
if len(parts) <= 0 {
return fmt.Errorf("must include at least one partition to compact")
}
2022-05-10 21:35:59 +00:00
fmt.Printf("compacting %d paritions\n", len(parts))
var makeMsgForPartitions func(partitionsBf bitfield.BitField) ([]*types.Message, error)
makeMsgForPartitions = func(partitionsBf bitfield.BitField) ([]*types.Message, error) {
params := miner.CompactPartitionsParams{
Deadline: deadline,
Partitions: partitionsBf,
}
sp, aerr := actors.SerializeParams(&params)
if aerr != nil {
return nil, xerrors.Errorf("serializing params: %w", err)
}
msg := &types.Message{
From: minfo.Worker,
To: maddr,
Method: builtin.MethodsMiner.CompactPartitions,
Value: big.Zero(),
Params: sp,
}
estimatedMsg, err := fullNodeAPI.GasEstimateMessageGas(ctx, msg, nil, types.EmptyTSK)
if err != nil && xerrors.Is(err, &api.ErrOutOfGas{}) {
// the message is too big -- split into 2
partitionsSlice, err := partitionsBf.All(math.MaxUint64)
if err != nil {
return nil, err
}
partitions1 := bitfield.New()
for i := 0; i < len(partitionsSlice)/2; i++ {
partitions1.Set(uint64(i))
}
msgs1, err := makeMsgForPartitions(partitions1)
if err != nil {
return nil, err
}
// time for the second half
partitions2 := bitfield.New()
for i := len(partitionsSlice) / 2; i < len(partitionsSlice); i++ {
partitions2.Set(uint64(i))
}
msgs2, err := makeMsgForPartitions(partitions2)
if err != nil {
return nil, err
}
return append(msgs1, msgs2...), nil
} else if err != nil {
return nil, err
}
return []*types.Message{estimatedMsg}, nil
}
2022-05-10 21:35:59 +00:00
partitions := bitfield.New()
for _, partition := range parts {
partitions.Set(uint64(partition))
}
msgs, err := makeMsgForPartitions(partitions)
if err != nil {
return xerrors.Errorf("failed to make messages: %w", err)
}
// Actually send the messages if really-do-it provided, simulate otherwise
if cctx.Bool("really-do-it") {
smsgs, err := fullNodeAPI.MpoolBatchPushMessage(ctx, msgs, nil)
if err != nil {
return xerrors.Errorf("mpool push: %w", err)
}
if len(smsgs) == 1 {
fmt.Printf("Requested compact partitions in message %s\n", smsgs[0].Cid())
} else {
fmt.Printf("Requested compact partitions in %d messages\n\n", len(smsgs))
for _, v := range smsgs {
fmt.Println(v.Cid())
}
}
2022-05-25 18:37:33 +00:00
for _, v := range smsgs {
wait, err := fullNodeAPI.StateWaitMsg(ctx, v.Cid(), 2)
if err != nil {
return err
}
// check it executed successfully
if wait.Receipt.ExitCode.IsError() {
fmt.Println(cctx.App.Writer, "compact partitions msg %s failed!", v.Cid())
return err
}
}
return nil
2022-05-25 18:37:33 +00:00
}
for i, v := range msgs {
fmt.Printf("total of %d CompactPartitions msgs would be sent\n", len(msgs))
estMsg, err := fullNodeAPI.GasEstimateMessageGas(ctx, v, nil, types.EmptyTSK)
if err != nil {
return err
}
fmt.Printf("msg %d would cost up to %s\n", i+1, types.FIL(estMsg.RequiredFunds()))
2022-05-25 18:37:33 +00:00
}
return nil
},
}
2022-08-18 14:37:22 +00:00
var sectorsNumbersCmd = &cli.Command{
Name: "numbers",
Usage: "manage sector number assignments",
Subcommands: []*cli.Command{
sectorsNumbersInfoCmd,
sectorsNumbersReservationsCmd,
sectorsNumbersReserveCmd,
sectorsNumbersFreeCmd,
},
}
var sectorsNumbersInfoCmd = &cli.Command{
Name: "info",
Usage: "view sector assigner state",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:51:18 +00:00
am, err := minerAPI.SectorNumAssignerMeta(ctx)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
2022-08-24 15:25:37 +00:00
alloc, err := strle.BitfieldToHumanRanges(am.Allocated)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
2022-08-24 15:25:37 +00:00
reserved, err := strle.BitfieldToHumanRanges(am.Reserved)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
fmt.Printf("Next free: %s\n", am.Next)
fmt.Printf("Allocated: %s\n", alloc)
fmt.Printf("Reserved: %s\n", reserved)
return nil
},
}
var sectorsNumbersReservationsCmd = &cli.Command{
Name: "reservations",
Usage: "list sector number reservations",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:51:18 +00:00
rs, err := minerAPI.SectorNumReservations(ctx)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
var out []string
for name, field := range rs {
2022-08-24 15:25:37 +00:00
hr, err := strle.BitfieldToHumanRanges(field)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
count, err := field.Count()
if err != nil {
return err
}
out = append(out, fmt.Sprintf("%s: count=%d %s", name, count, hr))
}
fmt.Printf("reservations: %d\n", len(out))
sort.Strings(out)
for _, s := range out {
fmt.Println(s)
}
return nil
},
}
var sectorsNumbersReserveCmd = &cli.Command{
Name: "reserve",
Usage: "create sector number reservations",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "force",
Usage: "skip duplicate reservation checks (note: can lead to damaging other reservations on free)",
},
},
ArgsUsage: "[reservation name] [reserved ranges]",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 2 {
return lcli.IncorrectNumArgs(cctx)
2022-08-18 14:37:22 +00:00
}
2022-08-24 15:25:37 +00:00
bf, err := strle.HumanRangesToBitField(cctx.Args().Get(1))
2022-08-18 14:37:22 +00:00
if err != nil {
return xerrors.Errorf("parsing ranges: %w", err)
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorNumReserve(ctx, cctx.Args().First(), bf, cctx.Bool("force"))
2022-08-18 14:37:22 +00:00
},
}
var sectorsNumbersFreeCmd = &cli.Command{
Name: "free",
Usage: "remove sector number reservations",
ArgsUsage: "[reservation name]",
Action: func(cctx *cli.Context) error {
2022-09-14 18:51:18 +00:00
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
2022-08-18 14:37:22 +00:00
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
2022-09-14 18:33:29 +00:00
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
2022-08-18 14:37:22 +00:00
}
2022-09-14 18:51:18 +00:00
return minerAPI.SectorNumFree(ctx, cctx.Args().First())
2022-08-18 14:37:22 +00:00
},
}
var sectorsUnsealCmd = &cli.Command{
Name: "unseal",
Usage: "unseal a sector",
ArgsUsage: "[sector number]",
Action: func(cctx *cli.Context) error {
minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
}
sectorNum, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
return minerAPI.SectorUnseal(ctx, abi.SectorNumber(sectorNum))
},
}