lotus/cmd/lotus-storage-miner/sealing.go

308 lines
7.2 KiB
Go
Raw Normal View History

2020-03-23 12:29:24 +00:00
package main
2020-03-23 14:56:22 +00:00
import (
2020-09-23 17:26:26 +00:00
"encoding/hex"
"encoding/json"
2020-03-23 14:56:22 +00:00
"fmt"
2020-07-21 18:07:49 +00:00
"os"
2020-03-23 14:56:22 +00:00
"sort"
2020-05-01 12:06:19 +00:00
"strings"
2020-07-21 18:07:49 +00:00
"text/tabwriter"
"time"
2020-05-01 12:06:19 +00:00
"github.com/fatih/color"
"github.com/google/uuid"
"github.com/urfave/cli/v2"
2020-09-23 17:26:26 +00:00
"golang.org/x/xerrors"
2020-03-23 14:56:22 +00:00
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
2020-05-01 12:06:19 +00:00
"github.com/filecoin-project/lotus/chain/types"
2020-03-23 14:56:22 +00:00
lcli "github.com/filecoin-project/lotus/cli"
)
2020-07-21 18:07:49 +00:00
var sealingCmd = &cli.Command{
Name: "sealing",
Usage: "interact with sealing pipeline",
2020-03-23 14:56:22 +00:00
Subcommands: []*cli.Command{
2020-07-21 18:07:49 +00:00
sealingJobsCmd,
sealingWorkersCmd,
sealingSchedDiagCmd,
sealingAbortCmd,
2020-03-23 14:56:22 +00:00
},
}
2020-07-21 18:07:49 +00:00
var sealingWorkersCmd = &cli.Command{
Name: "workers",
2020-03-23 14:56:22 +00:00
Usage: "list workers",
2020-05-01 12:06:19 +00:00
Flags: []cli.Flag{
&cli.BoolFlag{Name: "color"},
},
2020-03-23 14:56:22 +00:00
Action: func(cctx *cli.Context) error {
2020-05-01 12:06:19 +00:00
color.NoColor = !cctx.Bool("color")
2020-03-23 14:56:22 +00:00
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
stats, err := nodeApi.WorkerStats(ctx)
if err != nil {
return err
}
2020-03-24 23:49:45 +00:00
type sortableStat struct {
id uuid.UUID
2020-04-23 22:23:20 +00:00
storiface.WorkerStats
2020-03-24 23:49:45 +00:00
}
st := make([]sortableStat, 0, len(stats))
2020-03-23 14:56:22 +00:00
for id, stat := range stats {
2020-03-24 23:49:45 +00:00
st = append(st, sortableStat{id, stat})
2020-03-23 14:56:22 +00:00
}
sort.Slice(st, func(i, j int) bool {
return st[i].id.String() < st[j].id.String()
2020-03-23 14:56:22 +00:00
})
for _, stat := range st {
gpuUse := "not "
2020-05-01 12:06:19 +00:00
gpuCol := color.FgBlue
2020-03-23 14:56:22 +00:00
if stat.GpuUsed {
2020-05-01 12:06:19 +00:00
gpuCol = color.FgGreen
2020-03-23 14:56:22 +00:00
gpuUse = ""
}
2020-10-18 11:03:17 +00:00
var disabled string
if !stat.Enabled {
disabled = color.RedString(" (disabled)")
}
fmt.Printf("Worker %s, host %s%s\n", stat.id, color.MagentaString(stat.Info.Hostname), disabled)
2020-05-01 12:06:19 +00:00
var barCols = uint64(64)
cpuBars := int(stat.CpuUse * barCols / stat.Info.Resources.CPUs)
2020-05-01 19:51:31 +00:00
cpuBar := strings.Repeat("|", cpuBars) + strings.Repeat(" ", int(barCols)-cpuBars)
2020-05-01 12:06:19 +00:00
fmt.Printf("\tCPU: [%s] %d/%d core(s) in use\n",
color.GreenString(cpuBar), stat.CpuUse, stat.Info.Resources.CPUs)
2020-05-01 12:06:19 +00:00
2020-05-01 19:51:31 +00:00
ramBarsRes := int(stat.Info.Resources.MemReserved * barCols / stat.Info.Resources.MemPhysical)
ramBarsUsed := int(stat.MemUsedMin * barCols / stat.Info.Resources.MemPhysical)
2020-05-01 12:06:19 +00:00
ramBar := color.YellowString(strings.Repeat("|", ramBarsRes)) +
color.GreenString(strings.Repeat("|", ramBarsUsed)) +
2020-05-01 19:51:31 +00:00
strings.Repeat(" ", int(barCols)-ramBarsUsed-ramBarsRes)
2020-05-01 12:06:19 +00:00
2020-05-01 19:51:31 +00:00
vmem := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap
2020-05-01 12:06:19 +00:00
2020-05-01 19:51:31 +00:00
vmemBarsRes := int(stat.Info.Resources.MemReserved * barCols / vmem)
vmemBarsUsed := int(stat.MemUsedMax * barCols / vmem)
2020-05-01 12:06:19 +00:00
vmemBar := color.YellowString(strings.Repeat("|", vmemBarsRes)) +
color.GreenString(strings.Repeat("|", vmemBarsUsed)) +
2020-05-01 19:51:31 +00:00
strings.Repeat(" ", int(barCols)-vmemBarsUsed-vmemBarsRes)
2020-05-01 12:06:19 +00:00
fmt.Printf("\tRAM: [%s] %d%% %s/%s\n", ramBar,
2020-05-01 19:51:31 +00:00
(stat.Info.Resources.MemReserved+stat.MemUsedMin)*100/stat.Info.Resources.MemPhysical,
types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMin)),
2020-05-01 12:06:19 +00:00
types.SizeStr(types.NewInt(stat.Info.Resources.MemPhysical)))
fmt.Printf("\tVMEM: [%s] %d%% %s/%s\n", vmemBar,
2020-05-01 19:51:31 +00:00
(stat.Info.Resources.MemReserved+stat.MemUsedMax)*100/vmem,
types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMax)),
2020-05-01 12:06:19 +00:00
types.SizeStr(types.NewInt(vmem)))
2020-03-23 14:56:22 +00:00
for _, gpu := range stat.Info.Resources.GPUs {
2020-05-01 12:06:19 +00:00
fmt.Printf("\tGPU: %s\n", color.New(gpuCol).Sprintf("%s, %sused", gpu, gpuUse))
2020-03-23 14:56:22 +00:00
}
}
return nil
},
}
2020-07-21 18:07:49 +00:00
var sealingJobsCmd = &cli.Command{
Name: "jobs",
Usage: "list running jobs",
2020-07-21 18:07:49 +00:00
Flags: []cli.Flag{
&cli.BoolFlag{Name: "color"},
&cli.BoolFlag{
2020-11-11 16:39:31 +00:00
Name: "show-ret-done",
Usage: "show returned but not consumed calls",
},
2020-07-21 18:07:49 +00:00
},
Action: func(cctx *cli.Context) error {
color.NoColor = !cctx.Bool("color")
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
jobs, err := nodeApi.WorkerJobs(ctx)
if err != nil {
return xerrors.Errorf("getting worker jobs: %w", err)
}
type line struct {
storiface.WorkerJob
wid uuid.UUID
2020-07-21 18:07:49 +00:00
}
lines := make([]line, 0)
for wid, jobs := range jobs {
for _, job := range jobs {
lines = append(lines, line{
WorkerJob: job,
wid: wid,
})
}
}
// oldest first
sort.Slice(lines, func(i, j int) bool {
if lines[i].RunWait != lines[j].RunWait {
return lines[i].RunWait < lines[j].RunWait
}
if lines[i].Start.Equal(lines[j].Start) {
return lines[i].ID.ID.String() < lines[j].ID.ID.String()
}
2020-07-21 18:07:49 +00:00
return lines[i].Start.Before(lines[j].Start)
})
workerHostnames := map[uuid.UUID]string{}
2020-07-21 18:07:49 +00:00
wst, err := nodeApi.WorkerStats(ctx)
if err != nil {
return xerrors.Errorf("getting worker stats: %w", err)
}
for wid, st := range wst {
workerHostnames[wid] = st.Info.Hostname
2020-07-21 18:07:49 +00:00
}
tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
_, _ = fmt.Fprintf(tw, "ID\tSector\tWorker\tHostname\tTask\tState\tTime\n")
2020-07-21 18:07:49 +00:00
for _, l := range lines {
state := "running"
switch {
case l.RunWait > 0:
state = fmt.Sprintf("assigned(%d)", l.RunWait-1)
case l.RunWait == storiface.RWRetDone:
if !cctx.Bool("show-ret-done") {
continue
}
state = "ret-done"
case l.RunWait == storiface.RWReturned:
state = "returned"
case l.RunWait == storiface.RWRetWait:
2020-09-23 17:26:26 +00:00
state = "ret-wait"
}
dur := "n/a"
if !l.Start.IsZero() {
dur = time.Now().Sub(l.Start).Truncate(time.Millisecond * 100).String()
}
hostname, ok := workerHostnames[l.wid]
if !ok {
hostname = l.Hostname
}
2020-10-18 11:03:17 +00:00
_, _ = fmt.Fprintf(tw, "%s\t%d\t%s\t%s\t%s\t%s\t%s\n",
hex.EncodeToString(l.ID.ID[:4]),
2020-10-18 11:03:17 +00:00
l.Sector.Number,
hex.EncodeToString(l.wid[:4]),
hostname,
2020-10-18 11:03:17 +00:00
l.Task.Short(),
state,
dur)
2020-07-21 18:07:49 +00:00
}
return tw.Flush()
},
}
var sealingSchedDiagCmd = &cli.Command{
Name: "sched-diag",
Usage: "Dump internal scheduler state",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "force-sched",
},
},
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
st, err := nodeApi.SealingSchedDiag(ctx, cctx.Bool("force-sched"))
if err != nil {
return err
}
j, err := json.MarshalIndent(&st, "", " ")
if err != nil {
return err
}
fmt.Println(string(j))
return nil
},
}
var sealingAbortCmd = &cli.Command{
2020-11-11 16:39:31 +00:00
Name: "abort",
Usage: "Abort a running job",
2020-11-12 07:04:55 +00:00
ArgsUsage: "[callid]",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return xerrors.Errorf("expected 1 argument")
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
jobs, err := nodeApi.WorkerJobs(ctx)
if err != nil {
return xerrors.Errorf("getting worker jobs: %w", err)
}
var job *storiface.WorkerJob
outer:
for _, workerJobs := range jobs {
for _, j := range workerJobs {
if strings.HasPrefix(j.ID.ID.String(), cctx.Args().First()) {
2020-11-11 16:39:31 +00:00
j := j
job = &j
break outer
}
}
}
if job == nil {
return xerrors.Errorf("job with specified id prefix not found")
}
fmt.Printf("aborting job %s, task %s, sector %d, running on host %s\n", job.ID.String(), job.Task.Short(), job.Sector.Number, job.Hostname)
return nodeApi.SealingAbort(ctx, job.ID)
},
}