lp: more storage cli/rpc
This commit is contained in:
Łukasz Magiera
parent
f3e5497281
commit
e37b7f6083
@ -5,6 +5,9 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
)
|
||||
|
||||
@ -14,6 +17,11 @@ type LotusProvider interface {
|
||||
AllocatePieceToSector(ctx context.Context, maddr address.Address, piece PieceDealInfo, rawSize int64, source url.URL, header http.Header) (SectorOffset, error) //perm:write
|
||||
|
||||
StorageAddLocal(ctx context.Context, path string) error //perm:admin
|
||||
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
|
||||
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
|
||||
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
|
||||
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
|
||||
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
|
||||
|
||||
// Trigger shutdown
|
||||
Shutdown(context.Context) error //perm:admin
|
||||
|
||||
@ -838,6 +838,16 @@ type LotusProviderMethods struct {
|
||||
|
||||
StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"`
|
||||
|
||||
StorageDetachLocal func(p0 context.Context, p1 string) error `perm:"admin"`
|
||||
|
||||
StorageInfo func(p0 context.Context, p1 storiface.ID) (storiface.StorageInfo, error) `perm:"admin"`
|
||||
|
||||
StorageList func(p0 context.Context) (map[storiface.ID][]storiface.Decl, error) `perm:"admin"`
|
||||
|
||||
StorageLocal func(p0 context.Context) (map[storiface.ID]string, error) `perm:"admin"`
|
||||
|
||||
StorageStat func(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) `perm:"admin"`
|
||||
|
||||
Version func(p0 context.Context) (Version, error) `perm:"admin"`
|
||||
}
|
||||
|
||||
@ -5238,6 +5248,61 @@ func (s *LotusProviderStub) StorageAddLocal(p0 context.Context, p1 string) error
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) StorageDetachLocal(p0 context.Context, p1 string) error {
|
||||
if s.Internal.StorageDetachLocal == nil {
|
||||
return ErrNotSupported
|
||||
}
|
||||
return s.Internal.StorageDetachLocal(p0, p1)
|
||||
}
|
||||
|
||||
func (s *LotusProviderStub) StorageDetachLocal(p0 context.Context, p1 string) error {
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) StorageInfo(p0 context.Context, p1 storiface.ID) (storiface.StorageInfo, error) {
|
||||
if s.Internal.StorageInfo == nil {
|
||||
return *new(storiface.StorageInfo), ErrNotSupported
|
||||
}
|
||||
return s.Internal.StorageInfo(p0, p1)
|
||||
}
|
||||
|
||||
func (s *LotusProviderStub) StorageInfo(p0 context.Context, p1 storiface.ID) (storiface.StorageInfo, error) {
|
||||
return *new(storiface.StorageInfo), ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) StorageList(p0 context.Context) (map[storiface.ID][]storiface.Decl, error) {
|
||||
if s.Internal.StorageList == nil {
|
||||
return *new(map[storiface.ID][]storiface.Decl), ErrNotSupported
|
||||
}
|
||||
return s.Internal.StorageList(p0)
|
||||
}
|
||||
|
||||
func (s *LotusProviderStub) StorageList(p0 context.Context) (map[storiface.ID][]storiface.Decl, error) {
|
||||
return *new(map[storiface.ID][]storiface.Decl), ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) StorageLocal(p0 context.Context) (map[storiface.ID]string, error) {
|
||||
if s.Internal.StorageLocal == nil {
|
||||
return *new(map[storiface.ID]string), ErrNotSupported
|
||||
}
|
||||
return s.Internal.StorageLocal(p0)
|
||||
}
|
||||
|
||||
func (s *LotusProviderStub) StorageLocal(p0 context.Context) (map[storiface.ID]string, error) {
|
||||
return *new(map[storiface.ID]string), ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) StorageStat(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) {
|
||||
if s.Internal.StorageStat == nil {
|
||||
return *new(fsutil.FsStat), ErrNotSupported
|
||||
}
|
||||
return s.Internal.StorageStat(p0, p1)
|
||||
}
|
||||
|
||||
func (s *LotusProviderStub) StorageStat(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) {
|
||||
return *new(fsutil.FsStat), ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) Version(p0 context.Context) (Version, error) {
|
||||
if s.Internal.Version == nil {
|
||||
return *new(Version), ErrNotSupported
|
||||
|
||||
@ -190,5 +190,7 @@ var cliCmd = &cli.Command{
|
||||
|
||||
return nil
|
||||
},
|
||||
Subcommands: []*cli.Command{},
|
||||
Subcommands: []*cli.Command{
|
||||
storageCmd,
|
||||
},
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"github.com/filecoin-project/lotus/api/client"
|
||||
cliutil "github.com/filecoin-project/lotus/cli/util"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
@ -77,9 +78,76 @@ func LotusProviderHandler(
|
||||
|
||||
type ProviderAPI struct {
|
||||
*deps.Deps
|
||||
paths.SectorIndex
|
||||
ShutdownChan chan struct{}
|
||||
}
|
||||
|
||||
func (p *ProviderAPI) StorageDetachLocal(ctx context.Context, path string) error {
|
||||
path, err := homedir.Expand(path)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("expanding local path: %w", err)
|
||||
}
|
||||
|
||||
// check that we have the path opened
|
||||
lps, err := p.LocalStore.Local(ctx)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting local path list: %w", err)
|
||||
}
|
||||
|
||||
var localPath *storiface.StoragePath
|
||||
for _, lp := range lps {
|
||||
if lp.LocalPath == path {
|
||||
lp := lp // copy to make the linter happy
|
||||
localPath = &lp
|
||||
break
|
||||
}
|
||||
}
|
||||
if localPath == nil {
|
||||
return xerrors.Errorf("no local paths match '%s'", path)
|
||||
}
|
||||
|
||||
// drop from the persisted storage.json
|
||||
var found bool
|
||||
if err := p.LocalPaths.SetStorage(func(sc *storiface.StorageConfig) {
|
||||
out := make([]storiface.LocalPath, 0, len(sc.StoragePaths))
|
||||
for _, storagePath := range sc.StoragePaths {
|
||||
if storagePath.Path != path {
|
||||
out = append(out, storagePath)
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
}
|
||||
sc.StoragePaths = out
|
||||
}); err != nil {
|
||||
return xerrors.Errorf("set storage config: %w", err)
|
||||
}
|
||||
if !found {
|
||||
// maybe this is fine?
|
||||
return xerrors.Errorf("path not found in storage.json")
|
||||
}
|
||||
|
||||
// unregister locally, drop from sector index
|
||||
return p.LocalStore.ClosePath(ctx, localPath.ID)
|
||||
}
|
||||
|
||||
func (p *ProviderAPI) StorageLocal(ctx context.Context) (map[storiface.ID]string, error) {
|
||||
ps, err := p.LocalStore.Local(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out = make(map[storiface.ID]string)
|
||||
for _, path := range ps {
|
||||
out[path.ID] = path.LocalPath
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *ProviderAPI) StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) {
|
||||
return p.LocalStore.FsStat(ctx, id)
|
||||
}
|
||||
|
||||
func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
|
||||
return api.ProviderAPIVersion0, nil
|
||||
}
|
||||
@ -147,7 +215,7 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c
|
||||
Handler: LotusProviderHandler(
|
||||
authVerify,
|
||||
remoteHandler,
|
||||
&ProviderAPI{dependencies, shutdownChan},
|
||||
&ProviderAPI{dependencies, dependencies.Si, shutdownChan},
|
||||
permissioned),
|
||||
ReadHeaderTimeout: time.Minute * 3,
|
||||
BaseContext: func(listener net.Listener) context.Context {
|
||||
|
||||
@ -2,16 +2,24 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/docker/go-units"
|
||||
"github.com/fatih/color"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
"github.com/filecoin-project/lotus/cmd/lotus-provider/rpc"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
"github.com/google/uuid"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
"golang.org/x/xerrors"
|
||||
"math/bits"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const metaFile = "sectorstore.json"
|
||||
@ -25,9 +33,10 @@ long term for proving (references as 'store') as well as how sectors will be
|
||||
stored while moving through the sealing pipeline (references as 'seal').`,
|
||||
Subcommands: []*cli.Command{
|
||||
storageAttachCmd,
|
||||
storageDetachCmd,
|
||||
storageListCmd,
|
||||
/*storageDetachCmd,
|
||||
storageRedeclareCmd,
|
||||
storageListCmd,
|
||||
storageFindCmd,
|
||||
storageCleanupCmd,
|
||||
storageLocks,*/
|
||||
@ -156,3 +165,232 @@ over time
|
||||
return minerApi.StorageAddLocal(ctx, p)
|
||||
},
|
||||
}
|
||||
|
||||
var storageDetachCmd = &cli.Command{
|
||||
Name: "detach",
|
||||
Usage: "detach local storage path",
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "really-do-it",
|
||||
},
|
||||
},
|
||||
ArgsUsage: "[path]",
|
||||
Action: func(cctx *cli.Context) error {
|
||||
minerApi, closer, err := rpc.GetProviderAPI(cctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
ctx := lcli.ReqContext(cctx)
|
||||
|
||||
if cctx.NArg() != 1 {
|
||||
return lcli.IncorrectNumArgs(cctx)
|
||||
}
|
||||
|
||||
p, err := homedir.Expand(cctx.Args().First())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("expanding path: %w", err)
|
||||
}
|
||||
|
||||
if !cctx.Bool("really-do-it") {
|
||||
return xerrors.Errorf("pass --really-do-it to execute the action")
|
||||
}
|
||||
|
||||
return minerApi.StorageDetachLocal(ctx, p)
|
||||
},
|
||||
}
|
||||
|
||||
var storageListCmd = &cli.Command{
|
||||
Name: "list",
|
||||
Usage: "list local storage paths",
|
||||
Subcommands: []*cli.Command{
|
||||
//storageListSectorsCmd,
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
minerApi, closer, err := rpc.GetProviderAPI(cctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
ctx := lcli.ReqContext(cctx)
|
||||
|
||||
st, err := minerApi.StorageList(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
local, err := minerApi.StorageLocal(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type fsInfo struct {
|
||||
storiface.ID
|
||||
sectors []storiface.Decl
|
||||
stat fsutil.FsStat
|
||||
}
|
||||
|
||||
sorted := make([]fsInfo, 0, len(st))
|
||||
for id, decls := range st {
|
||||
st, err := minerApi.StorageStat(ctx, id)
|
||||
if err != nil {
|
||||
sorted = append(sorted, fsInfo{ID: id, sectors: decls})
|
||||
continue
|
||||
}
|
||||
|
||||
sorted = append(sorted, fsInfo{id, decls, st})
|
||||
}
|
||||
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].stat.Capacity != sorted[j].stat.Capacity {
|
||||
return sorted[i].stat.Capacity > sorted[j].stat.Capacity
|
||||
}
|
||||
return sorted[i].ID < sorted[j].ID
|
||||
})
|
||||
|
||||
for _, s := range sorted {
|
||||
|
||||
var cnt [5]int
|
||||
for _, decl := range s.sectors {
|
||||
for i := range cnt {
|
||||
if decl.SectorFileType&(1<<i) != 0 {
|
||||
cnt[i]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("%s:\n", s.ID)
|
||||
|
||||
pingStart := time.Now()
|
||||
st, err := minerApi.StorageStat(ctx, s.ID)
|
||||
if err != nil {
|
||||
fmt.Printf("\t%s: %s:\n", color.RedString("Error"), err)
|
||||
continue
|
||||
}
|
||||
ping := time.Now().Sub(pingStart)
|
||||
|
||||
safeRepeat := func(s string, count int) string {
|
||||
if count < 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Repeat(s, count)
|
||||
}
|
||||
|
||||
var barCols = int64(50)
|
||||
|
||||
// filesystem use bar
|
||||
{
|
||||
usedPercent := (st.Capacity - st.FSAvailable) * 100 / st.Capacity
|
||||
|
||||
percCol := color.FgGreen
|
||||
switch {
|
||||
case usedPercent > 98:
|
||||
percCol = color.FgRed
|
||||
case usedPercent > 90:
|
||||
percCol = color.FgYellow
|
||||
}
|
||||
|
||||
set := (st.Capacity - st.FSAvailable) * barCols / st.Capacity
|
||||
used := (st.Capacity - (st.FSAvailable + st.Reserved)) * barCols / st.Capacity
|
||||
reserved := set - used
|
||||
bar := safeRepeat("#", int(used)) + safeRepeat("*", int(reserved)) + safeRepeat(" ", int(barCols-set))
|
||||
|
||||
desc := ""
|
||||
if st.Max > 0 {
|
||||
desc = " (filesystem)"
|
||||
}
|
||||
|
||||
fmt.Printf("\t[%s] %s/%s %s%s\n", color.New(percCol).Sprint(bar),
|
||||
types.SizeStr(types.NewInt(uint64(st.Capacity-st.FSAvailable))),
|
||||
types.SizeStr(types.NewInt(uint64(st.Capacity))),
|
||||
color.New(percCol).Sprintf("%d%%", usedPercent), desc)
|
||||
}
|
||||
|
||||
// optional configured limit bar
|
||||
if st.Max > 0 {
|
||||
usedPercent := st.Used * 100 / st.Max
|
||||
|
||||
percCol := color.FgGreen
|
||||
switch {
|
||||
case usedPercent > 98:
|
||||
percCol = color.FgRed
|
||||
case usedPercent > 90:
|
||||
percCol = color.FgYellow
|
||||
}
|
||||
|
||||
set := st.Used * barCols / st.Max
|
||||
used := (st.Used + st.Reserved) * barCols / st.Max
|
||||
reserved := set - used
|
||||
bar := safeRepeat("#", int(used)) + safeRepeat("*", int(reserved)) + safeRepeat(" ", int(barCols-set))
|
||||
|
||||
fmt.Printf("\t[%s] %s/%s %s (limit)\n", color.New(percCol).Sprint(bar),
|
||||
types.SizeStr(types.NewInt(uint64(st.Used))),
|
||||
types.SizeStr(types.NewInt(uint64(st.Max))),
|
||||
color.New(percCol).Sprintf("%d%%", usedPercent))
|
||||
}
|
||||
|
||||
fmt.Printf("\t%s; %s; %s; %s; %s; Reserved: %s\n",
|
||||
color.YellowString("Unsealed: %d", cnt[0]),
|
||||
color.GreenString("Sealed: %d", cnt[1]),
|
||||
color.BlueString("Caches: %d", cnt[2]),
|
||||
color.GreenString("Updated: %d", cnt[3]),
|
||||
color.BlueString("Update-caches: %d", cnt[4]),
|
||||
types.SizeStr(types.NewInt(uint64(st.Reserved))))
|
||||
|
||||
si, err := minerApi.StorageInfo(ctx, s.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Print("\t")
|
||||
if si.CanSeal || si.CanStore {
|
||||
fmt.Printf("Weight: %d; Use: ", si.Weight)
|
||||
if si.CanSeal {
|
||||
fmt.Print(color.MagentaString("Seal "))
|
||||
}
|
||||
if si.CanStore {
|
||||
fmt.Print(color.CyanString("Store"))
|
||||
}
|
||||
} else {
|
||||
fmt.Print(color.HiYellowString("Use: ReadOnly"))
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
if len(si.Groups) > 0 {
|
||||
fmt.Printf("\tGroups: %s\n", strings.Join(si.Groups, ", "))
|
||||
}
|
||||
if len(si.AllowTo) > 0 {
|
||||
fmt.Printf("\tAllowTo: %s\n", strings.Join(si.AllowTo, ", "))
|
||||
}
|
||||
|
||||
if len(si.AllowTypes) > 0 || len(si.DenyTypes) > 0 {
|
||||
denied := storiface.FTAll.SubAllowed(si.AllowTypes, si.DenyTypes)
|
||||
allowed := storiface.FTAll ^ denied
|
||||
|
||||
switch {
|
||||
case bits.OnesCount64(uint64(allowed)) == 0:
|
||||
fmt.Printf("\tAllow Types: %s\n", color.RedString("None"))
|
||||
case bits.OnesCount64(uint64(allowed)) < bits.OnesCount64(uint64(denied)):
|
||||
fmt.Printf("\tAllow Types: %s\n", color.GreenString(strings.Join(allowed.Strings(), " ")))
|
||||
default:
|
||||
fmt.Printf("\tDeny Types: %s\n", color.RedString(strings.Join(denied.Strings(), " ")))
|
||||
}
|
||||
}
|
||||
|
||||
if localPath, ok := local[s.ID]; ok {
|
||||
fmt.Printf("\tLocal: %s\n", color.GreenString(localPath))
|
||||
}
|
||||
for i, l := range si.URLs {
|
||||
var rtt string
|
||||
if _, ok := local[s.ID]; !ok && i == 0 {
|
||||
rtt = " (latency: " + ping.Truncate(time.Microsecond*100).String() + ")"
|
||||
}
|
||||
|
||||
fmt.Printf("\tURL: %s%s\n", l, rtt) // TODO; try pinging maybe?? print latency?
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user