Merge pull request #5800 from filecoin-project/feat/allinfo-utils

shed command to unpack miner info dumps

cmd/lotus-shed/main.go

@@ -37,6 +37,7 @@ func main() {
 		mpoolCmd,
 		genesisVerifyCmd,
 		mathCmd,
+		minerCmd,
 		mpoolStatsCmd,
 		exportChainCmd,
 		consensusCmd,

cmd/lotus-shed/miner.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var minerCmd = &cli.Command{
+	Name:  "miner",
+	Usage: "miner-related utilities",
+	Subcommands: []*cli.Command{
+		minerUnpackInfoCmd,
+	},
+}
+
+var minerUnpackInfoCmd = &cli.Command{
+	Name:      "unpack-info",
+	Usage:     "unpack miner info all dump",
+	ArgsUsage: "[allinfo.txt] [dir]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 2 {
+			return xerrors.Errorf("expected 2 args")
+		}
+
+		src, err := homedir.Expand(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("expand src: %w", err)
+		}
+
+		f, err := os.Open(src)
+		if err != nil {
+			return xerrors.Errorf("open file: %w", err)
+		}
+		defer f.Close() // nolint
+
+		dest, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("expand dest: %w", err)
+		}
+
+		var outf *os.File
+
+		r := bufio.NewReader(f)
+		for {
+			l, _, err := r.ReadLine()
+			if err == io.EOF {
+				if outf != nil {
+					return outf.Close()
+				}
+			}
+			if err != nil {
+				return xerrors.Errorf("read line: %w", err)
+			}
+			sl := string(l)
+
+			if strings.HasPrefix(sl, "#") {
+				if strings.Contains(sl, "..") {
+					return xerrors.Errorf("bad name %s", sl)
+				}
+
+				if strings.HasPrefix(sl, "#: ") {
+					if outf != nil {
+						if err := outf.Close(); err != nil {
+							return xerrors.Errorf("close out file: %w", err)
+						}
+					}
+					p := filepath.Join(dest, sl[len("#: "):])
+					if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
+						return xerrors.Errorf("mkdir: %w", err)
+					}
+					outf, err = os.Create(p)
+					if err != nil {
+						return xerrors.Errorf("create out file: %w", err)
+					}
+					continue
+				}
+
+				if strings.HasPrefix(sl, "##: ") {
+					if outf != nil {
+						if err := outf.Close(); err != nil {
+							return xerrors.Errorf("close out file: %w", err)
+						}
+					}
+					p := filepath.Join(dest, "Per Sector Infos", sl[len("##: "):])
+					if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
+						return xerrors.Errorf("mkdir: %w", err)
+					}
+					outf, err = os.Create(p)
+					if err != nil {
+						return xerrors.Errorf("create out file: %w", err)
+					}
+					continue
+				}
+			}
+
+			if outf != nil {
+				if _, err := outf.Write(l); err != nil {
+					return xerrors.Errorf("write line: %w", err)
+				}
+				if _, err := outf.Write([]byte("\n")); err != nil {
+					return xerrors.Errorf("write line end: %w", err)
+				}
+			}
+		}
+	},
+}

cmd/lotus-storage-miner/info_all.go

@@ -35,80 +35,80 @@ var infoAllCmd = &cli.Command{
 
 		fmt.Println("#: Version")
 		if err := lcli.VersionCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Miner Info")
 		if err := infoCmdAct(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		// Verbose info
 
 		fmt.Println("\n#: Storage List")
 		if err := storageListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Worker List")
 		if err := sealingWorkersCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: PeerID")
 		if err := lcli.NetId.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Listen Addresses")
 		if err := lcli.NetListen.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Reachability")
 		if err := lcli.NetReachability.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		// Very Verbose info
 		fmt.Println("\n#: Peers")
 		if err := lcli.NetPeers.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sealing Jobs")
 		if err := sealingJobsCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sched Diag")
 		if err := sealingSchedDiagCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Storage Ask")
 		if err := getAskCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Storage Deals")
 		if err := dealsListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Retrieval Deals")
 		if err := retrievalDealsListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sector List")
 		if err := sectorsListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sector Refs")
 		if err := sectorsRefsCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		// Very Very Verbose info
@@ -116,7 +116,7 @@ var infoAllCmd = &cli.Command{
 
 		list, err := nodeApi.SectorsList(ctx)
 		if err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		sort.Slice(list, func(i, j int) bool {
@@ -129,11 +129,11 @@ var infoAllCmd = &cli.Command{
 			fs := &flag.FlagSet{}
 			for _, f := range sectorsStatusCmd.Flags {
 				if err := f.Apply(fs); err != nil {
-					return err
+					fmt.Println("ERROR: ", err)
 				}
 			}
 			if err := fs.Parse([]string{"--log", "--on-chain-info", fmt.Sprint(s)}); err != nil {
-				return err
+				fmt.Println("ERROR: ", err)
 			}
 
 			if err := sectorsStatusCmd.Action(cli.NewContext(cctx.App, fs, cctx)); err != nil {
@@ -144,7 +144,7 @@ var infoAllCmd = &cli.Command{
 
 			fs = &flag.FlagSet{}
 			if err := fs.Parse([]string{fmt.Sprint(s)}); err != nil {
-				return err
+				fmt.Println("ERROR: ", err)
 			}
 
 			if err := storageFindCmd.Action(cli.NewContext(cctx.App, fs, cctx)); err != nil {
@@ -155,7 +155,7 @@ var infoAllCmd = &cli.Command{
 		if !_test {
 			fmt.Println("\n#: Goroutines")
 			if err := lcli.PprofGoroutines.Action(cctx); err != nil {
-				return err
+				fmt.Println("ERROR: ", err)
 			}
 		}