feat: Catch panic to generate report and reraise

build/panic_reporter.go

@@ -0,0 +1,146 @@
+package build
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"runtime/pprof"
+	"syscall"
+	"time"
+
+	"github.com/icza/backscanner"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var panicLog = logging.Logger("panic_reporter")
+
+// PanicReportingPath is the name of the subdir created within the repoPath path
+// provided to GeneratePanicReport
+var PanicReportingPath = "panic_reports"
+
+// PanicReportJournalTail is the number of lines captured from the end of
+// the lotus journal to be included in the panic report.
+var PanicReportJournalTail = 500
+
+// GeneratePanicReport produces a timestamped dump of the application state
+// for inspection and debugging purposes. Call this function from any place
+// where a panic or severe error needs to be examined. `repoPath` is the path
+// where the report should be written. `label` is an optional string to include
+// next to the report timestamp.
+func GeneratePanicReport(repoPath string, label string) {
+	// make sure we always dump the latest logs on the way out
+	// especially since we're probably panicking
+	defer panicLog.Sync()
+
+	if repoPath == "" {
+		panicLog.Error("repo path is empty, aborting panic report generation")
+		return
+	}
+	reportPath := filepath.Join(repoPath, PanicReportingPath, generateReportName(label))
+	panicLog.Warnf("generating panic report at %s", reportPath)
+	syscall.Umask(0)
+	err := os.MkdirAll(reportPath, 0755)
+	if err != nil {
+		panicLog.Error(err.Error())
+		return
+	}
+
+	writeStackTrace(filepath.Join(reportPath, "stacktrace.dump"))
+	writeProfile("goroutines", filepath.Join(reportPath, "goroutines.pprof.gz"))
+	writeProfile("heap", filepath.Join(reportPath, "heap.pprof.gz"))
+	writeJournalTail(PanicReportJournalTail, repoPath, filepath.Join(reportPath, "journal.ndjson"))
+}
+
+func writeStackTrace(file string) {
+	f, err := os.Create(file)
+	if err != nil {
+		panicLog.Error(err.Error())
+	}
+	defer f.Close()
+
+	if _, err := f.Write(debug.Stack()); err != nil {
+		panicLog.Error(err.Error())
+	}
+
+}
+
+func writeProfile(profileType string, file string) {
+	p := pprof.Lookup(profileType)
+	if p == nil {
+		panicLog.Warnf("%s profile not available", profileType)
+		return
+	}
+	f, err := os.Create(file)
+	if err != nil {
+		panicLog.Error(err.Error())
+		return
+	}
+	defer f.Close()
+
+	if err := p.WriteTo(f, 0); err != nil {
+		panicLog.Error(err.Error())
+	}
+}
+
+func writeJournalTail(tailLen int, repoPath, file string) {
+	f, err := os.Create(file)
+	if err != nil {
+		panicLog.Error(err.Error())
+		return
+	}
+	defer f.Close()
+
+	jPath, err := getLatestJournalFilePath(repoPath)
+	if err != nil {
+		panicLog.Warnf("failed getting latest journal: %s", err.Error())
+		return
+	}
+	j, err := os.OpenFile(jPath, os.O_RDONLY, 0400)
+	if err != nil {
+		panicLog.Error(err.Error())
+		return
+	}
+	js, err := j.Stat()
+	if err != nil {
+		panicLog.Error(err.Error())
+		return
+	}
+	jScan := backscanner.New(j, int(js.Size()))
+	linesWritten := 0
+	for {
+		if linesWritten > tailLen {
+			break
+		}
+		line, _, err := jScan.LineBytes()
+		if err != nil {
+			if err != io.EOF {
+				panicLog.Error(err.Error())
+			}
+			break
+		}
+		if _, err := f.Write(line); err != nil {
+			panicLog.Error(err.Error())
+			break
+		}
+		if _, err := f.Write([]byte("\n")); err != nil {
+			panicLog.Error(err.Error())
+			break
+		}
+		linesWritten++
+	}
+}
+
+func getLatestJournalFilePath(repoPath string) (string, error) {
+	journalPath := filepath.Join(repoPath, "journal")
+	entries, err := os.ReadDir(journalPath)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(journalPath, entries[len(entries)-1].Name()), nil
+}
+
+func generateReportName(label string) string {
+	return fmt.Sprintf("report_%s_%s", label, time.Now().Format("2006-01-02T150405Z0700"))
+}

cli/helper.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/filecoin-project/lotus/build"
 	ufcli "github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 )
@@ -32,6 +33,13 @@ func ShowHelp(cctx *ufcli.Context, err error) error {
 }
 
 func RunApp(app *ufcli.App) {
+	defer func() {
+		if r := recover(); r != nil {
+			// Generate report in LOTUS_PATH and re-raise panic
+			build.GeneratePanicReport(os.Getenv("LOTUS_PATH"), "app_panic")
+			panic(r)
+		}
+	}()
 	if err := app.Run(os.Args); err != nil {
 		if os.Getenv("LOTUS_DEV") != "" {
 			log.Warnf("%+v", err)

go.mod

@@ -63,6 +63,7 @@ require (
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/golang-lru v0.5.4
+	github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
 	github.com/ipfs/bbloom v0.0.4
 	github.com/ipfs/go-bitswap v0.3.4

go.sum

@@ -571,6 +571,10 @@ github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3
 github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo=
+github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 h1:9tcYMdi+7Rb1y0E9Del1DRHui7Ne3za5lLw6CjMJv/M=
+github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94/go.mod h1:GYeBD1CF7AqnKZK+UCytLcY3G+UKo0ByXX/3xfdNyqQ=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=