147 lines
3.6 KiB
Go
147 lines
3.6 KiB
Go
package build
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime/debug"
|
|
"runtime/pprof"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/icza/backscanner"
|
|
logging "github.com/ipfs/go-log/v2"
|
|
)
|
|
|
|
var panicLog = logging.Logger("panic_reporter")
|
|
|
|
// PanicReportingPath is the name of the subdir created within the repoPath path
|
|
// provided to GeneratePanicReport
|
|
var PanicReportingPath = "panic_reports"
|
|
|
|
// PanicReportJournalTail is the number of lines captured from the end of
|
|
// the lotus journal to be included in the panic report.
|
|
var PanicReportJournalTail = 500
|
|
|
|
// GeneratePanicReport produces a timestamped dump of the application state
|
|
// for inspection and debugging purposes. Call this function from any place
|
|
// where a panic or severe error needs to be examined. `repoPath` is the path
|
|
// where the report should be written. `label` is an optional string to include
|
|
// next to the report timestamp.
|
|
func GeneratePanicReport(repoPath string, label string) {
|
|
// make sure we always dump the latest logs on the way out
|
|
// especially since we're probably panicking
|
|
defer panicLog.Sync()
|
|
|
|
if repoPath == "" {
|
|
panicLog.Error("repo path is empty, aborting panic report generation")
|
|
return
|
|
}
|
|
reportPath := filepath.Join(repoPath, PanicReportingPath, generateReportName(label))
|
|
panicLog.Warnf("generating panic report at %s", reportPath)
|
|
syscall.Umask(0)
|
|
err := os.MkdirAll(reportPath, 0755)
|
|
if err != nil {
|
|
panicLog.Error(err.Error())
|
|
return
|
|
}
|
|
|
|
writeStackTrace(filepath.Join(reportPath, "stacktrace.dump"))
|
|
writeProfile("goroutines", filepath.Join(reportPath, "goroutines.pprof.gz"))
|
|
writeProfile("heap", filepath.Join(reportPath, "heap.pprof.gz"))
|
|
writeJournalTail(PanicReportJournalTail, repoPath, filepath.Join(reportPath, "journal.ndjson"))
|
|
}
|
|
|
|
func writeStackTrace(file string) {
|
|
f, err := os.Create(file)
|
|
if err != nil {
|
|
panicLog.Error(err.Error())
|
|
}
|
|
defer f.Close()
|
|
|
|
if _, err := f.Write(debug.Stack()); err != nil {
|
|
panicLog.Error(err.Error())
|
|
}
|
|
|
|
}
|
|
|
|
func writeProfile(profileType string, file string) {
|
|
p := pprof.Lookup(profileType)
|
|
if p == nil {
|
|
panicLog.Warnf("%s profile not available", profileType)
|
|
return
|
|
}
|
|
f, err := os.Create(file)
|
|
if err != nil {
|
|
panicLog.Error(err.Error())
|
|
return
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := p.WriteTo(f, 0); err != nil {
|
|
panicLog.Error(err.Error())
|
|
}
|
|
}
|
|
|
|
func writeJournalTail(tailLen int, repoPath, file string) {
|
|
f, err := os.Create(file)
|
|
if err != nil {
|
|
panicLog.Error(err.Error())
|
|
return
|
|
}
|
|
defer f.Close()
|
|
|
|
jPath, err := getLatestJournalFilePath(repoPath)
|
|
if err != nil {
|
|
panicLog.Warnf("failed getting latest journal: %s", err.Error())
|
|
return
|
|
}
|
|
j, err := os.OpenFile(jPath, os.O_RDONLY, 0400)
|
|
if err != nil {
|
|
panicLog.Error(err.Error())
|
|
return
|
|
}
|
|
js, err := j.Stat()
|
|
if err != nil {
|
|
panicLog.Error(err.Error())
|
|
return
|
|
}
|
|
jScan := backscanner.New(j, int(js.Size()))
|
|
linesWritten := 0
|
|
for {
|
|
if linesWritten > tailLen {
|
|
break
|
|
}
|
|
line, _, err := jScan.LineBytes()
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
panicLog.Error(err.Error())
|
|
}
|
|
break
|
|
}
|
|
if _, err := f.Write(line); err != nil {
|
|
panicLog.Error(err.Error())
|
|
break
|
|
}
|
|
if _, err := f.Write([]byte("\n")); err != nil {
|
|
panicLog.Error(err.Error())
|
|
break
|
|
}
|
|
linesWritten++
|
|
}
|
|
}
|
|
|
|
func getLatestJournalFilePath(repoPath string) (string, error) {
|
|
journalPath := filepath.Join(repoPath, "journal")
|
|
entries, err := os.ReadDir(journalPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return filepath.Join(journalPath, entries[len(entries)-1].Name()), nil
|
|
}
|
|
|
|
func generateReportName(label string) string {
|
|
return fmt.Sprintf("report_%s_%s", label, time.Now().Format("2006-01-02T150405Z0700"))
|
|
}
|