package build import ( "fmt" "io" "os" "path/filepath" "runtime/debug" "runtime/pprof" "strconv" "strings" "time" "github.com/icza/backscanner" logging "github.com/ipfs/go-log/v2" ) var ( panicLog = logging.Logger("panic-reporter") defaultJournalTail = 500 ) // PanicReportingPath is the name of the subdir created within the repoPath // path provided to GeneratePanicReport var PanicReportingPath = "panic-reports" // PanicReportJournalTail is the number of lines captured from the end of // the lotus journal to be included in the panic report. var PanicReportJournalTail = defaultJournalTail // GenerateNodePanicReport produces a timestamped dump of the application state // for inspection and debugging purposes. Call this function from any place // where a panic or severe error needs to be examined. `persistPath` is the // path where the reports should be saved. `repoPath` is the path where the // journal should be read from. `label` is an optional string to include // next to the report timestamp. // // This function should be called for panics originating from the Lotus daemon. func GenerateNodePanicReport(persistPath, repoPath, label string) { generatePanicReport(NodeUserVersion(), persistPath, repoPath, label) } // GenerateMinerPanicReport produces a timestamped dump of the application state // for inspection and debugging purposes. Call this function from any place // where a panic or severe error needs to be examined. `persistPath` is the // path where the reports should be saved. `repoPath` is the path where the // journal should be read from. `label` is an optional string to include // next to the report timestamp. // // This function should be called for panics originating from the Lotus miner. func GenerateMinerPanicReport(persistPath, repoPath, label string) { generatePanicReport(MinerUserVersion(), persistPath, repoPath, label) } func generatePanicReport(buildVersion BuildVersion, persistPath, repoPath, label string) { // make sure we always dump the latest logs on the way out // especially since we're probably panicking defer panicLog.Sync() //nolint:errcheck if persistPath == "" && repoPath == "" { panicLog.Warn("missing persist and repo paths, aborting panic report creation") return } reportPath := filepath.Join(repoPath, PanicReportingPath, generateReportName(label)) if persistPath != "" { reportPath = filepath.Join(persistPath, generateReportName(label)) } panicLog.Warnf("generating panic report at %s", reportPath) tl := os.Getenv("LOTUS_PANIC_JOURNAL_LOOKBACK") if tl != "" && PanicReportJournalTail == defaultJournalTail { i, err := strconv.Atoi(tl) if err == nil { PanicReportJournalTail = i } } err := os.MkdirAll(reportPath, 0755) if err != nil { panicLog.Error(err.Error()) return } writeAppVersion(buildVersion, filepath.Join(reportPath, "version")) writeStackTrace(filepath.Join(reportPath, "stacktrace.dump")) writeProfile("goroutines", filepath.Join(reportPath, "goroutines.pprof.gz")) writeProfile("heap", filepath.Join(reportPath, "heap.pprof.gz")) writeJournalTail(PanicReportJournalTail, repoPath, filepath.Join(reportPath, "journal.ndjson")) } func writeAppVersion(buildVersion BuildVersion, file string) { f, err := os.Create(file) if err != nil { panicLog.Error(err.Error()) } defer f.Close() //nolint:errcheck versionString := []byte(string(buildVersion) + BuildTypeString() + CurrentCommit + "\n") if _, err := f.Write(versionString); err != nil { panicLog.Error(err.Error()) } } func writeStackTrace(file string) { f, err := os.Create(file) if err != nil { panicLog.Error(err.Error()) } defer f.Close() //nolint:errcheck if _, err := f.Write(debug.Stack()); err != nil { panicLog.Error(err.Error()) } } func writeProfile(profileType string, file string) { p := pprof.Lookup(profileType) if p == nil { panicLog.Warnf("%s profile not available", profileType) return } f, err := os.Create(file) if err != nil { panicLog.Error(err.Error()) return } defer f.Close() //nolint:errcheck if err := p.WriteTo(f, 0); err != nil { panicLog.Error(err.Error()) } } func writeJournalTail(tailLen int, repoPath, file string) { if repoPath == "" { panicLog.Warn("repo path is empty, aborting copy of journal log") return } f, err := os.Create(file) if err != nil { panicLog.Error(err.Error()) return } defer f.Close() //nolint:errcheck jPath, err := getLatestJournalFilePath(repoPath) if err != nil { panicLog.Warnf("failed getting latest journal: %s", err.Error()) return } j, err := os.OpenFile(jPath, os.O_RDONLY, 0400) if err != nil { panicLog.Error(err.Error()) return } js, err := j.Stat() if err != nil { panicLog.Error(err.Error()) return } jScan := backscanner.New(j, int(js.Size())) linesWritten := 0 for { if linesWritten > tailLen { break } line, _, err := jScan.LineBytes() if err != nil { if err != io.EOF { panicLog.Error(err.Error()) } break } if _, err := f.Write(line); err != nil { panicLog.Error(err.Error()) break } if _, err := f.Write([]byte("\n")); err != nil { panicLog.Error(err.Error()) break } linesWritten++ } } func getLatestJournalFilePath(repoPath string) (string, error) { journalPath := filepath.Join(repoPath, "journal") entries, err := os.ReadDir(journalPath) if err != nil { return "", err } return filepath.Join(journalPath, entries[len(entries)-1].Name()), nil } func generateReportName(label string) string { label = strings.ReplaceAll(label, " ", "") return fmt.Sprintf("report_%s_%s", label, time.Now().Format("2006-01-02T150405")) }