lotus/build/panic_reporter.go

package build

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"syscall"
	"time"

	"github.com/icza/backscanner"
	logging "github.com/ipfs/go-log/v2"
)

var (
	panicLog           = logging.Logger("panic-reporter")
	defaultJournalTail = 500
)

// PanicReportingPath is the name of the subdir created within the repoPath path
// provided to GeneratePanicReport
var PanicReportingPath = "lotus-panic-reports"

// PanicReportJournalTail is the number of lines captured from the end of
// the lotus journal to be included in the panic report.
var PanicReportJournalTail = defaultJournalTail

// GeneratePanicReport produces a timestamped dump of the application state
// for inspection and debugging purposes. Call this function from any place
// where a panic or severe error needs to be examined. `persistPath` is the
// path where the reports should be saved. `repoPath` is the path where the
// journal should be read from. `label` is an optional string to include
// next to the report timestamp.
func GeneratePanicReport(persistPath, repoPath, label string) {
	// make sure we always dump the latest logs on the way out
	// especially since we're probably panicking
	defer panicLog.Sync()

	if persistPath == "" {
		panicLog.Error("persist path is empty, aborting panic report generation")
		return
	}
	tl := os.Getenv("LOTUS_PANIC_JOURNAL_LOOKBACK")
	if tl != "" && PanicReportJournalTail == defaultJournalTail {
		i, err := strconv.Atoi(tl)
		if err == nil {
			PanicReportJournalTail = i
		}
	}

	reportPath := filepath.Join(persistPath, PanicReportingPath, generateReportName(label))
	panicLog.Warnf("generating panic report at %s", reportPath)
	syscall.Umask(0)
	err := os.MkdirAll(reportPath, 0755)
	if err != nil {
		panicLog.Error(err.Error())
		return
	}

	writeStackTrace(filepath.Join(reportPath, "stacktrace.dump"))
	writeProfile("goroutines", filepath.Join(reportPath, "goroutines.pprof.gz"))
	writeProfile("heap", filepath.Join(reportPath, "heap.pprof.gz"))
	writeJournalTail(PanicReportJournalTail, repoPath, filepath.Join(reportPath, "journal.ndjson"))
}

func writeStackTrace(file string) {
	f, err := os.Create(file)
	if err != nil {
		panicLog.Error(err.Error())
	}
	defer f.Close()

	if _, err := f.Write(debug.Stack()); err != nil {
		panicLog.Error(err.Error())
	}

}

func writeProfile(profileType string, file string) {
	p := pprof.Lookup(profileType)
	if p == nil {
		panicLog.Warnf("%s profile not available", profileType)
		return
	}
	f, err := os.Create(file)
	if err != nil {
		panicLog.Error(err.Error())
		return
	}
	defer f.Close()

	if err := p.WriteTo(f, 0); err != nil {
		panicLog.Error(err.Error())
	}
}

func writeJournalTail(tailLen int, repoPath, file string) {
	if repoPath == "" {
		panicLog.Warn("repo path is empty, aborting copy of journal log")
		return
	}

	f, err := os.Create(file)
	if err != nil {
		panicLog.Error(err.Error())
		return
	}
	defer f.Close()

	jPath, err := getLatestJournalFilePath(repoPath)
	if err != nil {
		panicLog.Warnf("failed getting latest journal: %s", err.Error())
		return
	}
	j, err := os.OpenFile(jPath, os.O_RDONLY, 0400)
	if err != nil {
		panicLog.Error(err.Error())
		return
	}
	js, err := j.Stat()
	if err != nil {
		panicLog.Error(err.Error())
		return
	}
	jScan := backscanner.New(j, int(js.Size()))
	linesWritten := 0
	for {
		if linesWritten > tailLen {
			break
		}
		line, _, err := jScan.LineBytes()
		if err != nil {
			if err != io.EOF {
				panicLog.Error(err.Error())
			}
			break
		}
		if _, err := f.Write(line); err != nil {
			panicLog.Error(err.Error())
			break
		}
		if _, err := f.Write([]byte("\n")); err != nil {
			panicLog.Error(err.Error())
			break
		}
		linesWritten++
	}
}

func getLatestJournalFilePath(repoPath string) (string, error) {
	journalPath := filepath.Join(repoPath, "journal")
	entries, err := os.ReadDir(journalPath)
	if err != nil {
		return "", err
	}
	return filepath.Join(journalPath, entries[len(entries)-1].Name()), nil
}

func generateReportName(label string) string {
	return fmt.Sprintf("report_%s_%s", label, time.Now().Format("2006-01-02T150405Z0700"))
}