track sync errors in status

api/api_full.go

@@ -277,6 +277,7 @@ const (
 	StagePersistHeaders
 	StageMessages
 	StageSyncComplete
+	StageSyncErrored
 )
 
 type MpoolChange int

chain/sync.go

@@ -1115,6 +1115,7 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error
 
 	headers, err := syncer.collectHeaders(ctx, ts, syncer.store.GetHeaviestTipSet())
 	if err != nil {
+		ss.Error(err)
 		return err
 	}
 
@@ -1131,14 +1132,18 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error
 		toPersist = append(toPersist, ts.Blocks()...)
 	}
 	if err := syncer.store.PersistBlockHeaders(toPersist...); err != nil {
-		return xerrors.Errorf("failed to persist synced blocks to the chainstore: %w", err)
+		err = xerrors.Errorf("failed to persist synced blocks to the chainstore: %w", err)
+		ss.Error(err)
+		return err
 	}
 	toPersist = nil
 
 	ss.SetStage(api.StageMessages)
 
 	if err := syncer.syncMessagesAndCheckState(ctx, headers); err != nil {
-		return xerrors.Errorf("collectChain syncMessages: %w", err)
+		err = xerrors.Errorf("collectChain syncMessages: %w", err)
+		ss.Error(err)
+		return err
 	}
 
 	ss.SetStage(api.StageSyncComplete)

chain/syncstate.go

@@ -3,6 +3,7 @@ package chain
 import (
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -18,17 +19,22 @@ func SyncStageString(v api.SyncStateStage) string {
 		return "message sync"
 	case api.StageSyncComplete:
 		return "complete"
+	case api.StageSyncErrored:
+		return "error"
 	default:
 		return fmt.Sprintf("<unknown: %d>", v)
 	}
 }
 
 type SyncerState struct {
-	lk     sync.Mutex
-	Target *types.TipSet
-	Base   *types.TipSet
-	Stage  api.SyncStateStage
-	Height uint64
+	lk      sync.Mutex
+	Target  *types.TipSet
+	Base    *types.TipSet
+	Stage   api.SyncStateStage
+	Height  uint64
+	Message string
+	Start   time.Time
+	End     time.Time
 }
 
 func (ss *SyncerState) SetStage(v api.SyncStateStage) {
@@ -39,6 +45,9 @@ func (ss *SyncerState) SetStage(v api.SyncStateStage) {
 	ss.lk.Lock()
 	defer ss.lk.Unlock()
 	ss.Stage = v
+	if v == api.StageSyncComplete {
+		ss.End = time.Now()
+	}
 }
 
 func (ss *SyncerState) Init(base, target *types.TipSet) {
@@ -52,6 +61,9 @@ func (ss *SyncerState) Init(base, target *types.TipSet) {
 	ss.Base = base
 	ss.Stage = api.StageHeaders
 	ss.Height = 0
+	ss.Message = ""
+	ss.Start = time.Now()
+	ss.End = time.Time{}
 }
 
 func (ss *SyncerState) SetHeight(h uint64) {
@@ -64,13 +76,28 @@ func (ss *SyncerState) SetHeight(h uint64) {
 	ss.Height = h
 }
 
+func (ss *SyncerState) Error(err error) {
+	if ss == nil {
+		return
+	}
+
+	ss.lk.Lock()
+	defer ss.lk.Unlock()
+	ss.Message = err.Error()
+	ss.Stage = api.StageSyncErrored
+	ss.End = time.Now()
+}
+
 func (ss *SyncerState) Snapshot() SyncerState {
 	ss.lk.Lock()
 	defer ss.lk.Unlock()
 	return SyncerState{
-		Base:   ss.Base,
-		Target: ss.Target,
-		Stage:  ss.Stage,
-		Height: ss.Height,
+		Base:    ss.Base,
+		Target:  ss.Target,
+		Stage:   ss.Stage,
+		Height:  ss.Height,
+		Message: ss.Message,
+		Start:   ss.Start,
+		End:     ss.End,
 	}
 }