storageminer: States for common failure modes

api/api_storage.go

@@ -21,6 +21,11 @@ const (
 	Committing
 	Proving
 
+	SealFailed
+	PreCommitFailed
+	SealCommitFailed
+	CommitFailed
+
 	FailedUnrecoverable
 )
 
@@ -33,6 +38,12 @@ var SectorStates = []string{
 	PreCommitted:         "PreCommitted",
 	Committing:           "Committing",
 	Proving:              "Proving",
+
+	SealFailed:       "SealFailed",
+	PreCommitFailed:  "PreCommitFailed",
+	SealCommitFailed: "SealCommitFailed",
+	CommitFailed:     "CommitFailed",
+
 	FailedUnrecoverable: "FailedUnrecoverable",
 }
 

build/params_debug.go

@@ -4,6 +4,8 @@ package build
 
 import "os"
 
+var SectorSizes = []uint64{1024}
+
 // Seconds
 const BlockDelay = 6
 

cmd/lotus-storage-miner/info.go

@@ -128,7 +128,7 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) (map[string]int, er
 			return nil, err
 		}
 
-		out[api.SectorStateStr(st.State)]++
+		out[api.SectorStates[st.State]]++
 	}
 
 	return out, nil

cmd/lotus-storage-miner/sectors.go

@@ -61,7 +61,7 @@ var sectorsStatusCmd = &cli.Command{
 		}
 
 		fmt.Printf("SectorID:\t%d\n", status.SectorID)
-		fmt.Printf("Status:\t%s\n", api.SectorStateStr(status.State))
+		fmt.Printf("Status:\t%s\n", api.SectorStates[status.State])
 		fmt.Printf("CommD:\t\t%x\n", status.CommD)
 		fmt.Printf("CommR:\t\t%x\n", status.CommR)
 		fmt.Printf("Ticket:\t\t%x\n", status.Ticket.TicketBytes)
@@ -132,7 +132,7 @@ var sectorsListCmd = &cli.Command{
 
 			fmt.Printf("%d: %s\tsSet: %s\tpSet: %s\ttktH: %d\tseedH: %d\tdeals: %v\n",
 				s,
-				api.SectorStateStr(st.State),
+				api.SectorStates[st.State],
 				yesno(inSSet),
 				yesno(inPSet),
 				st.Ticket.BlockHeight,

storage/sector_states.go

@@ -71,7 +71,7 @@ func (m *Miner) handleUnsealed(ctx context.Context, sector SectorInfo) *sectorUp
 
 	rspco, err := m.sb.SealPreCommit(sector.SectorID, *ticket, sector.pieceInfos())
 	if err != nil {
-		return sector.upd().fatal(xerrors.Errorf("seal pre commit failed: %w", err))
+		return sector.upd().to(api.SealFailed).error(xerrors.Errorf("seal pre commit failed: %w", err))
 	}
 
 	return sector.upd().to(api.PreCommitting).state(func(info *SectorInfo) {
@@ -97,7 +97,7 @@ func (m *Miner) handlePreCommitting(ctx context.Context, sector SectorInfo) *sec
 	}
 	enc, aerr := actors.SerializeParams(params)
 	if aerr != nil {
-		return sector.upd().fatal(xerrors.Errorf("could not serialize commit sector parameters: %w", aerr))
+		return sector.upd().to(api.PreCommitFailed).error(xerrors.Errorf("could not serialize commit sector parameters: %w", aerr))
 	}
 
 	msg := &types.Message{
@@ -113,7 +113,7 @@ func (m *Miner) handlePreCommitting(ctx context.Context, sector SectorInfo) *sec
 	log.Info("submitting precommit for sector: ", sector.SectorID)
 	smsg, err := m.api.MpoolPushMessage(ctx, msg)
 	if err != nil {
-		return sector.upd().fatal(xerrors.Errorf("pushing message to mpool: %w", err))
+		return sector.upd().to(api.PreCommitFailed).error(xerrors.Errorf("pushing message to mpool: %w", err))
 	}
 
 	return sector.upd().to(api.PreCommitted).state(func(info *SectorInfo) {
@@ -127,12 +127,12 @@ func (m *Miner) handlePreCommitted(ctx context.Context, sector SectorInfo) *sect
 	log.Info("Sector precommitted: ", sector.SectorID)
 	mw, err := m.api.StateWaitMsg(ctx, *sector.PreCommitMessage)
 	if err != nil {
-		return sector.upd().fatal(err)
+		return sector.upd().to(api.PreCommitFailed).error(err)
 	}
 
 	if mw.Receipt.ExitCode != 0 {
 		log.Error("sector precommit failed: ", mw.Receipt.ExitCode)
-		return sector.upd().fatal(err)
+		return sector.upd().to(api.PreCommitFailed).error(err)
 	}
 	log.Info("precommit message landed on chain: ", sector.SectorID)
 
@@ -172,7 +172,7 @@ func (m *Miner) handleCommitting(ctx context.Context, sector SectorInfo) *sector
 
 	proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.rspco())
 	if err != nil {
-		return sector.upd().fatal(xerrors.Errorf("computing seal proof failed: %w", err))
+		return sector.upd().to(api.SealCommitFailed).error(xerrors.Errorf("computing seal proof failed: %w", err))
 	}
 
 	// TODO: Consider splitting states and persist proof for faster recovery
@@ -185,7 +185,7 @@ func (m *Miner) handleCommitting(ctx context.Context, sector SectorInfo) *sector
 
 	enc, aerr := actors.SerializeParams(params)
 	if aerr != nil {
-		return sector.upd().fatal(xerrors.Errorf("could not serialize commit sector parameters: %w", aerr))
+		return sector.upd().to(api.CommitFailed).error(xerrors.Errorf("could not serialize commit sector parameters: %w", aerr))
 	}
 
 	msg := &types.Message{
@@ -200,14 +200,14 @@ func (m *Miner) handleCommitting(ctx context.Context, sector SectorInfo) *sector
 
 	smsg, err := m.api.MpoolPushMessage(ctx, msg)
 	if err != nil {
-		log.Error(xerrors.Errorf("pushing message to mpool: %w", err))
+		return sector.upd().to(api.CommitFailed).error(xerrors.Errorf("pushing message to mpool: %w", err))
 	}
 
 	// TODO: Separate state before this wait, so we persist message cid?
 
 	mw, err := m.api.StateWaitMsg(ctx, smsg.Cid())
 	if err != nil {
-		return sector.upd().fatal(xerrors.Errorf("failed to wait for porep inclusion: %w", err))
+		return sector.upd().to(api.CommitFailed).error(xerrors.Errorf("failed to wait for porep inclusion: %w", err))
 	}
 
 	if mw.Receipt.ExitCode != 0 {

storage/sectors.go

@@ -27,6 +27,15 @@ func (u *sectorUpdate) fatal(err error) *sectorUpdate {
 	}
 }
 
+func (u *sectorUpdate) error(err error) *sectorUpdate {
+	return &sectorUpdate{
+		newState: u.newState,
+		id:       u.id,
+		err:      err,
+		mut:      u.mut,
+	}
+}
+
 func (u *sectorUpdate) state(m func(*SectorInfo)) *sectorUpdate {
 	return &sectorUpdate{
 		newState: u.newState,
@@ -161,23 +170,23 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) {
 
 	/*
 
-		*<- Empty
+		*   Empty
 		|   |
 		|   v
 		*<- Packing <- incoming
 		|   |
 		|   v
-		*<- Unsealed
+		*<- Unsealed <--> SealFailed
 		|   |
 		|   v
-		*<- PreCommitting
+		*   PreCommitting <--> PreCommitFailed
+		|   |                  ^
+		|   v                  |
+		*<- PreCommitted ------/
 		|   |
-		|   v
+		|   v        v--> SealCommitFailed
-		*<- PreCommitted
-		|   |
-		|   v
 		*<- Committing
-		|   |
+		|   |        ^--> CommitFailed
 		|   v
 		*<- Proving
 		|
@@ -191,6 +200,7 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) {
 	*/
 
 	switch update.newState {
+	// Happy path
 	case api.Packing:
 		m.handleSectorUpdate(ctx, sector, m.handlePacking)
 	case api.Unsealed:
@@ -204,6 +214,18 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) {
 	case api.Proving:
 		// TODO: track sector health / expiration
 		log.Infof("Proving sector %d", update.id)
+
+	// Handled failure modes
+	case api.SealFailed:
+		log.Warn("sector %d entered unimplemented state 'SealFailed'", update.id)
+	case api.PreCommitFailed:
+		log.Warn("sector %d entered unimplemented state 'PreCommitFailed'", update.id)
+	case api.SealCommitFailed:
+		log.Warn("sector %d entered unimplemented state 'SealCommitFailed'", update.id)
+	case api.CommitFailed:
+		log.Warn("sector %d entered unimplemented state 'CommitFailed'", update.id)
+
+	// Fatal errors
 	case api.UndefinedSectorState:
 		log.Error("sector update with undefined state!")
 	case api.FailedUnrecoverable: