2020-01-15 20:49:11 +00:00
|
|
|
package sealing
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2020-01-22 20:29:19 +00:00
|
|
|
"fmt"
|
2020-01-16 01:25:49 +00:00
|
|
|
"reflect"
|
2020-01-22 20:29:19 +00:00
|
|
|
"time"
|
2020-01-15 20:49:11 +00:00
|
|
|
|
|
|
|
"golang.org/x/xerrors"
|
|
|
|
|
|
|
|
"github.com/filecoin-project/lotus/api"
|
|
|
|
"github.com/filecoin-project/lotus/lib/statemachine"
|
|
|
|
)
|
|
|
|
|
|
|
|
func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, error) {
|
|
|
|
next, err := m.plan(events, user.(*SectorInfo))
|
|
|
|
if err != nil || next == nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return func(ctx statemachine.Context, si SectorInfo) error {
|
|
|
|
err := next(ctx, si)
|
|
|
|
if err != nil {
|
2020-01-23 16:11:58 +00:00
|
|
|
log.Errorf("unhandled sector error (%d): %+v", si.SectorID, err)
|
|
|
|
return nil
|
2020-01-15 20:49:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2020-01-16 02:54:57 +00:00
|
|
|
var fsmPlanners = []func(events []statemachine.Event, state *SectorInfo) error{
|
2020-01-16 01:25:49 +00:00
|
|
|
api.UndefinedSectorState: planOne(on(SectorStart{}, api.Packing)),
|
2020-01-16 02:54:57 +00:00
|
|
|
api.Packing: planOne(on(SectorPacked{}, api.Unsealed)),
|
2020-01-16 02:53:59 +00:00
|
|
|
api.Unsealed: planOne(
|
|
|
|
on(SectorSealed{}, api.PreCommitting),
|
|
|
|
on(SectorSealFailed{}, api.SealFailed),
|
2020-01-22 19:47:29 +00:00
|
|
|
on(SectorPackingFailed{}, api.PackingFailed),
|
2020-01-16 02:53:59 +00:00
|
|
|
),
|
|
|
|
api.PreCommitting: planOne(
|
2020-01-22 19:47:29 +00:00
|
|
|
on(SectorSealFailed{}, api.SealFailed),
|
2020-01-20 22:04:46 +00:00
|
|
|
on(SectorPreCommitted{}, api.WaitSeed),
|
2020-01-16 02:53:59 +00:00
|
|
|
on(SectorPreCommitFailed{}, api.PreCommitFailed),
|
|
|
|
),
|
2020-01-20 22:04:46 +00:00
|
|
|
api.WaitSeed: planOne(
|
2020-01-16 02:53:59 +00:00
|
|
|
on(SectorSeedReady{}, api.Committing),
|
|
|
|
on(SectorPreCommitFailed{}, api.PreCommitFailed),
|
|
|
|
),
|
2020-01-16 01:25:49 +00:00
|
|
|
api.Committing: planCommitting,
|
2020-01-20 08:23:56 +00:00
|
|
|
api.CommitWait: planOne(
|
2020-01-29 21:25:06 +00:00
|
|
|
on(SectorProving{}, api.FinalizeSector),
|
2020-01-20 08:23:56 +00:00
|
|
|
on(SectorCommitFailed{}, api.CommitFailed),
|
|
|
|
),
|
2020-01-16 01:25:49 +00:00
|
|
|
|
2020-01-29 21:25:06 +00:00
|
|
|
api.FinalizeSector: planOne(
|
|
|
|
on(SectorFinalized{}, api.Proving),
|
|
|
|
),
|
|
|
|
|
2020-01-16 02:53:59 +00:00
|
|
|
api.Proving: planOne(
|
|
|
|
on(SectorFaultReported{}, api.FaultReported),
|
|
|
|
on(SectorFaulty{}, api.Faulty),
|
|
|
|
),
|
|
|
|
|
2020-01-23 15:38:01 +00:00
|
|
|
api.SealFailed: planOne(
|
|
|
|
on(SectorRetrySeal{}, api.Unsealed),
|
|
|
|
),
|
2020-01-23 17:34:04 +00:00
|
|
|
api.PreCommitFailed: planOne(
|
|
|
|
on(SectorRetryPreCommit{}, api.PreCommitting),
|
|
|
|
on(SectorRetryWaitSeed{}, api.WaitSeed),
|
2020-01-23 17:45:57 +00:00
|
|
|
on(SectorSealFailed{}, api.SealFailed),
|
2020-01-23 17:34:04 +00:00
|
|
|
),
|
2020-01-23 15:38:01 +00:00
|
|
|
|
2020-01-16 02:53:59 +00:00
|
|
|
api.Faulty: planOne(
|
|
|
|
on(SectorFaultReported{}, api.FaultReported),
|
|
|
|
),
|
|
|
|
api.FaultedFinal: final,
|
2020-01-16 01:25:49 +00:00
|
|
|
}
|
|
|
|
|
2020-01-15 20:49:11 +00:00
|
|
|
func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(statemachine.Context, SectorInfo) error, error) {
|
|
|
|
/////
|
|
|
|
// First process all events
|
|
|
|
|
2020-01-22 20:29:19 +00:00
|
|
|
for _, event := range events {
|
|
|
|
l := Log{
|
|
|
|
Timestamp: uint64(time.Now().Unix()),
|
|
|
|
Message: fmt.Sprintf("%+v", event),
|
|
|
|
Kind: fmt.Sprintf("event;%T", event.User),
|
|
|
|
}
|
|
|
|
|
|
|
|
if err, iserr := event.User.(xerrors.Formatter); iserr {
|
|
|
|
l.Trace = fmt.Sprintf("%+v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
state.Log = append(state.Log, l)
|
|
|
|
}
|
|
|
|
|
2020-01-16 01:25:49 +00:00
|
|
|
p := fsmPlanners[state.State]
|
|
|
|
if p == nil {
|
2020-01-23 15:47:33 +00:00
|
|
|
return nil, xerrors.Errorf("planner for state %s not found", api.SectorStates[state.State])
|
2020-01-16 01:25:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := p(events, state); err != nil {
|
|
|
|
return nil, xerrors.Errorf("running planner for state %s failed: %w", api.SectorStates[state.State], err)
|
|
|
|
}
|
|
|
|
|
2020-01-15 20:49:11 +00:00
|
|
|
/////
|
|
|
|
// Now decide what to do next
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
|
|
|
* Empty
|
|
|
|
| |
|
|
|
|
| v
|
|
|
|
*<- Packing <- incoming
|
|
|
|
| |
|
|
|
|
| v
|
|
|
|
*<- Unsealed <--> SealFailed
|
|
|
|
| |
|
|
|
|
| v
|
|
|
|
* PreCommitting <--> PreCommitFailed
|
|
|
|
| | ^
|
|
|
|
| v |
|
2020-01-20 22:04:46 +00:00
|
|
|
*<- WaitSeed ----------/
|
2020-01-15 20:49:11 +00:00
|
|
|
| |||
|
|
|
|
| vvv v--> SealCommitFailed
|
|
|
|
*<- Committing
|
|
|
|
| | ^--> CommitFailed
|
|
|
|
| v ^
|
|
|
|
*<- CommitWait ---/
|
|
|
|
| |
|
|
|
|
| v
|
|
|
|
*<- Proving
|
|
|
|
|
|
|
|
|
v
|
|
|
|
FailedUnrecoverable
|
|
|
|
|
|
|
|
UndefinedSectorState <- ¯\_(ツ)_/¯
|
|
|
|
| ^
|
|
|
|
*---------------------/
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
switch state.State {
|
|
|
|
// Happy path
|
|
|
|
case api.Packing:
|
|
|
|
return m.handlePacking, nil
|
|
|
|
case api.Unsealed:
|
|
|
|
return m.handleUnsealed, nil
|
|
|
|
case api.PreCommitting:
|
|
|
|
return m.handlePreCommitting, nil
|
2020-01-20 22:04:46 +00:00
|
|
|
case api.WaitSeed:
|
|
|
|
return m.handleWaitSeed, nil
|
2020-01-15 20:49:11 +00:00
|
|
|
case api.Committing:
|
|
|
|
return m.handleCommitting, nil
|
|
|
|
case api.CommitWait:
|
|
|
|
return m.handleCommitWait, nil
|
2020-01-29 21:25:06 +00:00
|
|
|
case api.FinalizeSector:
|
2020-01-29 22:37:31 +00:00
|
|
|
return m.handleFinalizeSector, nil
|
2020-01-15 20:49:11 +00:00
|
|
|
case api.Proving:
|
|
|
|
// TODO: track sector health / expiration
|
|
|
|
log.Infof("Proving sector %d", state.SectorID)
|
|
|
|
|
|
|
|
// Handled failure modes
|
|
|
|
case api.SealFailed:
|
2020-01-23 15:47:33 +00:00
|
|
|
return m.handleSealFailed, nil
|
2020-01-15 20:49:11 +00:00
|
|
|
case api.PreCommitFailed:
|
2020-01-23 17:34:04 +00:00
|
|
|
return m.handlePreCommitFailed, nil
|
2020-01-15 20:49:11 +00:00
|
|
|
case api.SealCommitFailed:
|
|
|
|
log.Warnf("sector %d entered unimplemented state 'SealCommitFailed'", state.SectorID)
|
|
|
|
case api.CommitFailed:
|
|
|
|
log.Warnf("sector %d entered unimplemented state 'CommitFailed'", state.SectorID)
|
|
|
|
|
|
|
|
// Faults
|
|
|
|
case api.Faulty:
|
|
|
|
return m.handleFaulty, nil
|
|
|
|
case api.FaultReported:
|
|
|
|
return m.handleFaultReported, nil
|
|
|
|
|
|
|
|
// Fatal errors
|
|
|
|
case api.UndefinedSectorState:
|
|
|
|
log.Error("sector update with undefined state!")
|
|
|
|
case api.FailedUnrecoverable:
|
|
|
|
log.Errorf("sector %d failed unrecoverably", state.SectorID)
|
|
|
|
default:
|
|
|
|
log.Errorf("unexpected sector update state: %d", state.State)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2020-01-16 01:25:49 +00:00
|
|
|
func planCommitting(events []statemachine.Event, state *SectorInfo) error {
|
|
|
|
for _, event := range events {
|
|
|
|
switch e := event.User.(type) {
|
2020-01-16 02:53:59 +00:00
|
|
|
case globalMutator:
|
|
|
|
if e.applyGlobal(state) {
|
|
|
|
return nil
|
|
|
|
}
|
2020-01-16 01:25:49 +00:00
|
|
|
case SectorCommitted: // the normal case
|
|
|
|
e.apply(state)
|
|
|
|
state.State = api.CommitWait
|
|
|
|
case SectorSeedReady: // seed changed :/
|
|
|
|
if e.seed.Equals(&state.Seed) {
|
|
|
|
log.Warnf("planCommitting: got SectorSeedReady, but the seed didn't change")
|
|
|
|
continue // or it didn't!
|
|
|
|
}
|
|
|
|
log.Warnf("planCommitting: commit Seed changed")
|
|
|
|
e.apply(state)
|
|
|
|
state.State = api.Committing
|
|
|
|
return nil
|
2020-01-20 22:04:46 +00:00
|
|
|
case SectorComputeProofFailed:
|
2020-01-16 02:53:59 +00:00
|
|
|
state.State = api.SealCommitFailed
|
|
|
|
case SectorSealFailed:
|
|
|
|
state.State = api.CommitFailed
|
2020-01-28 19:46:26 +00:00
|
|
|
case SectorCommitFailed:
|
|
|
|
state.State = api.CommitFailed
|
2020-01-16 01:25:49 +00:00
|
|
|
default:
|
|
|
|
return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-01-15 20:49:11 +00:00
|
|
|
func (m *Sealing) restartSectors(ctx context.Context) error {
|
|
|
|
trackedSectors, err := m.ListSectors()
|
|
|
|
if err != nil {
|
|
|
|
log.Errorf("loading sector list: %+v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, sector := range trackedSectors {
|
|
|
|
if err := m.sectors.Send(sector.SectorID, SectorRestart{}); err != nil {
|
|
|
|
log.Errorf("restarting sector %d: %+v", sector.SectorID, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: Grab on-chain sector set and diff with trackedSectors
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Sealing) ForceSectorState(ctx context.Context, id uint64, state api.SectorState) error {
|
|
|
|
return m.sectors.Send(id, SectorForceState{state})
|
|
|
|
}
|
2020-01-16 01:25:49 +00:00
|
|
|
|
|
|
|
func final(events []statemachine.Event, state *SectorInfo) error {
|
|
|
|
return xerrors.Errorf("didn't expect any events in state %s, got %+v", api.SectorStates[state.State], events)
|
|
|
|
}
|
|
|
|
|
|
|
|
func on(mut mutator, next api.SectorState) func() (mutator, api.SectorState) {
|
|
|
|
return func() (mutator, api.SectorState) {
|
|
|
|
return mut, next
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func planOne(ts ...func() (mut mutator, next api.SectorState)) func(events []statemachine.Event, state *SectorInfo) error {
|
|
|
|
return func(events []statemachine.Event, state *SectorInfo) error {
|
|
|
|
if len(events) != 1 {
|
2020-01-16 02:53:59 +00:00
|
|
|
for _, event := range events {
|
2020-01-22 19:47:29 +00:00
|
|
|
if gm, ok := event.User.(globalMutator); ok {
|
2020-01-16 02:53:59 +00:00
|
|
|
gm.applyGlobal(state)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2020-01-16 01:25:49 +00:00
|
|
|
return xerrors.Errorf("planner for state %s only has a plan for a single event only, got %+v", api.SectorStates[state.State], events)
|
|
|
|
}
|
|
|
|
|
2020-01-22 19:47:29 +00:00
|
|
|
if gm, ok := events[0].User.(globalMutator); ok {
|
2020-01-22 18:30:56 +00:00
|
|
|
gm.applyGlobal(state)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-01-16 01:25:49 +00:00
|
|
|
for _, t := range ts {
|
|
|
|
mut, next := t()
|
|
|
|
|
|
|
|
if reflect.TypeOf(events[0].User) != reflect.TypeOf(mut) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-01-21 16:05:10 +00:00
|
|
|
if err, iserr := events[0].User.(error); iserr {
|
|
|
|
log.Warnf("sector %d got error event %T: %+v", state.SectorID, events[0].User, err)
|
|
|
|
}
|
|
|
|
|
2020-01-16 01:25:49 +00:00
|
|
|
events[0].User.(mutator).apply(state)
|
|
|
|
state.State = next
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-01-22 18:30:56 +00:00
|
|
|
return xerrors.Errorf("planner for state %s received unexpected event %T (%+v)", api.SectorStates[state.State], events[0].User, events[0])
|
2020-01-16 01:25:49 +00:00
|
|
|
}
|
|
|
|
}
|