lotus/storage/pipeline/fsm_test.go

package sealing

import (
	"context"
	"testing"

	logging "github.com/ipfs/go-log/v2"
	"github.com/stretchr/testify/require"
	"golang.org/x/xerrors"

	"github.com/filecoin-project/go-address"
	"github.com/filecoin-project/go-state-types/abi"
	"github.com/filecoin-project/go-statemachine"

	"github.com/filecoin-project/lotus/storage/sealer/storiface"
)

func init() {
	_ = logging.SetLogLevel("*", "INFO")
}

func (t *test) planSingle(evt interface{}) {
	_, _, err := t.s.plan([]statemachine.Event{{User: evt}}, t.state)
	require.NoError(t.t, err)
}

type test struct {
	s     *Sealing
	t     *testing.T
	state *SectorInfo
}

func TestHappyPath(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: Packing},
	}

	m.planSingle(SectorPacked{})
	require.Equal(m.t, m.state.State, GetTicket)

	m.planSingle(SectorTicket{})
	require.Equal(m.t, m.state.State, PreCommit1)

	m.planSingle(SectorPreCommit1{})
	require.Equal(m.t, m.state.State, PreCommit2)

	m.planSingle(SectorPreCommit2{})
	require.Equal(m.t, m.state.State, SubmitPreCommitBatch)

	m.planSingle(SectorPreCommitBatchSent{})
	require.Equal(m.t, m.state.State, PreCommitBatchWait)

	m.planSingle(SectorPreCommitLanded{})
	require.Equal(m.t, m.state.State, WaitSeed)

	m.planSingle(SectorSeedReady{})
	require.Equal(m.t, m.state.State, Committing)

	m.planSingle(SectorCommitted{})
	require.Equal(m.t, m.state.State, SubmitCommit)

	m.planSingle(SectorCommitSubmitted{})
	require.Equal(m.t, m.state.State, CommitWait)

	m.planSingle(SectorProving{})
	require.Equal(m.t, m.state.State, FinalizeSector)

	m.planSingle(SectorFinalized{})
	require.Equal(m.t, m.state.State, Proving)

	expected := []SectorState{Packing, GetTicket, PreCommit1, PreCommit2, SubmitPreCommitBatch, PreCommitBatchWait, WaitSeed, Committing, SubmitCommit, CommitWait, FinalizeSector, Proving}
	for i, n := range notif {
		if n.before.State != expected[i] {
			t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State)
		}
		if n.after.State != expected[i+1] {
			t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State)
		}
	}
}

func TestHappyPathFinalizeEarly(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: Packing},
	}

	m.planSingle(SectorPacked{})
	require.Equal(m.t, m.state.State, GetTicket)

	m.planSingle(SectorTicket{})
	require.Equal(m.t, m.state.State, PreCommit1)

	m.planSingle(SectorPreCommit1{})
	require.Equal(m.t, m.state.State, PreCommit2)

	m.planSingle(SectorPreCommit2{})
	require.Equal(m.t, m.state.State, SubmitPreCommitBatch)

	m.planSingle(SectorPreCommitBatchSent{})
	require.Equal(m.t, m.state.State, PreCommitBatchWait)

	m.planSingle(SectorPreCommitLanded{})
	require.Equal(m.t, m.state.State, WaitSeed)

	m.planSingle(SectorSeedReady{})
	require.Equal(m.t, m.state.State, Committing)

	m.planSingle(SectorProofReady{})
	require.Equal(m.t, m.state.State, CommitFinalize)

	m.planSingle(SectorFinalized{})
	require.Equal(m.t, m.state.State, SubmitCommit)

	m.planSingle(SectorSubmitCommitAggregate{})
	require.Equal(m.t, m.state.State, SubmitCommitAggregate)

	m.planSingle(SectorCommitAggregateSent{})
	require.Equal(m.t, m.state.State, CommitAggregateWait)

	m.planSingle(SectorProving{})
	require.Equal(m.t, m.state.State, FinalizeSector)

	m.planSingle(SectorFinalized{})
	require.Equal(m.t, m.state.State, Proving)

	expected := []SectorState{Packing, GetTicket, PreCommit1, PreCommit2, SubmitPreCommitBatch, PreCommitBatchWait, WaitSeed, Committing, CommitFinalize, SubmitCommit, SubmitCommitAggregate, CommitAggregateWait, FinalizeSector, Proving}
	for i, n := range notif {
		if n.before.State != expected[i] {
			t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State)
		}
		if n.after.State != expected[i+1] {
			t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State)
		}
	}
}

func TestCommitFinalizeFailed(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: Committing},
	}

	m.planSingle(SectorProofReady{})
	require.Equal(m.t, m.state.State, CommitFinalize)

	m.planSingle(SectorFinalizeFailed{})
	require.Equal(m.t, m.state.State, CommitFinalizeFailed)

	m.planSingle(SectorRetryFinalize{})
	require.Equal(m.t, m.state.State, CommitFinalize)

	m.planSingle(SectorFinalized{})
	require.Equal(m.t, m.state.State, SubmitCommit)

	expected := []SectorState{Committing, CommitFinalize, CommitFinalizeFailed, CommitFinalize, SubmitCommit}
	for i, n := range notif {
		if n.before.State != expected[i] {
			t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State)
		}
		if n.after.State != expected[i+1] {
			t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State)
		}
	}
}
func TestSeedRevert(t *testing.T) {
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
		},
		t:     t,
		state: &SectorInfo{State: Packing},
	}

	m.planSingle(SectorPacked{})
	require.Equal(m.t, m.state.State, GetTicket)

	m.planSingle(SectorTicket{})
	require.Equal(m.t, m.state.State, PreCommit1)

	m.planSingle(SectorPreCommit1{})
	require.Equal(m.t, m.state.State, PreCommit2)

	m.planSingle(SectorPreCommit2{})
	require.Equal(m.t, m.state.State, SubmitPreCommitBatch)

	m.planSingle(SectorPreCommitBatchSent{})
	require.Equal(m.t, m.state.State, PreCommitBatchWait)

	m.planSingle(SectorPreCommitLanded{})
	require.Equal(m.t, m.state.State, WaitSeed)

	m.planSingle(SectorSeedReady{})
	require.Equal(m.t, m.state.State, Committing)

	_, _, err := m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state)
	require.NoError(t, err)
	require.Equal(m.t, m.state.State, Committing)

	// not changing the seed this time
	_, _, err = m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state)
	require.NoError(t, err)
	require.Equal(m.t, m.state.State, SubmitCommit)

	m.planSingle(SectorCommitSubmitted{})
	require.Equal(m.t, m.state.State, CommitWait)

	m.planSingle(SectorProving{})
	require.Equal(m.t, m.state.State, FinalizeSector)

	m.planSingle(SectorFinalized{})
	require.Equal(m.t, m.state.State, Proving)
}

func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) {
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
		},
		t:     t,
		state: &SectorInfo{State: Committing},
	}

	events := []statemachine.Event{{User: SectorCommitFailed{}}}

	_, err := planCommitting(events, m.state)
	require.NoError(t, err)

	require.Equal(t, CommitFailed, m.state.State)
}

func TestPlannerList(t *testing.T) {
	for state := range ExistSectorStateList {
		_, ok := fsmPlanners[state]
		require.True(t, ok, "state %s", state)
	}

	for state := range fsmPlanners {
		if state == UndefinedSectorState {
			continue
		}
		_, ok := ExistSectorStateList[state]
		require.True(t, ok, "state %s", state)
	}
}

func TestBrokenState(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: "not a state"},
	}

	_, _, err := m.s.plan([]statemachine.Event{{User: SectorPacked{}}}, m.state)
	require.Error(t, err)
	require.Equal(m.t, m.state.State, SectorState("not a state"))

	m.planSingle(SectorRemove{})
	require.Equal(m.t, m.state.State, Removing)

	expected := []SectorState{"not a state", "not a state", Removing}
	for i, n := range notif {
		if n.before.State != expected[i] {
			t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State)
		}
		if n.after.State != expected[i+1] {
			t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State)
		}
	}
}

func TestBadEvent(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: Proving},
	}

	_, processed, err := m.s.Plan([]statemachine.Event{{User: SectorPacked{}}}, m.state)
	require.NoError(t, err)
	require.Equal(t, uint64(1), processed)
	require.Equal(m.t, m.state.State, Proving)

	require.Len(t, m.state.Log, 2)
	require.Contains(t, m.state.Log[1].Message, "received unexpected event")
}

func TestTicketExpired(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: Packing},
	}

	m.planSingle(SectorPacked{})
	require.Equal(m.t, m.state.State, GetTicket)

	m.planSingle(SectorTicket{})
	require.Equal(m.t, m.state.State, PreCommit1)

	expired := checkTicketExpired(0, MaxTicketAge+1)
	require.True(t, expired)

	m.planSingle(SectorOldTicket{})
	require.Equal(m.t, m.state.State, GetTicket)

	expected := []SectorState{Packing, GetTicket, PreCommit1, GetTicket}
	for i, n := range notif {
		if n.before.State != expected[i] {
			t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State)
		}
		if n.after.State != expected[i+1] {
			t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State)
		}
	}
}

func TestCreationTimeCleared(t *testing.T) {
	var notif []struct{ before, after SectorInfo }
	ma, _ := address.NewIDAddress(55151)
	m := test{
		s: &Sealing{
			maddr: ma,
			stats: SectorStats{
				bySector: map[abi.SectorID]SectorState{},
				byState:  map[SectorState]int64{},
			},
			notifee: func(before, after SectorInfo) {
				notif = append(notif, struct{ before, after SectorInfo }{before, after})
			},
		},
		t:     t,
		state: &SectorInfo{State: Available},
	}

	// sector starts with zero CreationTime
	m.planSingle(SectorStartCCUpdate{})
	require.Equal(m.t, m.state.State, SnapDealsWaitDeals)

	require.Equal(t, int64(0), m.state.CreationTime)

	// First AddPiece will set CreationTime
	m.planSingle(SectorAddPiece{})
	require.Equal(m.t, m.state.State, SnapDealsAddPiece)

	require.NotEqual(t, int64(0), m.state.CreationTime)

	m.planSingle(SectorPieceAdded{})
	require.Equal(m.t, m.state.State, SnapDealsWaitDeals)

	// abort shoult clean up CreationTime
	m.planSingle(SectorAbortUpgrade{})
	require.Equal(m.t, m.state.State, AbortUpgrade)

	require.NotEqual(t, int64(0), m.state.CreationTime)

	m.planSingle(SectorRevertUpgradeToProving{})
	require.Equal(m.t, m.state.State, Proving)

	require.Equal(t, int64(0), m.state.CreationTime)

	m.planSingle(SectorMarkForUpdate{})

	// in case CreationTime was set for whatever reason (lotus bug / manual sector state change)
	// make sure we clean it up when starting upgrade
	m.state.CreationTime = 325
	m.planSingle(SectorStartCCUpdate{})
	require.Equal(m.t, m.state.State, SnapDealsWaitDeals)

	require.Equal(t, int64(0), m.state.CreationTime)

	// "First" AddPiece will set CreationTime
	m.planSingle(SectorAddPiece{})
	require.Equal(m.t, m.state.State, SnapDealsAddPiece)

	require.NotEqual(t, int64(0), m.state.CreationTime)
}

func TestRetrySoftErr(t *testing.T) {
	i := 0

	tf := func() error {
		i++
		switch i {
		case 1:
			return storiface.Err(storiface.ErrTempAllocateSpace, xerrors.New("foo"))
		case 2:
			return nil
		default:
			t.Fatalf("what")
			return xerrors.Errorf("this error didn't ever happen, and will never happen")
		}
	}

	err := retrySoftErr(context.Background(), tf)
	require.NoError(t, err)
	require.Equal(t, 2, i)
}