lotus/storage/wdpost_changehandler_test.go

package storage

import (
	"context"
	"fmt"
	"sync"
	"testing"
	"time"

	tutils "github.com/filecoin-project/specs-actors/support/testing"

	"github.com/filecoin-project/go-state-types/crypto"

	"github.com/ipfs/go-cid"
	"github.com/stretchr/testify/require"

	"github.com/filecoin-project/go-address"
	"github.com/filecoin-project/go-state-types/abi"
	"github.com/filecoin-project/go-state-types/dline"
	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
	"github.com/filecoin-project/lotus/chain/types"
)

var dummyCid cid.Cid

func init() {
	dummyCid, _ = cid.Parse("bafkqaaa")
}

type proveRes struct {
	posts []miner.SubmitWindowedPoStParams
	err   error
}

type postStatus string

const (
	postStatusStart    postStatus = "postStatusStart"
	postStatusProving  postStatus = "postStatusProving"
	postStatusComplete postStatus = "postStatusComplete"
)

type mockAPI struct {
	ch            *changeHandler
	deadline      *dline.Info
	proveResult   chan *proveRes
	submitResult  chan error
	onStateChange chan struct{}

	tsLock sync.RWMutex
	ts     map[types.TipSetKey]*types.TipSet

	abortCalledLock sync.RWMutex
	abortCalled     bool

	statesLk   sync.RWMutex
	postStates map[abi.ChainEpoch]postStatus
}

func newMockAPI() *mockAPI {
	return &mockAPI{
		proveResult:   make(chan *proveRes),
		onStateChange: make(chan struct{}),
		submitResult:  make(chan error),
		postStates:    make(map[abi.ChainEpoch]postStatus),
		ts:            make(map[types.TipSetKey]*types.TipSet),
	}
}

func (m *mockAPI) makeTs(t *testing.T, h abi.ChainEpoch) *types.TipSet {
	m.tsLock.Lock()
	defer m.tsLock.Unlock()

	ts := makeTs(t, h)
	m.ts[ts.Key()] = ts
	return ts
}

func (m *mockAPI) setDeadline(di *dline.Info) {
	m.tsLock.Lock()
	defer m.tsLock.Unlock()

	m.deadline = di
}

func (m *mockAPI) getDeadline(currentEpoch abi.ChainEpoch) *dline.Info {
	close := miner.WPoStChallengeWindow - 1
	dlIdx := uint64(0)
	for close < currentEpoch {
		close += miner.WPoStChallengeWindow
		dlIdx++
	}
	return NewDeadlineInfo(0, dlIdx, currentEpoch)
}

func (m *mockAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) {
	m.tsLock.RLock()
	defer m.tsLock.RUnlock()

	ts, ok := m.ts[key]
	if !ok {
		panic(fmt.Sprintf("unexpected tipset key %s", key))
	}

	if m.deadline != nil {
		m.deadline.CurrentEpoch = ts.Height()
		return m.deadline, nil
	}

	return m.getDeadline(ts.Height()), nil
}

func (m *mockAPI) startGeneratePoST(
	ctx context.Context,
	ts *types.TipSet,
	deadline *dline.Info,
	completeGeneratePoST CompleteGeneratePoSTCb,
) context.CancelFunc {
	ctx, cancel := context.WithCancel(ctx)

	m.statesLk.Lock()
	defer m.statesLk.Unlock()
	m.postStates[deadline.Open] = postStatusProving

	go func() {
		defer cancel()

		select {
		case psRes := <-m.proveResult:
			m.statesLk.Lock()
			{
				if psRes.err == nil {
					m.postStates[deadline.Open] = postStatusComplete
				} else {
					m.postStates[deadline.Open] = postStatusStart
				}
			}
			m.statesLk.Unlock()
			completeGeneratePoST(psRes.posts, psRes.err)
		case <-ctx.Done():
			completeGeneratePoST(nil, ctx.Err())
		}
	}()

	return cancel
}

func (m *mockAPI) getPostStatus(di *dline.Info) postStatus {
	m.statesLk.RLock()
	defer m.statesLk.RUnlock()

	status, ok := m.postStates[di.Open]
	if ok {
		return status
	}
	return postStatusStart
}

func (m *mockAPI) startSubmitPoST(
	ctx context.Context,
	ts *types.TipSet,
	deadline *dline.Info,
	posts []miner.SubmitWindowedPoStParams,
	completeSubmitPoST CompleteSubmitPoSTCb,
) context.CancelFunc {
	ctx, cancel := context.WithCancel(ctx)

	go func() {
		defer cancel()

		select {
		case err := <-m.submitResult:
			completeSubmitPoST(err)
		case <-ctx.Done():
			completeSubmitPoST(ctx.Err())
		}
	}()

	return cancel
}

func (m *mockAPI) onAbort(ts *types.TipSet, deadline *dline.Info) {
	m.abortCalledLock.Lock()
	defer m.abortCalledLock.Unlock()
	m.abortCalled = true
}

func (m *mockAPI) wasAbortCalled() bool {
	m.abortCalledLock.RLock()
	defer m.abortCalledLock.RUnlock()
	return m.abortCalled
}

func (m *mockAPI) recordPoStFailure(err error, ts *types.TipSet, deadline *dline.Info) {
}

func (m *mockAPI) setChangeHandler(ch *changeHandler) {
	m.ch = ch
}

// TestChangeHandlerBasic verifies we can generate a proof and submit it
func TestChangeHandlerBasic(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Move to the correct height to submit the proof
	currentEpoch = 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state
	<-s.ch.submitHdlr.processedHeadChanges
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Send a response to the submit call
	mock.submitResult <- nil

	// Should move to the complete state
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(di))
}

// TestChangeHandlerFromProvingToSubmittingNoHeadChange tests that when the
// chain is already advanced past the confidence interval, we should move from
// proving to submitting without a head change in between.
func TestChangeHandlerFromProvingToSubmittingNoHeadChange(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Monitor submit handler's processing of incoming postInfo
	s.ch.submitHdlr.processedPostReady = make(chan *postInfo)

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Trigger a head change that advances the chain beyond the submit
	// confidence
	currentEpoch = 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should be no change to state yet
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Should move directly to submitting state with no further head changes
	<-s.ch.submitHdlr.processedPostReady
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))
}

// TestChangeHandlerFromProvingEmptyProofsToComplete tests that when there are no
// proofs generated we should not submit anything to chain but submit state
// should move to completed
func TestChangeHandlerFromProvingEmptyProofsToComplete(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Monitor submit handler's processing of incoming postInfo
	s.ch.submitHdlr.processedPostReady = make(chan *postInfo)

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Trigger a head change that advances the chain beyond the submit
	// confidence
	currentEpoch = 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should be no change to state yet
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs with an empty proofs array
	posts := []miner.SubmitWindowedPoStParams{}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Should move directly to submitting complete state
	<-s.ch.submitHdlr.processedPostReady
	require.Equal(t, SubmitStateComplete, s.submitState(di))
}

// TestChangeHandlerDontStartUntilProvingPeriod tests that the handler
// ignores updates until the proving period has been reached.
func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	periodStart := miner.WPoStProvingPeriod
	dlIdx := uint64(1)
	currentEpoch := abi.ChainEpoch(10)
	di := NewDeadlineInfo(periodStart, dlIdx, currentEpoch)
	mock.setDeadline(di)

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	go triggerHeadAdvance(t, s, currentEpoch)

	// Nothing should happen because the proving period has not started
	select {
	case <-s.ch.proveHdlr.processedHeadChanges:
		require.Fail(t, "unexpected prove change")
	case <-s.ch.submitHdlr.processedHeadChanges:
		require.Fail(t, "unexpected submit change")
	case <-time.After(10 * time.Millisecond):
	}

	// Advance the head to the next proving period's first epoch
	currentEpoch = periodStart + miner.WPoStChallengeWindow
	di = NewDeadlineInfo(periodStart, dlIdx, currentEpoch)
	mock.setDeadline(di)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))
}

// TestChangeHandlerStartProvingNextDeadline verifies that the proof handler
// starts proving the next deadline after the current one
func TestChangeHandlerStartProvingNextDeadline(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch+ChallengeConfidence)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Trigger a head change that advances the chain beyond the submit
	// confidence
	currentEpoch = 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch+ChallengeConfidence)

	// Should be no change to state yet
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Trigger head change that advances the chain to the Challenge epoch for
	// the next deadline
	go func() {
		di = nextDeadline(di)
		currentEpoch = di.Challenge + ChallengeConfidence
		triggerHeadAdvance(t, s, currentEpoch)
	}()

	// Should start generating next window's proof
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))
}

// TestChangeHandlerProvingRounds verifies we can generate several rounds of
// proofs as the chain head advances
func TestChangeHandlerProvingRounds(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	completeProofIndex := abi.ChainEpoch(10)
	for currentEpoch := abi.ChainEpoch(1); currentEpoch < miner.WPoStChallengeWindow*5; currentEpoch++ {
		// Trigger a head change
		di := mock.getDeadline(currentEpoch)
		go triggerHeadAdvance(t, s, currentEpoch+ChallengeConfidence)

		// Wait for prover to process head change
		<-s.ch.proveHdlr.processedHeadChanges

		completeProofEpoch := di.Open + completeProofIndex
		next := nextDeadline(di)
		//fmt.Println("epoch", currentEpoch, s.mock.getPostStatus(di), "next", s.mock.getPostStatus(next))
		if currentEpoch >= next.Challenge {
			require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))
			// At the next deadline's challenge epoch, should start proving
			// for that epoch
			require.Equal(t, postStatusProving, s.mock.getPostStatus(next))
		} else if currentEpoch > completeProofEpoch {
			// After proving for the round is complete, should be in complete state
			require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))
			require.Equal(t, postStatusStart, s.mock.getPostStatus(next))
		} else {
			// Until proving completes, should be in the proving state
			require.Equal(t, postStatusProving, s.mock.getPostStatus(di))
			require.Equal(t, postStatusStart, s.mock.getPostStatus(next))
		}

		// Wait for submitter to process head change
		<-s.ch.submitHdlr.processedHeadChanges

		completeSubmitEpoch := completeProofEpoch + 1
		//fmt.Println("epoch", currentEpoch, s.submitState(di))
		if currentEpoch > completeSubmitEpoch {
			require.Equal(t, SubmitStateComplete, s.submitState(di))
		} else if currentEpoch > completeProofEpoch {
			require.Equal(t, SubmitStateSubmitting, s.submitState(di))
		} else {
			require.Equal(t, SubmitStateStart, s.submitState(di))
		}

		if currentEpoch == completeProofEpoch {
			// Send a response to the call to generate proofs
			posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
			mock.proveResult <- &proveRes{posts: posts}

			// Should move to proving complete
			<-s.ch.proveHdlr.processedPostResults
			require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))
		}

		if currentEpoch == completeSubmitEpoch {
			// Send a response to the submit call
			mock.submitResult <- nil

			// Should move to the complete state
			<-s.ch.submitHdlr.processedSubmitResults
			require.Equal(t, SubmitStateComplete, s.submitState(di))
		}
	}
}

// TestChangeHandlerProvingErrorRecovery verifies that the proof handler
// recovers correctly from an error
func TestChangeHandlerProvingErrorRecovery(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Send an error response to the call to generate proofs
	mock.proveResult <- &proveRes{err: fmt.Errorf("err")}

	// Should abort and then move to start state
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusStart, s.mock.getPostStatus(di))

	// Trigger a head change
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Send a success response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))
}

// TestChangeHandlerSubmitErrorRecovery verifies that the submit handler
// recovers correctly from an error
func TestChangeHandlerSubmitErrorRecovery(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Move to the correct height to submit the proof
	currentEpoch = 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Read from prover incoming channel (so as not to block)
	<-s.ch.proveHdlr.processedHeadChanges

	// Should move to submitting state
	<-s.ch.submitHdlr.processedHeadChanges
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Send an error response to the call to submit
	mock.submitResult <- fmt.Errorf("err")

	// Should abort and then move back to the start state
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateStart, s.submitState(di))
	require.True(t, mock.wasAbortCalled())

	// Trigger another head change
	go triggerHeadAdvance(t, s, currentEpoch)

	// Read from prover incoming channel (so as not to block)
	<-s.ch.proveHdlr.processedHeadChanges

	// Should move to submitting state
	<-s.ch.submitHdlr.processedHeadChanges
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Send a response to the submit call
	mock.submitResult <- nil

	// Should move to the complete state
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(di))
}

// TestChangeHandlerProveExpiry verifies that the prove handler
// behaves correctly on expiry
func TestChangeHandlerProveExpiry(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Move to a height that expires the current proof
	currentEpoch = miner.WPoStChallengeWindow
	di = mock.getDeadline(currentEpoch)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should trigger an abort and start proving for the new deadline
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))
	<-s.ch.proveHdlr.processedPostResults
	require.True(t, mock.wasAbortCalled())

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))
}

// TestChangeHandlerSubmitExpiry verifies that the submit handler
// behaves correctly on expiry
func TestChangeHandlerSubmitExpiry(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Ignore prove handler head change processing for this test
	s.ch.proveHdlr.processedHeadChanges = nil

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := abi.ChainEpoch(1)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Move to the correct height to submit the proof
	currentEpoch = 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state
	<-s.ch.submitHdlr.processedHeadChanges
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Move to a height that expires the submit
	currentEpoch = miner.WPoStChallengeWindow
	di = mock.getDeadline(currentEpoch)
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should trigger an abort and move back to start state
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedSubmitResults
		require.True(t, mock.wasAbortCalled())
	}()

	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedHeadChanges
		require.Equal(t, SubmitStateStart, s.submitState(di))
	}()

	wg.Wait()
}

// TestChangeHandlerProveRevert verifies that the prove handler
// behaves correctly on revert
func TestChangeHandlerProveRevert(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := miner.WPoStChallengeWindow
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should start proving
	<-s.ch.proveHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Trigger a revert to the previous epoch
	revertEpoch := di.Open - 5
	go triggerHeadChange(t, s, revertEpoch, currentEpoch)

	// Should be no change
	<-s.ch.proveHdlr.processedHeadChanges
	require.Equal(t, postStatusProving, s.mock.getPostStatus(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))
	require.False(t, mock.wasAbortCalled())
}

// TestChangeHandlerSubmittingRevert verifies that the submit handler
// behaves correctly when there's a revert from the submitting state
func TestChangeHandlerSubmittingRevert(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Ignore prove handler head change processing for this test
	s.ch.proveHdlr.processedHeadChanges = nil

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := miner.WPoStChallengeWindow
	go triggerHeadAdvance(t, s, currentEpoch)

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Move to the correct height to submit the proof
	currentEpoch = currentEpoch + 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state
	<-s.ch.submitHdlr.processedHeadChanges
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Trigger a revert to the previous epoch
	revertEpoch := di.Open - 5
	go triggerHeadChange(t, s, revertEpoch, currentEpoch)

	var wg sync.WaitGroup
	wg.Add(2)

	// Should trigger an abort
	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedSubmitResults
		require.True(t, mock.wasAbortCalled())
	}()

	// Should resubmit current epoch
	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedHeadChanges
		require.Equal(t, SubmitStateSubmitting, s.submitState(di))
	}()

	wg.Wait()

	// Send a response to the resubmit call
	mock.submitResult <- nil

	// Should move to the complete state
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(di))
}

// TestChangeHandlerSubmitCompleteRevert verifies that the submit handler
// behaves correctly when there's a revert from the submit complete state
func TestChangeHandlerSubmitCompleteRevert(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Ignore prove handler head change processing for this test
	s.ch.proveHdlr.processedHeadChanges = nil

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := miner.WPoStChallengeWindow
	go triggerHeadAdvance(t, s, currentEpoch)

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	di := mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateStart, s.submitState(di))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(di))

	// Move to the correct height to submit the proof
	currentEpoch = currentEpoch + 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state
	<-s.ch.submitHdlr.processedHeadChanges
	di = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Send a response to the resubmit call
	mock.submitResult <- nil

	// Should move to the complete state
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(di))

	// Trigger a revert to the previous epoch
	revertEpoch := di.Open - 5
	go triggerHeadChange(t, s, revertEpoch, currentEpoch)

	// Should resubmit current epoch
	<-s.ch.submitHdlr.processedHeadChanges
	require.Equal(t, SubmitStateSubmitting, s.submitState(di))

	// Send a response to the resubmit call
	mock.submitResult <- nil

	// Should move to the complete state
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(di))
}

// TestChangeHandlerSubmitRevertTwoEpochs verifies that the submit handler
// behaves correctly when the revert is two epochs deep
func TestChangeHandlerSubmitRevertTwoEpochs(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Ignore prove handler head change processing for this test
	s.ch.proveHdlr.processedHeadChanges = nil

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := miner.WPoStChallengeWindow
	go triggerHeadAdvance(t, s, currentEpoch)

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	diE1 := mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateStart, s.submitState(diE1))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: diE1.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE1))

	// Move to the challenge epoch for the next deadline
	diE2 := nextDeadline(diE1)
	currentEpoch = diE2.Challenge + ChallengeConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state for epoch 1
	<-s.ch.submitHdlr.processedHeadChanges
	diE1 = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(diE1))

	// Send a response to the submit call for epoch 1
	mock.submitResult <- nil

	// Should move to the complete state for epoch 1
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(diE1))

	// Should start proving epoch 2
	// Send a response to the call to generate proofs
	postsE2 := []miner.SubmitWindowedPoStParams{{Deadline: diE2.Index}}
	mock.proveResult <- &proveRes{posts: postsE2}

	// Should move to proving complete for epoch 2
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE2))

	// Move to the correct height to submit the proof for epoch 2
	currentEpoch = diE2.Open + 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state for epoch 2
	<-s.ch.submitHdlr.processedHeadChanges
	diE2 = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(diE2))

	// Trigger a revert through two epochs (from epoch 2 to epoch 0)
	revertEpoch := diE1.Open - 5
	go triggerHeadChange(t, s, revertEpoch, currentEpoch)

	var wg sync.WaitGroup
	wg.Add(2)

	// Should trigger an abort
	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedSubmitResults
		require.True(t, mock.wasAbortCalled())
	}()

	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedHeadChanges

		// Should reset epoch 1 (that is expired) to start state
		require.Equal(t, SubmitStateStart, s.submitState(diE1))
		// Should resubmit epoch 2
		require.Equal(t, SubmitStateSubmitting, s.submitState(diE2))
	}()

	wg.Wait()

	// Send a response to the resubmit call for epoch 2
	mock.submitResult <- nil

	// Should move to the complete state for epoch 2
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(diE2))
}

// TestChangeHandlerSubmitRevertAdvanceLess verifies that the submit handler
// behaves correctly when the revert is two epochs deep and the advance is
// to a lower height than before
func TestChangeHandlerSubmitRevertAdvanceLess(t *testing.T) {
	s := makeScaffolding(t)
	mock := s.mock

	// Ignore prove handler head change processing for this test
	s.ch.proveHdlr.processedHeadChanges = nil

	defer s.ch.shutdown()
	s.ch.start()

	// Trigger a head change
	currentEpoch := miner.WPoStChallengeWindow
	go triggerHeadAdvance(t, s, currentEpoch)

	// Submitter doesn't have anything to do yet
	<-s.ch.submitHdlr.processedHeadChanges
	diE1 := mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateStart, s.submitState(diE1))

	// Send a response to the call to generate proofs
	posts := []miner.SubmitWindowedPoStParams{{Deadline: diE1.Index}}
	mock.proveResult <- &proveRes{posts: posts}

	// Should move to proving complete
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE1))

	// Move to the challenge epoch for the next deadline
	diE2 := nextDeadline(diE1)
	currentEpoch = diE2.Challenge + ChallengeConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state for epoch 1
	<-s.ch.submitHdlr.processedHeadChanges
	diE1 = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(diE1))

	// Send a response to the submit call for epoch 1
	mock.submitResult <- nil

	// Should move to the complete state for epoch 1
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(diE1))

	// Should start proving epoch 2
	// Send a response to the call to generate proofs
	postsE2 := []miner.SubmitWindowedPoStParams{{Deadline: diE2.Index}}
	mock.proveResult <- &proveRes{posts: postsE2}

	// Should move to proving complete for epoch 2
	<-s.ch.proveHdlr.processedPostResults
	require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE2))

	// Move to the correct height to submit the proof for epoch 2
	currentEpoch = diE2.Open + 1 + SubmitConfidence
	go triggerHeadAdvance(t, s, currentEpoch)

	// Should move to submitting state for epoch 2
	<-s.ch.submitHdlr.processedHeadChanges
	diE2 = mock.getDeadline(currentEpoch)
	require.Equal(t, SubmitStateSubmitting, s.submitState(diE2))

	// Trigger a revert through two epochs (from epoch 2 to epoch 0)
	// then advance to the previous epoch (to epoch 1)
	revertEpoch := diE1.Open - 5
	currentEpoch = diE2.Open - 1
	go triggerHeadChange(t, s, revertEpoch, currentEpoch)

	var wg sync.WaitGroup
	wg.Add(2)

	// Should trigger an abort
	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedSubmitResults
		require.True(t, mock.wasAbortCalled())
	}()

	go func() {
		defer wg.Done()

		<-s.ch.submitHdlr.processedHeadChanges

		// Should resubmit epoch 1
		require.Equal(t, SubmitStateSubmitting, s.submitState(diE1))
		// Should reset epoch 2 to start state
		require.Equal(t, SubmitStateStart, s.submitState(diE2))
	}()

	wg.Wait()

	// Send a response to the resubmit call for epoch 1
	mock.submitResult <- nil

	// Should move to the complete state for epoch 1
	<-s.ch.submitHdlr.processedSubmitResults
	require.Equal(t, SubmitStateComplete, s.submitState(diE1))
}

type smScaffolding struct {
	ctx  context.Context
	mock *mockAPI
	ch   *changeHandler
}

func makeScaffolding(t *testing.T) *smScaffolding {
	ctx := context.Background()
	actor := tutils.NewActorAddr(t, "actor")
	mock := newMockAPI()
	ch := newChangeHandler(mock, actor)
	mock.setChangeHandler(ch)

	ch.proveHdlr.processedHeadChanges = make(chan *headChange)
	ch.proveHdlr.processedPostResults = make(chan *postResult)

	ch.submitHdlr.processedHeadChanges = make(chan *headChange)
	ch.submitHdlr.processedSubmitResults = make(chan *submitResult)

	return &smScaffolding{
		ctx:  ctx,
		mock: mock,
		ch:   ch,
	}
}

func triggerHeadAdvance(t *testing.T, s *smScaffolding, height abi.ChainEpoch) {
	ts := s.mock.makeTs(t, height)
	err := s.ch.update(s.ctx, nil, ts)
	require.NoError(t, err)
}

func triggerHeadChange(t *testing.T, s *smScaffolding, revertHeight, advanceHeight abi.ChainEpoch) {
	tsRev := s.mock.makeTs(t, revertHeight)
	tsAdv := s.mock.makeTs(t, advanceHeight)
	err := s.ch.update(s.ctx, tsRev, tsAdv)
	require.NoError(t, err)
}

func (s *smScaffolding) submitState(di *dline.Info) SubmitState {
	return s.ch.submitHdlr.getPostWindow(di).submitState
}

func makeTs(t *testing.T, h abi.ChainEpoch) *types.TipSet {
	var parents []cid.Cid
	msgcid := dummyCid

	a, _ := address.NewFromString("t00")
	b, _ := address.NewFromString("t02")
	var ts, err = types.NewTipSet([]*types.BlockHeader{
		{
			Height: h,
			Miner:  a,

			Parents: parents,

			Ticket: &types.Ticket{VRFProof: []byte{byte(h % 2)}},

			ParentStateRoot:       dummyCid,
			Messages:              msgcid,
			ParentMessageReceipts: dummyCid,

			BlockSig:     &crypto.Signature{Type: crypto.SigTypeBLS},
			BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
		},
		{
			Height: h,
			Miner:  b,

			Parents: parents,

			Ticket: &types.Ticket{VRFProof: []byte{byte((h + 1) % 2)}},

			ParentStateRoot:       dummyCid,
			Messages:              msgcid,
			ParentMessageReceipts: dummyCid,

			BlockSig:     &crypto.Signature{Type: crypto.SigTypeBLS},
			BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
		},
	})

	require.NoError(t, err)

	return ts
}