lotus/extern/sector-storage/sched_test.go

454 lines
12 KiB
Go
Raw Normal View History

package sectorstorage
2020-06-24 21:06:56 +00:00
import (
"context"
2020-07-16 23:26:55 +00:00
"fmt"
2020-07-16 21:41:04 +00:00
"io"
2020-07-16 23:26:55 +00:00
"runtime"
2020-07-16 21:41:04 +00:00
"sync"
2020-06-24 21:06:56 +00:00
"testing"
2020-07-16 21:41:04 +00:00
"time"
2020-06-24 21:06:56 +00:00
2020-07-16 21:41:04 +00:00
"github.com/ipfs/go-cid"
2020-06-24 21:06:56 +00:00
"github.com/stretchr/testify/require"
2020-07-16 21:41:04 +00:00
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
2020-07-16 21:41:04 +00:00
"github.com/filecoin-project/specs-storage/storage"
2020-06-24 21:06:56 +00:00
)
func TestWithPriority(t *testing.T) {
ctx := context.Background()
require.Equal(t, DefaultSchedPriority, getPriority(ctx))
ctx = WithPriority(ctx, 2222)
require.Equal(t, 2222, getPriority(ctx))
}
2020-07-16 21:41:04 +00:00
type schedTestWorker struct {
2020-07-16 21:41:15 +00:00
name string
2020-07-16 21:41:04 +00:00
taskTypes map[sealtasks.TaskType]struct{}
2020-07-16 21:41:15 +00:00
paths []stores.StoragePath
2020-07-16 21:41:04 +00:00
2020-07-16 21:41:15 +00:00
closed bool
2020-07-16 21:41:04 +00:00
closing chan struct{}
}
func (s *schedTestWorker) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
panic("implement me")
}
func (s *schedTestWorker) SealPreCommit2(ctx context.Context, sector abi.SectorID, pc1o storage.PreCommit1Out) (storage.SectorCids, error) {
panic("implement me")
}
func (s *schedTestWorker) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
panic("implement me")
}
func (s *schedTestWorker) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) {
panic("implement me")
}
func (s *schedTestWorker) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error {
panic("implement me")
}
func (s *schedTestWorker) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error {
panic("implement me")
}
func (s *schedTestWorker) Remove(ctx context.Context, sector abi.SectorID) error {
panic("implement me")
}
func (s *schedTestWorker) NewSector(ctx context.Context, sector abi.SectorID) error {
panic("implement me")
}
func (s *schedTestWorker) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
panic("implement me")
}
func (s *schedTestWorker) MoveStorage(ctx context.Context, sector abi.SectorID) error {
panic("implement me")
}
func (s *schedTestWorker) Fetch(ctx context.Context, id abi.SectorID, ft stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error {
panic("implement me")
}
func (s *schedTestWorker) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) error {
panic("implement me")
}
2020-07-30 20:38:05 +00:00
func (s *schedTestWorker) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
2020-07-16 21:41:04 +00:00
panic("implement me")
}
func (s *schedTestWorker) TaskTypes(ctx context.Context) (map[sealtasks.TaskType]struct{}, error) {
return s.taskTypes, nil
}
func (s *schedTestWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) {
return s.paths, nil
}
func (s *schedTestWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) {
return storiface.WorkerInfo{
2020-07-16 21:41:15 +00:00
Hostname: s.name,
2020-07-16 21:41:04 +00:00
Resources: storiface.WorkerResources{
MemPhysical: 128 << 30,
MemSwap: 200 << 30,
MemReserved: 2 << 30,
CPUs: 32,
GPUs: []string{"a GPU"},
},
}, nil
}
func (s *schedTestWorker) Closing(ctx context.Context) (<-chan struct{}, error) {
return s.closing, nil
}
func (s *schedTestWorker) Close() error {
if !s.closed {
2020-07-17 10:59:12 +00:00
log.Info("close schedTestWorker")
2020-07-16 21:41:04 +00:00
s.closed = true
close(s.closing)
}
return nil
}
var _ Worker = &schedTestWorker{}
func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}) {
w := &schedTestWorker{
2020-07-16 21:41:15 +00:00
name: name,
2020-07-16 21:41:04 +00:00
taskTypes: taskTypes,
2020-07-16 21:41:15 +00:00
paths: []stores.StoragePath{{ID: "bb-8", Weight: 2, LocalPath: "<octopus>food</octopus>", CanSeal: true, CanStore: true}},
2020-07-16 21:41:04 +00:00
closing: make(chan struct{}),
}
for _, path := range w.paths {
err := index.StorageAttach(context.TODO(), stores.StorageInfo{
2020-07-16 21:41:15 +00:00
ID: path.ID,
URLs: nil,
Weight: path.Weight,
CanSeal: path.CanSeal,
CanStore: path.CanStore,
2020-07-16 21:41:04 +00:00
}, fsutil.FsStat{
Capacity: 1 << 40,
Available: 1 << 40,
Reserved: 3,
})
require.NoError(t, err)
}
info, err := w.Info(context.TODO())
require.NoError(t, err)
sched.newWorkers <- &workerHandle{
2020-07-21 18:01:25 +00:00
w: w,
wt: &workTracker{
running: map[uint64]storiface.WorkerJob{},
},
2020-07-16 21:41:04 +00:00
info: info,
preparing: &activeResources{},
active: &activeResources{},
}
}
func TestSchedStartStop(t *testing.T) {
spt := abi.RegisteredSealProof_StackedDrg32GiBV1
sched := newScheduler(spt)
go sched.runSched()
addTestWorker(t, sched, stores.NewIndex(), "fred", nil)
2020-07-17 10:59:12 +00:00
require.NoError(t, sched.Close(context.TODO()))
2020-07-16 21:41:04 +00:00
}
func TestSched(t *testing.T) {
2020-07-17 10:59:12 +00:00
ctx, done := context.WithTimeout(context.Background(), 30*time.Second)
2020-07-16 23:26:55 +00:00
defer done()
2020-07-16 21:41:04 +00:00
2020-07-16 23:26:55 +00:00
spt := abi.RegisteredSealProof_StackedDrg32GiBV1
2020-07-16 21:41:04 +00:00
type workerSpec struct {
2020-07-16 21:41:15 +00:00
name string
2020-07-16 21:41:04 +00:00
taskTypes map[sealtasks.TaskType]struct{}
}
2020-07-16 23:32:49 +00:00
noopAction := func(ctx context.Context, w Worker) error {
2020-07-16 21:41:04 +00:00
return nil
}
type runMeta struct {
done map[string]chan struct{}
wg sync.WaitGroup
}
type task func(*testing.T, *scheduler, *stores.Index, *runMeta)
2020-07-16 23:26:55 +00:00
sched := func(taskName, expectWorker string, sid abi.SectorNumber, taskType sealtasks.TaskType) task {
_, _, l, _ := runtime.Caller(1)
_, _, l2, _ := runtime.Caller(2)
2020-07-16 21:41:04 +00:00
return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) {
done := make(chan struct{})
rm.done[taskName] = done
2020-08-05 12:36:49 +00:00
sel := newAllocSelector(index, stores.FTCache, stores.PathSealing)
2020-07-16 21:41:04 +00:00
rm.wg.Add(1)
go func() {
defer rm.wg.Done()
2020-07-16 23:26:55 +00:00
sectorNum := abi.SectorID{
Miner: 8,
Number: sid,
}
2020-07-16 23:32:49 +00:00
err := sched.Schedule(ctx, sectorNum, taskType, sel, func(ctx context.Context, w Worker) error {
2020-07-16 21:41:04 +00:00
wi, err := w.Info(ctx)
require.NoError(t, err)
require.Equal(t, expectWorker, wi.Hostname)
log.Info("IN ", taskName)
for {
_, ok := <-done
if !ok {
break
}
}
log.Info("OUT ", taskName)
return nil
2020-07-16 23:32:49 +00:00
}, noopAction)
2020-07-16 23:26:55 +00:00
require.NoError(t, err, fmt.Sprint(l, l2))
2020-07-16 21:41:04 +00:00
}()
2020-07-16 23:26:55 +00:00
<-sched.testSync
2020-07-16 21:41:04 +00:00
}
}
taskStarted := func(name string) task {
2020-07-16 23:26:55 +00:00
_, _, l, _ := runtime.Caller(1)
_, _, l2, _ := runtime.Caller(2)
2020-07-16 21:41:04 +00:00
return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) {
2020-07-16 23:26:55 +00:00
select {
case rm.done[name] <- struct{}{}:
case <-ctx.Done():
t.Fatal("ctx error", ctx.Err(), l, l2)
}
2020-07-16 21:41:04 +00:00
}
}
taskDone := func(name string) task {
2020-07-16 23:26:55 +00:00
_, _, l, _ := runtime.Caller(1)
_, _, l2, _ := runtime.Caller(2)
2020-07-16 21:41:04 +00:00
return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) {
2020-07-16 23:26:55 +00:00
select {
case rm.done[name] <- struct{}{}:
case <-ctx.Done():
t.Fatal("ctx error", ctx.Err(), l, l2)
}
2020-07-16 21:41:04 +00:00
close(rm.done[name])
}
}
taskNotScheduled := func(name string) task {
2020-07-16 23:26:55 +00:00
_, _, l, _ := runtime.Caller(1)
_, _, l2, _ := runtime.Caller(2)
2020-07-16 21:41:04 +00:00
return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) {
select {
case rm.done[name] <- struct{}{}:
2020-07-16 23:26:55 +00:00
t.Fatal("not expected", l, l2)
2020-07-16 21:41:04 +00:00
case <-time.After(10 * time.Millisecond): // TODO: better synchronization thingy
}
}
}
testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) {
return func(t *testing.T) {
index := stores.NewIndex()
sched := newScheduler(spt)
2020-07-16 23:26:55 +00:00
sched.testSync = make(chan struct{})
2020-07-16 21:41:04 +00:00
go sched.runSched()
for _, worker := range workers {
addTestWorker(t, sched, index, worker.name, worker.taskTypes)
}
rm := runMeta{
done: map[string]chan struct{}{},
}
for _, task := range tasks {
task(t, sched, index, &rm)
}
log.Info("wait for async stuff")
rm.wg.Wait()
2020-07-17 10:59:12 +00:00
require.NoError(t, sched.Close(context.TODO()))
2020-07-16 21:41:04 +00:00
}
}
multTask := func(tasks ...task) task {
return func(t *testing.T, s *scheduler, index *stores.Index, meta *runMeta) {
for _, tsk := range tasks {
tsk(t, s, index, meta)
}
}
}
t.Run("one-pc1", testFunc([]workerSpec{
2020-07-16 21:41:15 +00:00
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
2020-07-16 21:41:04 +00:00
}, []task{
2020-07-16 23:26:55 +00:00
sched("pc1-1", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskDone("pc1-1"),
}))
t.Run("pc1-2workers-1", testFunc([]workerSpec{
2020-07-16 21:41:15 +00:00
{name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}},
{name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
2020-07-16 21:41:04 +00:00
}, []task{
2020-07-16 23:26:55 +00:00
sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskDone("pc1-1"),
}))
t.Run("pc1-2workers-2", testFunc([]workerSpec{
2020-07-16 21:41:15 +00:00
{name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
{name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}},
2020-07-16 21:41:04 +00:00
}, []task{
2020-07-16 23:26:55 +00:00
sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskDone("pc1-1"),
}))
t.Run("pc1-block-pc2", testFunc([]workerSpec{
2020-07-16 21:41:15 +00:00
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
2020-07-16 21:41:04 +00:00
}, []task{
2020-07-16 23:26:55 +00:00
sched("pc1", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskStarted("pc1"),
2020-07-16 23:26:55 +00:00
sched("pc2", "fred", 8, sealtasks.TTPreCommit2),
2020-07-16 21:41:04 +00:00
taskNotScheduled("pc2"),
taskDone("pc1"),
taskDone("pc2"),
}))
t.Run("pc2-block-pc1", testFunc([]workerSpec{
2020-07-16 21:41:15 +00:00
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
2020-07-16 21:41:04 +00:00
}, []task{
2020-07-16 23:26:55 +00:00
sched("pc2", "fred", 8, sealtasks.TTPreCommit2),
2020-07-16 21:41:04 +00:00
taskStarted("pc2"),
2020-07-16 23:26:55 +00:00
sched("pc1", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskNotScheduled("pc1"),
taskDone("pc2"),
taskDone("pc1"),
}))
t.Run("pc1-batching", testFunc([]workerSpec{
2020-07-16 21:41:15 +00:00
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}},
2020-07-16 21:41:04 +00:00
}, []task{
2020-07-16 23:26:55 +00:00
sched("t1", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskStarted("t1"),
2020-07-16 23:26:55 +00:00
sched("t2", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskStarted("t2"),
// with worker settings, we can only run 2 parallel PC1s
// start 2 more to fill fetch buffer
2020-07-16 23:26:55 +00:00
sched("t3", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskNotScheduled("t3"),
2020-07-16 23:26:55 +00:00
sched("t4", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 21:41:04 +00:00
taskNotScheduled("t4"),
taskDone("t1"),
taskDone("t2"),
taskStarted("t3"),
taskStarted("t4"),
taskDone("t3"),
taskDone("t4"),
}))
2020-07-16 23:26:55 +00:00
twoPC1 := func(prefix string, sid abi.SectorNumber, schedAssert func(name string) task) task {
2020-07-16 21:41:04 +00:00
return multTask(
2020-07-16 23:26:55 +00:00
sched(prefix+"-a", "fred", sid, sealtasks.TTPreCommit1),
2020-07-16 21:41:15 +00:00
schedAssert(prefix+"-a"),
2020-07-16 21:41:04 +00:00
2020-07-16 23:32:49 +00:00
sched(prefix+"-b", "fred", sid+1, sealtasks.TTPreCommit1),
2020-07-16 21:41:15 +00:00
schedAssert(prefix+"-b"),
)
2020-07-16 21:41:04 +00:00
}
2020-07-16 23:26:55 +00:00
twoPC1Act := func(prefix string, schedAssert func(name string) task) task {
2020-07-16 21:41:04 +00:00
return multTask(
2020-07-16 23:26:55 +00:00
schedAssert(prefix+"-a"),
schedAssert(prefix+"-b"),
2020-07-16 21:41:15 +00:00
)
2020-07-16 21:41:04 +00:00
}
2020-07-16 23:32:49 +00:00
// run this one a bunch of times, it had a very annoying tendency to fail randomly
for i := 0; i < 40; i++ {
2020-07-16 23:26:55 +00:00
t.Run("pc1-pc2-prio", testFunc([]workerSpec{
{name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}},
}, []task{
2020-07-16 23:32:49 +00:00
// fill queues
2020-07-16 23:26:55 +00:00
twoPC1("w0", 0, taskStarted),
twoPC1("w1", 2, taskNotScheduled),
2020-07-16 21:41:04 +00:00
2020-07-16 23:26:55 +00:00
// windowed
2020-07-16 21:41:04 +00:00
2020-07-16 23:32:49 +00:00
sched("t1", "fred", 8, sealtasks.TTPreCommit1),
2020-07-16 23:26:55 +00:00
taskNotScheduled("t1"),
2020-07-16 21:41:04 +00:00
2020-07-16 23:32:49 +00:00
sched("t2", "fred", 9, sealtasks.TTPreCommit1),
2020-07-16 23:26:55 +00:00
taskNotScheduled("t2"),
2020-07-16 21:41:04 +00:00
2020-07-16 23:32:49 +00:00
sched("t3", "fred", 10, sealtasks.TTPreCommit2),
2020-07-16 23:26:55 +00:00
taskNotScheduled("t3"),
2020-07-16 21:41:04 +00:00
2020-07-16 23:26:55 +00:00
twoPC1Act("w0", taskDone),
twoPC1Act("w1", taskStarted),
2020-07-16 21:41:04 +00:00
2020-07-16 23:26:55 +00:00
twoPC1Act("w1", taskDone),
2020-07-16 21:41:04 +00:00
2020-07-16 23:26:55 +00:00
taskStarted("t3"),
taskNotScheduled("t1"),
taskNotScheduled("t2"),
taskDone("t3"),
taskStarted("t1"),
taskStarted("t2"),
taskDone("t1"),
taskDone("t2"),
}))
}
2020-07-16 21:41:04 +00:00
}