sectorstorage: Make trySched less very slow

extern/sector-storage/sched.go

@@ -287,9 +287,27 @@ func (sh *scheduler) trySched() {
 
 	sh.workersLk.RLock()
 	defer sh.workersLk.RUnlock()
+	if len(sh.openWindows) == 0 {
+		// nothing to schedule on
+		return
+	}
 
 	// Step 1
-	for sqi := 0; sqi < sh.schedQueue.Len(); sqi++ {
+	concurrency := len(sh.openWindows)
+	throttle := make(chan struct{}, concurrency)
+
+	var wg sync.WaitGroup
+	wg.Add(sh.schedQueue.Len())
+
+	for i := 0; i < sh.schedQueue.Len(); i++ {
+		throttle <- struct{}{}
+
+		go func(sqi int) {
+			defer wg.Done()
+			defer func() {
+				<-throttle
+			}()
+
 			task := (*sh.schedQueue)[sqi]
 			needRes := ResourceTable[task.taskType][sh.spt]
 
@@ -323,7 +341,7 @@ func (sh *scheduler) trySched() {
 			}
 
 			if len(acceptableWindows[sqi]) == 0 {
-			continue
+				return
 			}
 
 			// Pick best worker (shuffle in case some workers are equally as good)
@@ -351,8 +369,11 @@ func (sh *scheduler) trySched() {
 				}
 				return r
 			})
+		}(i)
 	}
 
+	wg.Wait()
+
 	log.Debugf("SCHED windows: %+v", windows)
 	log.Debugf("SCHED Acceptable win: %+v", acceptableWindows)
 

extern/sector-storage/sched_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/specs-actors/actors/abi"
@@ -100,16 +101,18 @@ func (s *schedTestWorker) Paths(ctx context.Context) ([]stores.StoragePath, erro
 	return s.paths, nil
 }
 
-func (s *schedTestWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) {
+var decentWorkerResources = storiface.WorkerResources{
-	return storiface.WorkerInfo{
-		Hostname: s.name,
-		Resources: storiface.WorkerResources{
 	MemPhysical: 128 << 30,
 	MemSwap:     200 << 30,
 	MemReserved: 2 << 30,
 	CPUs:        32,
 	GPUs:        []string{"a GPU"},
-		},
+}
+
+func (s *schedTestWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) {
+	return storiface.WorkerInfo{
+		Hostname:  s.name,
+		Resources: decentWorkerResources,
 	}, nil
 }
 
@@ -451,3 +454,67 @@ func TestSched(t *testing.T) {
 		}))
 	}
 }
+
+type slowishSelector bool
+
+func (s slowishSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.RegisteredSealProof, a *workerHandle) (bool, error) {
+	time.Sleep(200 * time.Microsecond)
+	return bool(s), nil
+}
+
+func (s slowishSelector) Cmp(ctx context.Context, task sealtasks.TaskType, a, b *workerHandle) (bool, error) {
+	time.Sleep(100 * time.Microsecond)
+	return true, nil
+}
+
+var _ WorkerSelector = slowishSelector(true)
+
+func BenchmarkTrySched(b *testing.B) {
+	spt := abi.RegisteredSealProof_StackedDrg32GiBV1
+	logging.SetAllLoggers(logging.LevelInfo)
+	defer logging.SetAllLoggers(logging.LevelDebug)
+	ctx := context.Background()
+
+	test := func(windows, queue int) func(b *testing.B) {
+		return func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				b.StopTimer()
+
+				sched := newScheduler(spt)
+				sched.workers[0] = &workerHandle{
+					w: nil,
+					info: storiface.WorkerInfo{
+						Hostname:  "t",
+						Resources: decentWorkerResources,
+					},
+					preparing: &activeResources{},
+					active:    &activeResources{},
+				}
+
+				for i := 0; i < windows; i++ {
+					sched.openWindows = append(sched.openWindows, &schedWindowRequest{
+						worker: 0,
+						done:   make(chan *schedWindow, 1000),
+					})
+				}
+
+				for i := 0; i < queue; i++ {
+					sched.schedQueue.Push(&workerRequest{
+						taskType: sealtasks.TTCommit2,
+						sel:      slowishSelector(true),
+						ctx:      ctx,
+					})
+				}
+
+				b.StartTimer()
+
+				sched.trySched()
+			}
+		}
+	}
+
+	b.Run("1w-1q", test(1, 1))
+	b.Run("500w-1q", test(500, 1))
+	b.Run("1w-500q", test(1, 500))
+	b.Run("200w-400q", test(200, 400))
+}