package sealer import ( "context" "sync" "time" "github.com/google/uuid" "github.com/ipfs/go-cid" "go.opencensus.io/stats" "go.opencensus.io/tag" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/storage/sealer/sealtasks" "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type trackedWork struct { job storiface.WorkerJob worker storiface.WorkerID workerHostname string } type workTracker struct { lk sync.Mutex done map[storiface.CallID]struct{} running map[storiface.CallID]trackedWork prepared map[uuid.UUID]trackedWork // TODO: done, aggregate stats, queue stats, scheduler feedback } func (wt *workTracker) onDone(ctx context.Context, callID storiface.CallID) { wt.lk.Lock() defer wt.lk.Unlock() t, ok := wt.running[callID] if !ok { wt.done[callID] = struct{}{} stats.Record(ctx, metrics.WorkerUntrackedCallsReturned.M(1)) return } took := metrics.SinceInMilliseconds(t.job.Start) ctx, _ = tag.New( ctx, tag.Upsert(metrics.TaskType, string(t.job.Task)), tag.Upsert(metrics.WorkerHostname, t.workerHostname), ) stats.Record(ctx, metrics.WorkerCallsReturnedCount.M(1), metrics.WorkerCallsReturnedDuration.M(took)) delete(wt.running, callID) } func (wt *workTracker) track(ctx context.Context, ready chan struct{}, wid storiface.WorkerID, wi storiface.WorkerInfo, sid storiface.SectorRef, task sealtasks.TaskType, cb func() (storiface.CallID, error)) (storiface.CallID, error) { tracked := func(rw int, callID storiface.CallID) trackedWork { return trackedWork{ job: storiface.WorkerJob{ ID: callID, Sector: sid.ID, Task: task, Start: time.Now(), RunWait: rw, }, worker: wid, workerHostname: wi.Hostname, } } wt.lk.Lock() defer wt.lk.Unlock() select { case <-ready: case <-ctx.Done(): return storiface.UndefCall, ctx.Err() default: prepID := uuid.New() wt.prepared[prepID] = tracked(storiface.RWPrepared, storiface.UndefCall) wt.lk.Unlock() select { case <-ready: case <-ctx.Done(): wt.lk.Lock() delete(wt.prepared, prepID) return storiface.UndefCall, ctx.Err() } wt.lk.Lock() delete(wt.prepared, prepID) } wt.lk.Unlock() callID, err := cb() wt.lk.Lock() if err != nil { return callID, err } _, done := wt.done[callID] if done { delete(wt.done, callID) return callID, err } wt.running[callID] = tracked(storiface.RWRunning, callID) ctx, _ = tag.New( ctx, tag.Upsert(metrics.TaskType, string(task)), tag.Upsert(metrics.WorkerHostname, wi.Hostname), ) stats.Record(ctx, metrics.WorkerCallsStarted.M(1)) return callID, err } func (wt *workTracker) worker(wid storiface.WorkerID, wi storiface.WorkerInfo, w Worker) *trackedWorker { return &trackedWorker{ Worker: w, wid: wid, workerInfo: wi, execute: make(chan struct{}), tracker: wt, } } func (wt *workTracker) Running() ([]trackedWork, []trackedWork) { wt.lk.Lock() defer wt.lk.Unlock() running := make([]trackedWork, 0, len(wt.running)) for _, job := range wt.running { running = append(running, job) } prepared := make([]trackedWork, 0, len(wt.prepared)) for _, job := range wt.prepared { prepared = append(prepared, job) } return running, prepared } type trackedWorker struct { Worker wid storiface.WorkerID workerInfo storiface.WorkerInfo execute chan struct{} // channel blocking execution in case we're waiting for resources but the task is ready to execute tracker *workTracker } func (t *trackedWorker) start() { close(t.execute) } func (t *trackedWorker) SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTPreCommit1, func() (storiface.CallID, error) { return t.Worker.SealPreCommit1(ctx, sector, ticket, pieces) }) } func (t *trackedWorker) SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTPreCommit2, func() (storiface.CallID, error) { return t.Worker.SealPreCommit2(ctx, sector, pc1o) }) } func (t *trackedWorker) SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTCommit1, func() (storiface.CallID, error) { return t.Worker.SealCommit1(ctx, sector, ticket, seed, pieces, cids) }) } func (t *trackedWorker) SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTCommit2, func() (storiface.CallID, error) { return t.Worker.SealCommit2(ctx, sector, c1o) }) } func (t *trackedWorker) FinalizeSector(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTFinalize, func() (storiface.CallID, error) { return t.Worker.FinalizeSector(ctx, sector) }) } func (t *trackedWorker) ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTFinalizeUnsealed, func() (storiface.CallID, error) { return t.Worker.ReleaseUnsealed(ctx, sector, keepUnsealed) }) } func (t *trackedWorker) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, storiface.NoSectorRef, sealtasks.TTDataCid, func() (storiface.CallID, error) { return t.Worker.DataCid(ctx, pieceSize, pieceData) }) } func (t *trackedWorker) AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTAddPiece, func() (storiface.CallID, error) { return t.Worker.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData) }) } func (t *trackedWorker) Fetch(ctx context.Context, s storiface.SectorRef, ft storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, s, sealtasks.TTFetch, func() (storiface.CallID, error) { return t.Worker.Fetch(ctx, s, ft, ptype, am) }) } func (t *trackedWorker) UnsealPiece(ctx context.Context, id storiface.SectorRef, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, id, sealtasks.TTUnseal, func() (storiface.CallID, error) { return t.Worker.UnsealPiece(ctx, id, index, size, randomness, cid) }) } func (t *trackedWorker) ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTReplicaUpdate, func() (storiface.CallID, error) { return t.Worker.ReplicaUpdate(ctx, sector, pieces) }) } func (t *trackedWorker) ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTProveReplicaUpdate1, func() (storiface.CallID, error) { return t.Worker.ProveReplicaUpdate1(ctx, sector, sectorKey, newSealed, newUnsealed) }) } func (t *trackedWorker) ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTProveReplicaUpdate2, func() (storiface.CallID, error) { return t.Worker.ProveReplicaUpdate2(ctx, sector, sectorKey, newSealed, newUnsealed, vanillaProofs) }) } func (t *trackedWorker) FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) { return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTFinalizeReplicaUpdate, func() (storiface.CallID, error) { return t.Worker.FinalizeReplicaUpdate(ctx, sector) }) } var _ Worker = &trackedWorker{}