lotus/storage/sealer/worker_local.go

910 lines
28 KiB
Go
Raw Normal View History

package sealer
2020-03-23 11:40:02 +00:00
import (
"context"
"encoding/json"
2020-03-23 11:40:02 +00:00
"io"
"os"
2020-09-14 07:44:55 +00:00
"reflect"
"runtime"
2020-09-22 16:36:44 +00:00
"sync"
2020-10-30 17:01:37 +00:00
"sync/atomic"
2020-09-22 22:10:36 +00:00
"time"
2020-03-23 11:40:02 +00:00
"github.com/elastic/go-sysinfo"
2020-09-06 16:47:16 +00:00
"github.com/google/uuid"
"github.com/hashicorp/go-multierror"
2020-05-14 15:35:38 +00:00
"github.com/ipfs/go-cid"
2020-03-23 11:40:02 +00:00
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
2022-06-14 15:00:51 +00:00
"github.com/filecoin-project/go-state-types/proof"
2020-09-14 07:44:55 +00:00
"github.com/filecoin-project/go-statestore"
2020-03-23 11:40:02 +00:00
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
2020-03-23 11:40:02 +00:00
)
var pathTypes = []storiface.SectorFileType{storiface.FTUnsealed, storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache}
2020-03-23 11:40:02 +00:00
type WorkerConfig struct {
TaskTypes []sealtasks.TaskType
NoSwap bool
// os.Hostname if not set
Name string
// IgnoreResourceFiltering enables task distribution to happen on this
// worker regardless of its currently available resources. Used in testing
// with the local worker.
IgnoreResourceFiltering bool
MaxParallelChallengeReads int // 0 = no limit
ChallengeReadTimeout time.Duration // 0 = no timeout
2020-03-23 11:40:02 +00:00
}
2020-10-28 13:29:17 +00:00
// used do provide custom proofs impl (mostly used in testing)
2023-08-18 15:13:57 +00:00
type ExecutorFunc func(w *LocalWorker) (storiface.Storage, error)
2021-11-29 14:14:57 +00:00
type EnvFunc func(string) (string, bool)
2020-10-28 13:29:17 +00:00
2020-03-23 11:40:02 +00:00
type LocalWorker struct {
storage paths.Store
localStore *paths.Local
sindex paths.SectorIndex
2020-09-06 16:47:16 +00:00
ret storiface.WorkerReturn
2020-10-28 13:29:17 +00:00
executor ExecutorFunc
noSwap bool
2021-11-29 14:14:57 +00:00
envLookup EnvFunc
2020-03-23 11:40:02 +00:00
name string
// see equivalent field on WorkerConfig.
ignoreResources bool
ct *workerCallTracker
2020-03-23 11:40:02 +00:00
acceptTasks map[sealtasks.TaskType]struct{}
2020-09-22 16:36:44 +00:00
running sync.WaitGroup
taskLk sync.Mutex
challengeThrottle chan struct{}
challengeReadTimeout time.Duration
2020-10-30 17:01:37 +00:00
session uuid.UUID
testDisable int64
closing chan struct{}
2020-03-23 11:40:02 +00:00
}
2023-08-18 15:13:57 +00:00
func NewLocalWorkerWithExecutor(executor ExecutorFunc, wcfg WorkerConfig, envLookup EnvFunc, store paths.Store, local *paths.Local, sindex paths.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker {
2020-03-23 11:40:02 +00:00
acceptTasks := map[sealtasks.TaskType]struct{}{}
for _, taskType := range wcfg.TaskTypes {
acceptTasks[taskType] = struct{}{}
}
w := &LocalWorker{
2020-03-23 11:40:02 +00:00
storage: store,
localStore: local,
sindex: sindex,
2020-09-07 14:35:54 +00:00
ret: ret,
name: wcfg.Name,
2020-03-23 11:40:02 +00:00
ct: &workerCallTracker{
2020-09-14 07:44:55 +00:00
st: cst,
},
acceptTasks: acceptTasks,
executor: executor,
noSwap: wcfg.NoSwap,
envLookup: envLookup,
ignoreResources: wcfg.IgnoreResourceFiltering,
challengeReadTimeout: wcfg.ChallengeReadTimeout,
session: uuid.New(),
closing: make(chan struct{}),
}
if w.name == "" {
var err error
w.name, err = os.Hostname()
if err != nil {
panic(err)
}
}
if wcfg.MaxParallelChallengeReads > 0 {
w.challengeThrottle = make(chan struct{}, wcfg.MaxParallelChallengeReads)
2020-03-23 11:40:02 +00:00
}
if w.executor == nil {
2023-08-18 15:13:57 +00:00
w.executor = FFIExec()
}
unfinished, err := w.ct.unfinished()
if err != nil {
log.Errorf("reading unfinished tasks: %+v", err)
return w
}
go func() {
for _, call := range unfinished {
err := storiface.Err(storiface.ErrTempWorkerRestart, xerrors.Errorf("worker [name: %s] restarted", w.name))
// TODO: Handle restarting PC1 once support is merged
if doReturn(context.TODO(), call.RetType, call.ID, ret, nil, err) {
if err := w.ct.onReturned(call.ID); err != nil {
log.Errorf("marking call as returned failed: %s: %+v", call.RetType, err)
}
}
}
}()
return w
}
func NewLocalWorker(wcfg WorkerConfig, store paths.Store, local *paths.Local, sindex paths.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker {
2023-08-18 15:13:57 +00:00
return NewLocalWorkerWithExecutor(nil, wcfg, os.LookupEnv, store, local, sindex, ret, cst)
2020-03-23 11:40:02 +00:00
}
type localWorkerPathProvider struct {
2020-05-26 08:25:29 +00:00
w *LocalWorker
2020-09-06 16:54:00 +00:00
op storiface.AcquireMode
2020-03-23 11:40:02 +00:00
}
func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType) (storiface.SectorPaths, func(), error) {
paths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, existing, allocate, sealing, l.op)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:54:00 +00:00
return storiface.SectorPaths{}, nil, err
2020-03-23 11:40:02 +00:00
}
releaseStorage, err := l.w.localStore.Reserve(ctx, sector, allocate, storageIDs, storiface.FSOverheadSeal)
2020-07-06 16:36:44 +00:00
if err != nil {
2020-09-06 16:54:00 +00:00
return storiface.SectorPaths{}, nil, xerrors.Errorf("reserving storage space: %w", err)
2020-07-06 16:36:44 +00:00
}
2020-03-23 11:40:02 +00:00
log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, paths)
return paths, func() {
2020-07-06 16:36:44 +00:00
releaseStorage()
2020-03-23 11:40:02 +00:00
for _, fileType := range pathTypes {
if fileType&allocate == 0 {
continue
}
2020-09-06 16:54:00 +00:00
sid := storiface.PathByType(storageIDs, fileType)
2022-01-18 10:57:04 +00:00
if err := l.w.sindex.StorageDeclareSector(ctx, storiface.ID(sid), sector.ID, fileType, l.op == storiface.AcquireMove); err != nil {
2020-03-23 11:40:02 +00:00
log.Errorf("declare sector error: %+v", err)
}
}
}, nil
}
2023-08-18 15:13:57 +00:00
func FFIExec(opts ...ffiwrapper.FFIWrapperOpt) func(l *LocalWorker) (storiface.Storage, error) {
return func(l *LocalWorker) (storiface.Storage, error) {
return ffiwrapper.New(&localWorkerPathProvider{w: l}, opts...)
}
2023-09-21 15:37:02 +00:00
}
2023-08-18 15:13:57 +00:00
func (l *localWorkerPathProvider) AcquireSectorCopy(ctx context.Context, id storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error) {
return (&localWorkerPathProvider{w: l.w, op: storiface.AcquireCopy}).AcquireSector(ctx, id, existing, allocate, ptype)
2020-03-23 11:40:02 +00:00
}
type ReturnType string
2020-09-14 07:44:55 +00:00
const (
2022-04-26 16:22:52 +00:00
DataCid ReturnType = "DataCid"
2022-02-02 20:23:35 +00:00
AddPiece ReturnType = "AddPiece"
SealPreCommit1 ReturnType = "SealPreCommit1"
SealPreCommit2 ReturnType = "SealPreCommit2"
SealCommit1 ReturnType = "SealCommit1"
SealCommit2 ReturnType = "SealCommit2"
FinalizeSector ReturnType = "FinalizeSector"
FinalizeReplicaUpdate ReturnType = "FinalizeReplicaUpdate"
ReplicaUpdate ReturnType = "ReplicaUpdate"
ProveReplicaUpdate1 ReturnType = "ProveReplicaUpdate1"
ProveReplicaUpdate2 ReturnType = "ProveReplicaUpdate2"
GenerateSectorKey ReturnType = "GenerateSectorKey"
ReleaseUnsealed ReturnType = "ReleaseUnsealed"
MoveStorage ReturnType = "MoveStorage"
UnsealPiece ReturnType = "UnsealPiece"
DownloadSector ReturnType = "DownloadSector"
2022-02-02 20:23:35 +00:00
Fetch ReturnType = "Fetch"
)
2020-09-14 07:44:55 +00:00
// in: func(WorkerReturn, context.Context, CallID, err string)
// in: func(WorkerReturn, context.Context, CallID, ret T, err string)
2020-11-17 15:17:45 +00:00
func rfunc(in interface{}) func(context.Context, storiface.CallID, storiface.WorkerReturn, interface{}, *storiface.CallError) error {
2020-09-14 07:44:55 +00:00
rf := reflect.ValueOf(in)
ft := rf.Type()
2020-09-14 18:28:47 +00:00
withRet := ft.NumIn() == 5
2020-09-14 07:44:55 +00:00
2020-11-17 15:17:45 +00:00
return func(ctx context.Context, ci storiface.CallID, wr storiface.WorkerReturn, i interface{}, err *storiface.CallError) error {
2020-09-14 07:44:55 +00:00
rctx := reflect.ValueOf(ctx)
rwr := reflect.ValueOf(wr)
2020-11-17 15:17:45 +00:00
rerr := reflect.ValueOf(err)
2020-09-14 18:28:47 +00:00
rci := reflect.ValueOf(ci)
2020-09-14 07:44:55 +00:00
var ro []reflect.Value
if withRet {
ret := reflect.ValueOf(i)
if i == nil {
ret = reflect.Zero(rf.Type().In(3))
}
ro = rf.Call([]reflect.Value{rwr, rctx, rci, ret, rerr})
2020-09-14 07:44:55 +00:00
} else {
2020-09-14 18:28:47 +00:00
ro = rf.Call([]reflect.Value{rwr, rctx, rci, rerr})
2020-09-14 07:44:55 +00:00
}
2020-09-14 18:28:47 +00:00
if !ro[0].IsNil() {
return ro[0].Interface().(error)
}
return nil
2020-09-14 07:44:55 +00:00
}
}
2020-11-17 15:17:45 +00:00
var returnFunc = map[ReturnType]func(context.Context, storiface.CallID, storiface.WorkerReturn, interface{}, *storiface.CallError) error{
2022-04-26 16:22:52 +00:00
DataCid: rfunc(storiface.WorkerReturn.ReturnDataCid),
AddPiece: rfunc(storiface.WorkerReturn.ReturnAddPiece),
SealPreCommit1: rfunc(storiface.WorkerReturn.ReturnSealPreCommit1),
SealPreCommit2: rfunc(storiface.WorkerReturn.ReturnSealPreCommit2),
SealCommit1: rfunc(storiface.WorkerReturn.ReturnSealCommit1),
SealCommit2: rfunc(storiface.WorkerReturn.ReturnSealCommit2),
FinalizeSector: rfunc(storiface.WorkerReturn.ReturnFinalizeSector),
ReleaseUnsealed: rfunc(storiface.WorkerReturn.ReturnReleaseUnsealed),
ReplicaUpdate: rfunc(storiface.WorkerReturn.ReturnReplicaUpdate),
ProveReplicaUpdate1: rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate1),
ProveReplicaUpdate2: rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate2),
GenerateSectorKey: rfunc(storiface.WorkerReturn.ReturnGenerateSectorKeyFromData),
FinalizeReplicaUpdate: rfunc(storiface.WorkerReturn.ReturnFinalizeReplicaUpdate),
MoveStorage: rfunc(storiface.WorkerReturn.ReturnMoveStorage),
UnsealPiece: rfunc(storiface.WorkerReturn.ReturnUnsealPiece),
DownloadSector: rfunc(storiface.WorkerReturn.ReturnDownloadSector),
Fetch: rfunc(storiface.WorkerReturn.ReturnFetch),
2020-09-14 07:44:55 +00:00
}
func (l *LocalWorker) asyncCall(ctx context.Context, sector storiface.SectorRef, rt ReturnType, work func(ctx context.Context, ci storiface.CallID) (interface{}, error)) (storiface.CallID, error) {
2020-09-06 16:47:16 +00:00
ci := storiface.CallID{
Sector: sector.ID,
2020-09-06 16:47:16 +00:00
ID: uuid.New(),
}
if err := l.ct.onStart(ci, rt); err != nil {
2020-09-14 07:44:55 +00:00
log.Errorf("tracking call (start): %+v", err)
}
2020-09-22 16:36:44 +00:00
l.running.Add(1)
2020-09-14 07:44:55 +00:00
go func() {
2020-09-22 16:36:44 +00:00
defer l.running.Done()
ctx := &wctx{
2020-09-22 22:10:36 +00:00
vals: ctx,
closing: l.closing,
}
res, err := work(ctx, ci)
if err != nil {
rb, err := json.Marshal(res)
if err != nil {
log.Errorf("tracking call (marshaling results): %+v", err)
} else {
if err := l.ct.onDone(ci, rb); err != nil {
log.Errorf("tracking call (done): %+v", err)
}
}
}
2021-12-23 09:44:43 +00:00
if err != nil {
err = xerrors.Errorf("%w [name: %s]", err, l.name)
2021-12-23 09:44:43 +00:00
}
2020-11-17 15:17:45 +00:00
if doReturn(ctx, rt, ci, l.ret, res, toCallError(err)) {
if err := l.ct.onReturned(ci); err != nil {
log.Errorf("tracking call (done): %+v", err)
}
}
}()
return ci, nil
}
2020-11-17 15:17:45 +00:00
func toCallError(err error) *storiface.CallError {
var serr *storiface.CallError
if err != nil && !xerrors.As(err, &serr) {
2020-11-17 15:28:41 +00:00
serr = storiface.Err(storiface.ErrUnknown, err)
2020-11-17 15:17:45 +00:00
}
return serr
}
// doReturn tries to send the result to manager, returns true if successful
2020-11-17 15:17:45 +00:00
func doReturn(ctx context.Context, rt ReturnType, ci storiface.CallID, ret storiface.WorkerReturn, res interface{}, rerr *storiface.CallError) bool {
for {
err := returnFunc[rt](ctx, ci, ret, res, rerr)
if err == nil {
break
2020-09-14 07:44:55 +00:00
}
log.Errorf("return error, will retry in 5s: %s: %+v", rt, err)
select {
case <-time.After(5 * time.Second):
case <-ctx.Done():
log.Errorf("failed to return results: %s", ctx.Err())
// fine to just return, worker is most likely shutting down, and
// we didn't mark the result as returned yet, so we'll try to
// re-submit it on restart
return false
}
}
2020-09-06 16:47:16 +00:00
return true
2020-09-06 16:47:16 +00:00
}
func (l *LocalWorker) NewSector(ctx context.Context, sector storiface.SectorRef) error {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-03-23 11:40:02 +00:00
if err != nil {
return err
}
return sb.NewSector(ctx, sector)
}
func (l *LocalWorker) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2022-04-26 16:22:52 +00:00
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, storiface.NoSectorRef, DataCid, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2022-04-26 16:22:52 +00:00
return sb.DataCid(ctx, pieceSize, pieceData)
})
}
func (l *LocalWorker) AddPiece(ctx context.Context, sector storiface.SectorRef, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:47:16 +00:00
return storiface.UndefCall, err
2020-03-23 11:40:02 +00:00
}
return l.asyncCall(ctx, sector, AddPiece, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2020-09-14 07:44:55 +00:00
return sb.AddPiece(ctx, sector, epcs, sz, r)
2020-09-06 16:47:16 +00:00
})
2020-04-27 12:55:37 +00:00
}
func (l *LocalWorker) Fetch(ctx context.Context, sector storiface.SectorRef, fileType storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) (storiface.CallID, error) {
return l.asyncCall(ctx, sector, Fetch, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2020-09-06 16:54:00 +00:00
_, done, err := (&localWorkerPathProvider{w: l, op: am}).AcquireSector(ctx, sector, fileType, storiface.FTNone, ptype)
2020-09-06 16:47:16 +00:00
if err == nil {
done()
2020-05-13 18:45:14 +00:00
}
2020-09-14 07:44:55 +00:00
return nil, err
2020-09-06 16:47:16 +00:00
})
}
2020-05-13 18:45:14 +00:00
func (l *LocalWorker) SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) {
return l.asyncCall(ctx, sector, SealPreCommit1, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2020-03-23 11:40:02 +00:00
2020-09-06 16:47:16 +00:00
{
// cleanup previous failed attempts if they exist
if err := l.storage.Remove(ctx, sector.ID, storiface.FTSealed, true, nil); err != nil {
2020-09-14 07:44:55 +00:00
return nil, xerrors.Errorf("cleaning up sealed data: %w", err)
2020-09-06 16:47:16 +00:00
}
if err := l.storage.Remove(ctx, sector.ID, storiface.FTCache, true, nil); err != nil {
2020-09-14 07:44:55 +00:00
return nil, xerrors.Errorf("cleaning up cache data: %w", err)
2020-09-06 16:47:16 +00:00
}
}
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-09-06 16:47:16 +00:00
if err != nil {
2020-09-14 07:44:55 +00:00
return nil, err
2020-09-06 16:47:16 +00:00
}
2020-09-14 07:44:55 +00:00
return sb.SealPreCommit1(ctx, sector, ticket, pieces)
2020-09-06 16:47:16 +00:00
})
2020-03-23 11:40:02 +00:00
}
func (l *LocalWorker) SealPreCommit2(ctx context.Context, sector storiface.SectorRef, phase1Out storiface.PreCommit1Out) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:47:16 +00:00
return storiface.UndefCall, err
2020-03-23 11:40:02 +00:00
}
return l.asyncCall(ctx, sector, SealPreCommit2, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2020-09-14 07:44:55 +00:00
return sb.SealPreCommit2(ctx, sector, phase1Out)
2020-09-06 16:47:16 +00:00
})
2020-03-23 11:40:02 +00:00
}
func (l *LocalWorker) SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:47:16 +00:00
return storiface.UndefCall, err
2020-03-23 11:40:02 +00:00
}
return l.asyncCall(ctx, sector, SealCommit1, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2020-09-14 07:44:55 +00:00
return sb.SealCommit1(ctx, sector, ticket, seed, pieces, cids)
2020-09-06 16:47:16 +00:00
})
2020-03-23 11:40:02 +00:00
}
func (l *LocalWorker) SealCommit2(ctx context.Context, sector storiface.SectorRef, phase1Out storiface.Commit1Out) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:47:16 +00:00
return storiface.UndefCall, err
2020-03-23 11:40:02 +00:00
}
return l.asyncCall(ctx, sector, SealCommit2, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2020-09-14 07:44:55 +00:00
return sb.SealCommit2(ctx, sector, phase1Out)
2020-09-06 16:47:16 +00:00
})
2020-03-23 11:40:02 +00:00
}
func (l *LocalWorker) ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, ReplicaUpdate, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
sealerOut, err := sb.ReplicaUpdate(ctx, sector, pieces)
return sealerOut, err
})
}
func (l *LocalWorker) ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, ProveReplicaUpdate1, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
return sb.ProveReplicaUpdate1(ctx, sector, sectorKey, newSealed, newUnsealed)
})
}
func (l *LocalWorker) ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, ProveReplicaUpdate2, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
return sb.ProveReplicaUpdate2(ctx, sector, sectorKey, newSealed, newUnsealed, vanillaProofs)
})
}
func (l *LocalWorker) GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, GenerateSectorKey, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
return nil, sb.GenerateSectorKeyFromData(ctx, sector, commD)
})
}
func (l *LocalWorker) FinalizeSector(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2020-03-23 11:40:02 +00:00
if err != nil {
2020-09-06 16:47:16 +00:00
return storiface.UndefCall, err
2020-03-23 11:40:02 +00:00
}
return l.asyncCall(ctx, sector, FinalizeSector, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
return nil, sb.FinalizeSector(ctx, sector)
})
}
func (l *LocalWorker) FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, FinalizeReplicaUpdate, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
return nil, sb.FinalizeReplicaUpdate(ctx, sector)
2020-09-06 16:47:16 +00:00
})
2020-06-03 21:44:59 +00:00
}
func (l *LocalWorker) ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
2022-02-02 20:23:35 +00:00
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, ReleaseUnsealed, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
if err := sb.ReleaseUnsealed(ctx, sector, keepUnsealed); err != nil {
2022-02-02 20:23:35 +00:00
return nil, xerrors.Errorf("finalizing sector: %w", err)
2020-09-06 16:47:16 +00:00
}
2020-09-06 16:47:16 +00:00
if len(keepUnsealed) == 0 {
if err := l.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true, nil); err != nil {
2020-09-14 07:44:55 +00:00
return nil, xerrors.Errorf("removing unsealed data: %w", err)
2020-09-06 16:47:16 +00:00
}
2020-07-03 19:52:31 +00:00
}
2020-09-14 07:44:55 +00:00
return nil, err
2020-09-06 16:47:16 +00:00
})
2020-06-03 21:44:59 +00:00
}
func (l *LocalWorker) Remove(ctx context.Context, sector abi.SectorID) error {
var err error
if rerr := l.storage.Remove(ctx, sector, storiface.FTSealed, true, nil); rerr != nil {
err = multierror.Append(err, xerrors.Errorf("removing sector (sealed): %w", rerr))
}
if rerr := l.storage.Remove(ctx, sector, storiface.FTCache, true, nil); rerr != nil {
err = multierror.Append(err, xerrors.Errorf("removing sector (cache): %w", rerr))
}
if rerr := l.storage.Remove(ctx, sector, storiface.FTUnsealed, true, nil); rerr != nil {
err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr))
}
return err
}
func (l *LocalWorker) MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) {
return l.asyncCall(ctx, sector, MoveStorage, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
if err := l.storage.MoveStorage(ctx, sector, types); err != nil {
return nil, xerrors.Errorf("move to storage: %w", err)
}
for _, fileType := range storiface.PathTypes {
if fileType&types == 0 {
continue
}
if err := l.storage.RemoveCopies(ctx, sector.ID, fileType); err != nil {
return nil, xerrors.Errorf("rm copies (t:%s, s:%v): %w", fileType, sector, err)
}
}
return nil, nil
2020-09-06 16:47:16 +00:00
})
2020-03-23 11:40:02 +00:00
}
func (l *LocalWorker) UnsealPiece(ctx context.Context, sector storiface.SectorRef, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
2020-09-06 16:47:16 +00:00
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, UnsealPiece, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
2021-05-21 09:46:20 +00:00
log.Debugf("worker will unseal piece now, sector=%+v", sector.ID)
2020-09-06 16:47:16 +00:00
if err = sb.UnsealPiece(ctx, sector, index, size, randomness, cid); err != nil {
2020-09-14 07:44:55 +00:00
return nil, xerrors.Errorf("unsealing sector: %w", err)
2020-09-06 16:47:16 +00:00
}
2020-05-29 16:57:44 +00:00
2023-09-21 15:37:02 +00:00
// note: the unsealed file is moved to long-term storage in Manager.SectorsUnsealPiece
2023-09-21 15:37:02 +00:00
storageTypes := []storiface.SectorFileType{storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache}
for _, fileType := range storageTypes {
if err = l.storage.RemoveCopies(ctx, sector.ID, fileType); err != nil {
return nil, xerrors.Errorf("removing source data: %w", err)
}
2020-09-06 16:47:16 +00:00
}
2020-09-14 07:44:55 +00:00
2023-09-21 15:37:02 +00:00
log.Debugf("unsealed piece, sector=%+v", sector.ID)
2021-05-21 13:32:37 +00:00
2020-09-14 07:44:55 +00:00
return nil, nil
2020-09-06 16:47:16 +00:00
})
2020-05-14 15:35:38 +00:00
}
2022-09-16 21:45:23 +00:00
func (l *LocalWorker) DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return storiface.UndefCall, err
}
return l.asyncCall(ctx, sector, DownloadSector, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
return nil, sb.DownloadSectorData(ctx, sector, finalized, src)
})
}
2022-01-14 13:11:04 +00:00
func (l *LocalWorker) GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
return nil, err
}
// don't throttle winningPoSt
// * Always want it done asap
// * It's usually just one sector
2022-01-14 13:11:04 +00:00
var wg sync.WaitGroup
wg.Add(len(sectors))
2022-01-14 13:11:04 +00:00
vproofs := make([][]byte, len(sectors))
var rerr error
2022-01-14 13:11:04 +00:00
for i, s := range sectors {
go func(i int, s storiface.PostSectorChallenge) {
defer wg.Done()
2022-01-14 13:11:04 +00:00
if l.challengeReadTimeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, l.challengeReadTimeout)
defer cancel()
}
2022-01-14 13:11:04 +00:00
vanilla, err := l.storage.GenerateSingleVanillaProof(ctx, mid, s, ppt)
if err != nil {
2022-01-14 13:11:04 +00:00
rerr = multierror.Append(rerr, xerrors.Errorf("get winning sector:%d,vanila failed: %w", s.SectorNumber, err))
return
}
if vanilla == nil {
2022-01-14 13:11:04 +00:00
rerr = multierror.Append(rerr, xerrors.Errorf("get winning sector:%d,vanila is nil", s.SectorNumber))
}
2022-01-14 13:11:04 +00:00
vproofs[i] = vanilla
}(i, s)
}
wg.Wait()
if rerr != nil {
return nil, rerr
}
2022-01-14 13:11:04 +00:00
return sb.GenerateWinningPoStWithVanilla(ctx, ppt, mid, randomness, vproofs)
}
2022-01-14 13:11:04 +00:00
func (l *LocalWorker) GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) {
2023-12-01 18:47:08 +00:00
return l.GenerateWindowPoStAdv(ctx, ppt, mid, sectors, partitionIdx, randomness, false)
}
func (l *LocalWorker) GenerateWindowPoStAdv(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness, allowSkip bool) (storiface.WindowPoStResult, error) {
2023-08-18 15:13:57 +00:00
sb, err := l.executor(l)
if err != nil {
2022-01-14 13:11:04 +00:00
return storiface.WindowPoStResult{}, err
}
var slk sync.Mutex
var skipped []abi.SectorID
var wg sync.WaitGroup
2022-01-14 13:11:04 +00:00
wg.Add(len(sectors))
vproofs := make([][]byte, len(sectors))
2022-01-14 13:11:04 +00:00
for i, s := range sectors {
if l.challengeThrottle != nil {
select {
2022-03-25 20:07:47 +00:00
case l.challengeThrottle <- struct{}{}:
case <-ctx.Done():
return storiface.WindowPoStResult{}, xerrors.Errorf("context error waiting on challengeThrottle %w", err)
}
}
2022-01-14 13:11:04 +00:00
go func(i int, s storiface.PostSectorChallenge) {
defer wg.Done()
defer func() {
if l.challengeThrottle != nil {
2022-03-25 20:07:47 +00:00
<-l.challengeThrottle
}
}()
if l.challengeReadTimeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, l.challengeReadTimeout)
defer cancel()
}
2022-01-14 13:11:04 +00:00
vanilla, err := l.storage.GenerateSingleVanillaProof(ctx, mid, s, ppt)
2022-01-31 20:53:25 +00:00
slk.Lock()
defer slk.Unlock()
if err != nil || vanilla == nil {
skipped = append(skipped, abi.SectorID{
Miner: mid,
2022-01-14 13:11:04 +00:00
Number: s.SectorNumber,
})
log.Errorf("reading PoSt challenge for sector %d, vlen:%d, err: %s", s.SectorNumber, len(vanilla), err)
return
}
2022-01-31 20:53:25 +00:00
vproofs[i] = vanilla
2022-01-14 13:11:04 +00:00
}(i, s)
}
wg.Wait()
2022-01-14 13:11:04 +00:00
2023-10-27 14:10:55 +00:00
if len(skipped) > 0 && !allowSkip {
// This should happen rarely because before entering GenerateWindowPoSt we check all sectors by reading challenges.
// When it does happen, window post runner logic will just re-check sectors, and retry with newly-discovered-bad sectors skipped
log.Errorf("couldn't read some challenges (skipped %d)", len(skipped))
// note: can't return an error as this in an jsonrpc call
return storiface.WindowPoStResult{Skipped: skipped}, nil
2022-01-31 20:53:25 +00:00
}
// compact skipped sectors
var skippedSoFar int
for i := range vproofs {
if len(vproofs[i]) == 0 {
skippedSoFar++
continue
}
if skippedSoFar > 0 {
vproofs[i-skippedSoFar] = vproofs[i]
}
}
vproofs = vproofs[:len(vproofs)-skippedSoFar]
// compute the PoSt!
2022-01-14 13:11:04 +00:00
res, err := sb.GenerateWindowPoStWithVanilla(ctx, ppt, mid, randomness, vproofs, partitionIdx)
r := storiface.WindowPoStResult{
2022-01-14 13:11:04 +00:00
PoStProofs: res,
Skipped: skipped,
}
if err != nil {
log.Errorw("generating window PoSt failed", "error", err)
return r, xerrors.Errorf("generate window PoSt with vanilla proofs: %w", err)
}
return r, nil
}
2020-03-23 11:40:02 +00:00
func (l *LocalWorker) TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) {
l.taskLk.Lock()
defer l.taskLk.Unlock()
2020-03-23 11:40:02 +00:00
return l.acceptTasks, nil
}
2020-11-30 22:16:30 +00:00
func (l *LocalWorker) TaskDisable(ctx context.Context, tt sealtasks.TaskType) error {
l.taskLk.Lock()
defer l.taskLk.Unlock()
delete(l.acceptTasks, tt)
return nil
}
2020-11-30 22:16:30 +00:00
func (l *LocalWorker) TaskEnable(ctx context.Context, tt sealtasks.TaskType) error {
l.taskLk.Lock()
defer l.taskLk.Unlock()
l.acceptTasks[tt] = struct{}{}
return nil
}
2022-01-18 10:57:04 +00:00
func (l *LocalWorker) Paths(ctx context.Context) ([]storiface.StoragePath, error) {
2020-03-23 11:40:02 +00:00
return l.localStore.Local(ctx)
}
func (l *LocalWorker) memInfo() (memPhysical, memUsed, memSwap, memSwapUsed uint64, err error) {
h, err := sysinfo.Host()
if err != nil {
return 0, 0, 0, 0, err
}
mem, err := h.Memory()
if err != nil {
return 0, 0, 0, 0, err
}
memPhysical = mem.Total
// mem.Available is memory available without swapping, it is more relevant for this calculation
memUsed = mem.Total - mem.Available
memSwap = mem.VirtualTotal
memSwapUsed = mem.VirtualUsed
if cgMemMax, cgMemUsed, cgSwapMax, cgSwapUsed, err := cgroupV1Mem(); err == nil {
if cgMemMax > 0 && cgMemMax < memPhysical {
memPhysical = cgMemMax
memUsed = cgMemUsed
}
if cgSwapMax > 0 && cgSwapMax < memSwap {
memSwap = cgSwapMax
memSwapUsed = cgSwapUsed
}
}
if cgMemMax, cgMemUsed, cgSwapMax, cgSwapUsed, err := cgroupV2Mem(); err == nil {
if cgMemMax > 0 && cgMemMax < memPhysical {
memPhysical = cgMemMax
memUsed = cgMemUsed
}
if cgSwapMax > 0 && cgSwapMax < memSwap {
memSwap = cgSwapMax
memSwapUsed = cgSwapUsed
}
}
if l.noSwap {
memSwap = 0
memSwapUsed = 0
}
return memPhysical, memUsed, memSwap, memSwapUsed, nil
}
func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) {
2020-03-23 11:40:02 +00:00
gpus, err := ffi.GetGPUDevices()
if err != nil {
log.Errorf("getting gpu devices failed: %+v", err)
}
2023-11-10 02:40:40 +00:00
log.Infow("Detected GPU devices.", "count", len(gpus))
2020-03-23 11:40:02 +00:00
memPhysical, memUsed, memSwap, memSwapUsed, err := l.memInfo()
2020-03-23 11:40:02 +00:00
if err != nil {
return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err)
2020-03-23 11:40:02 +00:00
}
2021-11-29 14:14:57 +00:00
resEnv, err := storiface.ParseResourceEnv(func(key, def string) (string, bool) {
return l.envLookup(key)
2021-11-29 13:42:20 +00:00
})
if err != nil {
return storiface.WorkerInfo{}, xerrors.Errorf("interpreting resource env vars: %w", err)
}
return storiface.WorkerInfo{
Hostname: l.name,
IgnoreResources: l.ignoreResources,
Resources: storiface.WorkerResources{
2021-11-29 13:46:41 +00:00
MemPhysical: memPhysical,
MemUsed: memUsed,
MemSwap: memSwap,
2021-11-29 13:46:41 +00:00
MemSwapUsed: memSwapUsed,
CPUs: uint64(runtime.NumCPU()),
2020-03-23 11:40:02 +00:00
GPUs: gpus,
2021-11-29 13:46:41 +00:00
Resources: resEnv,
2020-03-23 11:40:02 +00:00
},
}, nil
}
func (l *LocalWorker) Session(ctx context.Context) (uuid.UUID, error) {
2020-10-30 17:01:37 +00:00
if atomic.LoadInt64(&l.testDisable) == 1 {
return uuid.UUID{}, xerrors.Errorf("disabled")
}
select {
case <-l.closing:
return ClosedWorkerID, nil
default:
return l.session, nil
}
2020-05-01 18:00:17 +00:00
}
2020-03-24 23:49:45 +00:00
func (l *LocalWorker) Close() error {
close(l.closing)
2020-03-24 23:49:45 +00:00
return nil
}
2022-07-29 22:33:30 +00:00
func (l *LocalWorker) Done() <-chan struct{} {
return l.closing
}
2020-09-22 16:36:44 +00:00
// WaitQuiet blocks as long as there are tasks running
func (l *LocalWorker) WaitQuiet() {
l.running.Wait()
}
2020-09-22 22:10:36 +00:00
type wctx struct {
vals context.Context
closing chan struct{}
}
func (w *wctx) Deadline() (time.Time, bool) {
return time.Time{}, false
}
func (w *wctx) Done() <-chan struct{} {
return w.closing
}
func (w *wctx) Err() error {
select {
case <-w.closing:
return context.Canceled
default:
return nil
}
}
func (w *wctx) Value(key interface{}) interface{} {
return w.vals.Value(key)
}
var _ context.Context = &wctx{}
2020-03-23 11:40:02 +00:00
var _ Worker = &LocalWorker{}