cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Adjust resource table for filecoin-ffi 0.30.1

Browse Source
This commit is contained in:
Łukasz Magiera 2020-06-30 19:26:56 +02:00
parent c1ac986f20
commit cff68a14d8
2 changed files with 14 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
resources.go
View File

@ -20,8 +20,6 @@ func (r Resources) MultiThread() bool {
return r.Threads == -1
}
const MaxCachingOverhead = 32 << 30
var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
sealtasks.TTAddPiece: {
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ // This is probably a bit conservative
@ -68,27 +66,27 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
sealtasks.TTPreCommit1: {
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
MaxMemory: 128 << 30,
MinMemory: 96 << 30,
MinMemory: 112 << 30,
Threads: 1,
BaseMinMemory: 60 << 30,
BaseMinMemory: 10 << 20,
},
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
MaxMemory: 64 << 30,
MinMemory: 48 << 30,
MinMemory: 56 << 30,
Threads: 1,
BaseMinMemory: 30 << 30,
BaseMinMemory: 10 << 20,
},
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
MaxMemory: 3 << 29, // 1.5G
MinMemory: 1 << 30,
MaxMemory: 1 << 30,
MinMemory: 768 << 20,
Threads: 1,
BaseMinMemory: 1 << 30,
BaseMinMemory: 1 << 20,
},
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
MaxMemory: 2 << 10,
@ -195,7 +193,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
},
sealtasks.TTCommit2: {
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
MaxMemory: 260 << 30, // TODO: Confirm
MaxMemory: 190 << 30, // TODO: Confirm
MinMemory: 60 << 30,
Threads: -1,

17
sched.go
View File

@ -250,7 +250,7 @@ func (sh *scheduler) maybeSchedRequest(req *workerRequest) (bool, error) {
}
tried++
if !canHandleRequest(needRes, sh.spt, wid, worker.info.Resources, worker.preparing) {
if !canHandleRequest(needRes, wid, worker.info.Resources, worker.preparing) {
continue
}
@ -316,7 +316,7 @@ func (sh *scheduler) assignWorker(wid WorkerID, w *workerHandle, req *workerRequ
return
}
err = w.active.withResources(sh.spt, wid, w.info.Resources, needRes, &sh.workersLk, func() error {
err = w.active.withResources(wid, w.info.Resources, needRes, &sh.workersLk, func() error {
w.preparing.free(w.info.Resources, needRes)
sh.workersLk.Unlock()
defer sh.workersLk.Lock() // we MUST return locked from this function
@ -350,8 +350,8 @@ func (sh *scheduler) assignWorker(wid WorkerID, w *workerHandle, req *workerRequ
return nil
}
func (a *activeResources) withResources(spt abi.RegisteredSealProof, id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error {
for !canHandleRequest(r, spt, id, wr, a) {
func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error {
for !canHandleRequest(r, id, wr, a) {
if a.cond == nil {
a.cond = sync.NewCond(locker)
}
@ -396,7 +396,7 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) {
a.memUsedMax -= r.MaxMemory
}
func canHandleRequest(needRes Resources, spt abi.RegisteredSealProof, wid WorkerID, res storiface.WorkerResources, active *activeResources) bool {
func canHandleRequest(needRes Resources, wid WorkerID, res storiface.WorkerResources, active *activeResources) bool {
// TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running)
minNeedMem := res.MemReserved + active.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory
@ -406,12 +406,7 @@ func canHandleRequest(needRes Resources, spt abi.RegisteredSealProof, wid Worker
}
maxNeedMem := res.MemReserved + active.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory
if spt == abi.RegisteredSealProof_StackedDrg32GiBV1 {
maxNeedMem += MaxCachingOverhead
}
if spt == abi.RegisteredSealProof_StackedDrg64GiBV1 {
maxNeedMem += MaxCachingOverhead * 2 // ewwrhmwh
}
if maxNeedMem > res.MemSwap+res.MemPhysical {
log.Debugf("sched: not scheduling on worker %d; not enough virtual memory - need: %dM, have %dM", wid, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib)
return false