398 lines
9.2 KiB
Go
398 lines
9.2 KiB
Go
package sectorstorage
|
|
|
|
import (
|
|
"strconv"
|
|
|
|
"github.com/filecoin-project/go-state-types/abi"
|
|
|
|
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
|
|
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
|
)
|
|
|
|
type Resources struct {
|
|
MinMemory uint64 // What Must be in RAM for decent perf
|
|
MaxMemory uint64 // Memory required (swap + ram)
|
|
|
|
// GPUUtilization specifes the number of GPUs a task can use
|
|
GPUUtilization float64
|
|
|
|
// MaxParallelism specifies the number of CPU cores when GPU is NOT in use
|
|
MaxParallelism int // -1 = multithread
|
|
|
|
// MaxParallelismGPU specifies the number of CPU cores when GPU is in use
|
|
MaxParallelismGPU int // when 0, inherits MaxParallelism
|
|
|
|
BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads)
|
|
}
|
|
|
|
/*
|
|
|
|
Percent of threads to allocate to parallel tasks
|
|
|
|
12 * 0.92 = 11
|
|
16 * 0.92 = 14
|
|
24 * 0.92 = 22
|
|
32 * 0.92 = 29
|
|
64 * 0.92 = 58
|
|
128 * 0.92 = 117
|
|
|
|
*/
|
|
var ParallelNum uint64 = 92
|
|
var ParallelDenom uint64 = 100
|
|
|
|
// TODO: Take NUMA into account
|
|
func (r Resources) Threads(wcpus uint64, gpus int) uint64 {
|
|
mp := r.MaxParallelism
|
|
|
|
if r.GPUUtilization > 0 && gpus > 0 && r.MaxParallelismGPU != 0 { // task can use GPUs and worker has some
|
|
mp = r.MaxParallelismGPU
|
|
}
|
|
|
|
if mp == -1 {
|
|
n := (wcpus * ParallelNum) / ParallelDenom
|
|
if n == 0 {
|
|
return wcpus
|
|
}
|
|
return n
|
|
}
|
|
|
|
return uint64(mp)
|
|
}
|
|
|
|
func (r *Resources) customizeForWorker(taskShortName string, wid WorkerID, info storiface.WorkerInfo) {
|
|
// update needed resources with worker options
|
|
if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_MEMORY"]; ok {
|
|
i, err := strconv.ParseUint(o, 10, 64)
|
|
if err != nil {
|
|
log.Errorf("unable to parse %s_MAX_MEMORY value %s: %e", taskShortName, o, err)
|
|
} else {
|
|
r.MaxMemory = i
|
|
}
|
|
}
|
|
if o, ok := info.Resources.ResourceOpts[taskShortName+"_MIN_MEMORY"]; ok {
|
|
i, err := strconv.ParseUint(o, 10, 64)
|
|
if err != nil {
|
|
log.Errorf("unable to parse %s_MIN_MEMORY value %s: %e", taskShortName, o, err)
|
|
} else {
|
|
r.MinMemory = i
|
|
}
|
|
}
|
|
if o, ok := info.Resources.ResourceOpts[taskShortName+"_BASE_MIN_MEMORY"]; ok {
|
|
i, err := strconv.ParseUint(o, 10, 64)
|
|
if err != nil {
|
|
log.Errorf("unable to parse %s_BASE_MIN_MEMORY value %s: %e", taskShortName, o, err)
|
|
} else {
|
|
r.BaseMinMemory = i
|
|
}
|
|
}
|
|
if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM"]; ok {
|
|
i, err := strconv.Atoi(o)
|
|
if err != nil {
|
|
log.Errorf("unable to parse %s_MAX_PARALLELISM value %s: %e", taskShortName, o, err)
|
|
} else {
|
|
r.MaxParallelism = i
|
|
}
|
|
}
|
|
if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM_GPU"]; ok {
|
|
i, err := strconv.Atoi(o)
|
|
if err != nil {
|
|
log.Errorf("unable to parse %s_GPU_PARALLELISM value %s: %e", taskShortName, o, err)
|
|
} else {
|
|
r.MaxParallelismGPU = i
|
|
}
|
|
}
|
|
if o, ok := info.Resources.ResourceOpts[taskShortName+"_GPU_UTILIZATION"]; ok {
|
|
i, err := strconv.ParseFloat(o, 64)
|
|
if err != nil {
|
|
log.Errorf("unable to parse %s_GPU_UTILIZATION value %s: %e", taskShortName, o, err)
|
|
} else {
|
|
r.GPUUtilization = i
|
|
}
|
|
}
|
|
log.Debugf("resources required for %s on %s(%s): %+v", taskShortName, wid, info.Hostname, r)
|
|
}
|
|
|
|
var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
|
|
sealtasks.TTAddPiece: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 8 << 30,
|
|
MinMemory: 8 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 4 << 30,
|
|
MinMemory: 4 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTPreCommit1: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 128 << 30,
|
|
MinMemory: 112 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 10 << 20,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 64 << 30,
|
|
MinMemory: 56 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 10 << 20,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 768 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 20,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTPreCommit2: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 30 << 30,
|
|
MinMemory: 30 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
MaxParallelismGPU: 6,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 15 << 30,
|
|
MinMemory: 15 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
MaxParallelismGPU: 6,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 3 << 29, // 1.5G
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: -1,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: -1,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTCommit1: { // Very short (~100ms), so params are very light
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTCommit2: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 190 << 30, // TODO: Confirm
|
|
MinMemory: 60 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
MaxParallelismGPU: 6,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 64 << 30, // params
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory
|
|
MinMemory: 30 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
MaxParallelismGPU: 6,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 32 << 30, // params
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 3 << 29, // 1.5G
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 1, // This is fine
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 10 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTFetch: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
},
|
|
}
|
|
|
|
func init() {
|
|
ResourceTable[sealtasks.TTUnseal] = ResourceTable[sealtasks.TTPreCommit1] // TODO: measure accurately
|
|
|
|
// V1_1 is the same as V1
|
|
for _, m := range ResourceTable {
|
|
m[abi.RegisteredSealProof_StackedDrg2KiBV1_1] = m[abi.RegisteredSealProof_StackedDrg2KiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg8MiBV1_1] = m[abi.RegisteredSealProof_StackedDrg8MiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg512MiBV1_1] = m[abi.RegisteredSealProof_StackedDrg512MiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg32GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg32GiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg64GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg64GiBV1]
|
|
}
|
|
}
|