93e4656a27
Before this change workers can only be allocated one GPU task, regardless of how much of the GPU resources that task uses, or how many GPUs are in the system. This makes GPUUtilization a float which can represent that a task needs a portion, or multiple GPUs. GPUs are accounted for like RAM and CPUs so that workers with more GPUs can be allocated more tasks. A known issue is that PC2 cannot use multiple GPUs. And even if the worker has multiple GPUs and is allocated multiple PC2 tasks, those tasks will only run on the first GPU. This could result in unexpected behavior when a worker with multiple GPUs is assigned multiple PC2 tasks. But this should not suprise any existing users who upgrade, as any existing users who run workers with multiple GPUs should already know this and be running a worker per GPU for PC2. But now those users have the freedom to customize the GPU utilization of PC2 to be less than one and effectively run multiple PC2 processes in a single worker. C2 is capable of utilizing multiple GPUs, and now workers can be customized for C2 accordingly.
326 lines
6.8 KiB
Go
326 lines
6.8 KiB
Go
package sectorstorage
|
|
|
|
import (
|
|
"github.com/filecoin-project/go-state-types/abi"
|
|
|
|
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
|
|
)
|
|
|
|
type Resources struct {
|
|
MinMemory uint64 // What Must be in RAM for decent perf
|
|
MaxMemory uint64 // Memory required (swap + ram)
|
|
|
|
MaxParallelism int // -1 = multithread
|
|
GPUUtilization float64
|
|
|
|
BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads)
|
|
}
|
|
|
|
/*
|
|
|
|
Percent of threads to allocate to parallel tasks
|
|
|
|
12 * 0.92 = 11
|
|
16 * 0.92 = 14
|
|
24 * 0.92 = 22
|
|
32 * 0.92 = 29
|
|
64 * 0.92 = 58
|
|
128 * 0.92 = 117
|
|
|
|
*/
|
|
var ParallelNum uint64 = 92
|
|
var ParallelDenom uint64 = 100
|
|
|
|
// TODO: Take NUMA into account
|
|
func (r Resources) Threads(wcpus uint64) uint64 {
|
|
if r.MaxParallelism == -1 {
|
|
n := (wcpus * ParallelNum) / ParallelDenom
|
|
if n == 0 {
|
|
return wcpus
|
|
}
|
|
return n
|
|
}
|
|
|
|
return uint64(r.MaxParallelism)
|
|
}
|
|
|
|
var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
|
|
sealtasks.TTAddPiece: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 8 << 30,
|
|
MinMemory: 8 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 4 << 30,
|
|
MinMemory: 4 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTPreCommit1: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 128 << 30,
|
|
MinMemory: 112 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 10 << 20,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 64 << 30,
|
|
MinMemory: 56 << 30,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 10 << 20,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 768 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 1 << 20,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTPreCommit2: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 30 << 30,
|
|
MinMemory: 30 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 15 << 30,
|
|
MinMemory: 15 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 3 << 29, // 1.5G
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: -1,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: -1,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTCommit1: { // Very short (~100ms), so params are very light
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 30,
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 1 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTCommit2: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 190 << 30, // TODO: Confirm
|
|
MinMemory: 60 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 64 << 30, // params
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory
|
|
MinMemory: 30 << 30,
|
|
|
|
MaxParallelism: -1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 32 << 30, // params
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 3 << 29, // 1.5G
|
|
MinMemory: 1 << 30,
|
|
|
|
MaxParallelism: 1, // This is fine
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 10 << 30,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 2 << 10,
|
|
MinMemory: 2 << 10,
|
|
|
|
MaxParallelism: 1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 2 << 10,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 8 << 20,
|
|
MinMemory: 8 << 20,
|
|
|
|
MaxParallelism: 1,
|
|
GPUUtilization: 1.0,
|
|
|
|
BaseMinMemory: 8 << 20,
|
|
},
|
|
},
|
|
sealtasks.TTFetch: {
|
|
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
|
|
MaxMemory: 1 << 20,
|
|
MinMemory: 1 << 20,
|
|
|
|
MaxParallelism: 0,
|
|
GPUUtilization: 0,
|
|
|
|
BaseMinMemory: 0,
|
|
},
|
|
},
|
|
}
|
|
|
|
func init() {
|
|
ResourceTable[sealtasks.TTUnseal] = ResourceTable[sealtasks.TTPreCommit1] // TODO: measure accurately
|
|
|
|
// V1_1 is the same as V1
|
|
for _, m := range ResourceTable {
|
|
m[abi.RegisteredSealProof_StackedDrg2KiBV1_1] = m[abi.RegisteredSealProof_StackedDrg2KiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg8MiBV1_1] = m[abi.RegisteredSealProof_StackedDrg8MiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg512MiBV1_1] = m[abi.RegisteredSealProof_StackedDrg512MiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg32GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg32GiBV1]
|
|
m[abi.RegisteredSealProof_StackedDrg64GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg64GiBV1]
|
|
}
|
|
}
|