package sectorstorage import ( "strconv" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) type Resources struct { MinMemory uint64 // What Must be in RAM for decent perf MaxMemory uint64 // Memory required (swap + ram) MaxParallelism int // -1 = multithread GPUUtilization float64 BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) } /* Percent of threads to allocate to parallel tasks 12 * 0.92 = 11 16 * 0.92 = 14 24 * 0.92 = 22 32 * 0.92 = 29 64 * 0.92 = 58 128 * 0.92 = 117 */ var ParallelNum uint64 = 92 var ParallelDenom uint64 = 100 // TODO: Take NUMA into account func (r Resources) Threads(wcpus uint64) uint64 { if r.MaxParallelism == -1 { n := (wcpus * ParallelNum) / ParallelDenom if n == 0 { return wcpus } return n } return uint64(r.MaxParallelism) } func (r *Resources) customizeForWorker(taskShortName string, wid WorkerID, info storiface.WorkerInfo) { // update needed resources with worker options if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_MEMORY"]; ok { i, err := strconv.ParseUint(o, 10, 64) if err != nil { log.Errorf("unable to parse %s_MAX_MEMORY value %s: %e", taskShortName, o, err) } else { r.MaxMemory = i } } if o, ok := info.Resources.ResourceOpts[taskShortName+"_MIN_MEMORY"]; ok { i, err := strconv.ParseUint(o, 10, 64) if err != nil { log.Errorf("unable to parse %s_MIN_MEMORY value %s: %e", taskShortName, o, err) } else { r.MinMemory = i } } if o, ok := info.Resources.ResourceOpts[taskShortName+"_BASE_MIN_MEMORY"]; ok { i, err := strconv.ParseUint(o, 10, 64) if err != nil { log.Errorf("unable to parse %s_BASE_MIN_MEMORY value %s: %e", taskShortName, o, err) } else { r.BaseMinMemory = i } } if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM"]; ok { i, err := strconv.Atoi(o) if err != nil { log.Errorf("unable to parse %s_MAX_PARALLELISM value %s: %e", taskShortName, o, err) } else { r.MaxParallelism = i } } if o, ok := info.Resources.ResourceOpts[taskShortName+"_GPU_UTILIZATION"]; ok { i, err := strconv.ParseFloat(o, 64) if err != nil { log.Errorf("unable to parse %s_GPU_UTILIZATION value %s: %e", taskShortName, o, err) } else { r.GPUUtilization = i } } log.Debugf("resources required for %s on %s(%s): %+v", taskShortName, wid, info.Hostname, r) } var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{ sealtasks.TTAddPiece: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ MaxMemory: 8 << 30, MinMemory: 8 << 30, MaxParallelism: 1, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ MaxMemory: 4 << 30, MinMemory: 4 << 30, MaxParallelism: 1, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 1 << 30, MinMemory: 1 << 30, MaxParallelism: 1, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ MaxMemory: 2 << 10, MinMemory: 2 << 10, MaxParallelism: 1, BaseMinMemory: 2 << 10, }, abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ MaxMemory: 8 << 20, MinMemory: 8 << 20, MaxParallelism: 1, BaseMinMemory: 8 << 20, }, }, sealtasks.TTPreCommit1: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ MaxMemory: 128 << 30, MinMemory: 112 << 30, MaxParallelism: 1, BaseMinMemory: 10 << 20, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ MaxMemory: 64 << 30, MinMemory: 56 << 30, MaxParallelism: 1, BaseMinMemory: 10 << 20, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 1 << 30, MinMemory: 768 << 20, MaxParallelism: 1, BaseMinMemory: 1 << 20, }, abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ MaxMemory: 2 << 10, MinMemory: 2 << 10, MaxParallelism: 1, BaseMinMemory: 2 << 10, }, abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ MaxMemory: 8 << 20, MinMemory: 8 << 20, MaxParallelism: 1, BaseMinMemory: 8 << 20, }, }, sealtasks.TTPreCommit2: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ MaxMemory: 30 << 30, MinMemory: 30 << 30, MaxParallelism: -1, GPUUtilization: 1.0, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ MaxMemory: 15 << 30, MinMemory: 15 << 30, MaxParallelism: -1, GPUUtilization: 1.0, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 3 << 29, // 1.5G MinMemory: 1 << 30, MaxParallelism: -1, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ MaxMemory: 2 << 10, MinMemory: 2 << 10, MaxParallelism: -1, BaseMinMemory: 2 << 10, }, abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ MaxMemory: 8 << 20, MinMemory: 8 << 20, MaxParallelism: -1, BaseMinMemory: 8 << 20, }, }, sealtasks.TTCommit1: { // Very short (~100ms), so params are very light abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ MaxMemory: 1 << 30, MinMemory: 1 << 30, MaxParallelism: 0, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ MaxMemory: 1 << 30, MinMemory: 1 << 30, MaxParallelism: 0, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 1 << 30, MinMemory: 1 << 30, MaxParallelism: 0, BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ MaxMemory: 2 << 10, MinMemory: 2 << 10, MaxParallelism: 0, BaseMinMemory: 2 << 10, }, abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ MaxMemory: 8 << 20, MinMemory: 8 << 20, MaxParallelism: 0, BaseMinMemory: 8 << 20, }, }, sealtasks.TTCommit2: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ MaxMemory: 190 << 30, // TODO: Confirm MinMemory: 60 << 30, MaxParallelism: 2, GPUUtilization: 1.0, BaseMinMemory: 64 << 30, // params }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory MinMemory: 30 << 30, MaxParallelism: 2, GPUUtilization: 1.0, BaseMinMemory: 32 << 30, // params }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 3 << 29, // 1.5G MinMemory: 1 << 30, MaxParallelism: 1, // This is fine GPUUtilization: 1.0, BaseMinMemory: 10 << 30, }, abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ MaxMemory: 2 << 10, MinMemory: 2 << 10, MaxParallelism: 1, GPUUtilization: 1.0, BaseMinMemory: 2 << 10, }, abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ MaxMemory: 8 << 20, MinMemory: 8 << 20, MaxParallelism: 1, GPUUtilization: 1.0, BaseMinMemory: 8 << 20, }, }, sealtasks.TTFetch: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ MaxMemory: 1 << 20, MinMemory: 1 << 20, MaxParallelism: 0, GPUUtilization: 0, BaseMinMemory: 0, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ MaxMemory: 1 << 20, MinMemory: 1 << 20, MaxParallelism: 0, GPUUtilization: 0, BaseMinMemory: 0, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 1 << 20, MinMemory: 1 << 20, MaxParallelism: 0, GPUUtilization: 0, BaseMinMemory: 0, }, abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ MaxMemory: 1 << 20, MinMemory: 1 << 20, MaxParallelism: 0, GPUUtilization: 0, BaseMinMemory: 0, }, abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ MaxMemory: 1 << 20, MinMemory: 1 << 20, MaxParallelism: 0, GPUUtilization: 0, BaseMinMemory: 0, }, }, } func init() { ResourceTable[sealtasks.TTUnseal] = ResourceTable[sealtasks.TTPreCommit1] // TODO: measure accurately // V1_1 is the same as V1 for _, m := range ResourceTable { m[abi.RegisteredSealProof_StackedDrg2KiBV1_1] = m[abi.RegisteredSealProof_StackedDrg2KiBV1] m[abi.RegisteredSealProof_StackedDrg8MiBV1_1] = m[abi.RegisteredSealProof_StackedDrg8MiBV1] m[abi.RegisteredSealProof_StackedDrg512MiBV1_1] = m[abi.RegisteredSealProof_StackedDrg512MiBV1] m[abi.RegisteredSealProof_StackedDrg32GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg32GiBV1] m[abi.RegisteredSealProof_StackedDrg64GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg64GiBV1] } }