From e2a1ca7caa571460fb32c099f3cd0b568a41ef68 Mon Sep 17 00:00:00 2001 From: Clint Armstrong Date: Tue, 31 Aug 2021 16:52:47 -0400 Subject: [PATCH 01/13] Use cgroup limits in worker memory calculations Worker processes may have memory limitations imposed by Systemd. But /proc/meminfo shows the entire system memory regardless of these limits. This results in the scheduler believing the worker has the entire system memory avaliable and the worker being allocated too many tasks. This change attempts to read cgroup memory limits for the worker process. It supports cgroups v1 and v2, and compares cgroup limits against the system memory and returns the most conservative values to prevent the worker from being allocated too many tasks and potentially triggering an OOM event. --- extern/sector-storage/cgroups.go | 12 +++ extern/sector-storage/cgroups_linux.go | 117 +++++++++++++++++++++++++ extern/sector-storage/worker_local.go | 63 ++++++++++--- go.mod | 1 + go.sum | 1 + 5 files changed, 181 insertions(+), 13 deletions(-) create mode 100644 extern/sector-storage/cgroups.go create mode 100644 extern/sector-storage/cgroups_linux.go diff --git a/extern/sector-storage/cgroups.go b/extern/sector-storage/cgroups.go new file mode 100644 index 000000000..e2ec0564e --- /dev/null +++ b/extern/sector-storage/cgroups.go @@ -0,0 +1,12 @@ +//go:build !linux +// +build !linux + +package sectorstorage + +func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) { + return 0, 0, 0, 0, nil +} + +func cgroupV2Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) { + return 0, 0, 0, 0, nil +} diff --git a/extern/sector-storage/cgroups_linux.go b/extern/sector-storage/cgroups_linux.go new file mode 100644 index 000000000..0b6efea99 --- /dev/null +++ b/extern/sector-storage/cgroups_linux.go @@ -0,0 +1,117 @@ +//go:build linux +// +build linux + +package sectorstorage + +import ( + "bufio" + "bytes" + "math" + "os" + "path/filepath" + + "github.com/containerd/cgroups" + cgroupv2 "github.com/containerd/cgroups/v2" +) + +func cgroupV2MountPoint() (string, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return "", err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := bytes.Fields(scanner.Bytes()) + if len(fields) >= 9 && bytes.Equal(fields[8], []byte("cgroup2")) { + return string(fields[4]), nil + } + } + return "", cgroups.ErrMountPointNotExist +} + +func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) { + path := cgroups.NestedPath("") + if pid := os.Getpid(); pid == 1 { + path = cgroups.RootPath + } + c, err := cgroups.Load(cgroups.SingleSubsystem(cgroups.V1, cgroups.Memory), path) + if err != nil { + return 0, 0, 0, 0, err + } + stats, err := c.Stat() + if err != nil { + return 0, 0, 0, 0, err + } + if stats.Memory == nil { + return 0, 0, 0, 0, nil + } + if stats.Memory.Usage != nil { + memoryMax = stats.Memory.Usage.Limit + // Exclude cached files + memoryUsed = stats.Memory.Usage.Usage - stats.Memory.InactiveFile - stats.Memory.ActiveFile + } + if stats.Memory.Swap != nil { + swapMax = stats.Memory.Swap.Limit + swapUsed = stats.Memory.Swap.Usage + } + return memoryMax, memoryUsed, swapMax, swapUsed, nil +} + +func cgroupV2MemFromPath(mp, path string) (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) { + c, err := cgroupv2.LoadManager(mp, path) + if err != nil { + return 0, 0, 0, 0, err + } + + stats, err := c.Stat() + if err != nil { + return 0, 0, 0, 0, err + } + + if stats.Memory != nil { + memoryMax = stats.Memory.UsageLimit + // Exclude memory used caching files + memoryUsed = stats.Memory.Usage - stats.Memory.File + swapMax = stats.Memory.SwapLimit + swapUsed = stats.Memory.SwapUsage + } + + return memoryMax, memoryUsed, swapMax, swapUsed, nil +} + +func cgroupV2Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) { + memoryMax = math.MaxUint64 + swapMax = math.MaxUint64 + + path, err := cgroupv2.PidGroupPath(os.Getpid()) + if err != nil { + return 0, 0, 0, 0, err + } + + mp, err := cgroupV2MountPoint() + if err != nil { + return 0, 0, 0, 0, err + } + + for path != "/" { + cgMemoryMax, cgMemoryUsed, cgSwapMax, cgSwapUsed, err := cgroupV2MemFromPath(mp, path) + if err != nil { + return 0, 0, 0, 0, err + } + if cgMemoryMax != 0 && cgMemoryMax < memoryMax { + log.Debugf("memory limited by cgroup %s: %v", path, cgMemoryMax) + memoryMax = cgMemoryMax + memoryUsed = cgMemoryUsed + } + if cgSwapMax != 0 && cgSwapMax < swapMax { + log.Debugf("swap limited by cgroup %s: %v", path, cgSwapMax) + swapMax = cgSwapMax + swapUsed = cgSwapUsed + } + path = filepath.Dir(path) + } + + return memoryMax, memoryUsed, swapMax, swapUsed, nil +} diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index d45d140f8..c8a3d0e7c 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -482,6 +482,53 @@ func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { return l.localStore.Local(ctx) } +func (l *LocalWorker) memInfo() (memPhysical uint64, memVirtual uint64, memReserved uint64, err error) { + h, err := sysinfo.Host() + if err != nil { + return 0, 0, 0, err + } + + mem, err := h.Memory() + if err != nil { + return 0, 0, 0, err + } + memPhysical = mem.Total + memAvail := mem.Free + memSwap := mem.VirtualTotal + swapAvail := mem.VirtualFree + + if cgMemMax, cgMemUsed, cgSwapMax, cgSwapUsed, err := cgroupV1Mem(); err == nil { + if cgMemMax > 0 && cgMemMax < memPhysical { + memPhysical = cgMemMax + memAvail = cgMemMax - cgMemUsed + } + if cgSwapMax > 0 && cgSwapMax < memSwap { + memSwap = cgSwapMax + swapAvail = cgSwapMax - cgSwapUsed + } + } + + if cgMemMax, cgMemUsed, cgSwapMax, cgSwapUsed, err := cgroupV2Mem(); err == nil { + if cgMemMax > 0 && cgMemMax < memPhysical { + memPhysical = cgMemMax + memAvail = cgMemMax - cgMemUsed + } + if cgSwapMax > 0 && cgSwapMax < memSwap { + memSwap = cgSwapMax + swapAvail = cgSwapMax - cgSwapUsed + } + } + + if l.noSwap { + memSwap = 0 + swapAvail = 0 + } + + memReserved = memPhysical + memSwap - memAvail - swapAvail + + return memPhysical, memSwap, memReserved, nil +} + func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { hostname, err := os.Hostname() // TODO: allow overriding from config if err != nil { @@ -493,28 +540,18 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { log.Errorf("getting gpu devices failed: %+v", err) } - h, err := sysinfo.Host() - if err != nil { - return storiface.WorkerInfo{}, xerrors.Errorf("getting host info: %w", err) - } - - mem, err := h.Memory() + memPhysical, memSwap, memReserved, err := l.memInfo() if err != nil { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } - memSwap := mem.VirtualTotal - if l.noSwap { - memSwap = 0 - } - return storiface.WorkerInfo{ Hostname: hostname, IgnoreResources: l.ignoreResources, Resources: storiface.WorkerResources{ - MemPhysical: mem.Total, + MemPhysical: memPhysical, MemSwap: memSwap, - MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process + MemReserved: memReserved, CPUs: uint64(runtime.NumCPU()), GPUs: gpus, }, diff --git a/go.mod b/go.mod index 3866fb7de..ba29f8cdf 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/buger/goterm v1.0.3 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07 + github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327 github.com/coreos/go-systemd/v22 v22.3.2 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/dgraph-io/badger/v2 v2.2007.2 diff --git a/go.sum b/go.sum index 03d858d27..c0ceb01ae 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5O github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.2.0 h1:Fv93L3KKckEcEHR3oApXVzyBTDA8WAm6VXhPE00N3f8= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= From c4f46171aeec50d176fdf178979336f0adcb69dd Mon Sep 17 00:00:00 2001 From: Clint Armstrong Date: Thu, 9 Sep 2021 17:41:59 -0400 Subject: [PATCH 02/13] Report memory used and swap used in worker res Attempting to report "memory used by other processes" in the MemReserved field fails to take into account the fact that the system's memory used includes memory used by ongoing tasks. To properly account for this, worker should report the memory and swap used, then the scheduler that is aware of the memory requirements for a task can determine if there is sufficient memory available for a task. --- api/docgen/docgen.go | 3 +- api/version.go | 2 +- cmd/lotus-miner/sealing.go | 75 ++++++++++------------- cmd/lotus-seal-worker/info.go | 7 ++- extern/sector-storage/sched_resources.go | 34 +++++++--- extern/sector-storage/sched_test.go | 6 +- extern/sector-storage/storiface/worker.go | 4 +- extern/sector-storage/testworker_test.go | 2 +- extern/sector-storage/worker_local.go | 32 +++++----- 9 files changed, 91 insertions(+), 74 deletions(-) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 11440619c..a9b256a7b 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -231,8 +231,9 @@ func init() { Hostname: "host", Resources: storiface.WorkerResources{ MemPhysical: 256 << 30, + MemUsed: 2 << 30, MemSwap: 120 << 30, - MemReserved: 2 << 30, + MemSwapUsed: 2 << 30, CPUs: 64, GPUs: []string{"aGPU 1337"}, }, diff --git a/api/version.go b/api/version.go index 2c87fe0a4..4cd8bf51b 100644 --- a/api/version.go +++ b/api/version.go @@ -58,7 +58,7 @@ var ( FullAPIVersion1 = newVer(2, 1, 0) MinerAPIVersion0 = newVer(1, 2, 0) - WorkerAPIVersion0 = newVer(1, 1, 0) + WorkerAPIVersion0 = newVer(1, 2, 0) ) //nolint:varcheck,deadcode diff --git a/cmd/lotus-miner/sealing.go b/cmd/lotus-miner/sealing.go index 472af8da6..6bc4ad3c0 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cmd/lotus-miner/sealing.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "math" "os" "sort" "strings" @@ -32,6 +33,17 @@ var sealingCmd = &cli.Command{ }, } +var barCols = float64(64) + +func barString(total, y, g float64) string { + yBars := int(math.Round(y / total * barCols)) + gBars := int(math.Round(g / total * barCols)) + eBars := int(barCols) - yBars - gBars + return color.YellowString(strings.Repeat("|", yBars)) + + color.GreenString(strings.Repeat("|", gBars)) + + strings.Repeat(" ", eBars) +} + var sealingWorkersCmd = &cli.Command{ Name: "workers", Usage: "list workers", @@ -89,55 +101,36 @@ var sealingWorkersCmd = &cli.Command{ fmt.Printf("Worker %s, host %s%s\n", stat.id, color.MagentaString(stat.Info.Hostname), disabled) - var barCols = uint64(64) - cpuBars := int(stat.CpuUse * barCols / stat.Info.Resources.CPUs) - cpuBar := strings.Repeat("|", cpuBars) - if int(barCols)-cpuBars >= 0 { - cpuBar += strings.Repeat(" ", int(barCols)-cpuBars) - } - fmt.Printf("\tCPU: [%s] %d/%d core(s) in use\n", - color.GreenString(cpuBar), stat.CpuUse, stat.Info.Resources.CPUs) + barString(float64(stat.Info.Resources.CPUs), 0, float64(stat.CpuUse)), stat.CpuUse, stat.Info.Resources.CPUs) - ramBarsRes := int(stat.Info.Resources.MemReserved * barCols / stat.Info.Resources.MemPhysical) - ramBarsUsed := int(stat.MemUsedMin * barCols / stat.Info.Resources.MemPhysical) - ramRepeatSpace := int(barCols) - (ramBarsUsed + ramBarsRes) - - colorFunc := color.YellowString - if ramRepeatSpace < 0 { - ramRepeatSpace = 0 - colorFunc = color.RedString + ramTotal := stat.Info.Resources.MemPhysical + ramTasks := stat.MemUsedMin + ramUsed := stat.Info.Resources.MemUsed + var ramReserved uint64 = 0 + if ramUsed > ramTasks { + ramReserved = ramUsed - ramTasks } - - ramBar := colorFunc(strings.Repeat("|", ramBarsRes)) + - color.GreenString(strings.Repeat("|", ramBarsUsed)) + - strings.Repeat(" ", ramRepeatSpace) - - vmem := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap - - vmemBarsRes := int(stat.Info.Resources.MemReserved * barCols / vmem) - vmemBarsUsed := int(stat.MemUsedMax * barCols / vmem) - vmemRepeatSpace := int(barCols) - (vmemBarsUsed + vmemBarsRes) - - colorFunc = color.YellowString - if vmemRepeatSpace < 0 { - vmemRepeatSpace = 0 - colorFunc = color.RedString - } - - vmemBar := colorFunc(strings.Repeat("|", vmemBarsRes)) + - color.GreenString(strings.Repeat("|", vmemBarsUsed)) + - strings.Repeat(" ", vmemRepeatSpace) + ramBar := barString(float64(ramTotal), float64(ramReserved), float64(ramTasks)) fmt.Printf("\tRAM: [%s] %d%% %s/%s\n", ramBar, - (stat.Info.Resources.MemReserved+stat.MemUsedMin)*100/stat.Info.Resources.MemPhysical, - types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMin)), + (ramTasks+ramReserved)*100/stat.Info.Resources.MemPhysical, + types.SizeStr(types.NewInt(ramTasks+ramUsed)), types.SizeStr(types.NewInt(stat.Info.Resources.MemPhysical))) + vmemTotal := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap + vmemTasks := stat.MemUsedMax + vmemUsed := stat.Info.Resources.MemUsed + stat.Info.Resources.MemSwapUsed + var vmemReserved uint64 = 0 + if vmemUsed > vmemTasks { + vmemReserved = vmemUsed - vmemTasks + } + vmemBar := barString(float64(vmemTotal), float64(vmemReserved), float64(vmemTasks)) + fmt.Printf("\tVMEM: [%s] %d%% %s/%s\n", vmemBar, - (stat.Info.Resources.MemReserved+stat.MemUsedMax)*100/vmem, - types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMax)), - types.SizeStr(types.NewInt(vmem))) + (vmemTasks+vmemReserved)*100/vmemTotal, + types.SizeStr(types.NewInt(vmemTasks+vmemReserved)), + types.SizeStr(types.NewInt(vmemTotal))) for _, gpu := range stat.Info.Resources.GPUs { fmt.Printf("\tGPU: %s\n", color.New(gpuCol).Sprintf("%s, %sused", gpu, gpuUse)) diff --git a/cmd/lotus-seal-worker/info.go b/cmd/lotus-seal-worker/info.go index 6d5c2d64e..057e1303c 100644 --- a/cmd/lotus-seal-worker/info.go +++ b/cmd/lotus-seal-worker/info.go @@ -58,8 +58,11 @@ var infoCmd = &cli.Command{ fmt.Printf("Hostname: %s\n", info.Hostname) fmt.Printf("CPUs: %d; GPUs: %v\n", info.Resources.CPUs, info.Resources.GPUs) - fmt.Printf("RAM: %s; Swap: %s\n", types.SizeStr(types.NewInt(info.Resources.MemPhysical)), types.SizeStr(types.NewInt(info.Resources.MemSwap))) - fmt.Printf("Reserved memory: %s\n", types.SizeStr(types.NewInt(info.Resources.MemReserved))) + fmt.Printf("RAM: %s/%s; Swap: %s/%s\n", + types.SizeStr(types.NewInt(info.Resources.MemUsed)), + types.SizeStr(types.NewInt(info.Resources.MemPhysical)), + types.SizeStr(types.NewInt(info.Resources.MemSwapUsed)), + types.SizeStr(types.NewInt(info.Resources.MemSwap))) fmt.Printf("Task types: ") for _, t := range ttList(tt) { diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index 7c16120c2..82e69c5f9 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -61,17 +61,26 @@ func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, call } res := info.Resources + // TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running) - minNeedMem := res.MemReserved + a.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory - if minNeedMem > res.MemPhysical { - log.Debugf("sched: not scheduling on worker %s for %s; not enough physical memory - need: %dM, have %dM", wid, caller, minNeedMem/mib, res.MemPhysical/mib) + memNeeded := needRes.MinMemory + needRes.BaseMinMemory + memUsed := a.memUsedMin + // assume that MemUsed can be swapped, so only check it in the vmem Check + memAvail := res.MemPhysical - memUsed + if memNeeded > memAvail { + log.Debugf("sched: not scheduling on worker %s for %s; not enough physical memory - need: %dM, have %dM available", wid, caller, memNeeded/mib, memAvail/mib) return false } - maxNeedMem := res.MemReserved + a.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory + vmemNeeded := needRes.MaxMemory + needRes.BaseMinMemory + vmemUsed := a.memUsedMax + if vmemUsed < res.MemUsed+res.MemSwapUsed { + vmemUsed = res.MemUsed + res.MemSwapUsed + } + vmemAvail := res.MemPhysical + res.MemSwap - vmemUsed - if maxNeedMem > res.MemSwap+res.MemPhysical { - log.Debugf("sched: not scheduling on worker %s for %s; not enough virtual memory - need: %dM, have %dM", wid, caller, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib) + if vmemNeeded > vmemAvail { + log.Debugf("sched: not scheduling on worker %s for %s; not enough virtual memory - need: %dM, have %dM available", wid, caller, vmemNeeded/mib, vmemAvail/mib) return false } @@ -96,12 +105,21 @@ func (a *activeResources) utilization(wr storiface.WorkerResources) float64 { cpu := float64(a.cpuUse) / float64(wr.CPUs) max = cpu - memMin := float64(a.memUsedMin+wr.MemReserved) / float64(wr.MemPhysical) + memUsed := a.memUsedMin + if memUsed < wr.MemUsed { + memUsed = wr.MemUsed + } + memMin := float64(memUsed) / float64(wr.MemPhysical) if memMin > max { max = memMin } - memMax := float64(a.memUsedMax+wr.MemReserved) / float64(wr.MemPhysical+wr.MemSwap) + vmemUsed := a.memUsedMax + if a.memUsedMax < wr.MemUsed+wr.MemSwapUsed { + vmemUsed = wr.MemUsed + wr.MemSwapUsed + } + memMax := float64(vmemUsed) / float64(wr.MemPhysical+wr.MemSwap) + if memMax > max { max = memMax } diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index fbc4d83ee..b98c93031 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -41,14 +41,16 @@ func TestWithPriority(t *testing.T) { var decentWorkerResources = storiface.WorkerResources{ MemPhysical: 128 << 30, MemSwap: 200 << 30, - MemReserved: 2 << 30, + MemUsed: 1 << 30, + MemSwapUsed: 1 << 30, CPUs: 32, GPUs: []string{"a GPU"}, } var constrainedWorkerResources = storiface.WorkerResources{ MemPhysical: 1 << 30, - MemReserved: 2 << 30, + MemUsed: 1 << 30, + MemSwapUsed: 1 << 30, CPUs: 1, } diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index e3374d6cf..d998cdce5 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -28,9 +28,9 @@ type WorkerInfo struct { type WorkerResources struct { MemPhysical uint64 + MemUsed uint64 MemSwap uint64 - - MemReserved uint64 // Used by system / other processes + MemSwapUsed uint64 CPUs uint64 // Logical cores GPUs []string diff --git a/extern/sector-storage/testworker_test.go b/extern/sector-storage/testworker_test.go index 2fe99f3d4..57c3b53ee 100644 --- a/extern/sector-storage/testworker_test.go +++ b/extern/sector-storage/testworker_test.go @@ -108,8 +108,8 @@ func (t *testWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) { Hostname: "testworkerer", Resources: storiface.WorkerResources{ MemPhysical: res.MinMemory * 3, + MemUsed: res.MinMemory, MemSwap: 0, - MemReserved: res.MinMemory, CPUs: 32, GPUs: nil, }, diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index c8a3d0e7c..1a1b3627f 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -482,51 +482,50 @@ func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { return l.localStore.Local(ctx) } -func (l *LocalWorker) memInfo() (memPhysical uint64, memVirtual uint64, memReserved uint64, err error) { +func (l *LocalWorker) memInfo() (memPhysical, memUsed, memSwap, memSwapUsed uint64, err error) { h, err := sysinfo.Host() if err != nil { - return 0, 0, 0, err + return 0, 0, 0, 0, err } mem, err := h.Memory() if err != nil { - return 0, 0, 0, err + return 0, 0, 0, 0, err } memPhysical = mem.Total - memAvail := mem.Free - memSwap := mem.VirtualTotal - swapAvail := mem.VirtualFree + // mem.Available is memory available without swapping, it is more relevant for this calculation + memUsed = mem.Total - mem.Available + memSwap = mem.VirtualTotal + memSwapUsed = mem.VirtualUsed if cgMemMax, cgMemUsed, cgSwapMax, cgSwapUsed, err := cgroupV1Mem(); err == nil { if cgMemMax > 0 && cgMemMax < memPhysical { memPhysical = cgMemMax - memAvail = cgMemMax - cgMemUsed + memUsed = cgMemUsed } if cgSwapMax > 0 && cgSwapMax < memSwap { memSwap = cgSwapMax - swapAvail = cgSwapMax - cgSwapUsed + memSwapUsed = cgSwapUsed } } if cgMemMax, cgMemUsed, cgSwapMax, cgSwapUsed, err := cgroupV2Mem(); err == nil { if cgMemMax > 0 && cgMemMax < memPhysical { memPhysical = cgMemMax - memAvail = cgMemMax - cgMemUsed + memUsed = cgMemUsed } if cgSwapMax > 0 && cgSwapMax < memSwap { memSwap = cgSwapMax - swapAvail = cgSwapMax - cgSwapUsed + memSwapUsed = cgSwapUsed } } if l.noSwap { memSwap = 0 - swapAvail = 0 + memSwapUsed = 0 } - memReserved = memPhysical + memSwap - memAvail - swapAvail - - return memPhysical, memSwap, memReserved, nil + return memPhysical, memUsed, memSwap, memSwapUsed, nil } func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { @@ -540,7 +539,7 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { log.Errorf("getting gpu devices failed: %+v", err) } - memPhysical, memSwap, memReserved, err := l.memInfo() + memPhysical, memUsed, memSwap, memSwapUsed, err := l.memInfo() if err != nil { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } @@ -550,8 +549,9 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { IgnoreResources: l.ignoreResources, Resources: storiface.WorkerResources{ MemPhysical: memPhysical, + MemUsed: memUsed, MemSwap: memSwap, - MemReserved: memReserved, + MemSwapUsed: memSwapUsed, CPUs: uint64(runtime.NumCPU()), GPUs: gpus, }, From 93e4656a2710dacee647fa99a7da264cc720457c Mon Sep 17 00:00:00 2001 From: Clint Armstrong Date: Tue, 31 Aug 2021 21:59:25 -0400 Subject: [PATCH 03/13] Use a float to represent GPU utilization Before this change workers can only be allocated one GPU task, regardless of how much of the GPU resources that task uses, or how many GPUs are in the system. This makes GPUUtilization a float which can represent that a task needs a portion, or multiple GPUs. GPUs are accounted for like RAM and CPUs so that workers with more GPUs can be allocated more tasks. A known issue is that PC2 cannot use multiple GPUs. And even if the worker has multiple GPUs and is allocated multiple PC2 tasks, those tasks will only run on the first GPU. This could result in unexpected behavior when a worker with multiple GPUs is assigned multiple PC2 tasks. But this should not suprise any existing users who upgrade, as any existing users who run workers with multiple GPUs should already know this and be running a worker per GPU for PC2. But now those users have the freedom to customize the GPU utilization of PC2 to be less than one and effectively run multiple PC2 processes in a single worker. C2 is capable of utilizing multiple GPUs, and now workers can be customized for C2 accordingly. --- api/docgen/docgen.go | 2 +- api/version.go | 2 +- cmd/lotus-miner/sealing.go | 8 ++++++- extern/sector-storage/resources.go | 26 +++++++++++------------ extern/sector-storage/sched.go | 2 +- extern/sector-storage/sched_resources.go | 14 ++++++------ extern/sector-storage/storiface/worker.go | 4 ++-- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index a9b256a7b..9402f50d8 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -241,7 +241,7 @@ func init() { Enabled: true, MemUsedMin: 0, MemUsedMax: 0, - GpuUsed: false, + GpuUsed: 0, CpuUse: 0, }, }) diff --git a/api/version.go b/api/version.go index 4cd8bf51b..6dfcc3f7f 100644 --- a/api/version.go +++ b/api/version.go @@ -58,7 +58,7 @@ var ( FullAPIVersion1 = newVer(2, 1, 0) MinerAPIVersion0 = newVer(1, 2, 0) - WorkerAPIVersion0 = newVer(1, 2, 0) + WorkerAPIVersion0 = newVer(1, 3, 0) ) //nolint:varcheck,deadcode diff --git a/cmd/lotus-miner/sealing.go b/cmd/lotus-miner/sealing.go index 6bc4ad3c0..16b02f7bb 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cmd/lotus-miner/sealing.go @@ -89,7 +89,7 @@ var sealingWorkersCmd = &cli.Command{ for _, stat := range st { gpuUse := "not " gpuCol := color.FgBlue - if stat.GpuUsed { + if stat.GpuUsed > 0 { gpuCol = color.FgGreen gpuUse = "" } @@ -132,6 +132,12 @@ var sealingWorkersCmd = &cli.Command{ types.SizeStr(types.NewInt(vmemTasks+vmemReserved)), types.SizeStr(types.NewInt(vmemTotal))) + if len(stat.Info.Resources.GPUs) > 0 { + gpuBar := barString(float64(len(stat.Info.Resources.GPUs)), 0, stat.GpuUsed) + fmt.Printf("\tGPU: [%s] %.f%% %.2f/%d gpu(s) in use\n", color.GreenString(gpuBar), + stat.GpuUsed*100/float64(len(stat.Info.Resources.GPUs)), + stat.GpuUsed, len(stat.Info.Resources.GPUs)) + } for _, gpu := range stat.Info.Resources.GPUs { fmt.Printf("\tGPU: %s\n", color.New(gpuCol).Sprintf("%s, %sused", gpu, gpuUse)) } diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index 2e989fdf4..d3c7bd7a5 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -11,7 +11,7 @@ type Resources struct { MaxMemory uint64 // Memory required (swap + ram) MaxParallelism int // -1 = multithread - CanGPU bool + GPUUtilization float64 BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) } @@ -135,7 +135,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 30 << 30, MaxParallelism: -1, - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 1 << 30, }, @@ -144,7 +144,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 15 << 30, MaxParallelism: -1, - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 1 << 30, }, @@ -221,7 +221,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 60 << 30, MaxParallelism: -1, - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 64 << 30, // params }, @@ -230,7 +230,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 30 << 30, MaxParallelism: -1, - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 32 << 30, // params }, @@ -239,7 +239,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 1 << 30, MaxParallelism: 1, // This is fine - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 10 << 30, }, @@ -248,7 +248,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 2 << 10, MaxParallelism: 1, - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 2 << 10, }, @@ -257,7 +257,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 8 << 20, MaxParallelism: 1, - CanGPU: true, + GPUUtilization: 1.0, BaseMinMemory: 8 << 20, }, @@ -268,7 +268,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 1 << 20, MaxParallelism: 0, - CanGPU: false, + GPUUtilization: 0, BaseMinMemory: 0, }, @@ -277,7 +277,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 1 << 20, MaxParallelism: 0, - CanGPU: false, + GPUUtilization: 0, BaseMinMemory: 0, }, @@ -286,7 +286,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 1 << 20, MaxParallelism: 0, - CanGPU: false, + GPUUtilization: 0, BaseMinMemory: 0, }, @@ -295,7 +295,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 1 << 20, MaxParallelism: 0, - CanGPU: false, + GPUUtilization: 0, BaseMinMemory: 0, }, @@ -304,7 +304,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MinMemory: 1 << 20, MaxParallelism: 0, - CanGPU: false, + GPUUtilization: 0, BaseMinMemory: 0, }, diff --git a/extern/sector-storage/sched.go b/extern/sector-storage/sched.go index 1ffb15e5b..e6e3121f3 100644 --- a/extern/sector-storage/sched.go +++ b/extern/sector-storage/sched.go @@ -114,7 +114,7 @@ type workerDisableReq struct { type activeResources struct { memUsedMin uint64 memUsedMax uint64 - gpuUsed bool + gpuUsed float64 cpuUse uint64 cond *sync.Cond diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index 82e69c5f9..636fc02d1 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -31,8 +31,8 @@ func (a *activeResources) hasWorkWaiting() bool { } func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { - if r.CanGPU { - a.gpuUsed = true + if r.GPUUtilization > 0 { + a.gpuUsed += r.GPUUtilization } a.cpuUse += r.Threads(wr.CPUs) a.memUsedMin += r.MinMemory @@ -40,8 +40,8 @@ func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { } func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { - if r.CanGPU { - a.gpuUsed = false + if r.GPUUtilization > 0 { + a.gpuUsed -= r.GPUUtilization } a.cpuUse -= r.Threads(wr.CPUs) a.memUsedMin -= r.MinMemory @@ -89,9 +89,9 @@ func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, call return false } - if len(res.GPUs) > 0 && needRes.CanGPU { - if a.gpuUsed { - log.Debugf("sched: not scheduling on worker %s for %s; GPU in use", wid, caller) + if len(res.GPUs) > 0 && needRes.GPUUtilization > 0 { + if a.gpuUsed+needRes.GPUUtilization > float64(len(res.GPUs)) { + log.Debugf("sched: not scheduling on worker %s for %s; GPU(s) in use", wid, caller) return false } } diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index d998cdce5..50d8b2159 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -42,8 +42,8 @@ type WorkerStats struct { MemUsedMin uint64 MemUsedMax uint64 - GpuUsed bool // nolint - CpuUse uint64 // nolint + GpuUsed float64 // nolint + CpuUse uint64 // nolint } const ( From 4ef8543128ec3ee2ed828d093763bf599548f779 Mon Sep 17 00:00:00 2001 From: Clint Armstrong Date: Tue, 31 Aug 2021 23:08:10 -0400 Subject: [PATCH 04/13] Permit workers to override resource table In an environment with heterogenious worker nodes, a universal resource table for all workers does not allow effective scheduling of tasks. Some workers may have different proof cache settings, changing the required memory for different tasks. Some workers may have a different count of CPUs per core-complex, changing the max parallelism of PC1. This change allows workers to customize these parameters with environment variables. A worker could set the environment variable PC1_MIN_MEMORY for example to customize the minimum memory requirement for PC1 tasks. If no environment variables are specified, the resource table on the miner is used, except for PC1 parallelism. If PC1_MAX_PARALLELISM is not specified, and FIL_PROOFS_USE_MULTICORE_SDR is set, PC1_MAX_PARALLELSIM will automatically be set to FIL_PROOFS_MULTICORE_SDR_PRODUCERS + 1. --- api/version.go | 2 +- extern/sector-storage/resources.go | 48 +++++++++++++++++++++++ extern/sector-storage/sched_worker.go | 3 ++ extern/sector-storage/storiface/worker.go | 5 ++- extern/sector-storage/worker_local.go | 39 +++++++++++++++--- 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/api/version.go b/api/version.go index 6dfcc3f7f..ff1115e1d 100644 --- a/api/version.go +++ b/api/version.go @@ -58,7 +58,7 @@ var ( FullAPIVersion1 = newVer(2, 1, 0) MinerAPIVersion0 = newVer(1, 2, 0) - WorkerAPIVersion0 = newVer(1, 3, 0) + WorkerAPIVersion0 = newVer(1, 4, 0) ) //nolint:varcheck,deadcode diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index d3c7bd7a5..c05bca62b 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -1,9 +1,12 @@ package sectorstorage import ( + "strconv" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) type Resources struct { @@ -44,6 +47,51 @@ func (r Resources) Threads(wcpus uint64) uint64 { return uint64(r.MaxParallelism) } +func (r *Resources) customizeForWorker(taskShortName string, wid WorkerID, info storiface.WorkerInfo) { + // update needed resources with worker options + if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_MEMORY"]; ok { + i, err := strconv.ParseUint(o, 10, 64) + if err != nil { + log.Errorf("unable to parse %s_MAX_MEMORY value %s: %e", taskShortName, o, err) + } else { + r.MaxMemory = i + } + } + if o, ok := info.Resources.ResourceOpts[taskShortName+"_MIN_MEMORY"]; ok { + i, err := strconv.ParseUint(o, 10, 64) + if err != nil { + log.Errorf("unable to parse %s_MIN_MEMORY value %s: %e", taskShortName, o, err) + } else { + r.MinMemory = i + } + } + if o, ok := info.Resources.ResourceOpts[taskShortName+"_BASE_MIN_MEMORY"]; ok { + i, err := strconv.ParseUint(o, 10, 64) + if err != nil { + log.Errorf("unable to parse %s_BASE_MIN_MEMORY value %s: %e", taskShortName, o, err) + } else { + r.BaseMinMemory = i + } + } + if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM"]; ok { + i, err := strconv.Atoi(o) + if err != nil { + log.Errorf("unable to parse %s_MAX_PARALLELISM value %s: %e", taskShortName, o, err) + } else { + r.MaxParallelism = i + } + } + if o, ok := info.Resources.ResourceOpts[taskShortName+"_GPU_UTILIZATION"]; ok { + i, err := strconv.ParseFloat(o, 64) + if err != nil { + log.Errorf("unable to parse %s_GPU_UTILIZATION value %s: %e", taskShortName, o, err) + } else { + r.GPUUtilization = i + } + } + log.Debugf("resources required for %s on %s(%s): %+v", taskShortName, wid, info.Hostname, r) +} + var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{ sealtasks.TTAddPiece: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ diff --git a/extern/sector-storage/sched_worker.go b/extern/sector-storage/sched_worker.go index e717e58e2..bb6ba627b 100644 --- a/extern/sector-storage/sched_worker.go +++ b/extern/sector-storage/sched_worker.go @@ -297,6 +297,7 @@ func (sw *schedWorker) workerCompactWindows() { for ti, todo := range window.todo { needRes := ResourceTable[todo.taskType][todo.sector.ProofType] + needRes.customizeForWorker(todo.taskType.Short(), sw.wid, worker.info) if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info) { continue } @@ -358,6 +359,7 @@ assignLoop: worker.lk.Lock() for t, todo := range firstWindow.todo { needRes := ResourceTable[todo.taskType][todo.sector.ProofType] + needRes.customizeForWorker(todo.taskType.Short(), sw.wid, worker.info) if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) { tidx = t break @@ -457,6 +459,7 @@ func (sw *schedWorker) startProcessingTask(req *workerRequest) error { w, sh := sw.worker, sw.sched needRes := ResourceTable[req.taskType][req.sector.ProofType] + needRes.customizeForWorker(req.taskType.Short(), sw.wid, w.info) w.lk.Lock() w.preparing.add(w.info.Resources, needRes) diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index 50d8b2159..f28f106b1 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -32,8 +32,9 @@ type WorkerResources struct { MemSwap uint64 MemSwapUsed uint64 - CPUs uint64 // Logical cores - GPUs []string + CPUs uint64 // Logical cores + GPUs []string + ResourceOpts map[string]string } type WorkerStats struct { diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index 1a1b3627f..de69cea80 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -3,10 +3,12 @@ package sectorstorage import ( "context" "encoding/json" + "fmt" "io" "os" "reflect" "runtime" + "strconv" "sync" "sync/atomic" "time" @@ -544,16 +546,41 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } + resourceOpts := make(map[string]string) + for tt := range l.acceptTasks { + ttShort := tt.Short() + for _, res_opt := range []string{"_MAX_MEMORY", "_MIN_MEMORY", "_MAX_PARALLELISM", "_BASE_MIN_MEMORY", "_GPU_UTILIZATION"} { + n := ttShort + res_opt + if val, ok := os.LookupEnv(n); ok { + resourceOpts[n] = val + } + } + } + if _, ok := resourceOpts["PC1_MAX_PARALLELISM"]; !ok { + if os.Getenv("FIL_PROOFS_USE_MULTICORE_SDR") == "1" { + pc1MulticoreSDRProducers := 3 + if pc1MulticoreSDRProducersEnv := os.Getenv("FIL_PROOFS_MULTICORE_SDR_PRODUCERS"); pc1MulticoreSDRProducersEnv != "" { + pc1MulticoreSDRProducers, err = strconv.Atoi(pc1MulticoreSDRProducersEnv) + if err != nil { + log.Errorf("FIL_PROOFS_MULTICORE_SDR_PRODUCERS is not an integer: %+v", err) + pc1MulticoreSDRProducers = 3 + } + } + resourceOpts["PC1_MAX_PARALLELISM"] = fmt.Sprintf("%d", 1+pc1MulticoreSDRProducers) + } + } + return storiface.WorkerInfo{ Hostname: hostname, IgnoreResources: l.ignoreResources, Resources: storiface.WorkerResources{ - MemPhysical: memPhysical, - MemUsed: memUsed, - MemSwap: memSwap, - MemSwapUsed: memSwapUsed, - CPUs: uint64(runtime.NumCPU()), - GPUs: gpus, + MemPhysical: memPhysical, + MemUsed: memUsed, + MemSwap: memSwap, + MemSwapUsed: memSwapUsed, + CPUs: uint64(runtime.NumCPU()), + GPUs: gpus, + ResourceOpts: resourceOpts, }, }, nil } From 36868a87497cb115c273b763682e41faea68e3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 25 Nov 2021 13:13:16 +0100 Subject: [PATCH 05/13] sched: C2 is not all-core load --- extern/sector-storage/resources.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index c05bca62b..5ba785fc2 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -268,7 +268,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 190 << 30, // TODO: Confirm MinMemory: 60 << 30, - MaxParallelism: -1, + MaxParallelism: 2, GPUUtilization: 1.0, BaseMinMemory: 64 << 30, // params @@ -277,7 +277,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory MinMemory: 30 << 30, - MaxParallelism: -1, + MaxParallelism: 2, GPUUtilization: 1.0, BaseMinMemory: 32 << 30, // params From b961e1aab5abb0899a532afd51c62dc3de8bd943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 12:40:54 +0100 Subject: [PATCH 06/13] sched resources: Separate Parallelism defaults depending on GPU presence --- extern/sector-storage/resources.go | 48 ++++++++++++++++++------ extern/sector-storage/sched_resources.go | 8 ++-- extern/sector-storage/worker_local.go | 2 +- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index 5ba785fc2..e8e3e60cb 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -13,9 +13,15 @@ type Resources struct { MinMemory uint64 // What Must be in RAM for decent perf MaxMemory uint64 // Memory required (swap + ram) - MaxParallelism int // -1 = multithread + // GPUUtilization specifes the number of GPUs a task can use GPUUtilization float64 + // MaxParallelism specifies the number of CPU cores when GPU is NOT in use + MaxParallelism int // -1 = multithread + + // MaxParallelismGPU specifies the number of CPU cores when GPU is in use + MaxParallelismGPU int // when 0, inherits MaxParallelism + BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) } @@ -35,8 +41,14 @@ var ParallelNum uint64 = 92 var ParallelDenom uint64 = 100 // TODO: Take NUMA into account -func (r Resources) Threads(wcpus uint64) uint64 { - if r.MaxParallelism == -1 { +func (r Resources) Threads(wcpus uint64, gpus int) uint64 { + mp := r.MaxParallelism + + if r.GPUUtilization > 0 && gpus > 0 && r.MaxParallelismGPU != 0 { // task can use GPUs and worker has some + mp = r.MaxParallelismGPU + } + + if mp == -1 { n := (wcpus * ParallelNum) / ParallelDenom if n == 0 { return wcpus @@ -44,7 +56,7 @@ func (r Resources) Threads(wcpus uint64) uint64 { return n } - return uint64(r.MaxParallelism) + return uint64(mp) } func (r *Resources) customizeForWorker(taskShortName string, wid WorkerID, info storiface.WorkerInfo) { @@ -81,6 +93,14 @@ func (r *Resources) customizeForWorker(taskShortName string, wid WorkerID, info r.MaxParallelism = i } } + if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM_GPU"]; ok { + i, err := strconv.Atoi(o) + if err != nil { + log.Errorf("unable to parse %s_GPU_PARALLELISM value %s: %e", taskShortName, o, err) + } else { + r.MaxParallelismGPU = i + } + } if o, ok := info.Resources.ResourceOpts[taskShortName+"_GPU_UTILIZATION"]; ok { i, err := strconv.ParseFloat(o, 64) if err != nil { @@ -182,8 +202,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 30 << 30, MinMemory: 30 << 30, - MaxParallelism: -1, - GPUUtilization: 1.0, + MaxParallelism: -1, + MaxParallelismGPU: 6, + GPUUtilization: 1.0, BaseMinMemory: 1 << 30, }, @@ -191,8 +212,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 15 << 30, MinMemory: 15 << 30, - MaxParallelism: -1, - GPUUtilization: 1.0, + MaxParallelism: -1, + MaxParallelismGPU: 6, + GPUUtilization: 1.0, BaseMinMemory: 1 << 30, }, @@ -268,8 +290,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 190 << 30, // TODO: Confirm MinMemory: 60 << 30, - MaxParallelism: 2, - GPUUtilization: 1.0, + MaxParallelism: -1, + MaxParallelismGPU: 6, + GPUUtilization: 1.0, BaseMinMemory: 64 << 30, // params }, @@ -277,8 +300,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory MinMemory: 30 << 30, - MaxParallelism: 2, - GPUUtilization: 1.0, + MaxParallelism: -1, + MaxParallelismGPU: 6, + GPUUtilization: 1.0, BaseMinMemory: 32 << 30, // params }, diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index 636fc02d1..cbd8fb625 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -34,7 +34,7 @@ func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { if r.GPUUtilization > 0 { a.gpuUsed += r.GPUUtilization } - a.cpuUse += r.Threads(wr.CPUs) + a.cpuUse += r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin += r.MinMemory a.memUsedMax += r.MaxMemory } @@ -43,7 +43,7 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { if r.GPUUtilization > 0 { a.gpuUsed -= r.GPUUtilization } - a.cpuUse -= r.Threads(wr.CPUs) + a.cpuUse -= r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin -= r.MinMemory a.memUsedMax -= r.MaxMemory @@ -84,8 +84,8 @@ func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, call return false } - if a.cpuUse+needRes.Threads(res.CPUs) > res.CPUs { - log.Debugf("sched: not scheduling on worker %s for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs), a.cpuUse, res.CPUs) + if a.cpuUse+needRes.Threads(res.CPUs, len(res.GPUs)) > res.CPUs { + log.Debugf("sched: not scheduling on worker %s for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs, len(res.GPUs)), a.cpuUse, res.CPUs) return false } diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index de69cea80..7e5ce8f57 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -549,7 +549,7 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { resourceOpts := make(map[string]string) for tt := range l.acceptTasks { ttShort := tt.Short() - for _, res_opt := range []string{"_MAX_MEMORY", "_MIN_MEMORY", "_MAX_PARALLELISM", "_BASE_MIN_MEMORY", "_GPU_UTILIZATION"} { + for _, res_opt := range []string{"_MAX_MEMORY", "_MIN_MEMORY", "_MAX_PARALLELISM", "_MAX_PARALLELISM_GPU", "_BASE_MIN_MEMORY", "_GPU_UTILIZATION"} { n := ttShort + res_opt if val, ok := os.LookupEnv(n); ok { resourceOpts[n] = val From c9a2ff4007de10d03b6905c241dfaae403c4afe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 14:42:20 +0100 Subject: [PATCH 07/13] cleanup worker resource overrides --- api/version.go | 2 +- extern/sector-storage/manager.go | 5 - extern/sector-storage/sched.go | 14 +- extern/sector-storage/sched_resources.go | 8 +- extern/sector-storage/sched_test.go | 4 +- extern/sector-storage/sched_worker.go | 24 ++- .../{ => storiface}/resources.go | 154 +++++++++++------- .../storiface/resources_test.go | 75 +++++++++ extern/sector-storage/storiface/worker.go | 30 +++- extern/sector-storage/worker_local.go | 31 +--- extern/sector-storage/worker_tracked.go | 8 +- 11 files changed, 232 insertions(+), 123 deletions(-) rename extern/sector-storage/{ => storiface}/resources.go (71%) create mode 100644 extern/sector-storage/storiface/resources_test.go diff --git a/api/version.go b/api/version.go index ff1115e1d..93148f28d 100644 --- a/api/version.go +++ b/api/version.go @@ -58,7 +58,7 @@ var ( FullAPIVersion1 = newVer(2, 1, 0) MinerAPIVersion0 = newVer(1, 2, 0) - WorkerAPIVersion0 = newVer(1, 4, 0) + WorkerAPIVersion0 = newVer(1, 5, 0) ) //nolint:varcheck,deadcode diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 430313730..fb081ee5d 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -51,13 +51,8 @@ type SectorManager interface { FaultTracker } -type WorkerID uuid.UUID // worker session UUID var ClosedWorkerID = uuid.UUID{} -func (w WorkerID) String() string { - return uuid.UUID(w).String() -} - type Manager struct { ls stores.LocalStorage storage *stores.Remote diff --git a/extern/sector-storage/sched.go b/extern/sector-storage/sched.go index e6e3121f3..d7d7d3265 100644 --- a/extern/sector-storage/sched.go +++ b/extern/sector-storage/sched.go @@ -53,7 +53,7 @@ type WorkerSelector interface { type scheduler struct { workersLk sync.RWMutex - workers map[WorkerID]*workerHandle + workers map[storiface.WorkerID]*workerHandle schedule chan *workerRequest windowRequests chan *schedWindowRequest @@ -95,7 +95,7 @@ type workerHandle struct { } type schedWindowRequest struct { - worker WorkerID + worker storiface.WorkerID done chan *schedWindow } @@ -107,7 +107,7 @@ type schedWindow struct { type workerDisableReq struct { activeWindows []*schedWindow - wid WorkerID + wid storiface.WorkerID done func() } @@ -145,7 +145,7 @@ type workerResponse struct { func newScheduler() *scheduler { return &scheduler{ - workers: map[WorkerID]*workerHandle{}, + workers: map[storiface.WorkerID]*workerHandle{}, schedule: make(chan *workerRequest), windowRequests: make(chan *schedWindowRequest, 20), @@ -378,7 +378,6 @@ func (sh *scheduler) trySched() { }() task := (*sh.schedQueue)[sqi] - needRes := ResourceTable[task.taskType][task.sector.ProofType] task.indexHeap = sqi for wnd, windowRequest := range sh.openWindows { @@ -394,6 +393,8 @@ func (sh *scheduler) trySched() { continue } + needRes := worker.info.Resources.ResourceSpec(task.sector.ProofType, task.taskType) + // TODO: allow bigger windows if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info) { continue @@ -457,7 +458,6 @@ func (sh *scheduler) trySched() { for sqi := 0; sqi < queueLen; sqi++ { task := (*sh.schedQueue)[sqi] - needRes := ResourceTable[task.taskType][task.sector.ProofType] selectedWindow := -1 for _, wnd := range acceptableWindows[task.indexHeap] { @@ -466,6 +466,8 @@ func (sh *scheduler) trySched() { log.Debugf("SCHED try assign sqi:%d sector %d to window %d", sqi, task.sector.ID.Number, wnd) + needRes := info.Resources.ResourceSpec(task.sector.ProofType, task.taskType) + // TODO: allow bigger windows if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", info) { continue diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index cbd8fb625..6e5d70508 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) -func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerInfo, r Resources, locker sync.Locker, cb func() error) error { +func (a *activeResources) withResources(id storiface.WorkerID, wr storiface.WorkerInfo, r storiface.Resources, locker sync.Locker, cb func() error) error { for !a.canHandleRequest(r, id, "withResources", wr) { if a.cond == nil { a.cond = sync.NewCond(locker) @@ -30,7 +30,7 @@ func (a *activeResources) hasWorkWaiting() bool { return a.waiting > 0 } -func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { +func (a *activeResources) add(wr storiface.WorkerResources, r storiface.Resources) { if r.GPUUtilization > 0 { a.gpuUsed += r.GPUUtilization } @@ -39,7 +39,7 @@ func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { a.memUsedMax += r.MaxMemory } -func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { +func (a *activeResources) free(wr storiface.WorkerResources, r storiface.Resources) { if r.GPUUtilization > 0 { a.gpuUsed -= r.GPUUtilization } @@ -54,7 +54,7 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { // canHandleRequest evaluates if the worker has enough available resources to // handle the request. -func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, info storiface.WorkerInfo) bool { +func (a *activeResources) canHandleRequest(needRes storiface.Resources, wid storiface.WorkerID, caller string, info storiface.WorkerInfo) bool { if info.IgnoreResources { // shortcircuit; if this worker is ignoring resources, it can always handle the request. return true diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index b98c93031..f64ed57e2 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -560,7 +560,7 @@ func BenchmarkTrySched(b *testing.B) { b.StopTimer() sched := newScheduler() - sched.workers[WorkerID{}] = &workerHandle{ + sched.workers[storiface.WorkerID{}] = &workerHandle{ workerRpc: nil, info: storiface.WorkerInfo{ Hostname: "t", @@ -572,7 +572,7 @@ func BenchmarkTrySched(b *testing.B) { for i := 0; i < windows; i++ { sched.openWindows = append(sched.openWindows, &schedWindowRequest{ - worker: WorkerID{}, + worker: storiface.WorkerID{}, done: make(chan *schedWindow, 1000), }) } diff --git a/extern/sector-storage/sched_worker.go b/extern/sector-storage/sched_worker.go index bb6ba627b..762c3fc3a 100644 --- a/extern/sector-storage/sched_worker.go +++ b/extern/sector-storage/sched_worker.go @@ -4,17 +4,18 @@ import ( "context" "time" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) type schedWorker struct { sched *scheduler worker *workerHandle - wid WorkerID + wid storiface.WorkerID heartbeatTimer *time.Ticker scheduledWindows chan *schedWindow @@ -50,7 +51,7 @@ func (sh *scheduler) runWorker(ctx context.Context, w Worker) error { closedMgr: make(chan struct{}), } - wid := WorkerID(sessID) + wid := storiface.WorkerID(sessID) sh.workersLk.Lock() _, exist := sh.workers[wid] @@ -237,7 +238,7 @@ func (sw *schedWorker) checkSession(ctx context.Context) bool { continue } - if WorkerID(curSes) != sw.wid { + if storiface.WorkerID(curSes) != sw.wid { if curSes != ClosedWorkerID { // worker restarted log.Warnw("worker session changed (worker restarted?)", "initial", sw.wid, "current", curSes) @@ -296,8 +297,7 @@ func (sw *schedWorker) workerCompactWindows() { var moved []int for ti, todo := range window.todo { - needRes := ResourceTable[todo.taskType][todo.sector.ProofType] - needRes.customizeForWorker(todo.taskType.Short(), sw.wid, worker.info) + needRes := worker.info.Resources.ResourceSpec(todo.sector.ProofType, todo.taskType) if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info) { continue } @@ -358,8 +358,7 @@ assignLoop: worker.lk.Lock() for t, todo := range firstWindow.todo { - needRes := ResourceTable[todo.taskType][todo.sector.ProofType] - needRes.customizeForWorker(todo.taskType.Short(), sw.wid, worker.info) + needRes := worker.info.Resources.ResourceSpec(todo.sector.ProofType, todo.taskType) if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) { tidx = t break @@ -420,7 +419,7 @@ assignLoop: continue } - needRes := ResourceTable[todo.taskType][todo.sector.ProofType] + needRes := storiface.ResourceTable[todo.taskType][todo.sector.ProofType] if worker.active.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) { tidx = t break @@ -458,8 +457,7 @@ assignLoop: func (sw *schedWorker) startProcessingTask(req *workerRequest) error { w, sh := sw.worker, sw.sched - needRes := ResourceTable[req.taskType][req.sector.ProofType] - needRes.customizeForWorker(req.taskType.Short(), sw.wid, w.info) + needRes := w.info.Resources.ResourceSpec(req.sector.ProofType, req.taskType) w.lk.Lock() w.preparing.add(w.info.Resources, needRes) @@ -542,7 +540,7 @@ func (sw *schedWorker) startProcessingTask(req *workerRequest) error { func (sw *schedWorker) startProcessingReadyTask(req *workerRequest) error { w, sh := sw.worker, sw.sched - needRes := ResourceTable[req.taskType][req.sector.ProofType] + needRes := w.info.Resources.ResourceSpec(req.sector.ProofType, req.taskType) w.active.add(w.info.Resources, needRes) @@ -582,7 +580,7 @@ func (sw *schedWorker) startProcessingReadyTask(req *workerRequest) error { return nil } -func (sh *scheduler) workerCleanup(wid WorkerID, w *workerHandle) { +func (sh *scheduler) workerCleanup(wid storiface.WorkerID, w *workerHandle) { select { case <-w.closingMgr: default: diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/storiface/resources.go similarity index 71% rename from extern/sector-storage/resources.go rename to extern/sector-storage/storiface/resources.go index e8e3e60cb..d634927ed 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/storiface/resources.go @@ -1,28 +1,31 @@ -package sectorstorage +package storiface import ( + "fmt" + "reflect" "strconv" + "strings" + + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) type Resources struct { - MinMemory uint64 // What Must be in RAM for decent perf - MaxMemory uint64 // Memory required (swap + ram) + MinMemory uint64 `envname:"MIN_MEMORY"` // What Must be in RAM for decent perf + MaxMemory uint64 `envname:"MAX_MEMORY"` // Memory required (swap + ram) // GPUUtilization specifes the number of GPUs a task can use - GPUUtilization float64 + GPUUtilization float64 `envname:"GPU_UTILIZATION"` // MaxParallelism specifies the number of CPU cores when GPU is NOT in use - MaxParallelism int // -1 = multithread + MaxParallelism int `envname:"MAX_PARALLELISM"` // -1 = multithread // MaxParallelismGPU specifies the number of CPU cores when GPU is in use - MaxParallelismGPU int // when 0, inherits MaxParallelism + MaxParallelismGPU int `envname:"MAX_PARALLELISM_GPU"` // when 0, inherits MaxParallelism - BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) + BaseMinMemory uint64 `envname:"BASE_MIN_MEMORY"` // What Must be in RAM for decent perf (shared between threads) } /* @@ -59,59 +62,6 @@ func (r Resources) Threads(wcpus uint64, gpus int) uint64 { return uint64(mp) } -func (r *Resources) customizeForWorker(taskShortName string, wid WorkerID, info storiface.WorkerInfo) { - // update needed resources with worker options - if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_MEMORY"]; ok { - i, err := strconv.ParseUint(o, 10, 64) - if err != nil { - log.Errorf("unable to parse %s_MAX_MEMORY value %s: %e", taskShortName, o, err) - } else { - r.MaxMemory = i - } - } - if o, ok := info.Resources.ResourceOpts[taskShortName+"_MIN_MEMORY"]; ok { - i, err := strconv.ParseUint(o, 10, 64) - if err != nil { - log.Errorf("unable to parse %s_MIN_MEMORY value %s: %e", taskShortName, o, err) - } else { - r.MinMemory = i - } - } - if o, ok := info.Resources.ResourceOpts[taskShortName+"_BASE_MIN_MEMORY"]; ok { - i, err := strconv.ParseUint(o, 10, 64) - if err != nil { - log.Errorf("unable to parse %s_BASE_MIN_MEMORY value %s: %e", taskShortName, o, err) - } else { - r.BaseMinMemory = i - } - } - if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM"]; ok { - i, err := strconv.Atoi(o) - if err != nil { - log.Errorf("unable to parse %s_MAX_PARALLELISM value %s: %e", taskShortName, o, err) - } else { - r.MaxParallelism = i - } - } - if o, ok := info.Resources.ResourceOpts[taskShortName+"_MAX_PARALLELISM_GPU"]; ok { - i, err := strconv.Atoi(o) - if err != nil { - log.Errorf("unable to parse %s_GPU_PARALLELISM value %s: %e", taskShortName, o, err) - } else { - r.MaxParallelismGPU = i - } - } - if o, ok := info.Resources.ResourceOpts[taskShortName+"_GPU_UTILIZATION"]; ok { - i, err := strconv.ParseFloat(o, 64) - if err != nil { - log.Errorf("unable to parse %s_GPU_UTILIZATION value %s: %e", taskShortName, o, err) - } else { - r.GPUUtilization = i - } - } - log.Debugf("resources required for %s on %s(%s): %+v", taskShortName, wid, info.Hostname, r) -} - var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{ sealtasks.TTAddPiece: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ @@ -395,3 +345,83 @@ func init() { m[abi.RegisteredSealProof_StackedDrg64GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg64GiBV1] } } + +func ParseResources(lookup func(key, def string) (string, bool)) (map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources, error) { + out := map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{} + + for taskType, defTT := range ResourceTable { + out[taskType] = map[abi.RegisteredSealProof]Resources{} + + for spt, defRes := range defTT { + r := defRes // copy + + spsz, err := spt.SectorSize() + if err != nil { + return nil, xerrors.Errorf("getting sector size: %w", err) + } + shortSize := strings.TrimSuffix(spsz.ShortString(), "iB") + + rr := reflect.ValueOf(&r) + for i := 0; i < rr.Elem().Type().NumField(); i++ { + f := rr.Elem().Type().Field(i) + + envname := f.Tag.Get("envname") + if envname == "" { + return nil, xerrors.Errorf("no envname for field '%s'", f.Name) + } + + envval, found := lookup(taskType.Short() + "_" + shortSize + "_" + envname, fmt.Sprint(rr.Elem().Field(i).Interface())) + if !found { + // special multicore SDR handling + if (taskType == sealtasks.TTPreCommit1 || taskType == sealtasks.TTUnseal) && envname == "MAX_PARALLELISM" { + v, ok := rr.Elem().Field(i).Addr().Interface().(*int) + if !ok { + // can't happen, but let's not panic + return nil, xerrors.Errorf("res.MAX_PARALLELISM is not int (!?): %w", err) + } + *v, err = getSDRThreads(lookup) + if err != nil { + return nil, err + } + } + + continue + } + + v := rr.Elem().Field(i).Addr().Interface() + switch fv := v.(type) { + case *uint64: + *fv, err = strconv.ParseUint(envval, 10, 64) + case *int: + *fv, err = strconv.Atoi(envval) + case *float64: + *fv, err = strconv.ParseFloat(envval, 64) + default: + return nil, xerrors.Errorf("unknown resource field type") + } + } + + out[taskType][spt] = r + } + } + + return out, nil +} + +func getSDRThreads(lookup func(key, def string) (string, bool)) (_ int, err error) { + producers := 0 + + if v, _ := lookup("FIL_PROOFS_USE_MULTICORE_SDR", ""); v == "1" { + producers = 3 + + if penv, found := lookup("FIL_PROOFS_MULTICORE_SDR_PRODUCERS", ""); found { + producers, err = strconv.Atoi(penv) + if err != nil { + return 0, xerrors.Errorf("parsing (atoi) FIL_PROOFS_MULTICORE_SDR_PRODUCERS: %w", err) + } + } + } + + // producers + the one core actually doing the work + return producers+1, nil +} diff --git a/extern/sector-storage/storiface/resources_test.go b/extern/sector-storage/storiface/resources_test.go new file mode 100644 index 000000000..f58f46e23 --- /dev/null +++ b/extern/sector-storage/storiface/resources_test.go @@ -0,0 +1,75 @@ +package storiface + +import ( + "fmt" + "testing" + + stabi "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" + "github.com/stretchr/testify/require" +) + +func TestListResourceVars(t *testing.T) { + _, err := ParseResources(func(key, def string) (string, bool) { + if def != "" { + fmt.Printf("%s=%s\n", key, def) + } + + return "", false + }) + + require.NoError(t, err) +} + +func TestListResourceOverride(t *testing.T) { + rt, err := ParseResources(func(key, def string) (string, bool) { + if key == "UNS_2K_MAX_PARALLELISM" { + return "2", true + } + if key == "PC2_2K_GPU_UTILIZATION" { + return "0.4", true + } + if key == "PC2_2K_MAX_MEMORY" { + return "2222", true + } + + return "", false + }) + + require.NoError(t, err) + require.Equal(t, 2, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) + require.Equal(t, 0.4, rt[sealtasks.TTPreCommit2][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].GPUUtilization) + require.Equal(t, uint64(2222), rt[sealtasks.TTPreCommit2][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxMemory) + + // check that defaults don't get mutated + require.Equal(t, 1, ResourceTable[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) +} + +func TestListResourceSDRMulticoreOverride(t *testing.T) { + rt, err := ParseResources(func(key, def string) (string, bool) { + if key == "FIL_PROOFS_USE_MULTICORE_SDR" { + return "1", true + } + + return "", false + }) + + require.NoError(t, err) + require.Equal(t, 4, rt[sealtasks.TTPreCommit1][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) + require.Equal(t, 4, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) + + rt, err = ParseResources(func(key, def string) (string, bool) { + if key == "FIL_PROOFS_USE_MULTICORE_SDR" { + return "1", true + } + if key == "FIL_PROOFS_MULTICORE_SDR_PRODUCERS" { + return "9000", true + } + + return "", false + }) + + require.NoError(t, err) + require.Equal(t, 9001, rt[sealtasks.TTPreCommit1][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) + require.Equal(t, 9001, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) +} diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index f28f106b1..380e968e1 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -15,6 +15,12 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" ) +type WorkerID uuid.UUID // worker session UUID + +func (w WorkerID) String() string { + return uuid.UUID(w).String() +} + type WorkerInfo struct { Hostname string @@ -34,7 +40,29 @@ type WorkerResources struct { CPUs uint64 // Logical cores GPUs []string - ResourceOpts map[string]string + + // if nil use the default resource table + Resources map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources +} + +func (wr WorkerResources) ResourceSpec(spt abi.RegisteredSealProof, tt sealtasks.TaskType) Resources { + res := ResourceTable[tt][spt] + + // if the worker specifies custom resource table, prefer that + if wr.Resources != nil { + tr, ok := wr.Resources[tt] + if !ok { + return res + } + + r, ok := tr[spt] + if ok { + return r + } + } + + // otherwise, use the default resource table + return res } type WorkerStats struct { diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index 7e5ce8f57..50bf6ada0 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -3,12 +3,10 @@ package sectorstorage import ( "context" "encoding/json" - "fmt" "io" "os" "reflect" "runtime" - "strconv" "sync" "sync/atomic" "time" @@ -546,28 +544,11 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } - resourceOpts := make(map[string]string) - for tt := range l.acceptTasks { - ttShort := tt.Short() - for _, res_opt := range []string{"_MAX_MEMORY", "_MIN_MEMORY", "_MAX_PARALLELISM", "_MAX_PARALLELISM_GPU", "_BASE_MIN_MEMORY", "_GPU_UTILIZATION"} { - n := ttShort + res_opt - if val, ok := os.LookupEnv(n); ok { - resourceOpts[n] = val - } - } - } - if _, ok := resourceOpts["PC1_MAX_PARALLELISM"]; !ok { - if os.Getenv("FIL_PROOFS_USE_MULTICORE_SDR") == "1" { - pc1MulticoreSDRProducers := 3 - if pc1MulticoreSDRProducersEnv := os.Getenv("FIL_PROOFS_MULTICORE_SDR_PRODUCERS"); pc1MulticoreSDRProducersEnv != "" { - pc1MulticoreSDRProducers, err = strconv.Atoi(pc1MulticoreSDRProducersEnv) - if err != nil { - log.Errorf("FIL_PROOFS_MULTICORE_SDR_PRODUCERS is not an integer: %+v", err) - pc1MulticoreSDRProducers = 3 - } - } - resourceOpts["PC1_MAX_PARALLELISM"] = fmt.Sprintf("%d", 1+pc1MulticoreSDRProducers) - } + resEnv, err := storiface.ParseResources(func(key, def string) (string, bool) { + return os.LookupEnv(key) + }) + if err != nil { + return storiface.WorkerInfo{}, xerrors.Errorf("interpreting resource env vars: %w", err) } return storiface.WorkerInfo{ @@ -580,7 +561,7 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { MemSwapUsed: memSwapUsed, CPUs: uint64(runtime.NumCPU()), GPUs: gpus, - ResourceOpts: resourceOpts, + Resources: resEnv, }, }, nil } diff --git a/extern/sector-storage/worker_tracked.go b/extern/sector-storage/worker_tracked.go index 5702426c3..7a88d9bd4 100644 --- a/extern/sector-storage/worker_tracked.go +++ b/extern/sector-storage/worker_tracked.go @@ -20,7 +20,7 @@ import ( type trackedWork struct { job storiface.WorkerJob - worker WorkerID + worker storiface.WorkerID workerHostname string } @@ -58,7 +58,7 @@ func (wt *workTracker) onDone(ctx context.Context, callID storiface.CallID) { delete(wt.running, callID) } -func (wt *workTracker) track(ctx context.Context, ready chan struct{}, wid WorkerID, wi storiface.WorkerInfo, sid storage.SectorRef, task sealtasks.TaskType, cb func() (storiface.CallID, error)) (storiface.CallID, error) { +func (wt *workTracker) track(ctx context.Context, ready chan struct{}, wid storiface.WorkerID, wi storiface.WorkerInfo, sid storage.SectorRef, task sealtasks.TaskType, cb func() (storiface.CallID, error)) (storiface.CallID, error) { tracked := func(rw int, callID storiface.CallID) trackedWork { return trackedWork{ job: storiface.WorkerJob{ @@ -122,7 +122,7 @@ func (wt *workTracker) track(ctx context.Context, ready chan struct{}, wid Worke return callID, err } -func (wt *workTracker) worker(wid WorkerID, wi storiface.WorkerInfo, w Worker) *trackedWorker { +func (wt *workTracker) worker(wid storiface.WorkerID, wi storiface.WorkerInfo, w Worker) *trackedWorker { return &trackedWorker{ Worker: w, wid: wid, @@ -152,7 +152,7 @@ func (wt *workTracker) Running() ([]trackedWork, []trackedWork) { type trackedWorker struct { Worker - wid WorkerID + wid storiface.WorkerID workerInfo storiface.WorkerInfo execute chan struct{} // channel blocking execution in case we're waiting for resources but the task is ready to execute From 6d52d8552bef647137946c22f6e1988f2d868719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 14:46:41 +0100 Subject: [PATCH 08/13] Fix docsgen --- api/docgen/docgen.go | 2 + build/openrpc/miner.json.gz | Bin 10500 -> 11151 bytes build/openrpc/worker.json.gz | Bin 2710 -> 3401 bytes documentation/en/api-v0-methods-miner.md | 583 ++++++++++++++++++- documentation/en/api-v0-methods-worker.md | 581 +++++++++++++++++- extern/sector-storage/storiface/resources.go | 4 +- extern/sector-storage/storiface/worker.go | 4 +- extern/sector-storage/worker_local.go | 14 +- 8 files changed, 1172 insertions(+), 16 deletions(-) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 9402f50d8..5478e5ea6 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -236,6 +236,7 @@ func init() { MemSwapUsed: 2 << 30, CPUs: 64, GPUs: []string{"aGPU 1337"}, + Resources: storiface.ResourceTable, }, }, Enabled: true, @@ -287,6 +288,7 @@ func init() { State: "ShardStateAvailable", Error: "", }) + addExample(storiface.ResourceTable) } func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) { diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 176c2814f1de81981f05838b1e374e22f416111a..f02a58bec816a99be0c2ecb0df65dbffb92dce48 100644 GIT binary patch literal 11151 zcmV;AD{#~wiwFP!00000|LlEhbK5r7@L$33{qQ6m*`Y3$CC~JOV<)M%PV2{Vw$E;y z*+L{FVNHQ52{~3X{_pPq;GF>PlpTk=oz^13g9G5)aB$ASH_hH4;%p4TC`A8bkrS+UUTr2}KX|@J#&oHFyQZ%R{eG4F;3tB|fqjiY{Jl^w+oK z@s|Ag=bu_nbGPd?bavW_ruDQ9bl_SHa_g`uo9_-bwu|}Hc(2)%e(7lra<>*`E91>> z0>9L}rr*c9c8wKULk15$!1cyLtRHQm_Xop~_NB)@=V(Cd%CR@dp_sK;K+8ovjqWyx zHMi&gMF!QrJTmuFoZ(oVD$r5pF;&6_1}a%kA=Hw%y64mOU>@Ooo0+I=&(?D?BDbSZM) zzzu%0aqNfPf0l?GF75w!IUJ9s+T&v{n`+N&vcSvQv@5{KDLB@oDE*4o8B`o=SDA(b zyj!8<>^V-p&8OmT+LeRO?DZN`HocB*X(wcBSv^hq@CG?+Odv&x=jRAoSKGOT-Bt1p zcH!JYzkemjc@HgX4vjmX?!9+^*v?(@{LHV%e&ZJ8NN`lkQ1MoY}_L*|t|qgRe?FM41z0H43O0<5>h*N8nI0w&&HgI8;~`5dml zSi&zZbuh`afnsQ7Fmnd@Qk0Ua8o@V!cKlZ#?EYkK-Vkqzxyx~tv0Eo!p$QlW2qx2FD zQ%l77vCx_@=n({bGp)LI$l%cwmf3BKYz_K@p?0Dfko*q?(BSw22SF29wsEI11UHZ} zuD^;A$Kg*A)70zm#o2h1fkKVg>&=!TU-DGYA`t*8CGz>pe2XoLiMO~HWMOj+!q%G! zzH3NKdrgGp7q*V5n+$ysK_Ek2NT8Ne6Y+8STB21AA)B|>h3gFQM)snZykyOhc_u`% z-<&T#q2R(qgyIE`e(G}=`!wYAr<0dym6oQ;YLi|?U#Apw#R1Bsv4jOONUnfE(h$Xk zJr$xj-t_H`ijUXG#2lG9CeYcvBZh4<`a03(&_&}>Wc!{a+pE&iGZpvKp+ej{2vYPG z8h6`GHE|ySpc40<5iVr5RAfhh$VGMuxg1AGy(^D(YVaQ}AhwCXTs*Uhi5x&z2)G+$ zFsWq%HvNE2;vu+Mp<5!3LV!mYE}$z&`6ql?4%Td{7)Lu>LNHl^ub8gb`&)AP`^~4> zf8L&+{rc(ZKW|TG@BVXjdh-iFoGP=>~ah+;W#pe;NRYoh7?o@^UP}t?xD3sT7=y^7}dqvZ(GZf z2v`0vvUv&wt7U!P%zdnn0v1*k*izD$Rr9^-l|=d z`;IM|WZTozenmU!^V!xgm?b3f;=g=-dfY$CX1NQ3DTWsQTdZO&{!75BtYsDO>MC3{ znrhvvLInrET>uH#lJJ2ET-N09mfVuyGss!hu!)OZ>H>QK7Kj=vrdPP$W?%e|3rt8M zcx57JfeqYQHZ)&@g=4RU_VoN!z}?Q>olDWW2bPE+2T}$H5`c+`9)!-_AcuA3cfjA= z5^945G_Zv!WF!YEa)5)VgV8;-T>j-7+hHnvBM#%(pFR}>*WNk?a{0(^R@eoMF~!KR z3ozR`naE_H7&hZ|ZX1Z+`80qvUW)d;RF=1LhFDgteuP+77$x5z>AQ!8H!>h0B4{X!{j&g6}aq=!&`De z!Q8TqI~QyT#a42H7MNgn#c8jz0Ka2(G(65$N0->8oIt9@WTn9rvIvD+FW(*v!4|g* z8SF|{VQdW9+9`sUfXAfq!gkhzdEngMQUEQ>my37A{z@3La~Tm@j^KA4+Bqa+Ml0EvuWZgaXPI$)m<*NjD0#Il_%XZINsf$ah4$ zAEKQoLgeC0L|PHJ5~xj(tpsc4FEV?^7!sFk7K5$s+F zcA_i73SuktiQFTHpG-GPv~su&5Uw0>8^o&|LPx;+A>h91=sdbg|7?YPWl$R+UK!Xn zNLPV=N4WbT+=1!plKm(hHXV~O{yZMg6ILz8U!4!&n3BdJ+ zoS%{*4S0>iI>RP#7;U+TP<}v*ti^$7Gsk2p=A2MJ7Y>;K2ha||1qIjz3)@*E6U=u2 zf`5v1{T_he%r=n$UVU9*V+AlVvB9I;AXslLiVgPmU;~XiWCp!3_IS-tDY>UUy6zDbu;TOP%ZyRvD3_fX3%aYq?b9QC`N!Flkt$M! z$y32_*nDf*#+_A5eWXCCbV*>v<|vXBz)Y>;K`pORWO%ovUqWLxsa$f++#*>{FUSN= z8Hg{8DUVb!CX6Cyh_-Y}od;R9@f7t0vbUOfup8Nc78pS5RY9IEnfr&^?ToV`InuPy zjJq%!ZnG?s?=!5;APM9<_INq$G0BP*iBdwNC+|6cn_`f{2k#PA}@zUrdg3G zDN<-&ffPCBug)H=st+0J9c#Q8)|l#9u`dxqXgZo87j#1`QYNUNI-4km(6P$PVU>PX z)J=pKnvGk?0o)Lq6c1mh&Lj%-I~I94EHcQ7>zN_ipr=^|Ob9;t$QVw3&08f>27knOf>+(}#~)O_0!w_4NcjBXoHK__SIFVB z(pb8qV*1-FI^9Y=U5`+TMK<*Z=fhuZ`;QNQKm7gczyEcQe)&JT`7l0polpOGYy9!> z%iH&}!7uJR`@_wH^YQZj*Z-r7$`Me{w8rcYC;LWA@=u&}|9(6umn zCi?6nCXu+KzCIlEw2$y1{Y(_U_OAx$X*YO{Tng74)}pWXd)m8)4Q5DrKWFH#-)!w< zGYLWPy z_-qvVA~vT5zo-_-#TT-4?3P%WB8Wv+xk&Q_g_T208oU)HoDvHxNUKs|*Pd3o^m=A2 z*7vL>#)=q^Kx_sFPqo!W6p})31QoJ8$yiibL_s&JH4;d-qdp|VPU{SLqhfbqiIB{< zJL)TiPsHW*yF)c`1$S#iD4&6QXfd&#dU!z$2SFEcnO{zVr@$8qfK_#0EP6IN>ar3v zHbE7o+zSyAQ^f(k;1Ms|20^QGVv!6`BiThVz|CxrQ^2Jkqav?+!-QX?oRscN94M2DJxyr(|)HbPY)F# z|Dlc8K+3TSn97Tj`y%HBZRx`HtAG=b|Fyz;D zmE2r$*;nx(rpy-S5|Je>sC|E5EL=p(IHZbUQRI}T>JrG!$~vJ)9&;(q1ChlCUUJ2* z;n#{=>AL4oGk(Eq(L*SkLTl!`6LOn39JKYqETK_mGSn3!Z0YOTKv6Jq>nGA8?srLb(hI!~8x^o03Foe;Pj=@Uh5ZNU?>x3Vk zBpwU_oa4i@6~u(^^?G14;*YMNX(BV&>-NCrNcKEelFTy7V3d^~!BC?X5yO~a*QykW z&RQ{juWNbEr)t~FEQ91ZVwtu-;sn>4&BqITG+_6QqytE>L5rgp4jLH5(RVwDPK8{K z`g&&i`mC`E7qi?77hI{x>Npfp1otr3$4W@aOYG7!Y%-#+78sQVQ{&d|G_b~R?3Uu& z-PZj=E>?N2On9US?+>X_{8f_$#b1pup!lmr_6u(mtI>y;QM4npNwV4E_$ zfCf4|gO=sRH1p&A(7CV~f}i5;-U%jC9@@tHu6tUW8cBXkbBOLAwsVIZ0iv-GGkq)lu$KwA7l7IRC*GjUZ*rWv;*8(vZ`Fy*E4;KPpf85j7m_&Xhw~DhW zS-S1Ykop%EWW`g(n5Za$M6M%h+VN5LsK2sPxv1m1fmu(-hsgw0(W404MB0fSDWedt zRA2TGfrBe_HIlMyLl=ZlL$^*8@$*6#yhoJB4^~yI{QFW`wUQ3y-87>>eT?^VQ%9#E zI}MqqAv;;!$!bqlPZennp5(#=TKMleZUh;iJxK3Ze+`N8jhlo+J;_XUX#n^G#tQFQx*POUr(wbIG-PNsV@eWb{a5;AOtpmk=i*O;aSO>uc1 zfl^AAdw0wodf)!OrPX%~5geKk@9{x(yM|7@KO6C$=9_vE++}M1q$>PW4Uz8lB%S^$ zpudh3Nn1%Fo#v7%`fE^0e_c7Yy=X#p4VqS6gHCmIs_QwbF3vKY*6Os@%hXz9MV4bI zpRXL$%*HDOR8e=KrnX-psOHpIY;>I_>onOj)MTjJhjkk5rE9cfMe49bRM*{#g@7t) zG$D!3u<16W&Klg6#fH|Yvre5oJ9QRvPeCdP&Vw5({WNP7UP;0WJ1>_{8J&i`6j|!@ zcBi)s=Fmwz>D+eG7d#YUwkUw2wo-;k6a`n;8cLwndYS0X2(ZZa9Ky@1VB`BM_i2k3NwMg`10FhmXc3GZKPnm1XZ;hN?~Ab*Bhp&1u)=> z_kdB1xwVi3TnZ^_tiIgS-Lf26q7+u~cLgM>S_TS~s%>qaDWeS)>5@XS`qc|>>{GI< z$ZIJ$Dt3c&eE5lMQkSrXCmH9ELVkw9U-Q!RWMRJsuS74RrhGP!rm)E{#iLM5+2Qg* z=O1sJnkL|v4ghKcNPxXT1h^d7A-GroVtc<_ae+M|_I9}f?gko&8|mzW-Yr3x-(L>_ zL0`Quz*lTpV2*&{AVz~dU^~K&H=t#CZ!O!n18iqtfwk?RTjKX-Koa@3oW-9dF{Df0 z;+>p>Q7gCNN9nT(8B^ck9b=H9cUFj7%FnRr->K$b{%|xJ4#Y2psqKn86q~=}A|wH? zn&dx`gBKW?{Hl0wCeAF32-Qxsh29?wM_Nz2a_oC-GCM7Nb_wUmO8R+v+KfVnic5G; z_-(m8?Hw`W4?HpE-{I)J6c<%`J)PN>1u1eER{VuPD)HF6wPWU~TYj37+y7%v`xT*0 zcznVMqWp~?MKA40u4)-Sn70yT@3%}(R#^z)oE5s4M9@OY2Uq6~stBnTyAaeT(XvC8 zkz83AQK~C+;fj<4%Jv#(-eC59YfpMrOOjtoP-8ioC^scCRO4Kl9$qgdQ1^2PDzlBF zR!mAyBwm*BH%!Aze3-B zGsPRfh16JGqfDQ73V7y)A*voKnBDR5sCLRp^)eELmXjp z`xY@=E*-RF^uq5%5KRwn$+hQ3X9g6&i|}LUlK-I$G_CC~S5KGZN|TJH6(Tds$YXv$ zC)2v`A-0ey)_^zNK+2ZzD)#;1Pg%!i5kdUXU6$A1k=~niU&dXOGJ*8Mjs}ht3Dr>^ zyF$biM=LJMjF`9+%vbuMksyFCKSa0~0~Z1#`4uKbW#)dw$nH2J4vVc{9^Q zkzgZ84F@qx}KsqpnPXh(7w38DBeXumr<&p3fd_(wk-XWz*6Qr3Yy5>M{|a$ z=Q|3TK7sovHFcRDBw6dHaw2tJpQjW#BV7@KonVmqj-Mo`W_&+MsBOA`BqJy!T=Jbj zp^(Y(195TF`G7hfP=tf~%8+ihaMx&U-`BCL1%SrvX#uVoJ32ruE0i+Qsl)%{rA7#c z%)+tPL0B5Npun^d*U%I>YfK>J8%~=X7%PZL4~W~?4M=Xu90AiNo;@J?x$!?b1iVYe z+mM)m*dajn%)<955&>pez<^s9fsk96Xn+=Djb?+ix=p~d=jWj^WSS?vH15+?KvZ#j z+%WcVro5hyjjf1J<&dbqglo@9|YsNJ^%XSS{}mY#B#0mo1|R_}Kx_vXBXE z5}2r%fOqQ+-2vMHe?UynIT!MzBlnh28&C%?m&ge+lMr;QU1zxdSZp}X3?K^jMm2aa zfYBHO1`wO!z9@outQU42>@{OsQ-9M5Ua@TKNfQyXGRJIv{?6ODGo^Pop; zf1WJg&kRX&uhF6wtF!Or>TeA<=ai992{G>8IhHFdpcEsmn~ zBxG5z`4NcUGj9(vgOyyS6}GjwTRe3D% zh}&rRP$xa@)S&oYoZ<&{njSXTi9cfF{{!LWih5e%47p;v>%eAfVKeU8re%n{IE8x_@1F7y zzpx=k_5(6OAT!#_Nz<3+Z5d^zlDIQ(yIIHGB0#A9sS$7g7fcEi_4YyL_)`;bD>+#L z+|}L{?-#&iVFP>7*wLb72vSdLx=o=9(4ITsq(9Je9H$Mi?n~pEqs$Z!x9-(~X)sy$ z<%Cz4@}mf^h`vbU_baZy-BENwO!UBO>Z{7{k38=uw5}mB?KMHJ+XI^;;D>?b2Ftn) zUQsBWg=l4chA~60^{aYfGhC3tgzcfcr-$1zH=HLXm2TpBHbFM=Es{T|77O_1mFHJb zev5YvjP&88iR%W^#r?m_TN6f#V}Ynxsd%pclxN}L0FSyj*#_feyWsoF(a|_FLCyWN zG3({Uco0gRdd6azauq*^-;~bxk=@Ic#NYX$W!Y${!1nJLwg$r*Tb3XO^>U{MHJ zn_A@hy@_y}%>XRiEydR1dmj2yfa-Z4P0_rMaJEA7G4~=iO!UxLWKDWfN^C^6a(m=BS0Foom6@nSTGM{uI`v`^QUL0E_H9~}*P+DG`{U!}}{pFuK< z-qQsMdmRK}d~odTCIED5S@zc(Tl_2=?%;nLaV4j`*~vqV06UorIK)-<*2la|cit&O z>Wn-xXGON&V(!ay9D1;yWUn3L$D-Hs!wobr-SN<*(|d?5IA@qod)oV1{P#6-k#o;* znexfx|NZ3h2W6=!1Y9U)CP1=C>lofeKHy_1(lDHXLiFPnYuQ~^TthQXRk@Rb)88VO zGLdD}QH}wS{7ce>qG4Fv>^9;8)o?T8IdzAT_E(UuZuoZo)#ooVHJfqtl0YuIU~dX^ zxzm^QMC9M7+W6J%@yTt^>DJ1oFir-oAwL`qkEX}N$#6Vr#s8Tm-GX}oub5Ah+W;9^ z%W5cBGt}y2(RNl+YgW?BN}L+&s=Q&b2It7Ipo4_qsH4MTfO7VE7v~AZ#UBi_)ZEC+ z$jI9R#?WYJE0`Qw@4Xy?W1=-9&hBtyeM9Vy%org^L6!4#SVGf-GfB#_)&s4A;sgm== zgjcWf8GfHt=FAf&Ujjqm92r&tnVM7XGXr=P;hI_Zvyp;lQdAXH&s1%Th_FJ^OSyO9 zMbZ++{mGz}vS@otC{iT+z7X~vtFj0^!mb-XtO_1ZF}RSuML1g>Ml4}%FFv^nPkWf} zfvJNC<1B725r$lpjkPx>r0~rK(v_COqNy0ebU(Rh-be{_wQK=}nhi`L52h{7p$Ang4TpNP-%1!T0F zh6H5-^SxveJ|_oEuFOg-Qqkg~6jI@$Z6=bxr%B3nmeduh(uY2-U8+FMRDs&%k2;Zk zIT}y1)9So5n#!V@eP6^+9?RP)u<2vXGWLA&w7d$NJrD06-aDSmr|Pn;UXloX8JqQE zJv#yF#@W@~uX`Zc-m<@%+jbFWwug>f0E%@fFY>}McKi5lAK&fcLnY`%#ymy1%ySjF zeMQdBAGULcoHLsc#sF0$ZVW=@DkCu{bvN%xl+;#XyjzMaOWci3Y!d-$v$2RbHu%b$ zy5RcitigH=U1gH}^TI%p0wIXuANIW3%|PMj>f^HQ7Mq8kJ_TpHLdFJPW|@uj^ini*`KLC;d6p561nuc`!PHM+b95pB%u2ezfQtgE2ZX!y4l`5O1J+ z$D{><)|-3%4dW?ZZ~YCkKd!gr4~VIDqI-4lwG(Zi5A=h6|DZp((WfVa!HItSZ>^{O zV!PCv_Lc2Yt(9~AuoHsi5KfQSCv+Nm0&GajI{cQ*qxk|7N13lIUp{OJY0d8oBn?F(Psy z^5mQL5v{LQI~NJ#V>hsVqsj^W}fN@$uJ|DF}ylU*Ot+GaC0 z&^Z?HA^zaQ)u&GsTljD9-YuQCxq=R~EM#GKt)29fk3L*|;tk(I7bSXl>}mZ($CKe? zGS=gL2hSSp-h=tr(*_9)`ug$kc+{T^MlmF!j~m1z1pC<2h6z+ggXw5GJ|0ZRF;t?D z+d^fOT$iKCbUGXzO$YI%i9Bu#m7}Mk&GD1d=JE5P&B^nj&FOQY&BtCKOua2fzgc0L zdbD3*TD1GNg=tYdGKJ|AqLLJ*Pl!r$!n7o9HZ4rc(`Gxuw9+ahh3ONbk`$&-hDutP z4r&O~lr-P3Wm=>A^1^i7*T?=FuTMwgqjAIdj3>wa>F9WTJRDb-tM$pWmRxOwOpD6Z zeUNEVus$`JCZ+4sl4(T5E)W$rO(x}QUms15j>ie%?>%c!^6L^2yU)JU!gYUrr%luS z^_{jw_t$q)Nba-mq@3Jm-)RZCzrNF^=KlIl+m`$4J0j`A^T38T=ij}jbD|#&_3>~# z*^e`cud_t@jrIOudNdmJM{$E8qPg0@L@Epi`rv4)Pey}DVp4|3qv2$FG#ni_?M(`h z>HM5eO_@nq`Gl025}i*+nNnF=XKyY+N@33KO_&z$ztP=!9YnHyo#FOIwe4wkmyzl2 zRBggy&s)Q$Ik8%nNSl_c6^OKfy?J9qnzBTnlt_~T_9=-pB4M}uD(=oueGcY8XQ+0D zYGV0;X9n>#Lzf8_D!3dmj+GE%gxd?|4aXS1{5M=86#A^pSc}AxS9=ndKP700 hi?4lFlX$M-+oP%W`1s}V{|5j7|Nk})^;@$C0suvdzRdst literal 10500 zcmV+fDf`wRiwFP!00000|LlEhbKADk@L$33{qQ6mS<%h1?3sRW`J2pB7 zCJSnud^H$mHZo3(3(7VgxVoHz)5~+CW85Lv!<2{~{b6@-V4*wXp<}EOTT#n1PCk7# z2wd~7Z?TO`ib?m>WYj&iEEjp6(J?IKnJ#vi1oPKle=X?^S!{?2UU=ZA8KW*-qW72} zcgEa}$zFmufEP{=n3OPd&$?6j-OWl{?>kYYKcmcfbIbp~p&`w|j ze*PH{Y|Cah%tdVD67Um2U(yg?cEB6g{f)BO#&M|2kj1+^@PGNeG@p3k^j_YOha2+O zUw;`L!`rOa(A^p*hS4z`=)$!ekoz_<8z017id8H(xnb^8Rjh((DqQrU|R?A=5+pF zWHRIP1IO?04R(=boG^ETI5LilOvv~X(d$9+tYe^iIb~(Q z({GEZ_^Wa0qA6XkG2_$gQrkEo8{6&}%7@p;U1I_nN)E3 zW6p8+3N10m4_TLV#x7k-7aiaXz^Bii1nV#H72?l_fQ$E+;Kdp`pQ81bOZeGiE+)A) zPy($SX3ha$iL=wwz=zF;$A_|C{#H-z-k795Yeb_MWWK5(f>M%-n_HhdfkOUNppld2 z)A?DU3Qu+rbg!I3U$gtaA3FXlmTCK-FcicJUW=y{0L0{QY>lL~QF)1mnI+=8Nl zP5lo(I~#8@(5Ml*b~X$JlBb3ii2=|lkx!rJ8*DR7{KY*d3o0}SUvDn>t{}1Knn=qp zY8_KIIr<`oK#salK&_@GlH&}t#9lRoYTouPTxUo$qKk6!k~c@zREp%FxmbL{z`2D8 z!wVem8gLlz8gcr!lb2bQmZi#ilU_$(XB2eJ0ji|2f(3F&u7p9-5XD7bDn)Uk>FX_% zo!7|10+~4`(A~ZvCbc+yofvcIq46lTeb1Bab?NAdiu>cCM%=pyGV~gnw;QLPxQ_wQ ziF@A&moi%^vSUEhBD;cIP9mh=l_x%GiXR>z)Iwk`pHX5V7qArqo`Xy-wJgA=AMi=+ zg6kE!A@V2$_=MpBwt`G_5zBJ0=2OKv+T9R>$r60QY{lQ-kc$u3A7}r0eLDU1W$RQeFm3&FNWStYq56JU44r2>2uIxwt}v8CDtL&GK&D_+GPr;l`OLa-fG^D zVd8Fx2bR#C!zBVHwQXebIRaGj$l2*TAewe7F^O8;Dn_PIxawB>Rk?5YqA9jL9phKD zRX(3>Op{we3NQZi*QZDQ!+e%ICzxSq z0yZRiU;&Re>E4hV5_W@vRZU7f>@g3}1y~?vuDD(idYkw79}ifNLGZ#t&;}0N+7wzZ z!NR3$sXcvv74SB5Z|gC%?tmpC$c2o9F(WFd=pF)8PD zVH?P=i)jFByp-(&qa^qZT;H9@w-jPBz=FB}-640&C-?6S0uDwd0@L%ejxC zoOldD%ZDz2#{l0DVa+VC>oEX@$Uzpz2YbLbz7~{EUwlRa!k~+A-1F@Xk=;24n2FVl zBkdv^LI0%>AA4Zqp~c4LBl-f}HP|@Pcr&57LLj;9oP5yV8@@QHwdW0eib4_DAWslZ zND(Wv4r#*Ps;FTBvF!o8h-jOmC|3dq2AXk9enVe?$N%WwkaGs+HZ^ZOuptcF=?Pk3 zg1wcXz19N!j@8ldC|@02V2=p`sTY%#1yjl*ly1FZdoTi9-Y(>@YgvVfG30Bf7+w+{ zm&Oa~t|jxpIo&V-Z99;Qw}gHnoY{qph%86&f!JFB?;y6tc8(IU&y043oS)awy+zE6 zv{2&)r|&JyH8&r{qgcb(gA!vJ3zICyX`(d%LF+o>>;V53#W9LG!YTeCu@43pl5U!ff1o0W5-aqP+#y$- zOgBrkYPby$t{U(@h*vv=mVh6Jfcv_m^Y|+Lz8CUUL2ZC|Rbcl)x*GIb!hIaV9q10A z`8u4h`C6Vq2J4e)1ViTxG_lp4VoL<>7CQ!$Fn9I|W>9>T09=2_#VHxefY&6fDYk&i zY0E=|i33_>FAijzIVMXv=Y$2haL58Ugw_S;3}6o|sJliMm~R0Dzsq#}4uD`vEo6ch zUsl*$0Zc4xis&{7)*G8)lfON1pm~d|@GG1>UW(I72;TEnE9kAd$&8V&#w!s=x!4Zo z|3DML;TO+ZjkiNM9JBe*yG3i~zx4iZXS1bi=fCsy0&N$=(E<-Q+wo?DMuWSN@le;B zm>Z+p`Zm9X6dI0mq|gQSvf)jo+;cd#OC9P#yJC*j)W(Rg>bcdj-;~-Sj;6$CGZ5Yo>y1OrRk`r-3phu%^3?GX zBNNriWDq??hHKdf_B(DCwG3c@i6Xvt*rHf5;iQKSfwXX*DqafkDAi^Z3 z@))=R8D2n-eRM1?Ijf9O<|tR485?@5((SV&eE!GTlaVe`h09aPaM*fnQ}fo|NquBM z>2yhG#TF=%48UBi;zO;jQdD^FNxwwKY+AV#nz?1NoSu;hoG}oe8B-qWVoW$i&Jf$v zDRmKK-NsYa6Ug6c7Qt?016pDLd#?)e?UK2FxZTb;FOnlm3(dF-^WiqjBE>$#of#yB ze9Ioshdm~Fu_AFwX!PX00B}i`gu_|F=A*oZlM5hLu^t$ ze4##*XwYw2szNK9m(K?bi~NW6uDYRa&?4w#PHEyy7A+jN7*=Vu~d%0bBOiob_NDpL2^ zUFAW}aEfc*YLPOP9gBu<{jH;yXm5=jUgFIVf8pSIkOd>5iJ|-(S(`z0}k7 z38h?QQ-5$a{FTx_-u-j`&#!;}`wso`f6RF|KK0y>|9WlyasSKfx3j@7-W&Sv`rdti zarf*0u|@3&s3%%u_J@;wBTW#NWYKqsoqD%B$U6x@eSH3tUpq$@FX+zjl7wdUsdt-q zs2&HdmShCmRgTY7*zV=fykNYUSuSp0rj9XhDdMoO$&u(8QS?mw*?UZ4aYy~$aL_T{ z!~5(rS^PS<8lYob<2CXaTsyo)zt`^=Z|)t;k@A1e(BHpN<76~==oowqP|n;}LsqsP zbPN$|d}92Y*u6hzuU97d*9!gb&p#Uv6)!^ZhgzkWgpX>O_?+Zyl>v1#*dn ztQ@-~SEdYNu~jb9JVjyE5K{(k#R;dv0!z}WRoJy-R4%=q8;kWlYl*QY#v>G);lb0+ z>LLzFqc_3|S-xZ}t1RN6o7EaAr1zsf6vNKy40WS&cVWqp%-35MD20#26ZN|zHE{{I zYeblsfjek(v7UK&PD~d;4+)uHO@e2@mkNN@eZE}uY<$#JC1_%TYD&3hA|j482l$dl z{A?Qp?b?Y&DnN~77pVX@vpvor+vXXyc}8uX(W6SHQLm?on~*$ufjt)YLDf6t(!kPJ zI*b}kSC9~tn~bD^UQqBb^NC6U<@yorDWY6~u(IHws?~mTm|jD#UMA1ECyNcUM8(3` zg_Y;t3=uzfC;Rq2@;tayl`iZUAJ_`Hmk6m#AUmS^LpwatlFWbFAV(+uGK5I*YD~QpEh2}eSYNWuGC&A zhEkpZx-nNus(tqPOQT+26Q&fCwY9@6QZRPQeoX<}M-FTj-!>Dr&4g9-Y%^ioOxWGM zug^@A3^Y-EkyJe=^1=tUL+G9snoheUPpky2@~S5Fja;Rp`A|&_lpWokbdd&a{XMvf zmiyIGrz}lIPHr<553_KY%?w)P(#;Hev%Q$2wp-YdU?F zA4pl=NswGQyShx`v2RuvezB&3{WwLzOW?1tw9E9_ugx9Eg0t^H7Icj1#&s*lX!!Q1 ziEur8wB$%H&=Q*4OP4ZgQjKYxXS`m@3m)qkTaO>RvYTNjuIs9}xe~Ik=0QxAEzSiZ zOIA|*epoC##Hu)?%3)FFl&9_zsLjezP?XqugYuOGNn6?`i|Z!KkgDSc{^YAhRHIY@8I_8um;I_q|v$ zDJ(NHoqb;>!qb=D+*Aqop_jngO8cLR&}ZT@MZ z&eP@_eZzdCRtQ{-dJ|1;ZOId}U5i-EbVV!#!}EFcy07u_oG@YRq#g_boa63v1u+qO zy$*0j;?X6vEM$dy-41XLRnH40$t?<`V@z8nF9D$^oRY}dSDUYJlA83TG6H1)J`q6hbi7)I!*;lwt^5H>(=rxVOsTM^+AcX3(`Nwe5`ww`R0NELQL_ zWBl3}@n;(!)gNhctfZ8@z#f}oixYjlz^E*k9d7N;0&D!nZY93$ZQal0VwI=Lgom2& z{)igoUpum({HqZLlz;7!{X$$@=4+X6ugo{rLw3m7~%!|_NH@O&kx>(r9x>$Mms2}0*t;F~hMfF|lrp>6vy&Ej}J zaxOSS@Kdtg8_8taL)+xHcRj7mjHEcGIYRdj>fRz(f@mxRIX!D7`Dzwss%kh)rh^;3 zBl-UW#lM388x`46ZqmYyYl#?)V!mC&`*R7fKkgW7Ok%&%Tg6$OEZzOeklimV$cv{+ zFi}|oiCst3wBw`dQGacxa#_c91GAou4^s)MvPTirLdHoiRz@LNslMtVf&f?QY9wvh zhBgRc2i-c+#Lr7z@D?!ivE?fYAqcqKGTc>4KRL`n>tzz*=ool4cW@-R#yA6 z`dE|Z;7cw{ppF08#f_i>^a#>B@x6w`czO)2Ja6TBE6-bb{)FWDps&d}kRt3oy+gi` z-a(khpzL3g$3E7K7*Ar~%J5c(w=%qy;ZI414>U;}Qhr?_xbxve2&0nxo>OYqKa>c+ z*^C&E525&2q7~$=Ab%u5{y2qNT3>vtOUmeyp(eR15TZ7M9+KIbC>u^xMWZAj?SEg8 z`e0K^B|VCEf6%IxC!tnancm8DU#5>V*-=u4%@DMwbiKwbD`?8g^B9y$vfRI8uIu;x zuwnIg3^5#<5%0-CwY!E^ygwQ7p5>d}BDky6{K-`KsT(5Q?nzqxRYHFqYLd3nLORPO zb@bPumj1eQDP1(7x&}?Fu0gB1TGjOwRhM9yR%^9d>v?Leu_ntglFyeeYG&gV0jjCH zP*dBl2vl=wEHS!PleL=c32HLb?!#J*_S`kvktTImA*y%Xiba5GX*4N`rr7ctQfCeB z$`V6s)mf{~o}4<1xThqQgy+GHm42Ev3a=vJMIW!0PZ^(vM=7$@>g`rFpgO1Y=P4AIlN@STlM#iG3Tx(JJI8p^#e{-pX)ah9B#4bZZ2)Mo_EarMwwvGI4#LoS7?Rong4# zMdpM+baF>tjQs+0fx!9l>qDNBPeW~_V7&rWy&TG5U~Se8SJV<1@Z@{IDaP8^$ORsQ z3^i6??%Un699yCcR>^lIB&uEpN|fqtZJsEj4K?YKQnCis3$N)Z+1BK>3>+Q1!8z`J zBu?fM*61YT3^FLrFvM$7nw~7^OYlN|Mb?zh=g|x{Ii`3VY9%{dJ?P@&jZ@14;?e=Y zD1Zb|2NB>2U|n#&0EGI#Jb8gVB6PD{0ndRZ5=J`j(77Q9i~H*#An1$V1AM`@4dw`# zF5)!U0o0Xtya{dFe`{0o7Vw>c4c62}HzfF)2}!JPHH$w_VknnC!uFaXT$~b^+(ePWyQ}#*9Ii$xC=o#BI49;|;Nr4}38e z+~MfIlowU{Uz$?eh77qJEAc`im3r*o+A;IhtvF37{Qg77_!S{1IzABuQT@h`vX^$G zSG9^C%wLJB_giKst15(O&Pv@&B55JxgWKf}su-!3yAU)W(W*mLkz7?6ajGkI;fj(2 zs(v-eyy5Hz)}HdJmNdUqpvG!6QEf_8s3y5IJG@RppgztasLD2uTQM0uk$PFh-*6mV z;=_e?MG;l~{+~+g^h~VtktPA%4@wY`GjHvUSFa>&28!-_5V3KTo+;7zHDu=M3g!B| zGr)5%4AFg|hS{AQk8Y=&Mh>sn$ik2zdkZ{7EKl5|!QX?mJj9VEcVH32<#iUMS=bgF&1iW>J~nG_%u9LhmpV~+v40FM90mra2Id=3GE?pY+I z0y)%x?8&{hCG&c}GUm~4Yk6ioJJuwT%7uG}*qeI?nT!i*XrFCd=#xfW?$tn4^D)C~ z)D4;xv#|zt4nrqCP=lZKqsilJ9iB_XH*h0pm0D`{Q z+XLSp?Q_Wu`2sN$_u>inHLwu0JupWL{(~z^zUTLXW3ZN)o;EX0Gzm76MBYR9_APZk zI!hN?^-G7TB4^aogkYx_WOv6;8dNjBpET5cx_=ZSC?#Bqoj{S0Dewacant&M zS|3o1gZtW$ZoY6=Xie{Sv8yG3#_VYct{FQzL@h6rGWJnd{Krp?kPexJOV?po8aQXb zqDW|HhTJtKkckZ^B?smTV$uQfHg*$|8!|`0qQtic`wcL3z8jW(}tY^)*hG+S%;d*>n;2W;Nd~qC0a9o6d_Y_>g zEl)ldo;=Krt`ZS~QSNAMo{WKM%o>gZZHO(CXcn?%9DyJ^0NOUP041S`iV1kLcGwnB z7yJP+I}=ZRehWp|O z7O`IR>2R-^*qR2LPVj(|;iea} z#Adf;&L4|8k8&f0LR#GQXekLu?c@E)#8!8Ttzz$P5?VuvxJoojF{=KQDDoB+o)X2l z_mR$tg89x?;)z}qu&nY_CJAhcE#Pv}=Qw*}yKFBGWShCLE_@i!vRveO$O1ToCR}e= z*!gQ@f%z6d@H=un5%&VYlv>CHFQoGg6APO{SVFMg*bJNe?STW$TV#b_;jH^oINu<6 z&s(jax9TP(itnusxZFQp`o8&mk{h}dlJf57IcZSMoadyW?(;lHF@jQJp!hr|@DxR? z-k#{RCpx8h_&7IW%Dcc`-5Nb%|6*>HD^kp{0$fQp^^NlCtQz2~=1+kEv~9j4U536R zuTgILO>ML)o0O<()gKHy{lTa$SXICahrQg8C_xwJWF|ZjyYfT;N4L5B^f~mIFtiJ| z1Rr`7u8ECGzateK6!Z2t&}G=%*w78VJPmCZ6O&l;R)Ms=R+HpgmVcTo-_H$6@~+Tg zCsybE)#`5zollaeK65gKHnOhJq6AZOkU7k(hFgcBecX4B6;_eH8q3T~YnzqHIX|BC zdc9co`T3bBa~ID*A59LA#=VXaln)WV9|V!-q?d+KfO}0{FI$VFY&{887Hn|@;`hwk zgWO;xk86d^PTVaal=Tmbs#Jb?IDd)Bf|lTT`I*9+Q=+U@4L%P?kzcj@9b(dBEU}>r z0KpR9g~g`8d<)LcejrzGZgr<`SJo~f22J6P3e}w=Npvi#pD_LbQCvp2bEb5?J`3tT z@gG;g4|y};iSch@_x_x{UYX=yEA+oV|7^%=Ene6TBNK(cuCaM5h2-y|^pgKLRq1Gk zWbKr^Heryfi1>~WOgy+k%oXB!H0T)DG)N2bXZr$r?2YSY%Rybx5Q4moMhtb*F-}c} z@8l_dSf}aU!EW*qAO9Z+FIUVj0~IfMlX!7XFvHNkw2>vskMn0A$TeQ02tIV}Z7}R= z#Z=SU%SxEt(vbjd?L0gRhG+_S@mWjtglz2TU(v2 zt2!2pvOlvS$Myp%L7+0) zt4Y)6=4~0}rjmFwf4jMhyG4S~`%`1y{?C|{DC!-6EbynM;MQ`ogt+UyEB-Hl$$|p9 zXzXZFF$ATjHQT0C1!&(LaMB<23LK{muy{AXmGS`A9Cbe$jc``vZ$t_Yas67_&Eh^8iq5M7GH8AQ8Crw;8 zkS*^2UEZ27(i{uK#Y)X{{cm{|9u4rYjgxIKPPPrczZe~ka}(6OkB+rlUW^Z+(y3=6 zmZ?zj3;0dx`~cZUxsv!hKeVhGEfw1SEyM1>ut&O7HpzmgmUTf*n4ObSESrR(H~#se z<*{ZG&d$u4LF*AN%5>$SbqTUg9Qd}hy#sAebeUj20Wb#5nkis$2zi@YWSQ0pyvuYDV=2=xnt@XEx%`5?8PL^EU$PTXe4}dP zSF97G&_11KG+bU{PI7if6(GN4nIPkQQ!tPd9>4v z*0uJ7xYoX(VYFu*7o+hcKmX_oae?v+f;QVJQzH(i+<~zFOLmdniv?u-GYtvK6y}eT zN%)i;FoiNJwMb=)i!#Whi?*3a{*Iw2*I7|l=t>_3xVEVRJEjWkT>hvP*%zbnaei8z zpGH$#RP+92{N#zeof4Zq-YjR&mrTp6vDx$S{^7mj>3pg-+v+)q(C4vPKkDTtKwZ1r zUH9ugh<$I_U(6{jP;6(&E+kZsGmu?e*hV3dzVzM3~0AuC3T5+E;)(Kintm9)iI2cdHhXDNMzYe}}VhnnN-a)^A&>vj)j!y=IlityvM#uPtdd#2p z74?|0mvjB-BMN6)<+7u)L$ejY*YN&a0_=}F#u}5@uXNl{v_4P9T+OY`A5ug40{2gI za3L{&H5SxeLr&HfWaxn5HB#LpAep4b6)?|jcRkc3ut?cBW038l-$eg13N9^_MaN3--IXXJ-jgLp84u3WK0v+R|KbTBL$G!e!GUz6N>KO0DU!M&4?(!q(4~Iv_=Ld0N5S-f)YraS8j~=q#VJ=cbjHwNfRc0krzdI^l8K>7(_0h!ZDpMksuFQnI z^IMh==1tIADgSMp)Jk@cB-@l;juT}`1VWmA%rmy9s?2zshyNb{0RR7j>D}|m G-~j+aD|@K` diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 7d79fe4403f11ef75f85e69b0ac3d3aabd624921..c6640c1404d5b2c9b33a424be1d53ce9d07fb007 100644 GIT binary patch literal 3401 zcmV-P4Yu+hiwFP!00000|Lk3DbDOxg|0){pn|6oT81UoFzUZdAy_spdzRl+Dizf50 z46>~a2v!nl+>F2be+1Y(J79zDq|DBAV3aIOZ0TCelZTdklP;UV?YiD@m3f5M|gQev6*gQ7=sNAgyJ|Mttd*OqzUsk_fa# z(?76<3_lRC|>KXhk*fe|N zD!_0EM~}R78T?-cZVxc-P`E&yYh-YTenE2?xGMO12WPl*LtJQ@DCpoC|3ETuHBj_K z-}dz$a`+a!5kK)98mMy#E$iyN3=pLbeG8s$$t}loz&cliBTAsP9Ub`fS3s~8RJr98 zagP%43qjvA-My8;@05Oa__gP{4&}%cZPwzSuvt$3Z(KzTTMSYRMzH<%maK2dw{PDh zSz?}TL%NbCQbaA7PeGI25=oX^NTD4}uSdla%~lt#!?3>`;p4;-y1mSrWoA7=Tl_GV z$c93O&?B^63k5x1y%$BFFhNwB^b|#rrO%!{6Til5S(;MEng6wNku({t?_>#@rq33% zrqpo}`X*2A7h9s?fpL zO%Hc#S)%A)4^w1H6HYxO7Q>~;fLxULgImkdYN?skD$s^fYeqcFx|1c$Q79Cd5hQ5^ za|JL15TpkAb<^fftnH=@QVLhqN2z5fR~+L$Byr^;<|u%r1&{2$B~rO8O{Yl>R{6SXLAyzfCnX8yVW5T+kPQ!iTfb!ztv; zuvm{3zcc)4|KymcBB=FQZ3@wD=Aj&%ZE@>Qh+B)~zBy1=1ER|U)2xP;k~)bYXkoID zVoWhk{gQfMX2w30h{;B{0k(D z%5FbFxiSBr9^VECv;g2Nl=K_`KuV7H+pwxIwSUg%4DoVv>+de@IWc=~f3?1|^_{QH zcMg?&-+4vmPDPI?n&j`#q+f=Jn!7xpcwWlyMcq~MeIcifY~SQDx5FqTDwQz||LZWm zB_)7Acz~8CZ1dHEI23(B%<-tP*?yL#SizI@3E9)-ieUp<(nMEfQEy)OOHc&zFDv6@_( zSaDOV^iL(5gOkhV@O)%*bUv~w^y$*%@iHin3g5qv@tEq$4+B< zLRQko^n|R`XG|-~X5GfLx@O)9R-@%ta!l+oX=jb=@n~nE>G5c1vFPz=r;X$>X{YVv zF==NluXR+n6XvZv_OHnhy`HSD_94UQG8EAviaW<3ubWwmXZg&<1VTtm_xT(*qRu$5^?dl#v zT0?AJn~-LZ=t+e%ZD3C+q%jNY5%E_GRL=npv_Q25sx44`K|poAu9!p_NC@sz&VVDKFEu|Yxv$>FSBz54s*H6mU`sVp49bWj%*pD;{Qv(QoP`OLB@lddo|1_`)nHc1wV?fL2J@4t?2rrhU&J5Og67WoQZLDYox<(xX zF?Nj@3xunZ(j^+_YoF&~vvYH^8w(~o+M#>f-l{2Fn$LXA3mOXNLLo678wf(cnS<~C zr~wPw%V2>BqGE-^iq;BzUm_CLyCrTw4=@fXuXeXzo=R^AomrmvlM?qslycaO+$s0G z3w|AH*3+5RjTzduN+8`ZYomCj3o;A*$eH0KlxIi z$eMgye4Hh@abg*U<%>iLTvYxX|Rl*MX1YVj#uhC8O=r1%DzaByWJ z!tor(B{|wKJwXf;o4NioMoZZBl2&WG%`$nCy24<>`AVJrfChrvcV;dJD-CJ`KGWDe zvfY22$A4XK<=E~&GkcC!b4{OP&07sT59#WI9>f53_E*tVM#QVFXM7r}w5$_kwE<}x zkiKNhtQUN%Ai{H5_gxk^k|Md9YFjs9J=Bn);?z!paIY8U25WhUQ*ZR5E z&%HK3*Dv^xiZJL?iuT_giW=9lLzIf9_V9oaLrprt#IWPrJ*_W13;jI@U-;^`dD^Xy z7Dcou;wM58gM#m^Bs}6i0dv%NznD34$NPoN5&ffphBkOUJ5@agz>q7=FAp=cp0D+M zug&v~3ci-IUt;~?8TC!2(m$6&{}JD#)VTl|%sdXZ$@L`6wTflT3oS4)^FN;3X3PIP z+4AFpL(SKS@Aoa}s9_OIoH>rQBik?P;Lwb}1S>nT{vv`lr1uQ)CE92OWB>)vx*P52 zfX@yGd9Q9pb%Ztd;JcLb|nrb32|A2A;v1J3|4qGaNlsCF9;bcpQtFzpj9N5zQ<$PF z+DPB`zOZMH-(=Ld!I7wS3?qB8=9T5oWDmfE*7Nz5$xU)e_H~I(Km~b*2p20xv;0fTl?iW5@(}+gk@*aWMgNhuX*lvlW2g{}By#_aV4+Ok{vJ-r7zT))r#V;Z?vmE1|0grHO0lq{O0Z0)b!CN{8s7HwZ fZtp;e{pmKNhGS{HzFYq{00960YlH}kK%oEtBT}BC literal 2710 zcmV;H3TgEpiwFP!00000|Lk2~Q`i-hVWZtR#ulb* zz;W*xE2><;J^YcYK%%S5GxYxQ0$cb2gruB>6o;Kg`@n$*Jh89`YQ!Chd*7ciBc3Mx zle%DY${Oz-=aPaA*uoA-TTov`P2Anx4fzcl1k6Tz68-j#i+d2)N(8prM^myrzlChh z6g;WMS@ge!{6i!a7i0*HYp@j;S1_=U-$vhWNuM^pGM_jOL=9ZipI{-sgUar7XKkGm zMfTC2zN7&!O(9E z9&^p@Tj&oVK67;)_&yg398ELPKQUQB`+I&%i>-y!!f3YlZ`kC9-QC?`3(LUsh#2D@ zPI`;RQ}-!v7`Cua1o5Kb4TPSe{`kP>l7`a}-_A^-f0&!I+^A>z)))2&Jfd|7o`~nW z5Vd%5u8TfV1_JjETdkIbKL=i4U!6`Yd?`2|{5AFg_f96a7AB4pvPIlW!F>>l>UIXi zm0)2%RWxncfFNcfQF0bBVqq0BFuk56EH1A*nU(PV53m)^3T4p1_h}2a6AKIYJD>s_ z+*2X|JsG|Ln0?7=;=$4w6c0$#xs@zgE1C= z+q}Ojzc8#8Xbi~b!H{7nDhmApi(%+Hlncd2*q$kRpC;Nh`_J<3v7 zapg$ICss=K?ekPCyPkZRwv;GvopZZjq(4wK3i?QZ#}y!#vU_adK)ASvM@sp!*K7{M zDh-=^%|Q$y4t&8iUrkrTk!IiJ{ia8x0?{NsZTf;gjsH6Y))?}CeC(cd-f5ZgK;#cb z`Yt$zI3%VjfFb*6ug|%=2pz=4#o!fQ4Ptze-D_cTgA*4nw{NW_DZi-c zfUsAtO5KxURr(W@Ju+0Jkj4znCXO~L*Zpfi5yIKz<|R?1xVW1tes71v{)sza2x>!C zH-%`Y<4_IG*0}Wz;?^>~-yEo01ER|V)4YXN(mE`GxRecXiYci?Sh60Nnez|T`11Gz ztjtu6G;5@}Ye@6xv;-bdMqK(6;<+%y2^nJ3^w+ffim;(^*w6I5vG_kP-xLVc0AMSO zbQ=J`oF1>1ure6hFN1*uRc&m;?9z$@vts(Ij-7Sv+_l*Gu2md6FW7(^ModjooS&J0 z7@}(~ih%llrMQ}mSh!l{C1{0Y3v(O6QOxVYDCTe{%-k@&NVE07=_w2tb6v&*kC`s+jZmv-(GLvNBW z{5&Q#S8)%5~Q~hrW;%7yy@%>f-iEY65uP4D*L+=`T?^;YdZ5PM1 zOQJ^7Fq(xl#o#+y#Di?ap+Hw%6qv%M7KEgt_Kd>Pl-G^G?;`F>@*25Hn@w(<5q1qR zTFgOQeN(ekd^hq6IZXPI+r$ldFS`2;=x|ginX<+d$(GvMX{%FWYl7P#9NS9XF}(uCw-^FgR@Ejdn23emWSZoFtfvMv6)?g z3q-lMA$EX~q;#!c-o$za^>|0Fh$&ZBfj$Kwv^9IJju-h=1=;7@=1!I*pg zkG?m6@u1Tk&`vNu4g%=5AG*;RV4dw5y2_+;v-wO9g9+t2K~}vWtrw)P88dfF4sj-U zE+4+DV&=B_z8hT{!^kgzS{dfHHxF~$bq;siY;zl+ZC;6AAKa_sTpj0jEzTX4oG{l8 z{Zhc{>7j&pEg7P@)zk_RFkxtuK`69={M*?re_C>=`5M$} zFUd>@%V1)kb9cdRzodbe?)b~FvS8O=(xFGhy+mpRQ>!2nL3FP@Zm zt&&5N@%GX8juW1WGc}wnb@?JoP&Qm-KiPI(Y(zP>$}MSaE21zbRP6GzMU^M!37uJI@Ei{SgqY}Lc2R9=XR!ZV){CV8C~%b z*dzY;{KVhNkotMDXkk2SdBM_nN%KYv=CmZmhA^#x4sL>0s{<%{alwlh!b_yc&B*fl zRT6GfyV6=ZHLc!rxaHL+N)O6qCg0TfZE1pPr&C8qt!~M&oM`**D&MS1QLg%Gujz+u z{yA1~!5ny6^69F=?HWhOrcUFGI)+bNQv7e~aZ7fH>hn1DbC*i53Co$q2EXsrZFNr8 z@rkFM{ZiGA?VrgXfQh~5@|h{z(adx-+3sEgDH1Rx>+vFItCL?U1;e75&OjeT>;sCTD$GP#}n QF8~1l{}9Ey)meG~0OJE!`~Uy| diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index c642854e7..3d27f0c75 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -2453,18 +2453,595 @@ Response: "IgnoreResources": false, "Resources": { "MemPhysical": 274877906944, + "MemUsed": 2147483648, "MemSwap": 128849018880, - "MemReserved": 2147483648, + "MemSwapUsed": 2147483648, "CPUs": 64, "GPUs": [ "aGPU 1337" - ] + ], + "Resources": { + "seal/v0/addpiece": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "3": { + "MinMemory": 4294967296, + "MaxMemory": 4294967296, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "4": { + "MinMemory": 8589934592, + "MaxMemory": 8589934592, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "8": { + "MinMemory": 4294967296, + "MaxMemory": 4294967296, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "9": { + "MinMemory": 8589934592, + "MaxMemory": 8589934592, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + } + }, + "seal/v0/commit/1": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "3": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "4": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "8": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "9": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + } + }, + "seal/v0/commit/2": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10737418240 + }, + "3": { + "MinMemory": 32212254720, + "MaxMemory": 161061273600, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 34359738368 + }, + "4": { + "MinMemory": 64424509440, + "MaxMemory": 204010946560, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 68719476736 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10737418240 + }, + "8": { + "MinMemory": 32212254720, + "MaxMemory": 161061273600, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 34359738368 + }, + "9": { + "MinMemory": 64424509440, + "MaxMemory": 204010946560, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 68719476736 + } + }, + "seal/v0/fetch": { + "0": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "1": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "2": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "3": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "4": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "5": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "6": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "7": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "8": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "9": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + } + }, + "seal/v0/precommit/1": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "3": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "4": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "8": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "9": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + } + }, + "seal/v0/precommit/2": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "3": { + "MinMemory": 16106127360, + "MaxMemory": 16106127360, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + }, + "4": { + "MinMemory": 32212254720, + "MaxMemory": 32212254720, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "8": { + "MinMemory": 16106127360, + "MaxMemory": 16106127360, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + }, + "9": { + "MinMemory": 32212254720, + "MaxMemory": 32212254720, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + } + }, + "seal/v0/unseal": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "3": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "4": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "8": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "9": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + } + } + } } }, "Enabled": true, "MemUsedMin": 0, "MemUsedMax": 0, - "GpuUsed": false, + "GpuUsed": 0, "CpuUse": 0 } } diff --git a/documentation/en/api-v0-methods-worker.md b/documentation/en/api-v0-methods-worker.md index c620113f4..7a1c2e2f2 100644 --- a/documentation/en/api-v0-methods-worker.md +++ b/documentation/en/api-v0-methods-worker.md @@ -92,10 +92,587 @@ Response: "IgnoreResources": true, "Resources": { "MemPhysical": 42, + "MemUsed": 42, "MemSwap": 42, - "MemReserved": 42, + "MemSwapUsed": 42, "CPUs": 42, - "GPUs": null + "GPUs": null, + "Resources": { + "seal/v0/addpiece": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "3": { + "MinMemory": 4294967296, + "MaxMemory": 4294967296, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "4": { + "MinMemory": 8589934592, + "MaxMemory": 8589934592, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "8": { + "MinMemory": 4294967296, + "MaxMemory": 4294967296, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "9": { + "MinMemory": 8589934592, + "MaxMemory": 8589934592, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + } + }, + "seal/v0/commit/1": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "3": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "4": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "8": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "9": { + "MinMemory": 1073741824, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + } + }, + "seal/v0/commit/2": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10737418240 + }, + "3": { + "MinMemory": 32212254720, + "MaxMemory": 161061273600, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 34359738368 + }, + "4": { + "MinMemory": 64424509440, + "MaxMemory": 204010946560, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 68719476736 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 1, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10737418240 + }, + "8": { + "MinMemory": 32212254720, + "MaxMemory": 161061273600, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 34359738368 + }, + "9": { + "MinMemory": 64424509440, + "MaxMemory": 204010946560, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 68719476736 + } + }, + "seal/v0/fetch": { + "0": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "1": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "2": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "3": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "4": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "5": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "6": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "7": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "8": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + }, + "9": { + "MinMemory": 1048576, + "MaxMemory": 1048576, + "GPUUtilization": 0, + "MaxParallelism": 0, + "MaxParallelismGPU": 0, + "BaseMinMemory": 0 + } + }, + "seal/v0/precommit/1": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "3": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "4": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "8": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "9": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + } + }, + "seal/v0/precommit/2": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "3": { + "MinMemory": 16106127360, + "MaxMemory": 16106127360, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + }, + "4": { + "MinMemory": 32212254720, + "MaxMemory": 32212254720, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 1073741824, + "MaxMemory": 1610612736, + "GPUUtilization": 0, + "MaxParallelism": -1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1073741824 + }, + "8": { + "MinMemory": 16106127360, + "MaxMemory": 16106127360, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + }, + "9": { + "MinMemory": 32212254720, + "MaxMemory": 32212254720, + "GPUUtilization": 1, + "MaxParallelism": -1, + "MaxParallelismGPU": 6, + "BaseMinMemory": 1073741824 + } + }, + "seal/v0/unseal": { + "0": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "1": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "2": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "3": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "4": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "5": { + "MinMemory": 2048, + "MaxMemory": 2048, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 2048 + }, + "6": { + "MinMemory": 8388608, + "MaxMemory": 8388608, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 8388608 + }, + "7": { + "MinMemory": 805306368, + "MaxMemory": 1073741824, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 1048576 + }, + "8": { + "MinMemory": 60129542144, + "MaxMemory": 68719476736, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + }, + "9": { + "MinMemory": 120259084288, + "MaxMemory": 137438953472, + "GPUUtilization": 0, + "MaxParallelism": 1, + "MaxParallelismGPU": 0, + "BaseMinMemory": 10485760 + } + } + } } } ``` diff --git a/extern/sector-storage/storiface/resources.go b/extern/sector-storage/storiface/resources.go index d634927ed..f12d0df05 100644 --- a/extern/sector-storage/storiface/resources.go +++ b/extern/sector-storage/storiface/resources.go @@ -370,7 +370,7 @@ func ParseResources(lookup func(key, def string) (string, bool)) (map[sealtasks. return nil, xerrors.Errorf("no envname for field '%s'", f.Name) } - envval, found := lookup(taskType.Short() + "_" + shortSize + "_" + envname, fmt.Sprint(rr.Elem().Field(i).Interface())) + envval, found := lookup(taskType.Short()+"_"+shortSize+"_"+envname, fmt.Sprint(rr.Elem().Field(i).Interface())) if !found { // special multicore SDR handling if (taskType == sealtasks.TTPreCommit1 || taskType == sealtasks.TTUnseal) && envname == "MAX_PARALLELISM" { @@ -423,5 +423,5 @@ func getSDRThreads(lookup func(key, def string) (string, bool)) (_ int, err erro } // producers + the one core actually doing the work - return producers+1, nil + return producers + 1, nil } diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index 380e968e1..5889701d0 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -38,8 +38,8 @@ type WorkerResources struct { MemSwap uint64 MemSwapUsed uint64 - CPUs uint64 // Logical cores - GPUs []string + CPUs uint64 // Logical cores + GPUs []string // if nil use the default resource table Resources map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index 50bf6ada0..de3a04f93 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -555,13 +555,13 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { Hostname: hostname, IgnoreResources: l.ignoreResources, Resources: storiface.WorkerResources{ - MemPhysical: memPhysical, - MemUsed: memUsed, - MemSwap: memSwap, - MemSwapUsed: memSwapUsed, - CPUs: uint64(runtime.NumCPU()), - GPUs: gpus, - Resources: resEnv, + MemPhysical: memPhysical, + MemUsed: memUsed, + MemSwap: memSwap, + MemSwapUsed: memSwapUsed, + CPUs: uint64(runtime.NumCPU()), + GPUs: gpus, + Resources: resEnv, }, }, nil } From f25efecb745213267d5e980f1669e4a10e0dcea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 15:14:57 +0100 Subject: [PATCH 09/13] worker: Test resource table overrides --- extern/sector-storage/manager_test.go | 126 +++++++++++++++++- extern/sector-storage/piece_provider_test.go | 3 +- extern/sector-storage/sched_test.go | 8 +- extern/sector-storage/storiface/resources.go | 2 +- .../storiface/resources_test.go | 8 +- extern/sector-storage/testworker_test.go | 2 +- extern/sector-storage/worker_local.go | 11 +- 7 files changed, 142 insertions(+), 18 deletions(-) diff --git a/extern/sector-storage/manager_test.go b/extern/sector-storage/manager_test.go index d4044bbae..cb03b2ef4 100644 --- a/extern/sector-storage/manager_test.go +++ b/extern/sector-storage/manager_test.go @@ -332,7 +332,7 @@ func TestRestartWorker(t *testing.T) { return &testExec{apch: arch}, nil }, WorkerConfig{ TaskTypes: localTasks, - }, stor, lstor, idx, m, statestore.New(wds)) + }, os.LookupEnv, stor, lstor, idx, m, statestore.New(wds)) err := m.AddWorker(ctx, w) require.NoError(t, err) @@ -368,7 +368,7 @@ func TestRestartWorker(t *testing.T) { return &testExec{apch: arch}, nil }, WorkerConfig{ TaskTypes: localTasks, - }, stor, lstor, idx, m, statestore.New(wds)) + }, os.LookupEnv, stor, lstor, idx, m, statestore.New(wds)) err = m.AddWorker(ctx, w) require.NoError(t, err) @@ -404,7 +404,7 @@ func TestReenableWorker(t *testing.T) { return &testExec{apch: arch}, nil }, WorkerConfig{ TaskTypes: localTasks, - }, stor, lstor, idx, m, statestore.New(wds)) + }, os.LookupEnv, stor, lstor, idx, m, statestore.New(wds)) err := m.AddWorker(ctx, w) require.NoError(t, err) @@ -453,3 +453,123 @@ func TestReenableWorker(t *testing.T) { i, _ = m.sched.Info(ctx) require.Len(t, i.(SchedDiagInfo).OpenWindows, 2) } + +func TestResUse(t *testing.T) { + logging.SetAllLoggers(logging.LevelDebug) + + ctx, done := context.WithCancel(context.Background()) + defer done() + + ds := datastore.NewMapDatastore() + + m, lstor, stor, idx, cleanup := newTestMgr(ctx, t, ds) + defer cleanup() + + localTasks := []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTCommit1, sealtasks.TTFinalize, sealtasks.TTFetch, + } + + wds := datastore.NewMapDatastore() + + arch := make(chan chan apres) + w := newLocalWorker(func() (ffiwrapper.Storage, error) { + return &testExec{apch: arch}, nil + }, WorkerConfig{ + TaskTypes: localTasks, + }, func(s string) (string, bool) { + return "", false + }, stor, lstor, idx, m, statestore.New(wds)) + + err := m.AddWorker(ctx, w) + require.NoError(t, err) + + sid := storage.SectorRef{ + ID: abi.SectorID{Miner: 1000, Number: 1}, + ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1, + } + + go func() { + _, err := m.AddPiece(ctx, sid, nil, 1016, strings.NewReader(strings.Repeat("testthis", 127))) + require.Error(t, err) + }() + +l: + for { + st := m.WorkerStats() + require.Len(t, st, 1) + for _, w := range st { + if w.MemUsedMax > 0 { + break l + } + time.Sleep(time.Millisecond) + } + } + + st := m.WorkerStats() + require.Len(t, st, 1) + for _, w := range st { + require.Equal(t, storiface.ResourceTable[sealtasks.TTAddPiece][abi.RegisteredSealProof_StackedDrg2KiBV1].MaxMemory, w.MemUsedMax) + } +} + +func TestResOverride(t *testing.T) { + logging.SetAllLoggers(logging.LevelDebug) + + ctx, done := context.WithCancel(context.Background()) + defer done() + + ds := datastore.NewMapDatastore() + + m, lstor, stor, idx, cleanup := newTestMgr(ctx, t, ds) + defer cleanup() + + localTasks := []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTCommit1, sealtasks.TTFinalize, sealtasks.TTFetch, + } + + wds := datastore.NewMapDatastore() + + arch := make(chan chan apres) + w := newLocalWorker(func() (ffiwrapper.Storage, error) { + return &testExec{apch: arch}, nil + }, WorkerConfig{ + TaskTypes: localTasks, + }, func(s string) (string, bool) { + if s == "AP_2K_MAX_MEMORY" { + return "99999", true + } + + return "", false + }, stor, lstor, idx, m, statestore.New(wds)) + + err := m.AddWorker(ctx, w) + require.NoError(t, err) + + sid := storage.SectorRef{ + ID: abi.SectorID{Miner: 1000, Number: 1}, + ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1, + } + + go func() { + _, err := m.AddPiece(ctx, sid, nil, 1016, strings.NewReader(strings.Repeat("testthis", 127))) + require.Error(t, err) + }() + +l: + for { + st := m.WorkerStats() + require.Len(t, st, 1) + for _, w := range st { + if w.MemUsedMax > 0 { + break l + } + time.Sleep(time.Millisecond) + } + } + + st := m.WorkerStats() + require.Len(t, st, 1) + for _, w := range st { + require.Equal(t, uint64(99999), w.MemUsedMax) + } +} diff --git a/extern/sector-storage/piece_provider_test.go b/extern/sector-storage/piece_provider_test.go index 0abba1bd8..1aad3d2d2 100644 --- a/extern/sector-storage/piece_provider_test.go +++ b/extern/sector-storage/piece_provider_test.go @@ -7,6 +7,7 @@ import ( "math/rand" "net" "net/http" + "os" "testing" "github.com/filecoin-project/go-state-types/abi" @@ -286,7 +287,7 @@ func (p *pieceProviderTestHarness) addRemoteWorker(t *testing.T, tasks []sealtas worker := newLocalWorker(nil, WorkerConfig{ TaskTypes: tasks, - }, remote, localStore, p.index, p.mgr, csts) + }, os.LookupEnv, remote, localStore, p.index, p.mgr, csts) p.servers = append(p.servers, svc) p.localStores = append(p.localStores, localStore) diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index f64ed57e2..a2191a8e2 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -301,8 +301,8 @@ func TestSched(t *testing.T) { } testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) { - ParallelNum = 1 - ParallelDenom = 1 + storiface.ParallelNum = 1 + storiface.ParallelDenom = 1 return func(t *testing.T) { index := stores.NewIndex() @@ -618,7 +618,7 @@ func TestWindowCompact(t *testing.T) { taskType: task, sector: storage.SectorRef{ProofType: spt}, }) - window.allocated.add(wh.info.Resources, ResourceTable[task][spt]) + window.allocated.add(wh.info.Resources, storiface.ResourceTable[task][spt]) } wh.activeWindows = append(wh.activeWindows, window) @@ -637,7 +637,7 @@ func TestWindowCompact(t *testing.T) { for ti, task := range tasks { require.Equal(t, task, wh.activeWindows[wi].todo[ti].taskType, "%d, %d", wi, ti) - expectRes.add(wh.info.Resources, ResourceTable[task][spt]) + expectRes.add(wh.info.Resources, storiface.ResourceTable[task][spt]) } require.Equal(t, expectRes.cpuUse, wh.activeWindows[wi].allocated.cpuUse, "%d", wi) diff --git a/extern/sector-storage/storiface/resources.go b/extern/sector-storage/storiface/resources.go index f12d0df05..dd43a37dd 100644 --- a/extern/sector-storage/storiface/resources.go +++ b/extern/sector-storage/storiface/resources.go @@ -346,7 +346,7 @@ func init() { } } -func ParseResources(lookup func(key, def string) (string, bool)) (map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources, error) { +func ParseResourceEnv(lookup func(key, def string) (string, bool)) (map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources, error) { out := map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{} for taskType, defTT := range ResourceTable { diff --git a/extern/sector-storage/storiface/resources_test.go b/extern/sector-storage/storiface/resources_test.go index f58f46e23..bf7425d24 100644 --- a/extern/sector-storage/storiface/resources_test.go +++ b/extern/sector-storage/storiface/resources_test.go @@ -10,7 +10,7 @@ import ( ) func TestListResourceVars(t *testing.T) { - _, err := ParseResources(func(key, def string) (string, bool) { + _, err := ParseResourceEnv(func(key, def string) (string, bool) { if def != "" { fmt.Printf("%s=%s\n", key, def) } @@ -22,7 +22,7 @@ func TestListResourceVars(t *testing.T) { } func TestListResourceOverride(t *testing.T) { - rt, err := ParseResources(func(key, def string) (string, bool) { + rt, err := ParseResourceEnv(func(key, def string) (string, bool) { if key == "UNS_2K_MAX_PARALLELISM" { return "2", true } @@ -46,7 +46,7 @@ func TestListResourceOverride(t *testing.T) { } func TestListResourceSDRMulticoreOverride(t *testing.T) { - rt, err := ParseResources(func(key, def string) (string, bool) { + rt, err := ParseResourceEnv(func(key, def string) (string, bool) { if key == "FIL_PROOFS_USE_MULTICORE_SDR" { return "1", true } @@ -58,7 +58,7 @@ func TestListResourceSDRMulticoreOverride(t *testing.T) { require.Equal(t, 4, rt[sealtasks.TTPreCommit1][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) require.Equal(t, 4, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism) - rt, err = ParseResources(func(key, def string) (string, bool) { + rt, err = ParseResourceEnv(func(key, def string) (string, bool) { if key == "FIL_PROOFS_USE_MULTICORE_SDR" { return "1", true } diff --git a/extern/sector-storage/testworker_test.go b/extern/sector-storage/testworker_test.go index 57c3b53ee..581c60d3b 100644 --- a/extern/sector-storage/testworker_test.go +++ b/extern/sector-storage/testworker_test.go @@ -102,7 +102,7 @@ func (t *testWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { } func (t *testWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) { - res := ResourceTable[sealtasks.TTPreCommit2][abi.RegisteredSealProof_StackedDrg2KiBV1] + res := storiface.ResourceTable[sealtasks.TTPreCommit2][abi.RegisteredSealProof_StackedDrg2KiBV1] return storiface.WorkerInfo{ Hostname: "testworkerer", diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index de3a04f93..3545c50c0 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -42,6 +42,7 @@ type WorkerConfig struct { // used do provide custom proofs impl (mostly used in testing) type ExecutorFunc func() (ffiwrapper.Storage, error) +type EnvFunc func(string) (string, bool) type LocalWorker struct { storage stores.Store @@ -50,6 +51,7 @@ type LocalWorker struct { ret storiface.WorkerReturn executor ExecutorFunc noSwap bool + envLookup EnvFunc // see equivalent field on WorkerConfig. ignoreResources bool @@ -64,7 +66,7 @@ type LocalWorker struct { closing chan struct{} } -func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { +func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, envLookup EnvFunc, store stores.Store, local *stores.Local, sindex stores.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { acceptTasks := map[sealtasks.TaskType]struct{}{} for _, taskType := range wcfg.TaskTypes { acceptTasks[taskType] = struct{}{} @@ -82,6 +84,7 @@ func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, store stores.Store acceptTasks: acceptTasks, executor: executor, noSwap: wcfg.NoSwap, + envLookup: envLookup, ignoreResources: wcfg.IgnoreResourceFiltering, session: uuid.New(), closing: make(chan struct{}), @@ -115,7 +118,7 @@ func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, store stores.Store } func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker { - return newLocalWorker(nil, wcfg, store, local, sindex, ret, cst) + return newLocalWorker(nil, wcfg, os.LookupEnv, store, local, sindex, ret, cst) } type localWorkerPathProvider struct { @@ -544,8 +547,8 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } - resEnv, err := storiface.ParseResources(func(key, def string) (string, bool) { - return os.LookupEnv(key) + resEnv, err := storiface.ParseResourceEnv(func(key, def string) (string, bool) { + return l.envLookup(key) }) if err != nil { return storiface.WorkerInfo{}, xerrors.Errorf("interpreting resource env vars: %w", err) From a597b072b8fb18f71342ce769c7519eba8636a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 15:25:01 +0100 Subject: [PATCH 10/13] fix sched tests --- extern/sector-storage/sched_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index a2191a8e2..667fabb66 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -44,7 +44,7 @@ var decentWorkerResources = storiface.WorkerResources{ MemUsed: 1 << 30, MemSwapUsed: 1 << 30, CPUs: 32, - GPUs: []string{"a GPU"}, + GPUs: []string{}, } var constrainedWorkerResources = storiface.WorkerResources{ @@ -190,6 +190,9 @@ func TestSchedStartStop(t *testing.T) { } func TestSched(t *testing.T) { + storiface.ParallelNum = 1 + storiface.ParallelDenom = 1 + ctx, done := context.WithTimeout(context.Background(), 30*time.Second) defer done() @@ -256,7 +259,9 @@ func TestSched(t *testing.T) { return nil }, noopAction) - require.NoError(t, err, fmt.Sprint(l, l2)) + if err != context.Canceled { + require.NoError(t, err, fmt.Sprint(l, l2)) + } }() <-sched.testSync @@ -301,9 +306,6 @@ func TestSched(t *testing.T) { } testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) { - storiface.ParallelNum = 1 - storiface.ParallelDenom = 1 - return func(t *testing.T) { index := stores.NewIndex() From 001ecbb5611f41581b3ad86c8076552ee01bb3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 15:26:16 +0100 Subject: [PATCH 11/13] fix lint --- extern/sector-storage/cgroups_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/sector-storage/cgroups_linux.go b/extern/sector-storage/cgroups_linux.go index 0b6efea99..38fe88f19 100644 --- a/extern/sector-storage/cgroups_linux.go +++ b/extern/sector-storage/cgroups_linux.go @@ -19,7 +19,7 @@ func cgroupV2MountPoint() (string, error) { if err != nil { return "", err } - defer f.Close() + defer f.Close() //nolint scanner := bufio.NewScanner(f) for scanner.Scan() { From cf20b0b2b830ee3ef62b2ee5b2ab09ab808f07da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 16:11:04 +0100 Subject: [PATCH 12/13] worker: Command to print resource-table env vars --- cmd/lotus-seal-worker/main.go | 1 + cmd/lotus-seal-worker/resources.go | 72 ++++++++++++++++++++++++++++ documentation/en/cli-lotus-worker.md | 16 +++++++ 3 files changed, 89 insertions(+) create mode 100644 cmd/lotus-seal-worker/resources.go diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index ce2a32cd4..5aec2f52f 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -60,6 +60,7 @@ func main() { storageCmd, setCmd, waitQuietCmd, + resourcesCmd, tasksCmd, } diff --git a/cmd/lotus-seal-worker/resources.go b/cmd/lotus-seal-worker/resources.go new file mode 100644 index 000000000..b3a01713c --- /dev/null +++ b/cmd/lotus-seal-worker/resources.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "os" + "sort" + + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" +) + +var resourcesCmd = &cli.Command{ + Name: "resources", + Usage: "Manage resource table overrides", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "all", + Usage: "print all resource envvars", + }, + &cli.BoolFlag{ + Name: "default", + Usage: "print all resource envvars", + }, + }, + Action: func(cctx *cli.Context) error { + def := map[string]string{} + set := map[string]string{} + all := map[string]string{} + + _, err := storiface.ParseResourceEnv(func(key, d string) (string, bool) { + if d != "" { + all[key] = d + def[key] = d + } + + s, ok := os.LookupEnv(key) + if ok { + all[key] = s + set[key] = s + } + + return s, ok + }) + if err != nil { + return err + } + + printMap := func(m map[string]string) { + var arr []string + for k, v := range m { + arr = append(arr, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(arr) + for _, s := range arr { + fmt.Println(s) + } + } + + if cctx.Bool("default") { + printMap(def) + } else { + if cctx.Bool("all") { + printMap(all) + } else { + printMap(set) + } + } + + return nil + }, +} diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index 3b0c7ae4f..bbfc7bb6c 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -15,6 +15,7 @@ COMMANDS: storage manage sector storage set Manage worker settings wait-quiet Block until all running tasks exit + resources Manage resource table overrides tasks Manage task processing help, h Shows a list of commands or help for one command @@ -127,6 +128,21 @@ OPTIONS: ``` +## lotus-worker resources +``` +NAME: + lotus-worker resources - Manage resource table overrides + +USAGE: + lotus-worker resources [command options] [arguments...] + +OPTIONS: + --all print all resource envvars (default: false) + --default print all resource envvars (default: false) + --help, -h show help (default: false) + +``` + ## lotus-worker tasks ``` NAME: From 330cfc33ee30e0373906d2bed8c9c86337d1e311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 29 Nov 2021 20:34:53 +0100 Subject: [PATCH 13/13] worker: Typo in resources cmd usage Co-authored-by: Aayush Rajasekaran --- cmd/lotus-seal-worker/resources.go | 2 +- documentation/en/cli-lotus-worker.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-seal-worker/resources.go b/cmd/lotus-seal-worker/resources.go index b3a01713c..539f141b9 100644 --- a/cmd/lotus-seal-worker/resources.go +++ b/cmd/lotus-seal-worker/resources.go @@ -20,7 +20,7 @@ var resourcesCmd = &cli.Command{ }, &cli.BoolFlag{ Name: "default", - Usage: "print all resource envvars", + Usage: "print default resource envvars", }, }, Action: func(cctx *cli.Context) error { diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index bbfc7bb6c..09a93f091 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -138,7 +138,7 @@ USAGE: OPTIONS: --all print all resource envvars (default: false) - --default print all resource envvars (default: false) + --default print default resource envvars (default: false) --help, -h show help (default: false) ```