sealing: fix: Make DataCid resource env vars make more sense

storage/sealer/sealtasks/task.go

@@ -96,6 +96,11 @@ func (a TaskType) WorkerType() string {
 	}
 }
 
+// SectorSized returns true if the task operates on a specific sector size
+func (a TaskType) SectorSized() bool {
+	return a != TTDataCid
+}
+
 func (a TaskType) MuchLess(b TaskType) (bool, bool) {
 	oa, ob := order[a], order[b]
 	oneNegative := oa^ob < 0

storage/sealer/storiface/resources.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
@@ -13,6 +14,8 @@ import (
 	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
 )
 
+var log = logging.Logger("resources")
+
 type Resources struct {
 	MinMemory uint64 `envname:"MIN_MEMORY"` // What Must be in RAM for decent perf
 	MaxMemory uint64 `envname:"MAX_MEMORY"` // Memory required (swap + ram; peak memory usage during task execution)
@@ -32,7 +35,6 @@ type Resources struct {
 }
 
 /*
-
 Percent of threads to allocate to parallel tasks
 
 12  * 0.92 = 11
@@ -41,7 +43,6 @@ type Resources struct {
 32  * 0.92 = 29
 64  * 0.92 = 58
 128 * 0.92 = 117
-
 */
 var ParallelNum uint64 = 92
 var ParallelDenom uint64 = 100
@@ -609,6 +610,9 @@ func ParseResourceEnv(lookup func(key, def string) (string, bool)) (map[sealtask
 				}
 
 				envval, found := lookup(taskType.Short()+"_"+shortSize+"_"+envname, fmt.Sprint(rr.Elem().Field(i).Interface()))
+				if !found {
+					// see if a non-size-specific envvar is set
+					envval, found = lookup(taskType.Short()+"_"+envname, fmt.Sprint(rr.Elem().Field(i).Interface()))
 					if !found {
 						// special multicore SDR handling
 						if (taskType == sealtasks.TTPreCommit1 || taskType == sealtasks.TTUnseal) && envname == "MAX_PARALLELISM" {
@@ -626,6 +630,12 @@ func ParseResourceEnv(lookup func(key, def string) (string, bool)) (map[sealtask
 						continue
 					}
 
+				} else {
+					if !taskType.SectorSized() {
+						log.Errorw("sector-size independent task resource var specified with sector-sized envvar", "env", taskType.Short()+"_"+shortSize+"_"+envname, "use", taskType.Short()+"_"+envname)
+					}
+				}
+
 				v := rr.Elem().Field(i).Addr().Interface()
 				switch fv := v.(type) {
 				case *uint64:

storage/sealer/storiface/resources_test.go

@@ -12,9 +12,12 @@ import (
 )
 
 func TestListResourceVars(t *testing.T) {
+	seen := map[string]struct{}{}
 	_, err := ParseResourceEnv(func(key, def string) (string, bool) {
-		if def != "" {
+		_, s := seen[key]
+		if !s && def != "" {
 			fmt.Printf("%s=%s\n", key, def)
+			seen[key] = struct{}{}
 		}
 
 		return "", false
@@ -75,3 +78,44 @@ func TestListResourceSDRMulticoreOverride(t *testing.T) {
 	require.Equal(t, 9001, rt[sealtasks.TTPreCommit1][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
 	require.Equal(t, 9001, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
 }
+
+func TestUnsizedSetAll(t *testing.T) {
+	rt, err := ParseResourceEnv(func(key, def string) (string, bool) {
+		if key == "UNS_MAX_PARALLELISM" {
+			return "2", true
+		}
+
+		return "", false
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, 2, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+	require.Equal(t, 2, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg32GiBV1].MaxParallelism)
+	require.Equal(t, 2, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg8MiBV1].MaxParallelism)
+
+	// check that defaults don't get mutated
+	require.Equal(t, 1, ResourceTable[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+}
+
+func TestUnsizedNotPreferred(t *testing.T) {
+	rt, err := ParseResourceEnv(func(key, def string) (string, bool) {
+		if key == "DC_MAX_PARALLELISM" {
+			return "2", true
+		}
+
+		// test should also print a warning for DataCid as it's not sector-size dependent
+		if key == "DC_64G_MAX_PARALLELISM" {
+			return "1", true
+		}
+
+		return "", false
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, 2, rt[sealtasks.TTDataCid][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+	require.Equal(t, 2, rt[sealtasks.TTDataCid][stabi.RegisteredSealProof_StackedDrg32GiBV1].MaxParallelism)
+	require.Equal(t, 1, rt[sealtasks.TTDataCid][stabi.RegisteredSealProof_StackedDrg64GiBV1_1].MaxParallelism)
+
+	// check that defaults don't get mutated
+	require.Equal(t, 1, ResourceTable[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+}