Merge pull request #31 from filecoin-project/feat/64g

Add entries for 64G sectors

ffiwrapper/config.go

@@ -30,6 +30,8 @@ func SealProofTypeFromSectorSize(ssize abi.SectorSize) (abi.RegisteredProof, err
 		return abi.RegisteredProof_StackedDRG512MiBSeal, nil
 	case 32 << 30:
 		return abi.RegisteredProof_StackedDRG32GiBSeal, nil
+	case 64 << 30:
+		return abi.RegisteredProof_StackedDRG64GiBSeal, nil
 	default:
 		return 0, xerrors.Errorf("unsupported sector size for miner: %v", ssize)
 	}

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/filecoin-project/filecoin-ffi v0.0.0-20200326153646-e899cc1dd072
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5
 	github.com/filecoin-project/go-paramfetch v0.0.1
-	github.com/filecoin-project/specs-actors v0.3.0
+	github.com/filecoin-project/specs-actors v0.4.1-0.20200508202406-42be6629284d
 	github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4

go.sum

@@ -28,6 +28,7 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj
 github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be h1:TooKBwR/g8jG0hZ3lqe9S5sy2vTUcLOZLlz3M5wGn2E=
 github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0=
 github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg=
+github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg=
 github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550 h1:aockulLU8Qjkdj4FQz53WQpNosAIYk8DxRediRLkE5c=
 github.com/filecoin-project/go-bitfield v0.0.0-20200309034705-8c7ac40bd550/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw=
 github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060 h1:/3qjGMn6ukXgZJHsIbuwGL7ipla8DOV3uHZDBJkBYfU=
@@ -40,8 +41,8 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC
 github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
 github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504 h1:mwuAaqxKThl70+7FkGdFKVLdwaQZQ8XmscKdhSBBtnc=
 github.com/filecoin-project/specs-actors v0.0.0-20200409043918-e569f4a2f504/go.mod h1:mdJraXq5vMy0+/FqVQIrnNlpQ/Em6zeu06G/ltQ0/lA=
-github.com/filecoin-project/specs-actors v0.3.0 h1:QxgAuTrZr5TPqjyprZk0nTYW5o0JWpzbb5v+4UHHvN0=
+github.com/filecoin-project/specs-actors v0.4.1-0.20200508202406-42be6629284d h1:vzuhvR+huV95QU+KSBCyQvLP6LUtwrPUyeUTzOx1B5I=
-github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
+github.com/filecoin-project/specs-actors v0.4.1-0.20200508202406-42be6629284d/go.mod h1:UW3ft23q6VS8wQoNqLWjENsu9gu1uh6lxOd+H8cwhT8=
 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102 h1:T3f/zkuvgtgqcXrb0NO3BicuveGOxxUAMPa/Yif2kuE=
 github.com/filecoin-project/specs-storage v0.0.0-20200417134612-61b2d91a6102/go.mod h1:xJ1/xl9+8zZeSSSFmDC3Wr6uusCTxyYPI0VeNVSFmPE=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

resources.go

@@ -24,6 +24,14 @@ const MaxCachingOverhead = 32 << 30
 
 var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{
 	sealtasks.TTAddPiece: {
+		abi.RegisteredProof_StackedDRG64GiBSeal: Resources{ // This is probably a bit conservative
+			MaxMemory: 64 << 30,
+			MinMemory: 64 << 30,
+
+			Threads: 1,
+
+			BaseMinMemory: 1 << 30,
+		},
 		abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ // This is probably a bit conservative
 			MaxMemory: 32 << 30,
 			MinMemory: 32 << 30,
@@ -58,6 +66,14 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{
 		},
 	},
 	sealtasks.TTPreCommit1: {
+		abi.RegisteredProof_StackedDRG64GiBSeal: Resources{
+			MaxMemory: 128 << 30,
+			MinMemory: 96 << 30,
+
+			Threads: 1,
+
+			BaseMinMemory: 60 << 30,
+		},
 		abi.RegisteredProof_StackedDRG32GiBSeal: Resources{
 			MaxMemory: 64 << 30,
 			MinMemory: 48 << 30,
@@ -92,6 +108,15 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{
 		},
 	},
 	sealtasks.TTPreCommit2: {
+		abi.RegisteredProof_StackedDRG64GiBSeal: Resources{
+			MaxMemory: 64 << 30,
+			MinMemory: 64 << 30,
+
+			Threads: -1,
+			CanGPU:  true,
+
+			BaseMinMemory: 60 << 30,
+		},
 		abi.RegisteredProof_StackedDRG32GiBSeal: Resources{
 			MaxMemory: 32 << 30,
 			MinMemory: 32 << 30,
@@ -127,6 +152,14 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{
 		},
 	},
 	sealtasks.TTCommit1: { // Very short (~100ms), so params are very light
+		abi.RegisteredProof_StackedDRG64GiBSeal: Resources{
+			MaxMemory: 1 << 30,
+			MinMemory: 1 << 30,
+
+			Threads: 0,
+
+			BaseMinMemory: 1 << 30,
+		},
 		abi.RegisteredProof_StackedDRG32GiBSeal: Resources{
 			MaxMemory: 1 << 30,
 			MinMemory: 1 << 30,
@@ -161,6 +194,15 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{
 		},
 	},
 	sealtasks.TTCommit2: {
+		abi.RegisteredProof_StackedDRG64GiBSeal: Resources{
+			MaxMemory: 260 << 30, // TODO: Confirm
+			MinMemory: 120 << 30,
+
+			Threads: -1,
+			CanGPU:  true,
+
+			BaseMinMemory: 128 << 30, // params
+		},
 		abi.RegisteredProof_StackedDRG32GiBSeal: Resources{
 			MaxMemory: 130 << 30,
 			MinMemory: 60 << 30,
@@ -199,6 +241,15 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{
 		},
 	},
 	sealtasks.TTFetch: {
+		abi.RegisteredProof_StackedDRG64GiBSeal: Resources{
+			MaxMemory: 1 << 20,
+			MinMemory: 1 << 20,
+
+			Threads: 0,
+			CanGPU:  false,
+
+			BaseMinMemory: 0,
+		},
 		abi.RegisteredProof_StackedDRG32GiBSeal: Resources{
 			MaxMemory: 1 << 20,
 			MinMemory: 1 << 20,

sched.go

@@ -378,6 +378,9 @@ func canHandleRequest(needRes Resources, spt abi.RegisteredProof, wid WorkerID,
 	if spt == abi.RegisteredProof_StackedDRG32GiBSeal {
 		maxNeedMem += MaxCachingOverhead
 	}
+	if spt == abi.RegisteredProof_StackedDRG64GiBSeal {
+		maxNeedMem += MaxCachingOverhead * 2 // ewwrhmwh
+	}
 	if maxNeedMem > res.MemSwap+res.MemPhysical {
 		log.Debugf("sched: not scheduling on worker %d; not enough virtual memory - need: %dM, have %dM", wid, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib)
 		return false