storage: Make remote Generate Vanilla more robust

provider/lpffi/sdr_funcs.go

@@ -192,7 +192,7 @@ func (sb *SealCalls) GenerateSynthPoRep() {
 }
 
 func (sb *SealCalls) PoRepSnark(ctx context.Context, sn storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
-	vproof, err := sb.sectors.storage.GenetartePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed)
+	vproof, err := sb.sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to generate vanilla proof: %w", err)
 	}

storage/paths/http_handler.go

@@ -329,7 +329,7 @@ func (handler *FetchHandler) generatePoRepVanillaProof(w http.ResponseWriter, r
 		return
 	}
 
-	vanilla, err := handler.Local.GenetartePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed)
+	vanilla, err := handler.Local.GeneratePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return

storage/paths/interface.go

@@ -50,5 +50,5 @@ type Store interface {
 	Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)
 
 	GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)
-	GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
+	GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
 }

storage/paths/local.go

@@ -810,7 +810,7 @@ func (st *Local) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
 	}
 }
 
-func (st *Local) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
+func (st *Local) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
 	src, _, err := st.AcquireSector(ctx, sr, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
 	if err != nil {
 		return nil, xerrors.Errorf("acquire sector: %w", err)

storage/paths/mocks/store.go

@@ -87,9 +87,9 @@ func (mr *MockStoreMockRecorder) GenerateSingleVanillaProof(arg0, arg1, arg2, ar
 }
 
 // GenetartePoRepVanillaProof mocks base method.
-func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) {
+func (m *MockStore) GeneratePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GenetartePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5)
+	ret := m.ctrl.Call(m, "GeneratePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5)
 	ret0, _ := ret[0].([]byte)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
@@ -98,7 +98,7 @@ func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storif
 // GenetartePoRepVanillaProof indicates an expected call of GenetartePoRepVanillaProof.
 func (mr *MockStoreMockRecorder) GenetartePoRepVanillaProof(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenetartePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GenetartePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GeneratePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GeneratePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5)
 }
 
 // MoveStorage mocks base method.

storage/paths/remote.go

@@ -784,13 +784,17 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
 		return nil, err
 	}
 
+	merr := xerrors.Errorf("sector not found")
+
 	for _, info := range si {
 		for _, u := range info.BaseURLs {
 			url := fmt.Sprintf("%s/vanilla/single", u)
 
 			req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq)))
 			if err != nil {
-				return nil, xerrors.Errorf("request: %w", err)
+				merr = multierror.Append(merr, xerrors.Errorf("request: %w", err))
+				log.Warnw("GenerateSingleVanillaProof request failed", "url", url, "error", err)
+				continue
 			}
 
 			if r.auth != nil {
@@ -800,7 +804,9 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
 
 			resp, err := http.DefaultClient.Do(req)
 			if err != nil {
-				return nil, xerrors.Errorf("do request: %w", err)
+				merr = multierror.Append(merr, xerrors.Errorf("do request: %w", err))
+				log.Warnw("GenerateSingleVanillaProof do request failed", "url", url, "error", err)
+				continue
 			}
 
 			if resp.StatusCode != http.StatusOK {
@@ -810,14 +816,18 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
 				}
 				body, err := io.ReadAll(resp.Body)
 				if err != nil {
-					return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
+					merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err))
+					log.Warnw("GenerateSingleVanillaProof read response body failed", "url", url, "error", err)
+					continue
 				}
 
 				if err := resp.Body.Close(); err != nil {
 					log.Error("response close: ", err)
 				}
 
-				return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))
+				merr = multierror.Append(merr, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body))))
+				log.Warnw("GenerateSingleVanillaProof non-200 code from remote", "code", resp.StatusCode, "url", url, "body", string(body))
+				continue
 			}
 
 			body, err := io.ReadAll(resp.Body)
@@ -826,7 +836,9 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
 					log.Error("response close: ", err)
 				}
 
-				return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
+				merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err))
+				log.Warnw("GenerateSingleVanillaProof read response body failed", "url", url, "error", err)
+				continue
 			}
 
 			_ = resp.Body.Close()
@@ -835,21 +847,26 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
 		}
 	}
 
-	return nil, xerrors.Errorf("sector not found")
+	return nil, merr
 }
 
-func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
+func (r *Remote) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
-	p, err := r.local.GenetartePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed)
+	// Attempt to generate the proof locally first
+	p, err := r.local.GeneratePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed)
 	if err != errPathNotFound {
 		return p, err
 	}
 
+	// Define the file types to look for based on the sector's state
 	ft := storiface.FTSealed | storiface.FTCache
+
+	// Find sector information
 	si, err := r.index.StorageFindSector(ctx, sr.ID, ft, 0, false)
 	if err != nil {
 		return nil, xerrors.Errorf("finding sector %d failed: %w", sr.ID, err)
 	}
 
+	// Prepare request parameters
 	requestParams := PoRepVanillaParams{
 		Sector:   sr,
 		Sealed:   sealed,
@@ -862,58 +879,65 @@ func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.Se
 		return nil, err
 	}
 
+	merr := xerrors.Errorf("sector not found")
+
+	// Iterate over all found sector locations
 	for _, info := range si {
 		for _, u := range info.BaseURLs {
 			url := fmt.Sprintf("%s/vanilla/porep", u)
 
+			// Create and send the request
 			req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq)))
 			if err != nil {
-				return nil, xerrors.Errorf("request: %w", err)
+				merr = multierror.Append(merr, xerrors.Errorf("request: %w", err))
+				log.Warnw("GeneratePoRepVanillaProof request failed", "url", url, "error", err)
+				continue
 			}
 
+			// Set auth headers if available
 			if r.auth != nil {
 				req.Header = r.auth.Clone()
 			}
 			req = req.WithContext(ctx)
 
+			// Execute the request
 			resp, err := http.DefaultClient.Do(req)
 			if err != nil {
-				return nil, xerrors.Errorf("do request: %w", err)
+				merr = multierror.Append(merr, xerrors.Errorf("do request: %w", err))
+				log.Warnw("GeneratePoRepVanillaProof do request failed", "url", url, "error", err)
+				continue
 			}
 
+			// Handle non-OK status codes
 			if resp.StatusCode != http.StatusOK {
+				body, _ := io.ReadAll(resp.Body)
+				_ = resp.Body.Close()
+
 				if resp.StatusCode == http.StatusNotFound {
 					log.Debugw("reading vanilla proof from remote not-found response", "url", url, "store", info.ID)
 					continue
 				}
+
+				merr = multierror.Append(merr, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body))))
+				log.Warnw("GeneratePoRepVanillaProof non-200 code from remote", "code", resp.StatusCode, "url", url, "body", string(body))
+				continue
+			}
+
+			// Read the response body
 			body, err := io.ReadAll(resp.Body)
 			if err != nil {
-					return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
+				merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err))
+				log.Warnw("GeneratePoRepVanillaProof read response body failed", "url", url, "error", err)
 			}
-
-				if err := resp.Body.Close(); err != nil {
-					log.Error("response close: ", err)
-				}
-
-				return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))
-			}
-
-			body, err := io.ReadAll(resp.Body)
-			if err != nil {
-				if err := resp.Body.Close(); err != nil {
-					log.Error("response close: ", err)
-				}
-
-				return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
-			}
-
 			_ = resp.Body.Close()
 
+			// Return the proof if successful
 			return body, nil
 		}
 	}
 
-	return nil, xerrors.Errorf("sector not found")
+	// Return the accumulated error if the proof was not generated
+	return nil, merr
 }
 
 type funcCloser func() error