From c293bcebb6bd6660d0e7966beec633eab0f60961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 17 Feb 2024 11:20:49 +0100 Subject: [PATCH] storage: Make remote Generate Vanilla more robust --- provider/lpffi/sdr_funcs.go | 2 +- storage/paths/http_handler.go | 2 +- storage/paths/interface.go | 2 +- storage/paths/local.go | 2 +- storage/paths/mocks/store.go | 6 +-- storage/paths/remote.go | 76 +++++++++++++++++++++++------------ 6 files changed, 57 insertions(+), 33 deletions(-) diff --git a/provider/lpffi/sdr_funcs.go b/provider/lpffi/sdr_funcs.go index ae402a036..daf7d5381 100644 --- a/provider/lpffi/sdr_funcs.go +++ b/provider/lpffi/sdr_funcs.go @@ -192,7 +192,7 @@ func (sb *SealCalls) GenerateSynthPoRep() { } func (sb *SealCalls) PoRepSnark(ctx context.Context, sn storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { - vproof, err := sb.sectors.storage.GenetartePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed) + vproof, err := sb.sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed) if err != nil { return nil, xerrors.Errorf("failed to generate vanilla proof: %w", err) } diff --git a/storage/paths/http_handler.go b/storage/paths/http_handler.go index 058dee71f..57de578a6 100644 --- a/storage/paths/http_handler.go +++ b/storage/paths/http_handler.go @@ -329,7 +329,7 @@ func (handler *FetchHandler) generatePoRepVanillaProof(w http.ResponseWriter, r return } - vanilla, err := handler.Local.GenetartePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed) + vanilla, err := handler.Local.GeneratePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed) if err != nil { http.Error(w, err.Error(), 500) return diff --git a/storage/paths/interface.go b/storage/paths/interface.go index a16821bd9..4ff206c6d 100644 --- a/storage/paths/interface.go +++ b/storage/paths/interface.go @@ -50,5 +50,5 @@ type Store interface { Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error) - GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) + GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) } diff --git a/storage/paths/local.go b/storage/paths/local.go index 28e620a99..71de0becf 100644 --- a/storage/paths/local.go +++ b/storage/paths/local.go @@ -810,7 +810,7 @@ func (st *Local) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act } } -func (st *Local) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { +func (st *Local) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { src, _, err := st.AcquireSector(ctx, sr, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) if err != nil { return nil, xerrors.Errorf("acquire sector: %w", err) diff --git a/storage/paths/mocks/store.go b/storage/paths/mocks/store.go index 19eebaab9..b7d46acbc 100644 --- a/storage/paths/mocks/store.go +++ b/storage/paths/mocks/store.go @@ -87,9 +87,9 @@ func (mr *MockStoreMockRecorder) GenerateSingleVanillaProof(arg0, arg1, arg2, ar } // GenetartePoRepVanillaProof mocks base method. -func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) { +func (m *MockStore) GeneratePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GenetartePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5) + ret := m.ctrl.Call(m, "GeneratePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5) ret0, _ := ret[0].([]byte) ret1, _ := ret[1].(error) return ret0, ret1 @@ -98,7 +98,7 @@ func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storif // GenetartePoRepVanillaProof indicates an expected call of GenetartePoRepVanillaProof. func (mr *MockStoreMockRecorder) GenetartePoRepVanillaProof(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenetartePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GenetartePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GeneratePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GeneratePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5) } // MoveStorage mocks base method. diff --git a/storage/paths/remote.go b/storage/paths/remote.go index 6433260b1..afa589683 100644 --- a/storage/paths/remote.go +++ b/storage/paths/remote.go @@ -784,13 +784,17 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act return nil, err } + merr := xerrors.Errorf("sector not found") + for _, info := range si { for _, u := range info.BaseURLs { url := fmt.Sprintf("%s/vanilla/single", u) req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq))) if err != nil { - return nil, xerrors.Errorf("request: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("request: %w", err)) + log.Warnw("GenerateSingleVanillaProof request failed", "url", url, "error", err) + continue } if r.auth != nil { @@ -800,7 +804,9 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, xerrors.Errorf("do request: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("do request: %w", err)) + log.Warnw("GenerateSingleVanillaProof do request failed", "url", url, "error", err) + continue } if resp.StatusCode != http.StatusOK { @@ -810,14 +816,18 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, xerrors.Errorf("resp.Body ReadAll: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err)) + log.Warnw("GenerateSingleVanillaProof read response body failed", "url", url, "error", err) + continue } if err := resp.Body.Close(); err != nil { log.Error("response close: ", err) } - return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body))) + merr = multierror.Append(merr, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))) + log.Warnw("GenerateSingleVanillaProof non-200 code from remote", "code", resp.StatusCode, "url", url, "body", string(body)) + continue } body, err := io.ReadAll(resp.Body) @@ -826,7 +836,9 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act log.Error("response close: ", err) } - return nil, xerrors.Errorf("resp.Body ReadAll: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err)) + log.Warnw("GenerateSingleVanillaProof read response body failed", "url", url, "error", err) + continue } _ = resp.Body.Close() @@ -835,21 +847,26 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act } } - return nil, xerrors.Errorf("sector not found") + return nil, merr } -func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { - p, err := r.local.GenetartePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed) +func (r *Remote) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { + // Attempt to generate the proof locally first + p, err := r.local.GeneratePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed) if err != errPathNotFound { return p, err } + // Define the file types to look for based on the sector's state ft := storiface.FTSealed | storiface.FTCache + + // Find sector information si, err := r.index.StorageFindSector(ctx, sr.ID, ft, 0, false) if err != nil { return nil, xerrors.Errorf("finding sector %d failed: %w", sr.ID, err) } + // Prepare request parameters requestParams := PoRepVanillaParams{ Sector: sr, Sealed: sealed, @@ -862,58 +879,65 @@ func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.Se return nil, err } + merr := xerrors.Errorf("sector not found") + + // Iterate over all found sector locations for _, info := range si { for _, u := range info.BaseURLs { url := fmt.Sprintf("%s/vanilla/porep", u) + // Create and send the request req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq))) if err != nil { - return nil, xerrors.Errorf("request: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("request: %w", err)) + log.Warnw("GeneratePoRepVanillaProof request failed", "url", url, "error", err) + continue } + // Set auth headers if available if r.auth != nil { req.Header = r.auth.Clone() } req = req.WithContext(ctx) + // Execute the request resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, xerrors.Errorf("do request: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("do request: %w", err)) + log.Warnw("GeneratePoRepVanillaProof do request failed", "url", url, "error", err) + continue } + // Handle non-OK status codes if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { log.Debugw("reading vanilla proof from remote not-found response", "url", url, "store", info.ID) continue } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, xerrors.Errorf("resp.Body ReadAll: %w", err) - } - if err := resp.Body.Close(); err != nil { - log.Error("response close: ", err) - } - - return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body))) + merr = multierror.Append(merr, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))) + log.Warnw("GeneratePoRepVanillaProof non-200 code from remote", "code", resp.StatusCode, "url", url, "body", string(body)) + continue } + // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { - if err := resp.Body.Close(); err != nil { - log.Error("response close: ", err) - } - - return nil, xerrors.Errorf("resp.Body ReadAll: %w", err) + merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err)) + log.Warnw("GeneratePoRepVanillaProof read response body failed", "url", url, "error", err) } - _ = resp.Body.Close() + // Return the proof if successful return body, nil } } - return nil, xerrors.Errorf("sector not found") + // Return the accumulated error if the proof was not generated + return nil, merr } type funcCloser func() error