cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

storage: Make remote Generate Vanilla more robust

Browse Source
This commit is contained in:
Łukasz Magiera 2024-02-17 11:20:49 +01:00
parent e22ab4c8fe
commit c293bcebb6
6 changed files with 57 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
provider/lpffi/sdr_funcs.go
View File

@ -192,7 +192,7 @@ func (sb *SealCalls) GenerateSynthPoRep() {
}
func (sb *SealCalls) PoRepSnark(ctx context.Context, sn storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
vproof, err := sb.sectors.storage.GenetartePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed)
vproof, err := sb.sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed)
if err != nil {
return nil, xerrors.Errorf("failed to generate vanilla proof: %w", err)
}

2
storage/paths/http_handler.go
View File

@ -329,7 +329,7 @@ func (handler *FetchHandler) generatePoRepVanillaProof(w http.ResponseWriter, r
return
}
vanilla, err := handler.Local.GenetartePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed)
vanilla, err := handler.Local.GeneratePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed)
if err != nil {
http.Error(w, err.Error(), 500)
return

2
storage/paths/interface.go
View File

@ -50,5 +50,5 @@ type Store interface {
Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)
GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)
GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
}

2
storage/paths/local.go
View File

@ -810,7 +810,7 @@ func (st *Local) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
}
}
func (st *Local) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
func (st *Local) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
src, _, err := st.AcquireSector(ctx, sr, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
if err != nil {
return nil, xerrors.Errorf("acquire sector: %w", err)

6
storage/paths/mocks/store.go
View File

@ -87,9 +87,9 @@ func (mr *MockStoreMockRecorder) GenerateSingleVanillaProof(arg0, arg1, arg2, ar
}
// GenetartePoRepVanillaProof mocks base method.
func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) {
func (m *MockStore) GeneratePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GenetartePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5)
ret := m.ctrl.Call(m, "GeneratePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5)
ret0, _ := ret[0].([]byte)
ret1, _ := ret[1].(error)
return ret0, ret1
@ -98,7 +98,7 @@ func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storif
// GenetartePoRepVanillaProof indicates an expected call of GenetartePoRepVanillaProof.
func (mr *MockStoreMockRecorder) GenetartePoRepVanillaProof(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenetartePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GenetartePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GeneratePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GeneratePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5)
}
// MoveStorage mocks base method.

76
storage/paths/remote.go
View File

@ -784,13 +784,17 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
return nil, err
}
merr := xerrors.Errorf("sector not found")
for _, info := range si {
for _, u := range info.BaseURLs {
url := fmt.Sprintf("%s/vanilla/single", u)
req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq)))
if err != nil {
return nil, xerrors.Errorf("request: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("request: %w", err))
log.Warnw("GenerateSingleVanillaProof request failed", "url", url, "error", err)
continue
}
if r.auth != nil {
@ -800,7 +804,9 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, xerrors.Errorf("do request: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("do request: %w", err))
log.Warnw("GenerateSingleVanillaProof do request failed", "url", url, "error", err)
continue
}
if resp.StatusCode != http.StatusOK {
@ -810,14 +816,18 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err))
log.Warnw("GenerateSingleVanillaProof read response body failed", "url", url, "error", err)
continue
}
if err := resp.Body.Close(); err != nil {
log.Error("response close: ", err)
}
return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))
merr = multierror.Append(merr, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body))))
log.Warnw("GenerateSingleVanillaProof non-200 code from remote", "code", resp.StatusCode, "url", url, "body", string(body))
continue
}
body, err := io.ReadAll(resp.Body)
@ -826,7 +836,9 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
log.Error("response close: ", err)
}
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err))
log.Warnw("GenerateSingleVanillaProof read response body failed", "url", url, "error", err)
continue
}
_ = resp.Body.Close()
@ -835,21 +847,26 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
}
}
return nil, xerrors.Errorf("sector not found")
return nil, merr
}
func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
p, err := r.local.GenetartePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed)
func (r *Remote) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
// Attempt to generate the proof locally first
p, err := r.local.GeneratePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed)
if err != errPathNotFound {
return p, err
}
// Define the file types to look for based on the sector's state
ft := storiface.FTSealed | storiface.FTCache
// Find sector information
si, err := r.index.StorageFindSector(ctx, sr.ID, ft, 0, false)
if err != nil {
return nil, xerrors.Errorf("finding sector %d failed: %w", sr.ID, err)
}
// Prepare request parameters
requestParams := PoRepVanillaParams{
Sector: sr,
Sealed: sealed,
@ -862,58 +879,65 @@ func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.Se
return nil, err
}
merr := xerrors.Errorf("sector not found")
// Iterate over all found sector locations
for _, info := range si {
for _, u := range info.BaseURLs {
url := fmt.Sprintf("%s/vanilla/porep", u)
// Create and send the request
req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq)))
if err != nil {
return nil, xerrors.Errorf("request: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("request: %w", err))
log.Warnw("GeneratePoRepVanillaProof request failed", "url", url, "error", err)
continue
}
// Set auth headers if available
if r.auth != nil {
req.Header = r.auth.Clone()
}
req = req.WithContext(ctx)
// Execute the request
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, xerrors.Errorf("do request: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("do request: %w", err))
log.Warnw("GeneratePoRepVanillaProof do request failed", "url", url, "error", err)
continue
}
// Handle non-OK status codes
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
log.Debugw("reading vanilla proof from remote not-found response", "url", url, "store", info.ID)
continue
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
}
if err := resp.Body.Close(); err != nil {
log.Error("response close: ", err)
}
return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))
merr = multierror.Append(merr, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body))))
log.Warnw("GeneratePoRepVanillaProof non-200 code from remote", "code", resp.StatusCode, "url", url, "body", string(body))
continue
}
// Read the response body
body, err := io.ReadAll(resp.Body)
if err != nil {
if err := resp.Body.Close(); err != nil {
log.Error("response close: ", err)
}
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
merr = multierror.Append(merr, xerrors.Errorf("resp.Body ReadAll: %w", err))
log.Warnw("GeneratePoRepVanillaProof read response body failed", "url", url, "error", err)
}
_ = resp.Body.Close()
// Return the proof if successful
return body, nil
}
}
return nil, xerrors.Errorf("sector not found")
// Return the accumulated error if the proof was not generated
return nil, merr
}
type funcCloser func() error