cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

storage: Support C1 directly from sector storage

Browse Source
This commit is contained in:
Łukasz Magiera 2024-01-11 16:46:47 +01:00
parent d9e903fa82
commit fe7558368a
4 changed files with 133 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
storage/paths/http_handler.go
View File

@ -3,6 +3,7 @@ package paths
import (
"bytes"
"encoding/json"
"github.com/ipfs/go-cid"
"io"
"net/http"
"os"
@ -55,6 +56,7 @@ func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
mux.HandleFunc("/remote/stat/{id}", handler.remoteStatFs).Methods("GET")
mux.HandleFunc("/remote/vanilla/single", handler.generateSingleVanillaProof).Methods("POST")
mux.HandleFunc("/remote/vanilla/porep", handler.generatePoRepVanillaProof).Methods("POST")
mux.HandleFunc("/remote/{type}/{id}/{spt}/allocated/{offset}/{size}", handler.remoteGetAllocated).Methods("GET")
mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET")
mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE")
@ -312,6 +314,31 @@ func (handler *FetchHandler) generateSingleVanillaProof(w http.ResponseWriter, r
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(vanilla))
}
type PoRepVanillaParams struct {
Sector storiface.SectorRef
Sealed cid.Cid
Unsealed cid.Cid
Ticket abi.SealRandomness
Seed abi.InteractiveSealRandomness
}
func (handler *FetchHandler) generatePoRepVanillaProof(w http.ResponseWriter, r *http.Request) {
var params PoRepVanillaParams
if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
http.Error(w, err.Error(), 500)
return
}
vanilla, err := handler.Local.GenetartePoRepVanillaProof(r.Context(), params.Sector, params.Sealed, params.Unsealed, params.Ticket, params.Seed)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(vanilla))
}
func FileTypeFromString(t string) (storiface.SectorFileType, error) {
switch t {
case storiface.FTUnsealed.String():

2
storage/paths/interface.go
View File

@ -2,6 +2,7 @@ package paths
import (
"context"
"github.com/ipfs/go-cid"
"io"
"github.com/filecoin-project/go-state-types/abi"
@ -48,4 +49,5 @@ type Store interface {
Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)
GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)
GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
}

24
storage/paths/local.go
View File

@ -3,6 +3,7 @@ package paths
import (
"context"
"encoding/json"
"github.com/ipfs/go-cid"
"math/bits"
"math/rand"
"os"
@ -809,4 +810,27 @@ func (st *Local) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
}
}
func (st *Local) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
src, _, err := st.AcquireSector(ctx, sr, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
if err != nil {
return nil, xerrors.Errorf("acquire sector: %w", err)
}
if src.Sealed == "" || src.Cache == "" {
return nil, errPathNotFound
}
ssize, err := sr.ProofType.SectorSize()
if err != nil {
return nil, xerrors.Errorf("getting sector size: %w", err)
}
secPiece := []abi.PieceInfo{{
Size: abi.PaddedPieceSize(ssize),
PieceCID: unsealed,
}}
return ffi.SealCommitPhase1(sr.ProofType, sealed, unsealed, src.Cache, src.Sealed, sr.ID.Number, sr.ID.Miner, ticket, seed, secPiece)
}
var _ Store = &Local{}

81
storage/paths/remote.go
View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/ipfs/go-cid"
"io"
"math/bits"
"net/http"
@ -827,6 +828,8 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
}
_ = resp.Body.Close()
return body, nil
}
}
@ -834,7 +837,83 @@ func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.Act
return nil, xerrors.Errorf("sector not found")
}
var _ Store = &Remote{}
func (r *Remote) GenetartePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
p, err := r.local.GenetartePoRepVanillaProof(ctx, sr, sealed, unsealed, ticket, seed)
if err != errPathNotFound {
return p, err
}
ft := storiface.FTSealed | storiface.FTCache
si, err := r.index.StorageFindSector(ctx, sr.ID, ft, 0, false)
if err != nil {
return nil, xerrors.Errorf("finding sector %d failed: %w", sr.ID, err)
}
requestParams := PoRepVanillaParams{
Sector: sr,
Sealed: sealed,
Unsealed: unsealed,
Ticket: ticket,
Seed: seed,
}
jreq, err := json.Marshal(requestParams)
if err != nil {
return nil, err
}
for _, info := range si {
for _, u := range info.BaseURLs {
url := fmt.Sprintf("%s/vanilla/porep", u)
req, err := http.NewRequest("POST", url, strings.NewReader(string(jreq)))
if err != nil {
return nil, xerrors.Errorf("request: %w", err)
}
if r.auth != nil {
req.Header = r.auth.Clone()
}
req = req.WithContext(ctx)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, xerrors.Errorf("do request: %w", err)
}
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
log.Debugw("reading vanilla proof from remote not-found response", "url", url, "store", info.ID)
continue
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
}
if err := resp.Body.Close(); err != nil {
log.Error("response close: ", err)
}
return nil, xerrors.Errorf("non-200 code from %s: '%s'", url, strings.TrimSpace(string(body)))
}
body, err := io.ReadAll(resp.Body)
if err != nil {
if err := resp.Body.Close(); err != nil {
log.Error("response close: ", err)
}
return nil, xerrors.Errorf("resp.Body ReadAll: %w", err)
}
_ = resp.Body.Close()
return body, nil
}
}
return nil, xerrors.Errorf("sector not found")
}
type funcCloser func() error