From ef834b988c5f1f2ab05c4f80e416f81a6f614a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Sep 2022 13:42:04 +0200 Subject: [PATCH] sector import: Fix test build --- itests/sector_import_full_test.go | 150 +++++++++++++++++++++++++++- itests/sector_import_simple_test.go | 20 +--- 2 files changed, 152 insertions(+), 18 deletions(-) diff --git a/itests/sector_import_full_test.go b/itests/sector_import_full_test.go index 3e67b94fc..b26a24145 100644 --- a/itests/sector_import_full_test.go +++ b/itests/sector_import_full_test.go @@ -3,7 +3,12 @@ package itests import ( "bytes" "context" + "encoding/json" "fmt" + lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/types" + spaths "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer/tarutil" "net/http" "net/http/httptest" "os" @@ -39,7 +44,7 @@ func TestSectorImport(t *testing.T) { makeTest := func(mut func(*testCase)) *testCase { tc := &testCase{ - c1handler: remoteCommit1, + c1handler: testRemoteCommit1, } mut(tc) return tc @@ -72,7 +77,11 @@ func TestSectorImport(t *testing.T) { mid, err := address.IDFromAddress(maddr) require.NoError(t, err) - spt, err := currentSealProof(ctx, client, maddr) + mi, err := client.StateMinerInfo(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + ver, err := client.StateNetworkVersion(ctx, types.EmptyTSK) + require.NoError(t, err) + spt, err := lminer.PreferredSealProofTypeFromWindowPoStType(ver, mi.WindowPoStProofType) require.NoError(t, err) ssize, err := spt.SectorSize() @@ -141,8 +150,9 @@ func TestSectorImport(t *testing.T) { // start http server serving sector data m := mux.NewRouter() - m.HandleFunc("/sectors/{type}/{id}", remoteGetSector(sectorDir)).Methods("GET") + m.HandleFunc("/sectors/{type}/{id}", testRemoteGetSector(sectorDir)).Methods("GET") m.HandleFunc("/sectors/{id}/commit1", tc.c1handler(sealer)).Methods("POST") + m.HandleFunc("/commit2", testRemoteCommit2(sealer)).Methods("POST") srv := httptest.NewServer(m) unsealedURL := fmt.Sprintf("%s/sectors/unsealed/s-t0%d-%d", srv.URL, mid, snum) @@ -288,3 +298,137 @@ func TestSectorImport(t *testing.T) { }))) } + +// note: stuff below is almost the same as in _simple version of this file; We need +// to copy it because on Circle we can't call those functions between test files, +// and for the _simple test we want everything in one file to make it easy to follow + +func testRemoteCommit1(s *ffiwrapper.Sealer) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + // validate sector id + id, err := storiface.ParseSectorID(vars["id"]) + if err != nil { + w.WriteHeader(500) + return + } + + var params api.RemoteCommit1Params + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + w.WriteHeader(500) + return + } + + sref := storiface.SectorRef{ + ID: id, + ProofType: params.ProofType, + } + + ssize, err := params.ProofType.SectorSize() + if err != nil { + w.WriteHeader(500) + return + } + + p, err := s.SealCommit1(r.Context(), sref, params.Ticket, params.Seed, []abi.PieceInfo{ + { + Size: abi.PaddedPieceSize(ssize), + PieceCID: params.Unsealed, + }, + }, storiface.SectorCids{ + Unsealed: params.Unsealed, + Sealed: params.Sealed, + }) + if err != nil { + w.WriteHeader(500) + return + } + + if _, err := w.Write(p); err != nil { + fmt.Println("c1 write error") + } + } +} + +func testRemoteCommit2(s *ffiwrapper.Sealer) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + var params api.RemoteCommit2Params + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + w.WriteHeader(500) + return + } + + sref := storiface.SectorRef{ + ID: params.Sector, + ProofType: params.ProofType, + } + + p, err := s.SealCommit2(r.Context(), sref, params.Commit1Out) + if err != nil { + fmt.Println("c2 error: ", err) + w.WriteHeader(500) + return + } + + if _, err := w.Write(p); err != nil { + fmt.Println("c2 write error") + } + } +} + +func testRemoteGetSector(sectorRoot string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + + // validate sector id + id, err := storiface.ParseSectorID(vars["id"]) + if err != nil { + w.WriteHeader(500) + return + } + + // validate type + _, err = spaths.FileTypeFromString(vars["type"]) + if err != nil { + w.WriteHeader(500) + return + } + + typ := vars["type"] + if typ == "cache" { + // if cache is requested, send the finalized cache we've created above + typ = "fin-cache" + } + + path := filepath.Join(sectorRoot, typ, vars["id"]) + + stat, err := os.Stat(path) + if err != nil { + w.WriteHeader(500) + return + } + + if stat.IsDir() { + if _, has := r.Header["Range"]; has { + w.WriteHeader(500) + return + } + + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(200) + + err := tarutil.TarDirectory(path, w, make([]byte, 1<<20)) + if err != nil { + return + } + } else { + w.Header().Set("Content-Type", "application/octet-stream") + // will do a ranged read over the file at the given path if the caller has asked for a ranged read in the request headers. + http.ServeFile(w, r, path) + } + + fmt.Printf("served sector file/dir, sectorID=%+v, fileType=%s, path=%s\n", id, vars["type"], path) + } +} diff --git a/itests/sector_import_simple_test.go b/itests/sector_import_simple_test.go index 419415c1b..f75d0d842 100644 --- a/itests/sector_import_simple_test.go +++ b/itests/sector_import_simple_test.go @@ -57,7 +57,11 @@ func TestSectorImportAfterPC2(t *testing.T) { mid, err := address.IDFromAddress(maddr) require.NoError(t, err) - spt, err := currentSealProof(ctx, client, maddr) + mi, err := client.StateMinerInfo(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + ver, err := client.StateNetworkVersion(ctx, types.EmptyTSK) + require.NoError(t, err) + spt, err := lminer.PreferredSealProofTypeFromWindowPoStType(ver, mi.WindowPoStProofType) require.NoError(t, err) ssize, err := spt.SectorSize() @@ -316,17 +320,3 @@ func remoteGetSector(sectorRoot string) func(w http.ResponseWriter, r *http.Requ fmt.Printf("served sector file/dir, sectorID=%+v, fileType=%s, path=%s\n", id, vars["type"], path) } } - -func currentSealProof(ctx context.Context, api api.FullNode, maddr address.Address) (abi.RegisteredSealProof, error) { - mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) - if err != nil { - return 0, err - } - - ver, err := api.StateNetworkVersion(ctx, types.EmptyTSK) - if err != nil { - return 0, err - } - - return lminer.PreferredSealProofTypeFromWindowPoStType(ver, mi.WindowPoStProofType) -}