sector import: FinalizeInto, send finalized cacde in itest

This commit is contained in:
Łukasz Magiera 2022-09-06 17:09:27 +02:00
parent 3086e8aa01
commit 830c2ffdf5
2 changed files with 52 additions and 2 deletions

View File

@ -108,7 +108,10 @@ func TestSectorImportAfterPC2(t *testing.T) {
scids, err := sealer.SealPreCommit2(ctx, sref, pc1out) scids, err := sealer.SealPreCommit2(ctx, sref, pc1out)
require.NoError(t, err) require.NoError(t, err)
// todo split-finalize! // make finalized cache, put it in [sectorDir]/fin-cache
finDst := filepath.Join(sectorDir, "fin-cache", fmt.Sprintf("s-t01000-%d", snum))
require.NoError(t, os.MkdirAll(finDst, 0777))
require.NoError(t, sealer.FinalizeSectorInto(ctx, sref, finDst))
//////// ////////
// start http server serving sector data // start http server serving sector data
@ -183,7 +186,13 @@ func remoteGetSector(sectorRoot string) func(w http.ResponseWriter, r *http.Requ
return return
} }
path := filepath.Join(sectorRoot, vars["type"], vars["id"]) typ := vars["type"]
if typ == "cache" {
// if cache is requested, send the finalized cache we've created above
typ = "fin-cache"
}
path := filepath.Join(sectorRoot, typ, vars["id"])
stat, err := os.Stat(path) stat, err := os.Stat(path)
if err != nil { if err != nil {

View File

@ -11,9 +11,12 @@ import (
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"io" "io"
"io/ioutil"
"math/bits" "math/bits"
"os" "os"
"path/filepath"
"runtime" "runtime"
"syscall"
"github.com/detailyang/go-fallocate" "github.com/detailyang/go-fallocate"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
@ -1069,6 +1072,44 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storiface.SectorRef
return ffi.ClearCache(uint64(ssize), paths.Cache) return ffi.ClearCache(uint64(ssize), paths.Cache)
} }
// FinalizeSectorInto is like FinalizeSector, but writes finalized sector cache into a new path
func (sb *Sealer) FinalizeSectorInto(ctx context.Context, sector storiface.SectorRef, dest string) error {
ssize, err := sector.ProofType.SectorSize()
if err != nil {
return err
}
paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache, 0, storiface.PathStorage)
if err != nil {
return xerrors.Errorf("acquiring sector cache path: %w", err)
}
defer done()
files, err := ioutil.ReadDir(paths.Cache)
if err != nil {
return err
}
for _, file := range files {
if file.Name() != "t_aux" && file.Name() != "p_aux" {
// link all the non-aux files
if err := syscall.Link(filepath.Join(paths.Cache, file.Name()), filepath.Join(dest, file.Name())); err != nil {
return xerrors.Errorf("link %s: %w", file.Name(), err)
}
continue
}
d, err := os.ReadFile(filepath.Join(paths.Cache, file.Name()))
if err != nil {
return xerrors.Errorf("read %s: %w", file.Name(), err)
}
if err := os.WriteFile(filepath.Join(dest, file.Name()), d, 0666); err != nil {
return xerrors.Errorf("write %s: %w", file.Name(), err)
}
}
return ffi.ClearCache(uint64(ssize), dest)
}
func (sb *Sealer) FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) error { func (sb *Sealer) FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) error {
ssize, err := sector.ProofType.SectorSize() ssize, err := sector.ProofType.SectorSize()
if err != nil { if err != nil {