cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

do not depend on filecoin-ffi in api package

Browse Source
This commit is contained in:
Anton Evangelatov 2021-06-07 11:51:25 +02:00
parent 042ac8240f
commit b0cb0c1a4a
5 changed files with 28 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
extern/sector-storage/ffiwrapper/sealer_cgo.go vendored
View File

@ -23,6 +23,7 @@ import (
commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper"
"github.com/filecoin-project/go-commp-utils/zerocomm"
"github.com/filecoin-project/lotus/extern/sector-storage/fr32"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
)
@ -66,7 +67,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
}
var done func()
var stagedFile *PartialFile
var stagedFile *partialfile.PartialFile
defer func() {
if done != nil {
@ -87,7 +88,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err)
}
stagedFile, err = createPartialFile(maxPieceSize, stagedPath.Unsealed)
stagedFile, err = partialfile.CreatePartialFile(maxPieceSize, stagedPath.Unsealed)
if err != nil {
return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err)
}
@ -97,7 +98,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err)
}
stagedFile, err = OpenPartialFile(maxPieceSize, stagedPath.Unsealed)
stagedFile, err = partialfile.OpenPartialFile(maxPieceSize, stagedPath.Unsealed)
if err != nil {
return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err)
}
@ -257,7 +258,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
// try finding existing
unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage)
var pf *PartialFile
var pf *partialfile.PartialFile
switch {
case xerrors.Is(err, storiface.ErrSectorNotFound):
@ -267,7 +268,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
}
defer done()
pf, err = createPartialFile(maxPieceSize, unsealedPath.Unsealed)
pf, err = partialfile.CreatePartialFile(maxPieceSize, unsealedPath.Unsealed)
if err != nil {
return xerrors.Errorf("create unsealed file: %w", err)
}
@ -275,7 +276,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
case err == nil:
defer done()
pf, err = OpenPartialFile(maxPieceSize, unsealedPath.Unsealed)
pf, err = partialfile.OpenPartialFile(maxPieceSize, unsealedPath.Unsealed)
if err != nil {
return xerrors.Errorf("opening partial file: %w", err)
}
@ -427,7 +428,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag
}
maxPieceSize := abi.PaddedPieceSize(ssize)
pf, err := OpenPartialFile(maxPieceSize, path.Unsealed)
pf, err := partialfile.OpenPartialFile(maxPieceSize, path.Unsealed)
if err != nil {
if xerrors.Is(err, os.ErrNotExist) {
return false, nil
@ -589,7 +590,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef,
if len(keepUnsealed) > 0 {
sr := pieceRun(0, maxPieceSize)
sr := partialfile.PieceRun(0, maxPieceSize)
for _, s := range keepUnsealed {
si := &rlepluslazy.RunSliceIterator{}
@ -611,7 +612,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef,
}
defer done()
pf, err := OpenPartialFile(maxPieceSize, paths.Unsealed)
pf, err := partialfile.OpenPartialFile(maxPieceSize, paths.Unsealed)
if err == nil {
var at uint64
for sr.HasNext() {

3
extern/sector-storage/ffiwrapper/unseal_ranges.go vendored
View File

@ -7,6 +7,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
)
@ -17,7 +18,7 @@ const mergeGaps = 32 << 20
// TODO const expandRuns = 16 << 20 // unseal more than requested for future requests
func computeUnsealRanges(unsealed rlepluslazy.RunIterator, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (rlepluslazy.RunIterator, error) {
todo := pieceRun(offset.Padded(), size.Padded())
todo := partialfile.PieceRun(offset.Padded(), size.Padded())
todo, err := rlepluslazy.Subtract(todo, unsealed)
if err != nil {
return nil, xerrors.Errorf("compute todo-unsealed: %w", err)

20
extern/sector-storage/ffiwrapper/partialfile.go → extern/sector-storage/partialfile/partialfile.go vendored
View File

@ -1,4 +1,4 @@
package ffiwrapper
package partialfile
import (
"encoding/binary"
@ -14,8 +14,12 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
logging "github.com/ipfs/go-log/v2"
)
var log = logging.Logger("partialfile")
const veryLargeRle = 1 << 20
// Sectors can be partially unsealed. We support this by appending a small
@ -57,7 +61,7 @@ func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) err
return w.Truncate(maxPieceSize + int64(rb) + 4)
}
func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) {
func CreatePartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint
if err != nil {
return nil, xerrors.Errorf("openning partial file '%s': %w", path, err)
@ -188,7 +192,7 @@ func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedP
return nil, err
}
and, err := rlepluslazy.And(have, pieceRun(offset, size))
and, err := rlepluslazy.And(have, PieceRun(offset, size))
if err != nil {
return nil, err
}
@ -212,7 +216,7 @@ func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.
return err
}
ored, err := rlepluslazy.Or(have, pieceRun(offset, size))
ored, err := rlepluslazy.Or(have, PieceRun(offset, size))
if err != nil {
return err
}
@ -234,7 +238,7 @@ func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPie
return xerrors.Errorf("deallocating: %w", err)
}
s, err := rlepluslazy.Subtract(have, pieceRun(offset, size))
s, err := rlepluslazy.Subtract(have, PieceRun(offset, size))
if err != nil {
return err
}
@ -257,7 +261,7 @@ func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedP
return nil, err
}
and, err := rlepluslazy.And(have, pieceRun(offset, size))
and, err := rlepluslazy.And(have, PieceRun(offset, size))
if err != nil {
return nil, err
}
@ -285,7 +289,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi
return false, err
}
u, err := rlepluslazy.And(have, pieceRun(offset.Padded(), size.Padded()))
u, err := rlepluslazy.And(have, PieceRun(offset.Padded(), size.Padded()))
if err != nil {
return false, err
}
@ -298,7 +302,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi
return abi.PaddedPieceSize(uc) == size.Padded(), nil
}
func pieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator {
func PieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator {
var runs []rlepluslazy.Run
if offset > 0 {
runs = append(runs, rlepluslazy.Run{

4
extern/sector-storage/stores/http_handler.go vendored
View File

@ -12,7 +12,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/tarutil"
@ -130,7 +130,7 @@ func (handler *FetchHandler) remoteGetAllocated(w http.ResponseWriter, r *http.R
return
}
pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
if err != nil {
log.Error("opening partial file: ", err)
w.WriteHeader(500)

4
extern/sector-storage/stores/remote.go vendored
View File

@ -16,8 +16,8 @@ import (
"sort"
"sync"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/tarutil"
@ -415,7 +415,7 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a
return nil, err
}
pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
if err != nil {
return nil, xerrors.Errorf("opening partial file: %w", err)
}