cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

do not depend on filecoin-ffi in api package

Browse Source
This commit is contained in:
Anton Evangelatov 2021-06-07 11:51:25 +02:00
parent 042ac8240f
commit b0cb0c1a4a
5 changed files with 28 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
extern/sector-storage/ffiwrapper/sealer_cgo.go vendored
View File

@ -23,6 +23,7 @@ import (
commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper"
"github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/go-commp-utils/zerocomm"
"github.com/filecoin-project/lotus/extern/sector-storage/fr32" "github.com/filecoin-project/lotus/extern/sector-storage/fr32"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
) )
@ -66,7 +67,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
} }
var done func() var done func()
var stagedFile *PartialFile var stagedFile *partialfile.PartialFile
defer func() { defer func() {
if done != nil { if done != nil {
@ -87,7 +88,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err)
} }
stagedFile, err = createPartialFile(maxPieceSize, stagedPath.Unsealed) stagedFile, err = partialfile.CreatePartialFile(maxPieceSize, stagedPath.Unsealed)
if err != nil { if err != nil {
return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err) return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err)
} }
@ -97,7 +98,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err)
} }
stagedFile, err = OpenPartialFile(maxPieceSize, stagedPath.Unsealed) stagedFile, err = partialfile.OpenPartialFile(maxPieceSize, stagedPath.Unsealed)
if err != nil { if err != nil {
return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err) return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err)
} }
@ -257,7 +258,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
// try finding existing // try finding existing
unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage) unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage)
var pf *PartialFile var pf *partialfile.PartialFile
switch { switch {
case xerrors.Is(err, storiface.ErrSectorNotFound): case xerrors.Is(err, storiface.ErrSectorNotFound):
@ -267,7 +268,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
} }
defer done() defer done()
pf, err = createPartialFile(maxPieceSize, unsealedPath.Unsealed) pf, err = partialfile.CreatePartialFile(maxPieceSize, unsealedPath.Unsealed)
if err != nil { if err != nil {
return xerrors.Errorf("create unsealed file: %w", err) return xerrors.Errorf("create unsealed file: %w", err)
} }
@ -275,7 +276,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
case err == nil: case err == nil:
defer done() defer done()
pf, err = OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) pf, err = partialfile.OpenPartialFile(maxPieceSize, unsealedPath.Unsealed)
if err != nil { if err != nil {
return xerrors.Errorf("opening partial file: %w", err) return xerrors.Errorf("opening partial file: %w", err)
} }
@ -427,7 +428,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag
} }
maxPieceSize := abi.PaddedPieceSize(ssize) maxPieceSize := abi.PaddedPieceSize(ssize)
pf, err := OpenPartialFile(maxPieceSize, path.Unsealed) pf, err := partialfile.OpenPartialFile(maxPieceSize, path.Unsealed)
if err != nil { if err != nil {
if xerrors.Is(err, os.ErrNotExist) { if xerrors.Is(err, os.ErrNotExist) {
return false, nil return false, nil
@ -589,7 +590,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef,
if len(keepUnsealed) > 0 { if len(keepUnsealed) > 0 {
sr := pieceRun(0, maxPieceSize) sr := partialfile.PieceRun(0, maxPieceSize)
for _, s := range keepUnsealed { for _, s := range keepUnsealed {
si := &rlepluslazy.RunSliceIterator{} si := &rlepluslazy.RunSliceIterator{}
@ -611,7 +612,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef,
} }
defer done() defer done()
pf, err := OpenPartialFile(maxPieceSize, paths.Unsealed) pf, err := partialfile.OpenPartialFile(maxPieceSize, paths.Unsealed)
if err == nil { if err == nil {
var at uint64 var at uint64
for sr.HasNext() { for sr.HasNext() {

3
extern/sector-storage/ffiwrapper/unseal_ranges.go vendored
View File

@ -7,6 +7,7 @@ import (
"github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
) )
@ -17,7 +18,7 @@ const mergeGaps = 32 << 20
// TODO const expandRuns = 16 << 20 // unseal more than requested for future requests // TODO const expandRuns = 16 << 20 // unseal more than requested for future requests
func computeUnsealRanges(unsealed rlepluslazy.RunIterator, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (rlepluslazy.RunIterator, error) { func computeUnsealRanges(unsealed rlepluslazy.RunIterator, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (rlepluslazy.RunIterator, error) {
todo := pieceRun(offset.Padded(), size.Padded()) todo := partialfile.PieceRun(offset.Padded(), size.Padded())
todo, err := rlepluslazy.Subtract(todo, unsealed) todo, err := rlepluslazy.Subtract(todo, unsealed)
if err != nil { if err != nil {
return nil, xerrors.Errorf("compute todo-unsealed: %w", err) return nil, xerrors.Errorf("compute todo-unsealed: %w", err)

20
extern/sector-storage/ffiwrapper/partialfile.go → extern/sector-storage/partialfile/partialfile.go vendored
View File

@ -1,4 +1,4 @@
package ffiwrapper package partialfile
import ( import (
"encoding/binary" "encoding/binary"
@ -14,8 +14,12 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
logging "github.com/ipfs/go-log/v2"
) )
var log = logging.Logger("partialfile")
const veryLargeRle = 1 << 20 const veryLargeRle = 1 << 20
// Sectors can be partially unsealed. We support this by appending a small // Sectors can be partially unsealed. We support this by appending a small
@ -57,7 +61,7 @@ func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) err
return w.Truncate(maxPieceSize + int64(rb) + 4) return w.Truncate(maxPieceSize + int64(rb) + 4)
} }
func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { func CreatePartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint
if err != nil { if err != nil {
return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) return nil, xerrors.Errorf("openning partial file '%s': %w", path, err)
@ -188,7 +192,7 @@ func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedP
return nil, err return nil, err
} }
and, err := rlepluslazy.And(have, pieceRun(offset, size)) and, err := rlepluslazy.And(have, PieceRun(offset, size))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -212,7 +216,7 @@ func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.
return err return err
} }
ored, err := rlepluslazy.Or(have, pieceRun(offset, size)) ored, err := rlepluslazy.Or(have, PieceRun(offset, size))
if err != nil { if err != nil {
return err return err
} }
@ -234,7 +238,7 @@ func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPie
return xerrors.Errorf("deallocating: %w", err) return xerrors.Errorf("deallocating: %w", err)
} }
s, err := rlepluslazy.Subtract(have, pieceRun(offset, size)) s, err := rlepluslazy.Subtract(have, PieceRun(offset, size))
if err != nil { if err != nil {
return err return err
} }
@ -257,7 +261,7 @@ func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedP
return nil, err return nil, err
} }
and, err := rlepluslazy.And(have, pieceRun(offset, size)) and, err := rlepluslazy.And(have, PieceRun(offset, size))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -285,7 +289,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi
return false, err return false, err
} }
u, err := rlepluslazy.And(have, pieceRun(offset.Padded(), size.Padded())) u, err := rlepluslazy.And(have, PieceRun(offset.Padded(), size.Padded()))
if err != nil { if err != nil {
return false, err return false, err
} }
@ -298,7 +302,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi
return abi.PaddedPieceSize(uc) == size.Padded(), nil return abi.PaddedPieceSize(uc) == size.Padded(), nil
} }
func pieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { func PieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator {
var runs []rlepluslazy.Run var runs []rlepluslazy.Run
if offset > 0 { if offset > 0 {
runs = append(runs, rlepluslazy.Run{ runs = append(runs, rlepluslazy.Run{

4
extern/sector-storage/stores/http_handler.go vendored
View File

@ -12,7 +12,7 @@ import (
"golang.org/x/xerrors" "golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/tarutil" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil"
@ -130,7 +130,7 @@ func (handler *FetchHandler) remoteGetAllocated(w http.ResponseWriter, r *http.R
return return
} }
pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
if err != nil { if err != nil {
log.Error("opening partial file: ", err) log.Error("opening partial file: ", err)
w.WriteHeader(500) w.WriteHeader(500)

4
extern/sector-storage/stores/remote.go vendored
View File

@ -16,8 +16,8 @@ import (
"sort" "sort"
"sync" "sync"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/tarutil" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil"
@ -415,7 +415,7 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a
return nil, err return nil, err
} }
pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
if err != nil { if err != nil {
return nil, xerrors.Errorf("opening partial file: %w", err) return nil, xerrors.Errorf("opening partial file: %w", err)
} }