cerc-io/lotus
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

ffiwrapper: Handle null padding in DataCid automatically

Browse Source
This commit is contained in:
Łukasz Magiera 2022-05-24 15:59:11 +02:00
parent 3fa2ada987
commit 7054475c00
3 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/lotus-miner/sealing.go
View File

@ -436,7 +436,7 @@ var sealingDataCidCmd = &cli.Command{
return xerrors.Errorf("file larger than the piece") return xerrors.Errorf("file larger than the piece")
} }
} else { } else {
psize = padreader.PaddedSize(uint64(sz)).Padded() psize = padreader.PaddedSize(sz).Padded()
} }
pc, err := nodeApi.ComputeDataCid(ctx, psize.Unpadded(), r) pc, err := nodeApi.ComputeDataCid(ctx, psize.Unpadded(), r)

7
extern/sector-storage/ffiwrapper/sealer_cgo.go vendored
View File

@ -32,6 +32,7 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
"github.com/filecoin-project/lotus/lib/nullreader"
) )
var _ Storage = &Sealer{} var _ Storage = &Sealer{}
@ -53,6 +54,11 @@ func (sb *Sealer) NewSector(ctx context.Context, sector storage.SectorRef) error
} }
func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) { func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
pieceData = io.LimitReader(io.MultiReader(
pieceData,
nullreader.Reader{},
), int64(pieceSize))
// TODO: allow tuning those: // TODO: allow tuning those:
chunk := abi.PaddedPieceSize(4 << 20) chunk := abi.PaddedPieceSize(4 << 20)
parallel := runtime.NumCPU() parallel := runtime.NumCPU()
@ -73,6 +79,7 @@ func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize,
for { for {
var read int var read int
for rbuf := buf; len(rbuf) > 0; { for rbuf := buf; len(rbuf) > 0; {
n, err := pieceData.Read(rbuf) n, err := pieceData.Read(rbuf)
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
return abi.PieceInfo{}, xerrors.Errorf("pr read error: %w", err) return abi.PieceInfo{}, xerrors.Errorf("pr read error: %w", err)

12
itests/worker_test.go
View File

@ -49,17 +49,23 @@ func TestWorkerDataCid(t *testing.T) {
e, err := worker.Enabled(ctx) e, err := worker.Enabled(ctx)
require.NoError(t, err) require.NoError(t, err)
require.True(t, e) require.True(t, e)
/*
pi, err := miner.ComputeDataCid(ctx, 1016, strings.NewReader(strings.Repeat("a", 1016))) pi, err := miner.ComputeDataCid(ctx, 1016, strings.NewReader(strings.Repeat("a", 1016)))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, abi.PaddedPieceSize(1024), pi.Size) require.Equal(t, abi.PaddedPieceSize(1024), pi.Size)
require.Equal(t, "baga6ea4seaqlhznlutptgfwhffupyer6txswamerq5fc2jlwf2lys2mm5jtiaeq", pi.PieceCID.String()) require.Equal(t, "baga6ea4seaqlhznlutptgfwhffupyer6txswamerq5fc2jlwf2lys2mm5jtiaeq", pi.PieceCID.String())
*/
bigPiece := abi.PaddedPieceSize(16 << 20).Unpadded() bigPiece := abi.PaddedPieceSize(16 << 20).Unpadded()
pi, err := miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(bigPiece)))) pi, err = miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(bigPiece))))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, bigPiece.Padded(), pi.Size) require.Equal(t, bigPiece.Padded(), pi.Size)
require.Equal(t, "baga6ea4seaqmhoxl2ybw5m2wyd3pt3h4zmp7j52yumzu2rar26twns3uocq7yfa", pi.PieceCID.String()) require.Equal(t, "baga6ea4seaqmhoxl2ybw5m2wyd3pt3h4zmp7j52yumzu2rar26twns3uocq7yfa", pi.PieceCID.String())
nonFullPiece := abi.PaddedPieceSize(10 << 20).Unpadded()
pi, err = miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(nonFullPiece))))
require.NoError(t, err)
require.Equal(t, bigPiece.Padded(), pi.Size)
require.Equal(t, "baga6ea4seaqbxib4pdxs5cqdn3fmtj4rcxk6rx6ztiqmrx7fcpo3ymuxbp2rodi", pi.PieceCID.String())
} }
func TestWinningPostWorker(t *testing.T) { func TestWinningPostWorker(t *testing.T) {