Merge pull request #4929 from filecoin-project/feat/update-markets-commp-extraction
Remove all temp file generation for deals
This commit is contained in:
Łukasz Magiera
GitHub
commit
0e39e760e5
@ -19,9 +19,9 @@ import (
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-commp-utils/zerocomm"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
|
||||
market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
|
||||
|
||||
53
extern/sector-storage/ffiwrapper/files.go
vendored
53
extern/sector-storage/ffiwrapper/files.go
vendored
@ -1,53 +0,0 @@
|
||||
package ffiwrapper
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func ToReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
|
||||
f, ok := r.(*os.File)
|
||||
if ok {
|
||||
return f, func() error { return nil }, nil
|
||||
}
|
||||
|
||||
var w *os.File
|
||||
|
||||
f, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var wait sync.Mutex
|
||||
var werr error
|
||||
|
||||
wait.Lock()
|
||||
go func() {
|
||||
defer wait.Unlock()
|
||||
|
||||
var copied int64
|
||||
copied, werr = io.CopyN(w, r, n)
|
||||
if werr != nil {
|
||||
log.Warnf("toReadableFile: copy error: %+v", werr)
|
||||
}
|
||||
|
||||
err := w.Close()
|
||||
if werr == nil && err != nil {
|
||||
werr = err
|
||||
log.Warnf("toReadableFile: close error: %+v", err)
|
||||
return
|
||||
}
|
||||
if copied != n {
|
||||
log.Warnf("copied different amount than expected: %d != %d", copied, n)
|
||||
werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
|
||||
}
|
||||
}()
|
||||
|
||||
return f, func() error {
|
||||
wait.Lock()
|
||||
return werr
|
||||
}, nil
|
||||
}
|
||||
19
extern/sector-storage/ffiwrapper/sealer_cgo.go
vendored
19
extern/sector-storage/ffiwrapper/sealer_cgo.go
vendored
@ -20,9 +20,10 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
|
||||
commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper"
|
||||
"github.com/filecoin-project/go-commp-utils/zerocomm"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/fr32"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
)
|
||||
|
||||
var _ Storage = &Sealer{}
|
||||
@ -175,7 +176,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
|
||||
}
|
||||
|
||||
func (sb *Sealer) pieceCid(spt abi.RegisteredSealProof, in []byte) (cid.Cid, error) {
|
||||
prf, werr, err := ToReadableFile(bytes.NewReader(in), int64(len(in)))
|
||||
prf, werr, err := commpffi.ToReadableFile(bytes.NewReader(in), int64(len(in)))
|
||||
if err != nil {
|
||||
return cid.Undef, xerrors.Errorf("getting tee reader pipe: %w", err)
|
||||
}
|
||||
@ -610,20 +611,6 @@ func (sb *Sealer) Remove(ctx context.Context, sector storage.SectorRef) error {
|
||||
return xerrors.Errorf("not supported at this layer") // happens in localworker
|
||||
}
|
||||
|
||||
func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader, pieceSize abi.UnpaddedPieceSize) (cid.Cid, error) {
|
||||
f, werr, err := ToReadableFile(piece, int64(pieceSize))
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
pieceCID, err := ffi.GeneratePieceCIDFromFile(proofType, f, pieceSize)
|
||||
if err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
|
||||
return pieceCID, werr()
|
||||
}
|
||||
|
||||
func GetRequiredPadding(oldLength abi.PaddedPieceSize, newPieceLength abi.PaddedPieceSize) ([]abi.PaddedPieceSize, abi.PaddedPieceSize) {
|
||||
|
||||
padPieces := make([]abi.PaddedPieceSize, 0)
|
||||
|
||||
@ -15,6 +15,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper"
|
||||
|
||||
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
@ -465,7 +467,7 @@ func BenchmarkWriteWithAlignment(b *testing.B) {
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StopTimer()
|
||||
rf, w, _ := ToReadableFile(bytes.NewReader(bytes.Repeat([]byte{0xff, 0}, int(bt/2))), int64(bt))
|
||||
rf, w, _ := commpffi.ToReadableFile(bytes.NewReader(bytes.Repeat([]byte{0xff, 0}, int(bt/2))), int64(bt))
|
||||
tf, _ := ioutil.TempFile("/tmp/", "scrb-")
|
||||
b.StartTimer()
|
||||
|
||||
@ -524,7 +526,7 @@ func TestGenerateUnsealedCID(t *testing.T) {
|
||||
ups := int(abi.PaddedPieceSize(2048).Unpadded())
|
||||
|
||||
commP := func(b []byte) cid.Cid {
|
||||
pf, werr, err := ToReadableFile(bytes.NewReader(b), int64(len(b)))
|
||||
pf, werr, err := commpffi.ToReadableFile(bytes.NewReader(b), int64(len(b)))
|
||||
require.NoError(t, err)
|
||||
|
||||
c, err := ffi.GeneratePieceCIDFromFile(pt, pf, abi.UnpaddedPieceSize(len(b)))
|
||||
|
||||
@ -7,11 +7,12 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/fr32"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
|
||||
commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -29,7 +30,7 @@ func TestWriteTwoPcs(t *testing.T) {
|
||||
buf := bytes.Repeat([]byte{0xab * byte(i)}, int(paddedSize.Unpadded()))
|
||||
rawBytes = append(rawBytes, buf...)
|
||||
|
||||
rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf)))
|
||||
rf, w, _ := commpffi.ToReadableFile(bytes.NewReader(buf), int64(len(buf)))
|
||||
|
||||
_, _, _, err := ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg32GiBV1, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil)
|
||||
if err != nil {
|
||||
|
||||
4
extern/sector-storage/fr32/fr32_test.go
vendored
4
extern/sector-storage/fr32/fr32_test.go
vendored
@ -9,15 +9,15 @@ import (
|
||||
"testing"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/fr32"
|
||||
)
|
||||
|
||||
func padFFI(buf []byte) []byte {
|
||||
rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf)))
|
||||
rf, w, _ := commpffi.ToReadableFile(bytes.NewReader(buf), int64(len(buf)))
|
||||
tf, _ := ioutil.TempFile("/tmp/", "scrb-")
|
||||
|
||||
_, _, _, err := ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg32GiBV1, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil)
|
||||
|
||||
3
extern/sector-storage/mock/mock.go
vendored
3
extern/sector-storage/mock/mock.go
vendored
@ -11,6 +11,7 @@ import (
|
||||
|
||||
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
|
||||
|
||||
ffiwrapper2 "github.com/filecoin-project/go-commp-utils/ffiwrapper"
|
||||
commcid "github.com/filecoin-project/go-fil-commcid"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
@ -76,7 +77,7 @@ func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorID storage.SectorRef,
|
||||
var b bytes.Buffer
|
||||
tr := io.TeeReader(r, &b)
|
||||
|
||||
c, err := ffiwrapper.GeneratePieceCIDFromFile(sectorID.ProofType, tr, size)
|
||||
c, err := ffiwrapper2.GeneratePieceCIDFromFile(sectorID.ProofType, tr, size)
|
||||
if err != nil {
|
||||
return abi.PieceInfo{}, xerrors.Errorf("failed to generate piece cid: %w", err)
|
||||
}
|
||||
|
||||
56
extern/sector-storage/zerocomm/zerocomm.go
vendored
56
extern/sector-storage/zerocomm/zerocomm.go
vendored
@ -1,56 +0,0 @@
|
||||
package zerocomm
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
|
||||
commcid "github.com/filecoin-project/go-fil-commcid"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
const Levels = 37
|
||||
const Skip = 2 // can't generate for 32, 64b
|
||||
|
||||
var PieceComms = [Levels - Skip][32]byte{
|
||||
{0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, 0x83, 0x33},
|
||||
{0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, 0x75, 0xf},
|
||||
{0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, 0x85, 0x26},
|
||||
{0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, 0x37, 0x2f},
|
||||
{0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, 0xf8, 0x33},
|
||||
{0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, 0xf6, 0x24},
|
||||
{0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, 0xa3, 0x2f},
|
||||
{0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, 0x51, 0x8},
|
||||
{0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, 0x40, 0x11},
|
||||
{0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, 0x4f, 0x2},
|
||||
{0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, 0xd, 0x3f},
|
||||
{0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, 0x2a, 0x7},
|
||||
{0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, 0x2d, 0x19},
|
||||
{0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, 0x16, 0x2b},
|
||||
{0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, 0x13, 0x26},
|
||||
{0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, 0xe8, 0x11},
|
||||
{0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, 0x1c},
|
||||
{0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, 0xe1, 0x3a},
|
||||
{0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, 0x75, 0x8},
|
||||
{0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, 0xe7, 0x6},
|
||||
{0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, 0x1a, 0x2f},
|
||||
{0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, 0x1c, 0x37},
|
||||
{0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, 0xda, 0x6},
|
||||
{0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, 0x5b, 0x3d},
|
||||
{0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, 0xc6, 0x2c},
|
||||
{0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, 0x51, 0x7},
|
||||
{0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, 0x9b, 0x1},
|
||||
{0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, 0x85, 0x3c},
|
||||
{0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, 0x3e},
|
||||
{0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, 0xce, 0x2},
|
||||
{0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, 0x5, 0x1d},
|
||||
{0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, 0x92, 0x39},
|
||||
{0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, 0x81, 0x13},
|
||||
{0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, 0xe8, 0x19},
|
||||
{0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, 0xff, 0x32},
|
||||
}
|
||||
|
||||
func ZeroPieceCommitment(sz abi.UnpaddedPieceSize) cid.Cid {
|
||||
level := bits.TrailingZeros64(uint64(sz.Padded())) - Skip - 5 // 2^5 = 32
|
||||
commP, _ := commcid.PieceCommitmentV1ToCID(PieceComms[level][:])
|
||||
return commP
|
||||
}
|
||||
115
extern/sector-storage/zerocomm/zerocomm_test.go
vendored
115
extern/sector-storage/zerocomm/zerocomm_test.go
vendored
@ -1,115 +0,0 @@
|
||||
package zerocomm_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
commcid "github.com/filecoin-project/go-fil-commcid"
|
||||
abi "github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
)
|
||||
|
||||
func TestComms(t *testing.T) {
|
||||
t.Skip("don't have enough ram") // no, but seriously, currently this needs like 3tb of /tmp
|
||||
|
||||
var expPieceComms [zerocomm.Levels - zerocomm.Skip]cid.Cid
|
||||
|
||||
{
|
||||
l2, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 127)), 127)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expPieceComms[0] = l2
|
||||
}
|
||||
|
||||
for i := 1; i < zerocomm.Levels-2; i++ {
|
||||
var err error
|
||||
sz := abi.UnpaddedPieceSize(127 << uint(i))
|
||||
fmt.Println(i, sz)
|
||||
r := io.LimitReader(&NullReader{}, int64(sz))
|
||||
|
||||
expPieceComms[i], err = ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, r, sz)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, comm := range expPieceComms {
|
||||
c, err := commcid.CIDToPieceCommitmentV1(comm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(c) != string(zerocomm.PieceComms[i][:]) {
|
||||
t.Errorf("zero commitment %d didn't match", i)
|
||||
}
|
||||
}
|
||||
|
||||
for _, comm := range expPieceComms { // Could do codegen, but this is good enough
|
||||
fmt.Printf("%#v,\n", comm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommsSmall(t *testing.T) {
|
||||
var expPieceComms [8]cid.Cid
|
||||
lvls := len(expPieceComms) + zerocomm.Skip
|
||||
|
||||
{
|
||||
l2, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 127)), 127)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expPieceComms[0] = l2
|
||||
}
|
||||
|
||||
for i := 1; i < lvls-2; i++ {
|
||||
var err error
|
||||
sz := abi.UnpaddedPieceSize(127 << uint(i))
|
||||
fmt.Println(i, sz)
|
||||
r := io.LimitReader(&NullReader{}, int64(sz))
|
||||
|
||||
expPieceComms[i], err = ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, r, sz)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, comm := range expPieceComms {
|
||||
c, err := commcid.CIDToPieceCommitmentV1(comm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(c) != string(zerocomm.PieceComms[i][:]) {
|
||||
t.Errorf("zero commitment %d didn't match", i)
|
||||
}
|
||||
}
|
||||
|
||||
for _, comm := range expPieceComms { // Could do codegen, but this is good enough
|
||||
fmt.Printf("%#v,\n", comm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForSise(t *testing.T) {
|
||||
exp, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 1016)), 1016)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
actual := zerocomm.ZeroPieceCommitment(1016)
|
||||
if !exp.Equals(actual) {
|
||||
t.Errorf("zero commitment didn't match")
|
||||
}
|
||||
}
|
||||
|
||||
type NullReader struct{}
|
||||
|
||||
func (NullReader) Read(out []byte) (int, error) {
|
||||
for i := range out {
|
||||
out[i] = 0
|
||||
}
|
||||
return len(out), nil
|
||||
}
|
||||
2
extern/storage-sealing/checks.go
vendored
2
extern/storage-sealing/checks.go
vendored
@ -12,9 +12,9 @@ import (
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-commp-utils/zerocomm"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
)
|
||||
|
||||
// TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting
|
||||
|
||||
2
extern/storage-sealing/states_failed.go
vendored
2
extern/storage-sealing/states_failed.go
vendored
@ -13,7 +13,7 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
"github.com/filecoin-project/go-statemachine"
|
||||
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
"github.com/filecoin-project/go-commp-utils/zerocomm"
|
||||
)
|
||||
|
||||
const minRetryTime = 1 * time.Minute
|
||||
|
||||
5
go.mod
5
go.mod
@ -27,10 +27,11 @@ require (
|
||||
github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect
|
||||
github.com/filecoin-project/go-bitfield v0.2.3-0.20201110211213-fe2c1862e816
|
||||
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
|
||||
github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
|
||||
github.com/filecoin-project/go-data-transfer v1.2.0
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
|
||||
github.com/filecoin-project/go-fil-markets v1.0.5
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
|
||||
github.com/filecoin-project/go-fil-markets v1.0.6
|
||||
github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49
|
||||
github.com/filecoin-project/go-multistore v0.0.3
|
||||
github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
|
||||
|
||||
12
go.sum
12
go.sum
@ -245,6 +245,8 @@ github.com/filecoin-project/go-bitfield v0.2.3-0.20201110211213-fe2c1862e816 h1:
|
||||
github.com/filecoin-project/go-bitfield v0.2.3-0.20201110211213-fe2c1862e816/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
|
||||
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
|
||||
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
|
||||
github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 h1:0kHszkYP3hgApcjl5x4rpwONhN9+j7XDobf6at5XfHs=
|
||||
github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U=
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
|
||||
github.com/filecoin-project/go-data-transfer v1.0.1 h1:5sYKDbstyDsdJpVP4UGUW6+BgCNfgnH8hQgf0E3ZAno=
|
||||
@ -255,8 +257,12 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl
|
||||
github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
|
||||
github.com/filecoin-project/go-fil-markets v1.0.5 h1:g1EEdSJfi7nbVS2MhhUTRwNU+FHMOgvytTmKQqu6p0M=
|
||||
github.com/filecoin-project/go-fil-markets v1.0.5/go.mod h1:lt96/KQb5r+YxV31kVGHUVIbQa7zLbxSP6ZBX+ruzDA=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
|
||||
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335 h1:DF8eu0WdEBnSVdu71+jfT4YMk6fO7AIJk2ZiWd3l15c=
|
||||
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
|
||||
github.com/filecoin-project/go-fil-markets v1.0.6 h1:JTZBMKJ19YpK/vQfE0rHvjELy83NeGor5d4dBnoIlK0=
|
||||
github.com/filecoin-project/go-fil-markets v1.0.6/go.mod h1:iVYc+VrHIP15F5COkHNM6ndTwKSJ7qPrHSKCfFUnve4=
|
||||
github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
|
||||
github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
|
||||
github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
|
||||
@ -669,6 +675,8 @@ github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo=
|
||||
github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg=
|
||||
github.com/ipfs/iptb-plugins v0.2.1 h1:au4HWn9/pRPbkxA08pDx2oRAs4cnbgQWgV0teYXuuGA=
|
||||
github.com/ipfs/iptb-plugins v0.2.1/go.mod h1:QXMbtIWZ+jRsW8a4h13qAKU7jcM7qaittO8wOsTP0Rs=
|
||||
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4 h1:6phjU3kXvCEWOZpu+Ob0w6DzgPFZmDLgLPxJhD8RxEY=
|
||||
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw=
|
||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
|
||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
|
||||
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
|
||||
|
||||
@ -1,113 +0,0 @@
|
||||
package commp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/bits"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
"github.com/filecoin-project/go-padreader"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
)
|
||||
|
||||
const commPBufPad = abi.PaddedPieceSize(8 << 20)
|
||||
const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128)) // can't use .Unpadded() for const
|
||||
|
||||
type Writer struct {
|
||||
len int64
|
||||
buf [CommPBuf]byte
|
||||
leaves []cid.Cid
|
||||
}
|
||||
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
for len(p) > 0 {
|
||||
buffered := int(w.len % int64(len(w.buf)))
|
||||
toBuffer := len(w.buf) - buffered
|
||||
if toBuffer > len(p) {
|
||||
toBuffer = len(p)
|
||||
}
|
||||
|
||||
copied := copy(w.buf[buffered:], p[:toBuffer])
|
||||
p = p[copied:]
|
||||
w.len += int64(copied)
|
||||
|
||||
if copied > 0 && w.len%int64(len(w.buf)) == 0 {
|
||||
leaf, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, bytes.NewReader(w.buf[:]), CommPBuf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
w.leaves = append(w.leaves, leaf)
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (w *Writer) Sum() (api.DataCIDSize, error) {
|
||||
// process last non-zero leaf if exists
|
||||
lastLen := w.len % int64(len(w.buf))
|
||||
rawLen := w.len
|
||||
|
||||
// process remaining bit of data
|
||||
if lastLen != 0 {
|
||||
if len(w.leaves) != 0 {
|
||||
copy(w.buf[lastLen:], make([]byte, int(int64(CommPBuf)-lastLen)))
|
||||
lastLen = int64(CommPBuf)
|
||||
}
|
||||
|
||||
r, sz := padreader.New(bytes.NewReader(w.buf[:lastLen]), uint64(lastLen))
|
||||
p, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, r, sz)
|
||||
if err != nil {
|
||||
return api.DataCIDSize{}, err
|
||||
}
|
||||
|
||||
if sz < CommPBuf { // special case for pieces smaller than 16MiB
|
||||
return api.DataCIDSize{
|
||||
PayloadSize: w.len,
|
||||
PieceSize: sz.Padded(),
|
||||
PieceCID: p,
|
||||
}, nil
|
||||
}
|
||||
|
||||
w.leaves = append(w.leaves, p)
|
||||
}
|
||||
|
||||
// pad with zero pieces to power-of-two size
|
||||
fillerLeaves := (1 << (bits.Len(uint(len(w.leaves) - 1)))) - len(w.leaves)
|
||||
for i := 0; i < fillerLeaves; i++ {
|
||||
w.leaves = append(w.leaves, zerocomm.ZeroPieceCommitment(CommPBuf))
|
||||
}
|
||||
|
||||
if len(w.leaves) == 1 {
|
||||
return api.DataCIDSize{
|
||||
PayloadSize: rawLen,
|
||||
PieceSize: abi.PaddedPieceSize(len(w.leaves)) * commPBufPad,
|
||||
PieceCID: w.leaves[0],
|
||||
}, nil
|
||||
}
|
||||
|
||||
pieces := make([]abi.PieceInfo, len(w.leaves))
|
||||
for i, leaf := range w.leaves {
|
||||
pieces[i] = abi.PieceInfo{
|
||||
Size: commPBufPad,
|
||||
PieceCID: leaf,
|
||||
}
|
||||
}
|
||||
|
||||
p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, pieces)
|
||||
if err != nil {
|
||||
return api.DataCIDSize{}, xerrors.Errorf("generating unsealed CID: %w", err)
|
||||
}
|
||||
|
||||
return api.DataCIDSize{
|
||||
PayloadSize: rawLen,
|
||||
PieceSize: abi.PaddedPieceSize(len(w.leaves)) * commPBufPad,
|
||||
PieceCID: p,
|
||||
}, nil
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
package commp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/go-padreader"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
)
|
||||
|
||||
func TestWriterZero(t *testing.T) {
|
||||
for i, s := range []struct {
|
||||
writes []int
|
||||
expect abi.PaddedPieceSize
|
||||
}{
|
||||
{writes: []int{200}, expect: 256},
|
||||
{writes: []int{200, 200}, expect: 512},
|
||||
|
||||
{writes: []int{int(CommPBuf)}, expect: commPBufPad},
|
||||
{writes: []int{int(CommPBuf) * 2}, expect: 2 * commPBufPad},
|
||||
{writes: []int{int(CommPBuf), int(CommPBuf), int(CommPBuf)}, expect: 4 * commPBufPad},
|
||||
{writes: []int{int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf)}, expect: 16 * commPBufPad},
|
||||
|
||||
{writes: []int{200, int(CommPBuf)}, expect: 2 * commPBufPad},
|
||||
} {
|
||||
s := s
|
||||
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
||||
w := &Writer{}
|
||||
var rawSum int64
|
||||
for _, write := range s.writes {
|
||||
rawSum += int64(write)
|
||||
_, err := w.Write(make([]byte, write))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
p, err := w.Sum()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, rawSum, p.PayloadSize)
|
||||
require.Equal(t, s.expect, p.PieceSize)
|
||||
require.Equal(t, zerocomm.ZeroPieceCommitment(s.expect.Unpadded()).String(), p.PieceCID.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterData(t *testing.T) {
|
||||
dataLen := float64(CommPBuf) * 6.78
|
||||
data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(dataLen)))
|
||||
|
||||
pr, sz := padreader.New(bytes.NewReader(data), uint64(dataLen))
|
||||
exp, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, pr, sz)
|
||||
require.NoError(t, err)
|
||||
|
||||
w := &Writer{}
|
||||
_, err = io.Copy(w, bytes.NewReader(data))
|
||||
require.NoError(t, err)
|
||||
|
||||
res, err := w.Sum()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, exp.String(), res.PieceCID.String())
|
||||
}
|
||||
|
||||
func BenchmarkWriterZero(b *testing.B) {
|
||||
buf := make([]byte, int(CommPBuf)*b.N)
|
||||
b.SetBytes(int64(CommPBuf))
|
||||
b.ResetTimer()
|
||||
|
||||
w := &Writer{}
|
||||
|
||||
_, err := w.Write(buf)
|
||||
require.NoError(b, err)
|
||||
o, err := w.Sum()
|
||||
|
||||
b.StopTimer()
|
||||
|
||||
require.NoError(b, err)
|
||||
require.Equal(b, zerocomm.ZeroPieceCommitment(o.PieceSize.Unpadded()).String(), o.PieceCID.String())
|
||||
require.Equal(b, int64(CommPBuf)*int64(b.N), o.PayloadSize)
|
||||
}
|
||||
@ -33,9 +33,10 @@ import (
|
||||
"go.uber.org/fx"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-commp-utils/ffiwrapper"
|
||||
"github.com/filecoin-project/go-commp-utils/writer"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-fil-markets/discovery"
|
||||
"github.com/filecoin-project/go-fil-markets/pieceio"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
rm "github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/shared"
|
||||
@ -49,7 +50,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/commp"
|
||||
"github.com/filecoin-project/lotus/markets/utils"
|
||||
"github.com/filecoin-project/lotus/node/impl/full"
|
||||
"github.com/filecoin-project/lotus/node/impl/paych"
|
||||
@ -642,7 +642,7 @@ func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Addre
|
||||
func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) {
|
||||
|
||||
// Hard-code the sector type to 32GiBV1_1, because:
|
||||
// - pieceio.GeneratePieceCommitment requires a RegisteredSealProof
|
||||
// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
|
||||
// - commP itself is sector-size independent, with rather low probability of that changing
|
||||
// ( note how the final rust call is identical for every RegSP type )
|
||||
// https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050
|
||||
@ -662,7 +662,8 @@ func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet
|
||||
return nil, err
|
||||
}
|
||||
|
||||
commP, pieceSize, err := pieceio.GeneratePieceCommitment(arbitraryProofType, rdr, uint64(stat.Size()))
|
||||
pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
|
||||
commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
|
||||
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("computing commP failed: %w", err)
|
||||
@ -702,8 +703,8 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e
|
||||
func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) {
|
||||
dag := merkledag.NewDAGService(blockservice.New(a.CombinedBstore, offline.Exchange(a.CombinedBstore)))
|
||||
|
||||
w := &commp.Writer{}
|
||||
bw := bufio.NewWriterSize(w, int(commp.CommPBuf))
|
||||
w := &writer.Writer{}
|
||||
bw := bufio.NewWriterSize(w, int(writer.CommPBuf))
|
||||
|
||||
err := car.WriteCar(ctx, dag, []cid.Cid{root}, w)
|
||||
if err != nil {
|
||||
@ -714,7 +715,8 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
|
||||
return api.DataCIDSize{}, err
|
||||
}
|
||||
|
||||
return w.Sum()
|
||||
dataCIDSize, err := w.Sum()
|
||||
return api.DataCIDSize(dataCIDSize), err
|
||||
}
|
||||
|
||||
func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error {
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-commp-utils/zerocomm"
|
||||
commcid "github.com/filecoin-project/go-fil-commcid"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
@ -13,7 +14,6 @@ import (
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/wallet"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
||||
"github.com/filecoin-project/lotus/genesis"
|
||||
)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user