diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index b52490928..48183690d 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -19,9 +19,9 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" "github.com/filecoin-project/specs-storage/storage" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" diff --git a/extern/sector-storage/ffiwrapper/files.go b/extern/sector-storage/ffiwrapper/files.go deleted file mode 100644 index a13776d2d..000000000 --- a/extern/sector-storage/ffiwrapper/files.go +++ /dev/null @@ -1,53 +0,0 @@ -package ffiwrapper - -import ( - "io" - "os" - "sync" - - "golang.org/x/xerrors" -) - -func ToReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { - f, ok := r.(*os.File) - if ok { - return f, func() error { return nil }, nil - } - - var w *os.File - - f, w, err := os.Pipe() - if err != nil { - return nil, nil, err - } - - var wait sync.Mutex - var werr error - - wait.Lock() - go func() { - defer wait.Unlock() - - var copied int64 - copied, werr = io.CopyN(w, r, n) - if werr != nil { - log.Warnf("toReadableFile: copy error: %+v", werr) - } - - err := w.Close() - if werr == nil && err != nil { - werr = err - log.Warnf("toReadableFile: close error: %+v", err) - return - } - if copied != n { - log.Warnf("copied different amount than expected: %d != %d", copied, n) - werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) - } - }() - - return f, func() error { - wait.Lock() - return werr - }, nil -} diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 0887bc329..1c8c7ee84 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -20,9 +20,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" + commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" + "github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/lotus/extern/sector-storage/fr32" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" ) var _ Storage = &Sealer{} @@ -175,7 +176,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi } func (sb *Sealer) pieceCid(spt abi.RegisteredSealProof, in []byte) (cid.Cid, error) { - prf, werr, err := ToReadableFile(bytes.NewReader(in), int64(len(in))) + prf, werr, err := commpffi.ToReadableFile(bytes.NewReader(in), int64(len(in))) if err != nil { return cid.Undef, xerrors.Errorf("getting tee reader pipe: %w", err) } @@ -610,20 +611,6 @@ func (sb *Sealer) Remove(ctx context.Context, sector storage.SectorRef) error { return xerrors.Errorf("not supported at this layer") // happens in localworker } -func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader, pieceSize abi.UnpaddedPieceSize) (cid.Cid, error) { - f, werr, err := ToReadableFile(piece, int64(pieceSize)) - if err != nil { - return cid.Undef, err - } - - pieceCID, err := ffi.GeneratePieceCIDFromFile(proofType, f, pieceSize) - if err != nil { - return cid.Undef, err - } - - return pieceCID, werr() -} - func GetRequiredPadding(oldLength abi.PaddedPieceSize, newPieceLength abi.PaddedPieceSize) ([]abi.PaddedPieceSize, abi.PaddedPieceSize) { padPieces := make([]abi.PaddedPieceSize, 0) diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index 686ef847b..1292a9513 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/ipfs/go-cid" @@ -465,7 +467,7 @@ func BenchmarkWriteWithAlignment(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - rf, w, _ := ToReadableFile(bytes.NewReader(bytes.Repeat([]byte{0xff, 0}, int(bt/2))), int64(bt)) + rf, w, _ := commpffi.ToReadableFile(bytes.NewReader(bytes.Repeat([]byte{0xff, 0}, int(bt/2))), int64(bt)) tf, _ := ioutil.TempFile("/tmp/", "scrb-") b.StartTimer() @@ -524,7 +526,7 @@ func TestGenerateUnsealedCID(t *testing.T) { ups := int(abi.PaddedPieceSize(2048).Unpadded()) commP := func(b []byte) cid.Cid { - pf, werr, err := ToReadableFile(bytes.NewReader(b), int64(len(b))) + pf, werr, err := commpffi.ToReadableFile(bytes.NewReader(b), int64(len(b))) require.NoError(t, err) c, err := ffi.GeneratePieceCIDFromFile(pt, pf, abi.UnpaddedPieceSize(len(b))) diff --git a/extern/sector-storage/fr32/fr32_ffi_cmp_test.go b/extern/sector-storage/fr32/fr32_ffi_cmp_test.go index 3d5679095..49eb11548 100644 --- a/extern/sector-storage/fr32/fr32_ffi_cmp_test.go +++ b/extern/sector-storage/fr32/fr32_ffi_cmp_test.go @@ -7,11 +7,12 @@ import ( "os" "testing" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/fr32" ffi "github.com/filecoin-project/filecoin-ffi" + commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" + "github.com/filecoin-project/go-state-types/abi" "github.com/stretchr/testify/require" @@ -29,7 +30,7 @@ func TestWriteTwoPcs(t *testing.T) { buf := bytes.Repeat([]byte{0xab * byte(i)}, int(paddedSize.Unpadded())) rawBytes = append(rawBytes, buf...) - rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) + rf, w, _ := commpffi.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) _, _, _, err := ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg32GiBV1, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil) if err != nil { diff --git a/extern/sector-storage/fr32/fr32_test.go b/extern/sector-storage/fr32/fr32_test.go index 415134272..0626f72a9 100644 --- a/extern/sector-storage/fr32/fr32_test.go +++ b/extern/sector-storage/fr32/fr32_test.go @@ -9,15 +9,15 @@ import ( "testing" ffi "github.com/filecoin-project/filecoin-ffi" + commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-state-types/abi" "github.com/stretchr/testify/require" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/fr32" ) func padFFI(buf []byte) []byte { - rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) + rf, w, _ := commpffi.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) tf, _ := ioutil.TempFile("/tmp/", "scrb-") _, _, _, err := ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg32GiBV1, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 747fcdf8b..6179f40d7 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -11,6 +11,7 @@ import ( proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" + ffiwrapper2 "github.com/filecoin-project/go-commp-utils/ffiwrapper" commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" @@ -76,7 +77,7 @@ func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorID storage.SectorRef, var b bytes.Buffer tr := io.TeeReader(r, &b) - c, err := ffiwrapper.GeneratePieceCIDFromFile(sectorID.ProofType, tr, size) + c, err := ffiwrapper2.GeneratePieceCIDFromFile(sectorID.ProofType, tr, size) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("failed to generate piece cid: %w", err) } diff --git a/extern/sector-storage/zerocomm/zerocomm.go b/extern/sector-storage/zerocomm/zerocomm.go deleted file mode 100644 index 9855a5821..000000000 --- a/extern/sector-storage/zerocomm/zerocomm.go +++ /dev/null @@ -1,56 +0,0 @@ -package zerocomm - -import ( - "math/bits" - - commcid "github.com/filecoin-project/go-fil-commcid" - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" -) - -const Levels = 37 -const Skip = 2 // can't generate for 32, 64b - -var PieceComms = [Levels - Skip][32]byte{ - {0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, 0x83, 0x33}, - {0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, 0x75, 0xf}, - {0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, 0x85, 0x26}, - {0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, 0x37, 0x2f}, - {0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, 0xf8, 0x33}, - {0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, 0xf6, 0x24}, - {0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, 0xa3, 0x2f}, - {0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, 0x51, 0x8}, - {0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, 0x40, 0x11}, - {0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, 0x4f, 0x2}, - {0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, 0xd, 0x3f}, - {0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, 0x2a, 0x7}, - {0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, 0x2d, 0x19}, - {0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, 0x16, 0x2b}, - {0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, 0x13, 0x26}, - {0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, 0xe8, 0x11}, - {0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, 0x1c}, - {0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, 0xe1, 0x3a}, - {0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, 0x75, 0x8}, - {0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, 0xe7, 0x6}, - {0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, 0x1a, 0x2f}, - {0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, 0x1c, 0x37}, - {0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, 0xda, 0x6}, - {0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, 0x5b, 0x3d}, - {0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, 0xc6, 0x2c}, - {0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, 0x51, 0x7}, - {0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, 0x9b, 0x1}, - {0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, 0x85, 0x3c}, - {0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, 0x3e}, - {0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, 0xce, 0x2}, - {0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, 0x5, 0x1d}, - {0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, 0x92, 0x39}, - {0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, 0x81, 0x13}, - {0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, 0xe8, 0x19}, - {0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, 0xff, 0x32}, -} - -func ZeroPieceCommitment(sz abi.UnpaddedPieceSize) cid.Cid { - level := bits.TrailingZeros64(uint64(sz.Padded())) - Skip - 5 // 2^5 = 32 - commP, _ := commcid.PieceCommitmentV1ToCID(PieceComms[level][:]) - return commP -} diff --git a/extern/sector-storage/zerocomm/zerocomm_test.go b/extern/sector-storage/zerocomm/zerocomm_test.go deleted file mode 100644 index 393f61d64..000000000 --- a/extern/sector-storage/zerocomm/zerocomm_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package zerocomm_test - -import ( - "bytes" - "fmt" - "io" - "testing" - - commcid "github.com/filecoin-project/go-fil-commcid" - abi "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" -) - -func TestComms(t *testing.T) { - t.Skip("don't have enough ram") // no, but seriously, currently this needs like 3tb of /tmp - - var expPieceComms [zerocomm.Levels - zerocomm.Skip]cid.Cid - - { - l2, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 127)), 127) - if err != nil { - t.Fatal(err) - } - expPieceComms[0] = l2 - } - - for i := 1; i < zerocomm.Levels-2; i++ { - var err error - sz := abi.UnpaddedPieceSize(127 << uint(i)) - fmt.Println(i, sz) - r := io.LimitReader(&NullReader{}, int64(sz)) - - expPieceComms[i], err = ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, r, sz) - if err != nil { - t.Fatal(err) - } - } - - for i, comm := range expPieceComms { - c, err := commcid.CIDToPieceCommitmentV1(comm) - if err != nil { - t.Fatal(err) - } - if string(c) != string(zerocomm.PieceComms[i][:]) { - t.Errorf("zero commitment %d didn't match", i) - } - } - - for _, comm := range expPieceComms { // Could do codegen, but this is good enough - fmt.Printf("%#v,\n", comm) - } -} - -func TestCommsSmall(t *testing.T) { - var expPieceComms [8]cid.Cid - lvls := len(expPieceComms) + zerocomm.Skip - - { - l2, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 127)), 127) - if err != nil { - t.Fatal(err) - } - expPieceComms[0] = l2 - } - - for i := 1; i < lvls-2; i++ { - var err error - sz := abi.UnpaddedPieceSize(127 << uint(i)) - fmt.Println(i, sz) - r := io.LimitReader(&NullReader{}, int64(sz)) - - expPieceComms[i], err = ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, r, sz) - if err != nil { - t.Fatal(err) - } - } - - for i, comm := range expPieceComms { - c, err := commcid.CIDToPieceCommitmentV1(comm) - if err != nil { - t.Fatal(err) - } - if string(c) != string(zerocomm.PieceComms[i][:]) { - t.Errorf("zero commitment %d didn't match", i) - } - } - - for _, comm := range expPieceComms { // Could do codegen, but this is good enough - fmt.Printf("%#v,\n", comm) - } -} - -func TestForSise(t *testing.T) { - exp, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 1016)), 1016) - if err != nil { - return - } - - actual := zerocomm.ZeroPieceCommitment(1016) - if !exp.Equals(actual) { - t.Errorf("zero commitment didn't match") - } -} - -type NullReader struct{} - -func (NullReader) Read(out []byte) (int, error) { - for i := range out { - out[i] = 0 - } - return len(out), nil -} diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 56a55bb61..249163d66 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -12,9 +12,9 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" ) // TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index 760afc0ba..be3f3d35a 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-statemachine" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" + "github.com/filecoin-project/go-commp-utils/zerocomm" ) const minRetryTime = 1 * time.Minute diff --git a/go.mod b/go.mod index 36041cffd..e907d8651 100644 --- a/go.mod +++ b/go.mod @@ -27,10 +27,11 @@ require ( github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect github.com/filecoin-project/go-bitfield v0.2.3-0.20201110211213-fe2c1862e816 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 + github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.2.0 - github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v1.0.5 + github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a + github.com/filecoin-project/go-fil-markets v1.0.6 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 63ab135f3..f2c7ddd4a 100644 --- a/go.sum +++ b/go.sum @@ -245,6 +245,8 @@ github.com/filecoin-project/go-bitfield v0.2.3-0.20201110211213-fe2c1862e816 h1: github.com/filecoin-project/go-bitfield v0.2.3-0.20201110211213-fe2c1862e816/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= +github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 h1:0kHszkYP3hgApcjl5x4rpwONhN9+j7XDobf6at5XfHs= +github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1 h1:5sYKDbstyDsdJpVP4UGUW6+BgCNfgnH8hQgf0E3ZAno= @@ -255,8 +257,12 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v1.0.5 h1:g1EEdSJfi7nbVS2MhhUTRwNU+FHMOgvytTmKQqu6p0M= -github.com/filecoin-project/go-fil-markets v1.0.5/go.mod h1:lt96/KQb5r+YxV31kVGHUVIbQa7zLbxSP6ZBX+ruzDA= +github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= +github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335 h1:DF8eu0WdEBnSVdu71+jfT4YMk6fO7AIJk2ZiWd3l15c= +github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= +github.com/filecoin-project/go-fil-markets v1.0.6 h1:JTZBMKJ19YpK/vQfE0rHvjELy83NeGor5d4dBnoIlK0= +github.com/filecoin-project/go-fil-markets v1.0.6/go.mod h1:iVYc+VrHIP15F5COkHNM6ndTwKSJ7qPrHSKCfFUnve4= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -669,6 +675,8 @@ github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo= github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg= github.com/ipfs/iptb-plugins v0.2.1 h1:au4HWn9/pRPbkxA08pDx2oRAs4cnbgQWgV0teYXuuGA= github.com/ipfs/iptb-plugins v0.2.1/go.mod h1:QXMbtIWZ+jRsW8a4h13qAKU7jcM7qaittO8wOsTP0Rs= +github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4 h1:6phjU3kXvCEWOZpu+Ob0w6DzgPFZmDLgLPxJhD8RxEY= +github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw= github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U= github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ= github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= diff --git a/lib/commp/writer.go b/lib/commp/writer.go deleted file mode 100644 index 4c5e3350c..000000000 --- a/lib/commp/writer.go +++ /dev/null @@ -1,113 +0,0 @@ -package commp - -import ( - "bytes" - "math/bits" - - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - - ffi "github.com/filecoin-project/filecoin-ffi" - "github.com/filecoin-project/go-padreader" - "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" -) - -const commPBufPad = abi.PaddedPieceSize(8 << 20) -const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128)) // can't use .Unpadded() for const - -type Writer struct { - len int64 - buf [CommPBuf]byte - leaves []cid.Cid -} - -func (w *Writer) Write(p []byte) (int, error) { - n := len(p) - for len(p) > 0 { - buffered := int(w.len % int64(len(w.buf))) - toBuffer := len(w.buf) - buffered - if toBuffer > len(p) { - toBuffer = len(p) - } - - copied := copy(w.buf[buffered:], p[:toBuffer]) - p = p[copied:] - w.len += int64(copied) - - if copied > 0 && w.len%int64(len(w.buf)) == 0 { - leaf, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, bytes.NewReader(w.buf[:]), CommPBuf) - if err != nil { - return 0, err - } - w.leaves = append(w.leaves, leaf) - } - } - return n, nil -} - -func (w *Writer) Sum() (api.DataCIDSize, error) { - // process last non-zero leaf if exists - lastLen := w.len % int64(len(w.buf)) - rawLen := w.len - - // process remaining bit of data - if lastLen != 0 { - if len(w.leaves) != 0 { - copy(w.buf[lastLen:], make([]byte, int(int64(CommPBuf)-lastLen))) - lastLen = int64(CommPBuf) - } - - r, sz := padreader.New(bytes.NewReader(w.buf[:lastLen]), uint64(lastLen)) - p, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, r, sz) - if err != nil { - return api.DataCIDSize{}, err - } - - if sz < CommPBuf { // special case for pieces smaller than 16MiB - return api.DataCIDSize{ - PayloadSize: w.len, - PieceSize: sz.Padded(), - PieceCID: p, - }, nil - } - - w.leaves = append(w.leaves, p) - } - - // pad with zero pieces to power-of-two size - fillerLeaves := (1 << (bits.Len(uint(len(w.leaves) - 1)))) - len(w.leaves) - for i := 0; i < fillerLeaves; i++ { - w.leaves = append(w.leaves, zerocomm.ZeroPieceCommitment(CommPBuf)) - } - - if len(w.leaves) == 1 { - return api.DataCIDSize{ - PayloadSize: rawLen, - PieceSize: abi.PaddedPieceSize(len(w.leaves)) * commPBufPad, - PieceCID: w.leaves[0], - }, nil - } - - pieces := make([]abi.PieceInfo, len(w.leaves)) - for i, leaf := range w.leaves { - pieces[i] = abi.PieceInfo{ - Size: commPBufPad, - PieceCID: leaf, - } - } - - p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, pieces) - if err != nil { - return api.DataCIDSize{}, xerrors.Errorf("generating unsealed CID: %w", err) - } - - return api.DataCIDSize{ - PayloadSize: rawLen, - PieceSize: abi.PaddedPieceSize(len(w.leaves)) * commPBufPad, - PieceCID: p, - }, nil -} diff --git a/lib/commp/writer_test.go b/lib/commp/writer_test.go deleted file mode 100644 index 284648e4e..000000000 --- a/lib/commp/writer_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package commp - -import ( - "bytes" - "crypto/rand" - "fmt" - "io" - "io/ioutil" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-padreader" - "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" -) - -func TestWriterZero(t *testing.T) { - for i, s := range []struct { - writes []int - expect abi.PaddedPieceSize - }{ - {writes: []int{200}, expect: 256}, - {writes: []int{200, 200}, expect: 512}, - - {writes: []int{int(CommPBuf)}, expect: commPBufPad}, - {writes: []int{int(CommPBuf) * 2}, expect: 2 * commPBufPad}, - {writes: []int{int(CommPBuf), int(CommPBuf), int(CommPBuf)}, expect: 4 * commPBufPad}, - {writes: []int{int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf), int(CommPBuf)}, expect: 16 * commPBufPad}, - - {writes: []int{200, int(CommPBuf)}, expect: 2 * commPBufPad}, - } { - s := s - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - w := &Writer{} - var rawSum int64 - for _, write := range s.writes { - rawSum += int64(write) - _, err := w.Write(make([]byte, write)) - require.NoError(t, err) - } - - p, err := w.Sum() - require.NoError(t, err) - require.Equal(t, rawSum, p.PayloadSize) - require.Equal(t, s.expect, p.PieceSize) - require.Equal(t, zerocomm.ZeroPieceCommitment(s.expect.Unpadded()).String(), p.PieceCID.String()) - }) - } -} - -func TestWriterData(t *testing.T) { - dataLen := float64(CommPBuf) * 6.78 - data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(dataLen))) - - pr, sz := padreader.New(bytes.NewReader(data), uint64(dataLen)) - exp, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, pr, sz) - require.NoError(t, err) - - w := &Writer{} - _, err = io.Copy(w, bytes.NewReader(data)) - require.NoError(t, err) - - res, err := w.Sum() - require.NoError(t, err) - - require.Equal(t, exp.String(), res.PieceCID.String()) -} - -func BenchmarkWriterZero(b *testing.B) { - buf := make([]byte, int(CommPBuf)*b.N) - b.SetBytes(int64(CommPBuf)) - b.ResetTimer() - - w := &Writer{} - - _, err := w.Write(buf) - require.NoError(b, err) - o, err := w.Sum() - - b.StopTimer() - - require.NoError(b, err) - require.Equal(b, zerocomm.ZeroPieceCommitment(o.PieceSize.Unpadded()).String(), o.PieceCID.String()) - require.Equal(b, int64(CommPBuf)*int64(b.N), o.PayloadSize) -} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index ef8e9e022..7ecdbc869 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -33,9 +33,10 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-commp-utils/ffiwrapper" + "github.com/filecoin-project/go-commp-utils/writer" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/discovery" - "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/retrievalmarket" rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" @@ -49,7 +50,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/commp" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" @@ -642,7 +642,7 @@ func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Addre func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) { // Hard-code the sector type to 32GiBV1_1, because: - // - pieceio.GeneratePieceCommitment requires a RegisteredSealProof + // - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof // - commP itself is sector-size independent, with rather low probability of that changing // ( note how the final rust call is identical for every RegSP type ) // https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050 @@ -662,7 +662,8 @@ func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet return nil, err } - commP, pieceSize, err := pieceio.GeneratePieceCommitment(arbitraryProofType, rdr, uint64(stat.Size())) + pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size())) + commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize) if err != nil { return nil, xerrors.Errorf("computing commP failed: %w", err) @@ -702,8 +703,8 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) { dag := merkledag.NewDAGService(blockservice.New(a.CombinedBstore, offline.Exchange(a.CombinedBstore))) - w := &commp.Writer{} - bw := bufio.NewWriterSize(w, int(commp.CommPBuf)) + w := &writer.Writer{} + bw := bufio.NewWriterSize(w, int(writer.CommPBuf)) err := car.WriteCar(ctx, dag, []cid.Cid{root}, w) if err != nil { @@ -714,7 +715,8 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID return api.DataCIDSize{}, err } - return w.Sum() + dataCIDSize, err := w.Sum() + return api.DataCIDSize(dataCIDSize), err } func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error { diff --git a/storage/mockstorage/preseal.go b/storage/mockstorage/preseal.go index d119c273f..66a2a5054 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/mockstorage/preseal.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-commp-utils/zerocomm" commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -13,7 +14,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" "github.com/filecoin-project/lotus/genesis" )