diff --git a/fr32/fr32.go b/fr32/fr32.go index cb2dcaed6..06579dd0d 100644 --- a/fr32/fr32.go +++ b/fr32/fr32.go @@ -10,7 +10,7 @@ import ( var mtTresh = 32 << 20 -func mt(in, out []byte, padLen int, op func(in, out []byte)) { +func mt(in, out []byte, padLen int, op func(unpadded, padded []byte)) { threads := padLen / mtTresh if threads > runtime.NumCPU() { threads = 1 << (32 - bits.LeadingZeros32(uint32(runtime.NumCPU()))) @@ -35,6 +35,15 @@ func mt(in, out []byte, padLen int, op func(in, out []byte)) { // Assumes len(in)%127==0 and len(out)%128==0 func Pad(in, out []byte) { + if len(out) > mtTresh { + mt(in, out, len(out), pad) + return + } + + pad(in, out) +} + +func pad(in, out []byte) { if len(out) > mtTresh { mt(in, out, len(out), Pad) return @@ -79,13 +88,19 @@ func Pad(in, out []byte) { } } + + // Assumes len(in)%128==0 and len(out)%127==0 func Unpad(in []byte, out []byte) { - if len(out) > mtTresh { - mt(in, out, len(in), Unpad) + if len(in) > mtTresh { + mt(out, in, len(in), unpad) return } + unpad(out, in) +} + +func unpad(out, in []byte) { chunks := len(in) / 128 for chunk := 0; chunk < chunks; chunk++ { inOffNext := chunk*128 + 1 diff --git a/fr32/fr32_ffi_cmp_test.go b/fr32/fr32_ffi_cmp_test.go new file mode 100644 index 000000000..24bb24d49 --- /dev/null +++ b/fr32/fr32_ffi_cmp_test.go @@ -0,0 +1,66 @@ +package fr32 + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/ffiwrapper" +) + +func TestWriteTwoPcs(t *testing.T) { + tf, _ := ioutil.TempFile("/tmp/", "scrb-") + + paddedSize := abi.PaddedPieceSize(16 << 20) + n := 2 + + var rawBytes []byte + + for i := 0; i < n; i++ { + buf := bytes.Repeat([]byte{0xab * byte(i)}, int(paddedSize.Unpadded())) + rawBytes = append(rawBytes, buf...) + + rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) + + _, _, _, err := ffi.WriteWithAlignment(abi.RegisteredProof_StackedDRG32GiBSeal, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil) + if err != nil { + panic(err) + } + if err := w(); err != nil { + panic(err) + } + } + + if _, err := tf.Seek(io.SeekStart, 0); err != nil { + panic(err) + } + + ffiBytes, err := ioutil.ReadAll(tf) + if err != nil { + panic(err) + } + + if err := tf.Close(); err != nil { + panic(err) + } + + if err := os.Remove(tf.Name()); err != nil { + panic(err) + } + + outBytes := make([]byte, int(paddedSize) * n) + Pad(rawBytes, outBytes) + require.Equal(t, ffiBytes, outBytes) + + unpadBytes := make([]byte, int(paddedSize.Unpadded()) * n) + Unpad(ffiBytes, unpadBytes) + require.Equal(t, rawBytes, unpadBytes) +} +