Merge pull request #59 from filecoin-project/feat/gencommd-align-pieces

ffiwrapper: Insert alignment between pieces

ffiwrapper/sealer_cgo.go

@@ -543,8 +543,35 @@ func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader
 }
 
 func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) {
+	allPieces := make([]abi.PieceInfo, 0, len(pieces))
 	var sum abi.PaddedPieceSize
+
+	padTo := func(s abi.PaddedPieceSize, trailing bool) {
+		// pad remaining space with 0 CommPs
+		toFill := uint64(-sum % s)
+		if trailing && sum == 0 {
+			toFill = uint64(s)
+		}
+
+		n := bits.OnesCount64(toFill)
+		for i := 0; i < n; i++ {
+			next := bits.TrailingZeros64(toFill)
+			psize := uint64(1) << uint(next)
+			toFill ^= psize
+
+			padded := abi.PaddedPieceSize(psize)
+			allPieces = append(allPieces, abi.PieceInfo{
+				Size:     padded,
+				PieceCID: zerocomm.ZeroPieceCommitment(padded.Unpadded()),
+			})
+			sum += padded
+		}
+	}
+
 	for _, p := range pieces {
+		padTo(p.Size, false)
+
+		allPieces = append(allPieces, p)
 		sum += p.Size
 	}
 
@@ -553,22 +580,7 @@ func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceIn
 		return cid.Undef, err
 	}
 
-	{
-		// pad remaining space with 0 CommPs
-		toFill := uint64(abi.PaddedPieceSize(ssize) - sum)
-		n := bits.OnesCount64(toFill)
-		for i := 0; i < n; i++ {
-			next := bits.TrailingZeros64(toFill)
-			psize := uint64(1) << uint(next)
-			toFill ^= psize
+	padTo(abi.PaddedPieceSize(ssize), true)
 
-			unpadded := abi.PaddedPieceSize(psize).Unpadded()
-			pieces = append(pieces, abi.PieceInfo{
-				Size:     unpadded.Padded(),
-				PieceCID: zerocomm.ZeroPieceCommitment(unpadded),
-			})
-		}
-	}
-
-	return ffi.GenerateUnsealedCID(proofType, pieces)
+	return ffi.GenerateUnsealedCID(proofType, allPieces)
 }

ffiwrapper/sealer_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/ipfs/go-cid"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -484,3 +485,105 @@ func requireFDsClosed(t *testing.T, start int) {
 	log.Infow("open FDs", "start", start, "now", openNow)
 	require.Equal(t, start, openNow, "FDs shouldn't leak")
 }
+
+func TestGenerateUnsealedCID(t *testing.T) {
+	pt := abi.RegisteredSealProof_StackedDrg2KiBV1
+	ups := int(abi.PaddedPieceSize(2048).Unpadded())
+
+	commP := func(b []byte) cid.Cid {
+		pf, werr, err := ToReadableFile(bytes.NewReader(b), int64(len(b)))
+		require.NoError(t, err)
+
+		c, err := ffi.GeneratePieceCIDFromFile(pt, pf, abi.UnpaddedPieceSize(len(b)))
+		require.NoError(t, err)
+
+		require.NoError(t, werr())
+
+		return c
+	}
+
+	testCommEq := func(name string, in [][]byte, expect [][]byte) {
+		t.Run(name, func(t *testing.T) {
+			upi := make([]abi.PieceInfo, len(in))
+			for i, b := range in {
+				upi[i] = abi.PieceInfo{
+					Size:     abi.UnpaddedPieceSize(len(b)).Padded(),
+					PieceCID: commP(b),
+				}
+			}
+
+			sectorPi := []abi.PieceInfo{
+				{
+					Size:     2048,
+					PieceCID: commP(bytes.Join(expect, nil)),
+				},
+			}
+
+			expectCid, err := GenerateUnsealedCID(pt, sectorPi)
+			require.NoError(t, err)
+
+			actualCid, err := GenerateUnsealedCID(pt, upi)
+			require.NoError(t, err)
+
+			require.Equal(t, expectCid, actualCid)
+		})
+	}
+
+	barr := func(b byte, den int) []byte {
+		return bytes.Repeat([]byte{b}, ups/den)
+	}
+
+	// 0000
+	testCommEq("zero",
+		nil,
+		[][]byte{barr(0, 1)},
+	)
+
+	// 1111
+	testCommEq("one",
+		[][]byte{barr(1, 1)},
+		[][]byte{barr(1, 1)},
+	)
+
+	// 11 00
+	testCommEq("one|2",
+		[][]byte{barr(1, 2)},
+		[][]byte{barr(1, 2), barr(0, 2)},
+	)
+
+	// 1 0 00
+	testCommEq("one|4",
+		[][]byte{barr(1, 4)},
+		[][]byte{barr(1, 4), barr(0, 4), barr(0, 2)},
+	)
+
+	// 11 2 0
+	testCommEq("one|2-two|4",
+		[][]byte{barr(1, 2), barr(2, 4)},
+		[][]byte{barr(1, 2), barr(2, 4), barr(0, 4)},
+	)
+
+	// 1 0 22
+	testCommEq("one|4-two|2",
+		[][]byte{barr(1, 4), barr(2, 2)},
+		[][]byte{barr(1, 4), barr(0, 4), barr(2, 2)},
+	)
+
+	// 1 0 22 0000
+	testCommEq("one|8-two|4",
+		[][]byte{barr(1, 8), barr(2, 4)},
+		[][]byte{barr(1, 8), barr(0, 8), barr(2, 4), barr(0, 2)},
+	)
+
+	// 11 2 0 0000
+	testCommEq("one|4-two|8",
+		[][]byte{barr(1, 4), barr(2, 8)},
+		[][]byte{barr(1, 4), barr(2, 8), barr(0, 8), barr(0, 2)},
+	)
+
+	// 1 0 22 3 0 00 4444 5 0 00
+	testCommEq("one|16-two|8-three|16-four|4-five|16",
+		[][]byte{barr(1, 16), barr(2, 8), barr(3, 16), barr(4, 4), barr(5, 16)},
+		[][]byte{barr(1, 16), barr(0, 16), barr(2, 8), barr(3, 16), barr(0, 16), barr(0, 8), barr(4, 4), barr(5, 16), barr(0, 16), barr(0, 8)},
+	)
+}