ffiwrapper: Insert alignment between pieces

This commit is contained in:
Łukasz Magiera 2020-06-25 21:53:51 +02:00
parent 98ef8e4ef2
commit a8997dc35c
2 changed files with 132 additions and 17 deletions

View File

@ -543,8 +543,35 @@ func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader
}
func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) {
allPieces := make([]abi.PieceInfo, 0, len(pieces))
var sum abi.PaddedPieceSize
padTo := func(s abi.PaddedPieceSize, trailing bool) {
// pad remaining space with 0 CommPs
toFill := uint64(-sum % s)
if trailing && sum == 0 {
toFill = uint64(s)
}
n := bits.OnesCount64(toFill)
for i := 0; i < n; i++ {
next := bits.TrailingZeros64(toFill)
psize := uint64(1) << uint(next)
toFill ^= psize
padded := abi.PaddedPieceSize(psize)
allPieces = append(allPieces, abi.PieceInfo{
Size: padded,
PieceCID: zerocomm.ZeroPieceCommitment(padded.Unpadded()),
})
sum += padded
}
}
for _, p := range pieces {
padTo(p.Size, false)
allPieces = append(allPieces, p)
sum += p.Size
}
@ -553,22 +580,7 @@ func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceIn
return cid.Undef, err
}
{
// pad remaining space with 0 CommPs
toFill := uint64(abi.PaddedPieceSize(ssize) - sum)
n := bits.OnesCount64(toFill)
for i := 0; i < n; i++ {
next := bits.TrailingZeros64(toFill)
psize := uint64(1) << uint(next)
toFill ^= psize
padTo(abi.PaddedPieceSize(ssize), true)
unpadded := abi.PaddedPieceSize(psize).Unpadded()
pieces = append(pieces, abi.PieceInfo{
Size: unpadded.Padded(),
PieceCID: zerocomm.ZeroPieceCommitment(unpadded),
})
}
}
return ffi.GenerateUnsealedCID(proofType, pieces)
return ffi.GenerateUnsealedCID(proofType, allPieces)
}

View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
"github.com/ipfs/go-cid"
"io"
"io/ioutil"
"math/rand"
@ -484,3 +485,105 @@ func requireFDsClosed(t *testing.T, start int) {
log.Infow("open FDs", "start", start, "now", openNow)
require.Equal(t, start, openNow, "FDs shouldn't leak")
}
func TestGenerateUnsealedCID(t *testing.T) {
pt := abi.RegisteredSealProof_StackedDrg2KiBV1
ups := int(abi.PaddedPieceSize(2048).Unpadded())
commP := func(b []byte) cid.Cid {
pf, werr, err := ToReadableFile(bytes.NewReader(b), int64(len(b)))
require.NoError(t, err)
c, err := ffi.GeneratePieceCIDFromFile(pt, pf, abi.UnpaddedPieceSize(len(b)))
require.NoError(t, err)
require.NoError(t, werr())
return c
}
testCommEq := func(name string, in [][]byte, expect [][]byte) {
t.Run(name, func(t *testing.T) {
upi := make([]abi.PieceInfo, len(in))
for i, b := range in {
upi[i] = abi.PieceInfo{
Size: abi.UnpaddedPieceSize(len(b)).Padded(),
PieceCID: commP(b),
}
}
sectorPi := []abi.PieceInfo{
{
Size: 2048,
PieceCID: commP(bytes.Join(expect, nil)),
},
}
expectCid, err := GenerateUnsealedCID(pt, sectorPi)
require.NoError(t, err)
actualCid, err := GenerateUnsealedCID(pt, upi)
require.NoError(t, err)
require.Equal(t, expectCid, actualCid)
})
}
barr := func(b byte, den int) []byte {
return bytes.Repeat([]byte{b}, ups/den)
}
// 0000
testCommEq("zero",
nil,
[][]byte{barr(0, 1)},
)
// 1111
testCommEq("one",
[][]byte{barr(1, 1)},
[][]byte{barr(1, 1)},
)
// 11 00
testCommEq("one|2",
[][]byte{barr(1, 2)},
[][]byte{barr(1, 2), barr(0, 2)},
)
// 1 0 00
testCommEq("one|4",
[][]byte{barr(1, 4)},
[][]byte{barr(1, 4), barr(0, 4), barr(0, 2)},
)
// 11 2 0
testCommEq("one|2-two|4",
[][]byte{barr(1, 2), barr(2, 4)},
[][]byte{barr(1, 2), barr(2, 4), barr(0, 4)},
)
// 1 0 22
testCommEq("one|4-two|2",
[][]byte{barr(1, 4), barr(2, 2)},
[][]byte{barr(1, 4), barr(0, 4), barr(2, 2)},
)
// 1 0 22 0000
testCommEq("one|8-two|4",
[][]byte{barr(1, 8), barr(2, 4)},
[][]byte{barr(1, 8), barr(0, 8), barr(2, 4), barr(0, 2)},
)
// 11 2 0 0000
testCommEq("one|4-two|8",
[][]byte{barr(1, 4), barr(2, 8)},
[][]byte{barr(1, 4), barr(2, 8), barr(0, 8), barr(0, 2)},
)
// 1 0 22 3 0 00 4444 5 0 00
testCommEq("one|16-two|8-three|16-four|4-five|16",
[][]byte{barr(1, 16), barr(2, 8), barr(3, 16), barr(4, 4), barr(5, 16)},
[][]byte{barr(1, 16), barr(0, 16), barr(2, 8), barr(3, 16), barr(0, 16), barr(0, 8), barr(4, 4), barr(5, 16), barr(0, 16), barr(0, 8)},
)
}