114 lines
2.9 KiB
Go
114 lines
2.9 KiB
Go
package commp
|
|
|
|
import (
|
|
"bytes"
|
|
"math/bits"
|
|
|
|
"github.com/ipfs/go-cid"
|
|
"golang.org/x/xerrors"
|
|
|
|
ffi "github.com/filecoin-project/filecoin-ffi"
|
|
"github.com/filecoin-project/go-padreader"
|
|
"github.com/filecoin-project/go-state-types/abi"
|
|
|
|
"github.com/filecoin-project/lotus/api"
|
|
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
|
"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
|
|
)
|
|
|
|
const commPBufPad = abi.PaddedPieceSize(8 << 20)
|
|
const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128)) // can't use .Unpadded() for const
|
|
|
|
type Writer struct {
|
|
len int64
|
|
buf [CommPBuf]byte
|
|
leaves []cid.Cid
|
|
}
|
|
|
|
func (w *Writer) Write(p []byte) (int, error) {
|
|
n := len(p)
|
|
for len(p) > 0 {
|
|
buffered := int(w.len % int64(len(w.buf)))
|
|
toBuffer := len(w.buf) - buffered
|
|
if toBuffer > len(p) {
|
|
toBuffer = len(p)
|
|
}
|
|
|
|
copied := copy(w.buf[buffered:], p[:toBuffer])
|
|
p = p[copied:]
|
|
w.len += int64(copied)
|
|
|
|
if copied > 0 && w.len%int64(len(w.buf)) == 0 {
|
|
leaf, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, bytes.NewReader(w.buf[:]), CommPBuf)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
w.leaves = append(w.leaves, leaf)
|
|
}
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
func (w *Writer) Sum() (api.DataCIDSize, error) {
|
|
// process last non-zero leaf if exists
|
|
lastLen := w.len % int64(len(w.buf))
|
|
rawLen := w.len
|
|
|
|
// process remaining bit of data
|
|
if lastLen != 0 {
|
|
if len(w.leaves) != 0 {
|
|
copy(w.buf[lastLen:], make([]byte, int(int64(CommPBuf)-lastLen)))
|
|
lastLen = int64(CommPBuf)
|
|
}
|
|
|
|
r, sz := padreader.New(bytes.NewReader(w.buf[:lastLen]), uint64(lastLen))
|
|
p, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, r, sz)
|
|
if err != nil {
|
|
return api.DataCIDSize{}, err
|
|
}
|
|
|
|
if sz < CommPBuf { // special case for pieces smaller than 16MiB
|
|
return api.DataCIDSize{
|
|
PayloadSize: w.len,
|
|
PieceSize: sz.Padded(),
|
|
PieceCID: p,
|
|
}, nil
|
|
}
|
|
|
|
w.leaves = append(w.leaves, p)
|
|
}
|
|
|
|
// pad with zero pieces to power-of-two size
|
|
fillerLeaves := (1 << (bits.Len(uint(len(w.leaves) - 1)))) - len(w.leaves)
|
|
for i := 0; i < fillerLeaves; i++ {
|
|
w.leaves = append(w.leaves, zerocomm.ZeroPieceCommitment(CommPBuf))
|
|
}
|
|
|
|
if len(w.leaves) == 1 {
|
|
return api.DataCIDSize{
|
|
PayloadSize: rawLen,
|
|
PieceSize: abi.PaddedPieceSize(len(w.leaves)) * commPBufPad,
|
|
PieceCID: w.leaves[0],
|
|
}, nil
|
|
}
|
|
|
|
pieces := make([]abi.PieceInfo, len(w.leaves))
|
|
for i, leaf := range w.leaves {
|
|
pieces[i] = abi.PieceInfo{
|
|
Size: commPBufPad,
|
|
PieceCID: leaf,
|
|
}
|
|
}
|
|
|
|
p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, pieces)
|
|
if err != nil {
|
|
return api.DataCIDSize{}, xerrors.Errorf("generating unsealed CID: %w", err)
|
|
}
|
|
|
|
return api.DataCIDSize{
|
|
PayloadSize: rawLen,
|
|
PieceSize: abi.PaddedPieceSize(len(w.leaves)) * commPBufPad,
|
|
PieceCID: p,
|
|
}, nil
|
|
}
|