lotus/lib/commp/writer.go

package commp

import (
	"bytes"
	"math/bits"

	"github.com/ipfs/go-cid"
	"golang.org/x/xerrors"

	ffi "github.com/filecoin-project/filecoin-ffi"
	"github.com/filecoin-project/go-padreader"
	"github.com/filecoin-project/go-state-types/abi"

	"github.com/filecoin-project/lotus/api"
	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
	"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
)

const commPBufPad = abi.PaddedPieceSize(8 << 20)
const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128)) // can't use .Unpadded() for const

type Writer struct {
	len    int64
	buf    [CommPBuf]byte
	leaves []cid.Cid
}

func (w *Writer) Write(p []byte) (int, error) {
	n := len(p)
	for len(p) > 0 {
		buffered := int(w.len % int64(len(w.buf)))
		toBuffer := len(w.buf) - buffered
		if toBuffer > len(p) {
			toBuffer = len(p)
		}

		copied := copy(w.buf[buffered:], p[:toBuffer])
		p = p[copied:]
		w.len += int64(copied)

		if copied > 0 && w.len%int64(len(w.buf)) == 0 {
			leaf, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, bytes.NewReader(w.buf[:]), CommPBuf)
			if err != nil {
				return 0, err
			}
			w.leaves = append(w.leaves, leaf)
		}
	}
	return n, nil
}

func (w *Writer) Sum() (api.DataCIDSize, error) {
	// process last non-zero leaf if exists
	lastLen := w.len % int64(len(w.buf))
	rawLen := w.len

	// process remaining bit of data
	if lastLen != 0 {
		if len(w.leaves) != 0 {
			copy(w.buf[lastLen:], make([]byte, int(int64(CommPBuf)-lastLen)))
			lastLen = int64(CommPBuf)
		}

		r, sz := padreader.New(bytes.NewReader(w.buf[:lastLen]), uint64(lastLen))
		p, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg32GiBV1, r, sz)
		if err != nil {
			return api.DataCIDSize{}, err
		}

		if sz < CommPBuf { // special case for pieces smaller than 16MiB
			return api.DataCIDSize{
				PayloadSize: w.len,
				PieceSize:   sz.Padded(),
				PieceCID:    p,
			}, nil
		}

		w.leaves = append(w.leaves, p)
	}

	// pad with zero pieces to power-of-two size
	fillerLeaves := (1 << (bits.Len(uint(len(w.leaves) - 1)))) - len(w.leaves)
	for i := 0; i < fillerLeaves; i++ {
		w.leaves = append(w.leaves, zerocomm.ZeroPieceCommitment(CommPBuf))
	}

	if len(w.leaves) == 1 {
		return api.DataCIDSize{
			PayloadSize: rawLen,
			PieceSize:   abi.PaddedPieceSize(len(w.leaves)) * commPBufPad,
			PieceCID:    w.leaves[0],
		}, nil
	}

	pieces := make([]abi.PieceInfo, len(w.leaves))
	for i, leaf := range w.leaves {
		pieces[i] = abi.PieceInfo{
			Size:     commPBufPad,
			PieceCID: leaf,
		}
	}

	p, err := ffi.GenerateUnsealedCID(abi.RegisteredSealProof_StackedDrg32GiBV1, pieces)
	if err != nil {
		return api.DataCIDSize{}, xerrors.Errorf("generating unsealed CID: %w", err)
	}

	return api.DataCIDSize{
		PayloadSize: rawLen,
		PieceSize:   abi.PaddedPieceSize(len(w.leaves)) * commPBufPad,
		PieceCID:    p,
	}, nil
}