From 8c94758285031af4e7e90e8381ab043ac7edb968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Jul 2019 17:14:16 +0200 Subject: [PATCH] Improt sectorbuilder bindings --- .gitmodules | 3 + Makefile | 2 + lib/rust-fil-sector-builder | 1 + .../libsectorbuilder/cgo_bindings.go | 405 ++++++++++++++++++ .../libsectorbuilder/cgo_transforms.go | 113 +++++ scripts/install-sectorbuilder.sh | 27 ++ 6 files changed, 551 insertions(+) create mode 160000 lib/rust-fil-sector-builder create mode 100644 lib/sectorbuilder/libsectorbuilder/cgo_bindings.go create mode 100644 lib/sectorbuilder/libsectorbuilder/cgo_transforms.go create mode 100755 scripts/install-sectorbuilder.sh diff --git a/.gitmodules b/.gitmodules index a300a714f..476370f2e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "lib/bls-signatures/bls-signatures"] path = lib/bls-signatures/bls-signatures url = https://github.com/filecoin-project/bls-signatures.git +[submodule "lib/sectorbuilder/rust-builder"] + path = lib/rust-fil-sector-builder + url = https://github.com/filecoin-project/rust-fil-sector-builder diff --git a/Makefile b/Makefile index 9011ca7a6..403fb0b61 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,8 @@ blssigs: lib/bls-signatures/include/libbls_signatures.h lib/bls-signatures/include/libbls_signatures.h: lib/bls-signatures/bls-signatures ; ./scripts/install-bls-signatures.sh +lib/sectorbuilder/include: + deps: blssigs build: deps diff --git a/lib/rust-fil-sector-builder b/lib/rust-fil-sector-builder new file mode 160000 index 000000000..394b5b1bd --- /dev/null +++ b/lib/rust-fil-sector-builder @@ -0,0 +1 @@ +Subproject commit 394b5b1bd87efbee6489a7d22dfb3b1dba697af0 diff --git a/lib/sectorbuilder/libsectorbuilder/cgo_bindings.go b/lib/sectorbuilder/libsectorbuilder/cgo_bindings.go new file mode 100644 index 000000000..f576fc571 --- /dev/null +++ b/lib/sectorbuilder/libsectorbuilder/cgo_bindings.go @@ -0,0 +1,405 @@ +// +build !windows + +package libsectorbuilder + +import ( + "time" + "unsafe" + + logging "github.com/ipfs/go-log" + "github.com/pkg/errors" +) + +// #cgo LDFLAGS: -L${SRCDIR}/../lib -lsector_builder_ffi +// #cgo pkg-config: ${SRCDIR}/../lib/pkgconfig/sector_builder_ffi.pc +// #include "../include/sector_builder_ffi.h" +import "C" + +var log = logging.Logger("libsectorbuilder") // nolint: deadcode + +func elapsed(what string) func() { + start := time.Now() + return func() { + log.Debugf("%s took %v\n", what, time.Since(start)) + } +} + +// StagedSectorMetadata is a sector into which we write user piece-data before +// sealing. Note: SectorID is unique across all staged and sealed sectors for a +// storage miner actor. +type StagedSectorMetadata struct { + SectorID uint64 +} + +// SectorSealingStatus communicates how far along in the sealing process a +// sector has progressed. +type SectorSealingStatus struct { + SectorID uint64 + SealStatusCode uint8 // Sealed = 0, Pending = 1, Failed = 2, Sealing = 3 + SealErrorMsg string // will be nil unless SealStatusCode == 2 + CommD [32]byte // will be empty unless SealStatusCode == 0 + CommR [32]byte // will be empty unless SealStatusCode == 0 + CommRStar [32]byte // will be empty unless SealStatusCode == 0 + Proof []byte // will be empty unless SealStatusCode == 0 + Pieces []PieceMetadata // will be empty unless SealStatusCode == 0 +} + +// PieceMetadata represents a piece stored by the sector builder. +type PieceMetadata struct { + Key string + Size uint64 + InclusionProof []byte +} + +// VerifySeal returns true if the sealing operation from which its inputs were +// derived was valid, and false if not. +func VerifySeal( + sectorSize uint64, + commR [32]byte, + commD [32]byte, + commRStar [32]byte, + proverID [31]byte, + sectorID [31]byte, + proof []byte, +) (bool, error) { + defer elapsed("VerifySeal")() + + commDCBytes := C.CBytes(commD[:]) + defer C.free(commDCBytes) + + commRCBytes := C.CBytes(commR[:]) + defer C.free(commRCBytes) + + commRStarCBytes := C.CBytes(commRStar[:]) + defer C.free(commRStarCBytes) + + proofCBytes := C.CBytes(proof[:]) + defer C.free(proofCBytes) + + proverIDCBytes := C.CBytes(proverID[:]) + defer C.free(proverIDCBytes) + + sectorIDCbytes := C.CBytes(sectorID[:]) + defer C.free(sectorIDCbytes) + + // a mutable pointer to a VerifySealResponse C-struct + resPtr := (*C.sector_builder_ffi_VerifySealResponse)(unsafe.Pointer(C.sector_builder_ffi_verify_seal( + C.uint64_t(sectorSize), + (*[32]C.uint8_t)(commRCBytes), + (*[32]C.uint8_t)(commDCBytes), + (*[32]C.uint8_t)(commRStarCBytes), + (*[31]C.uint8_t)(proverIDCBytes), + (*[31]C.uint8_t)(sectorIDCbytes), + (*C.uint8_t)(proofCBytes), + C.size_t(len(proof)), + ))) + defer C.sector_builder_ffi_destroy_verify_seal_response(resPtr) + + if resPtr.status_code != 0 { + return false, errors.New(C.GoString(resPtr.error_msg)) + } + + return bool(resPtr.is_valid), nil +} + +// VerifyPoSt returns true if the PoSt-generation operation from which its +// inputs were derived was valid, and false if not. +func VerifyPoSt( + sectorSize uint64, + sortedCommRs [][32]byte, + challengeSeed [32]byte, + proofs [][]byte, + faults []uint64, +) (bool, error) { + defer elapsed("VerifyPoSt")() + + // validate verification request + if len(proofs) == 0 { + return false, errors.New("must provide at least one proof to verify") + } + + // CommRs must be provided to C.verify_post in the same order that they were + // provided to the C.generate_post + commRs := sortedCommRs + + // flattening the byte slice makes it easier to copy into the C heap + flattened := make([]byte, 32*len(commRs)) + for idx, commR := range commRs { + copy(flattened[(32*idx):(32*(1+idx))], commR[:]) + } + + // copy bytes from Go to C heap + flattenedCommRsCBytes := C.CBytes(flattened) + defer C.free(flattenedCommRsCBytes) + + challengeSeedCBytes := C.CBytes(challengeSeed[:]) + defer C.free(challengeSeedCBytes) + + proofPartitions, proofsPtr, proofsLen := cPoStProofs(proofs) + defer C.free(unsafe.Pointer(proofsPtr)) + + // allocate fixed-length array of uint64s in C heap + faultsPtr, faultsSize := cUint64s(faults) + defer C.free(unsafe.Pointer(faultsPtr)) + + // a mutable pointer to a VerifyPoStResponse C-struct + resPtr := (*C.sector_builder_ffi_VerifyPoStResponse)(unsafe.Pointer(C.sector_builder_ffi_verify_post( + C.uint64_t(sectorSize), + proofPartitions, + (*C.uint8_t)(flattenedCommRsCBytes), + C.size_t(len(flattened)), + (*[32]C.uint8_t)(challengeSeedCBytes), + proofsPtr, + proofsLen, + faultsPtr, + faultsSize, + ))) + defer C.sector_builder_ffi_destroy_verify_post_response(resPtr) + + if resPtr.status_code != 0 { + return false, errors.New(C.GoString(resPtr.error_msg)) + } + + return bool(resPtr.is_valid), nil +} + +// GetMaxUserBytesPerStagedSector returns the number of user bytes that will fit +// into a staged sector. Due to bit-padding, the number of user bytes that will +// fit into the staged sector will be less than number of bytes in sectorSize. +func GetMaxUserBytesPerStagedSector(sectorSize uint64) uint64 { + defer elapsed("GetMaxUserBytesPerStagedSector")() + + return uint64(C.sector_builder_ffi_get_max_user_bytes_per_staged_sector(C.uint64_t(sectorSize))) +} + +// InitSectorBuilder allocates and returns a pointer to a sector builder. +func InitSectorBuilder( + sectorSize uint64, + poRepProofPartitions uint8, + poStProofPartitions uint8, + lastUsedSectorID uint64, + metadataDir string, + proverID [31]byte, + sealedSectorDir string, + stagedSectorDir string, + maxNumOpenStagedSectors uint8, +) (unsafe.Pointer, error) { + defer elapsed("InitSectorBuilder")() + + cMetadataDir := C.CString(metadataDir) + defer C.free(unsafe.Pointer(cMetadataDir)) + + proverIDCBytes := C.CBytes(proverID[:]) + defer C.free(proverIDCBytes) + + cStagedSectorDir := C.CString(stagedSectorDir) + defer C.free(unsafe.Pointer(cStagedSectorDir)) + + cSealedSectorDir := C.CString(sealedSectorDir) + defer C.free(unsafe.Pointer(cSealedSectorDir)) + + class, err := cSectorClass(sectorSize, poRepProofPartitions, poStProofPartitions) + if err != nil { + return nil, errors.Wrap(err, "failed to get sector class") + } + + resPtr := (*C.sector_builder_ffi_InitSectorBuilderResponse)(unsafe.Pointer(C.sector_builder_ffi_init_sector_builder( + class, + C.uint64_t(lastUsedSectorID), + cMetadataDir, + (*[31]C.uint8_t)(proverIDCBytes), + cSealedSectorDir, + cStagedSectorDir, + C.uint8_t(maxNumOpenStagedSectors), + ))) + defer C.sector_builder_ffi_destroy_init_sector_builder_response(resPtr) + + if resPtr.status_code != 0 { + return nil, errors.New(C.GoString(resPtr.error_msg)) + } + + return unsafe.Pointer(resPtr.sector_builder), nil +} + +// DestroySectorBuilder deallocates the sector builder associated with the +// provided pointer. This function will panic if the provided pointer is null +// or if the sector builder has been previously deallocated. +func DestroySectorBuilder(sectorBuilderPtr unsafe.Pointer) { + defer elapsed("DestroySectorBuilder")() + + C.sector_builder_ffi_destroy_sector_builder((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr)) +} + +// AddPiece writes the given piece into an unsealed sector and returns the id +// of that sector. +func AddPiece( + sectorBuilderPtr unsafe.Pointer, + pieceKey string, + pieceSize uint64, + piecePath string, +) (sectorID uint64, retErr error) { + defer elapsed("AddPiece")() + + cPieceKey := C.CString(pieceKey) + defer C.free(unsafe.Pointer(cPieceKey)) + + cPiecePath := C.CString(piecePath) + defer C.free(unsafe.Pointer(cPiecePath)) + + resPtr := (*C.sector_builder_ffi_AddPieceResponse)(unsafe.Pointer(C.sector_builder_ffi_add_piece( + (*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), + cPieceKey, + C.uint64_t(pieceSize), + cPiecePath, + ))) + defer C.sector_builder_ffi_destroy_add_piece_response(resPtr) + + if resPtr.status_code != 0 { + return 0, errors.New(C.GoString(resPtr.error_msg)) + } + + return uint64(resPtr.sector_id), nil +} + +// ReadPieceFromSealedSector produces a byte buffer containing the piece +// associated with the provided key. If the key is not associated with any piece +// yet sealed into a sector, an error will be returned. +func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string) ([]byte, error) { + defer elapsed("ReadPieceFromSealedSector")() + + cPieceKey := C.CString(pieceKey) + defer C.free(unsafe.Pointer(cPieceKey)) + + resPtr := (*C.sector_builder_ffi_ReadPieceFromSealedSectorResponse)(unsafe.Pointer(C.sector_builder_ffi_read_piece_from_sealed_sector((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), cPieceKey))) + defer C.sector_builder_ffi_destroy_read_piece_from_sealed_sector_response(resPtr) + + if resPtr.status_code != 0 { + return nil, errors.New(C.GoString(resPtr.error_msg)) + } + + return goBytes(resPtr.data_ptr, resPtr.data_len), nil +} + +// SealAllStagedSectors schedules sealing of all staged sectors. +func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error { + defer elapsed("SealAllStagedSectors")() + + resPtr := (*C.sector_builder_ffi_SealAllStagedSectorsResponse)(unsafe.Pointer(C.sector_builder_ffi_seal_all_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr)))) + defer C.sector_builder_ffi_destroy_seal_all_staged_sectors_response(resPtr) + + if resPtr.status_code != 0 { + return errors.New(C.GoString(resPtr.error_msg)) + } + + return nil +} + +// GetAllStagedSectors returns a slice of all staged sector metadata for the sector builder. +func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadata, error) { + defer elapsed("GetAllStagedSectors")() + + resPtr := (*C.sector_builder_ffi_GetStagedSectorsResponse)(unsafe.Pointer(C.sector_builder_ffi_get_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr)))) + defer C.sector_builder_ffi_destroy_get_staged_sectors_response(resPtr) + + if resPtr.status_code != 0 { + return nil, errors.New(C.GoString(resPtr.error_msg)) + } + + meta, err := goStagedSectorMetadata((*C.sector_builder_ffi_FFIStagedSectorMetadata)(unsafe.Pointer(resPtr.sectors_ptr)), resPtr.sectors_len) + if err != nil { + return nil, err + } + + return meta, nil +} + +// GetSectorSealingStatusByID produces sector sealing status (staged, sealing in +// progress, sealed, failed) for the provided sector id if it exists, otherwise +// an error. +func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64) (SectorSealingStatus, error) { + defer elapsed("GetSectorSealingStatusByID")() + + resPtr := (*C.sector_builder_ffi_GetSealStatusResponse)(unsafe.Pointer(C.sector_builder_ffi_get_seal_status((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), C.uint64_t(sectorID)))) + defer C.sector_builder_ffi_destroy_get_seal_status_response(resPtr) + + if resPtr.status_code != 0 { + return SectorSealingStatus{}, errors.New(C.GoString(resPtr.error_msg)) + } + + if resPtr.seal_status_code == C.Failed { + return SectorSealingStatus{SealStatusCode: 2, SealErrorMsg: C.GoString(resPtr.seal_error_msg)}, nil + } else if resPtr.seal_status_code == C.Pending { + return SectorSealingStatus{SealStatusCode: 1}, nil + } else if resPtr.seal_status_code == C.Sealing { + return SectorSealingStatus{SealStatusCode: 3}, nil + } else if resPtr.seal_status_code == C.Sealed { + commRSlice := goBytes(&resPtr.comm_r[0], 32) + var commR [32]byte + copy(commR[:], commRSlice) + + commDSlice := goBytes(&resPtr.comm_d[0], 32) + var commD [32]byte + copy(commD[:], commDSlice) + + commRStarSlice := goBytes(&resPtr.comm_r_star[0], 32) + var commRStar [32]byte + copy(commRStar[:], commRStarSlice) + + proof := goBytes(resPtr.proof_ptr, resPtr.proof_len) + + ps, err := goPieceMetadata(resPtr.pieces_ptr, resPtr.pieces_len) + if err != nil { + return SectorSealingStatus{}, errors.Wrap(err, "failed to marshal from string to cid") + } + + return SectorSealingStatus{ + SectorID: sectorID, + SealStatusCode: 0, + CommD: commD, + CommR: commR, + CommRStar: commRStar, + Proof: proof, + Pieces: ps, + }, nil + } else { + // unknown + return SectorSealingStatus{}, errors.New("unexpected seal status") + } +} + +// GeneratePoSt produces a proof-of-spacetime for the provided replica commitments. +func GeneratePoSt( + sectorBuilderPtr unsafe.Pointer, + sortedCommRs [][32]byte, + challengeSeed [32]byte, +) ([][]byte, []uint64, error) { + defer elapsed("GeneratePoSt")() + + // flattening the byte slice makes it easier to copy into the C heap + commRs := sortedCommRs + flattened := make([]byte, 32*len(commRs)) + for idx, commR := range commRs { + copy(flattened[(32*idx):(32*(1+idx))], commR[:]) + } + + // copy the Go byte slice into C memory + cflattened := C.CBytes(flattened) + defer C.free(cflattened) + + challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0]) + + // a mutable pointer to a GeneratePoStResponse C-struct + resPtr := (*C.sector_builder_ffi_GeneratePoStResponse)(unsafe.Pointer(C.sector_builder_ffi_generate_post((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), (*C.uint8_t)(cflattened), C.size_t(len(flattened)), (*[32]C.uint8_t)(challengeSeedPtr)))) + defer C.sector_builder_ffi_destroy_generate_post_response(resPtr) + + if resPtr.status_code != 0 { + return nil, nil, errors.New(C.GoString(resPtr.error_msg)) + } + + proofs, err := goPoStProofs(resPtr.proof_partitions, resPtr.flattened_proofs_ptr, resPtr.flattened_proofs_len) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to convert to []PoStProof") + } + + return proofs, goUint64s(resPtr.faults_ptr, resPtr.faults_len), nil +} diff --git a/lib/sectorbuilder/libsectorbuilder/cgo_transforms.go b/lib/sectorbuilder/libsectorbuilder/cgo_transforms.go new file mode 100644 index 000000000..2dde69496 --- /dev/null +++ b/lib/sectorbuilder/libsectorbuilder/cgo_transforms.go @@ -0,0 +1,113 @@ +package libsectorbuilder + +import ( + "unsafe" +) + +// #cgo LDFLAGS: -L${SRCDIR}/../lib -lsector_builder_ffi +// #cgo pkg-config: ${SRCDIR}/../lib/pkgconfig/sector_builder_ffi.pc +// #include "../include/sector_builder_ffi.h" +import "C" + +// SingleProofPartitionProofLen denotes the number of bytes in a proof generated +// with a single partition. The number of bytes in a proof increases linearly +// with the number of partitions used when creating that proof. +const SingleProofPartitionProofLen = 192 + +func cPoStProofs(src [][]byte) (C.uint8_t, *C.uint8_t, C.size_t) { + proofSize := len(src[0]) + + flattenedLen := C.size_t(proofSize * len(src)) + + // flattening the byte slice makes it easier to copy into the C heap + flattened := make([]byte, flattenedLen) + for idx, proof := range src { + copy(flattened[(proofSize*idx):(proofSize*(1+idx))], proof[:]) + } + + proofPartitions := proofSize / SingleProofPartitionProofLen + + return C.uint8_t(proofPartitions), (*C.uint8_t)(C.CBytes(flattened)), flattenedLen +} + +func cUint64s(src []uint64) (*C.uint64_t, C.size_t) { + srcCSizeT := C.size_t(len(src)) + + // allocate array in C heap + cUint64s := C.malloc(srcCSizeT * C.sizeof_uint64_t) + + // create a Go slice backed by the C-array + pp := (*[1 << 30]C.uint64_t)(cUint64s) + for i, v := range src { + pp[i] = C.uint64_t(v) + } + + return (*C.uint64_t)(cUint64s), srcCSizeT +} + +func cSectorClass(sectorSize uint64, poRepProofPartitions uint8, poStProofPartitions uint8) (C.sector_builder_ffi_FFISectorClass, error) { + return C.sector_builder_ffi_FFISectorClass{ + sector_size: C.uint64_t(sectorSize), + porep_proof_partitions: C.uint8_t(poRepProofPartitions), + post_proof_partitions: C.uint8_t(poStProofPartitions), + }, nil +} + +func goBytes(src *C.uint8_t, size C.size_t) []byte { + return C.GoBytes(unsafe.Pointer(src), C.int(size)) +} + +func goStagedSectorMetadata(src *C.sector_builder_ffi_FFIStagedSectorMetadata, size C.size_t) ([]StagedSectorMetadata, error) { + sectors := make([]StagedSectorMetadata, size) + if src == nil || size == 0 { + return sectors, nil + } + + sectorPtrs := (*[1 << 30]C.sector_builder_ffi_FFIStagedSectorMetadata)(unsafe.Pointer(src))[:size:size] + for i := 0; i < int(size); i++ { + sectors[i] = StagedSectorMetadata{ + SectorID: uint64(sectorPtrs[i].sector_id), + } + } + + return sectors, nil +} + +func goPieceMetadata(src *C.sector_builder_ffi_FFIPieceMetadata, size C.size_t) ([]PieceMetadata, error) { + ps := make([]PieceMetadata, size) + if src == nil || size == 0 { + return ps, nil + } + + ptrs := (*[1 << 30]C.sector_builder_ffi_FFIPieceMetadata)(unsafe.Pointer(src))[:size:size] + for i := 0; i < int(size); i++ { + ps[i] = PieceMetadata{ + Key: C.GoString(ptrs[i].piece_key), + Size: uint64(ptrs[i].num_bytes), + } + } + + return ps, nil +} + +func goPoStProofs(partitions C.uint8_t, src *C.uint8_t, size C.size_t) ([][]byte, error) { + tmp := goBytes(src, size) + + arraySize := len(tmp) + chunkSize := int(partitions) * SingleProofPartitionProofLen + + out := make([][]byte, arraySize/chunkSize) + for i := 0; i < len(out); i++ { + out[i] = append([]byte{}, tmp[i*chunkSize:(i+1)*chunkSize]...) + } + + return out, nil +} + +func goUint64s(src *C.uint64_t, size C.size_t) []uint64 { + out := make([]uint64, size) + if src != nil { + copy(out, (*(*[1 << 30]uint64)(unsafe.Pointer(src)))[:size:size]) + } + return out +} diff --git a/scripts/install-sectorbuilder.sh b/scripts/install-sectorbuilder.sh new file mode 100755 index 000000000..18b1133ce --- /dev/null +++ b/scripts/install-sectorbuilder.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -Eeo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/install-shared.bash" + +subm_dir="lib/rust-fil-sector-builder" + +git submodule update --init --recursive $subm_dir + +if download_release_tarball tarball_path "${subm_dir}"; then + tmp_dir=$(mktemp -d) + tar -C $tmp_dir -xzf $tarball_path + + cp -R "${tmp_dir}/include" lib/sectorbuilder + cp -R "${tmp_dir}/lib" lib/sectorbuilder +else + echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build" + build_from_source "${subm_dir}" + + mkdir -p lib/sectorbuilder/include + mkdir -p lib/sectorbuilder/lib/pkgconfig + + find "${subm_dir}" -type f -name sector_builder_ffi.h -exec mv -- "{}" ./lib/sectorbuilder/include/ \; + find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" ./lib/sectorbuilder/lib/ \; + find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" ./lib/sectorbuilder/pkgconfig/ \; +fi \ No newline at end of file