cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

use new sectorbuilder file interfaces

Browse Source
This commit is contained in:
Łukasz Magiera 2019-09-23 12:50:28 +02:00
parent 04c17cb214
commit 9a990dbd8a
8 changed files with 122 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -53,7 +53,7 @@ build/.update-modules:
bin/ipget:
./bin/dist_get /ipns/dist.ipfs.io ipget bin/ipget v0.4.0
PARAM_SECTOR_SIZES:=1024 268435456
PARAM_SECTOR_SIZES:=1024 16777216 268435456 1073741824
PARAM_SECTOR_SIZES:=$(addprefix build/.params-,$(PARAM_SECTOR_SIZES))
$(PARAM_SECTOR_SIZES): build/proof-params/parameters.json bin/ipget

42
build/proof-params/parameters.json
View File

@ -1,4 +1,24 @@
{
"v12-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": {
"cid": "QmZ6Y88jRbRjjYQzhh8o85bcUeChj7NGyo9yK6VbywhJ9F",
"digest": "3d245479d9b5fc668d58c493da5f3ee1",
"sector_size": 16777216
},
"v12-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": {
"cid": "QmbGgBLMTCnRc1E1fsUCPyZE4SYzrtAYEWCqCtL55AgxE2",
"digest": "29bd4c152096f878f41257b433159d81",
"sector_size": 16777216
},
"v12-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": {
"cid": "Qmdm8vhWeRsZUUaHdysDr91gv6u6RFeC18hHxGnnzPrwcW",
"digest": "c67fd415a65e6d1caf4278597cf3462e",
"sector_size": 1073741824
},
"v12-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": {
"cid": "Qmc3xssw1syaZDZKmiM7TbCRnoDPfmD6V8ec1eTESidJQS",
"digest": "870355c10000010b9a4ece80892acca2",
"sector_size": 1073741824
},
"v12-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": {
"cid": "QmZBvF2F9wTYKLBxWSCQKe34D3M7vkNNc7ou8mxnNhZkZc",
"digest": "5d854e0ecfbd12cb7fa1247a6e6a0315",
@ -19,6 +39,16 @@
"digest": "c83eca165ba94233861227578d658a22",
"sector_size": 268435456
},
"v12-zigzag-proof-of-replication-0ed875801b4a99e4a6b46e58703e6857277ce020510fc9080041f6cfd5e0d286.params": {
"cid": "QmTkeyz3mfec4MqCbbiwuVWAQicY231zpdxSjdxN5U84PD",
"digest": "cf7118ac2273e2ccb6b451a5acd5f6e0",
"sector_size": 1073741824
},
"v12-zigzag-proof-of-replication-0ed875801b4a99e4a6b46e58703e6857277ce020510fc9080041f6cfd5e0d286.vk": {
"cid": "QmaGzJCwJNpQAD63QAKb8y4MKS1AMHeKyYzg3C5WyRSRRM",
"digest": "288c792f4fe09c85f8f35673fb9d5ed0",
"sector_size": 1073741824
},
"v12-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.params": {
"cid": "QmNSuxq15JPFCTehxVpgJydNZ79rpLoNwnLzQMGA9EziXg",
"digest": "818cd9cc2e0e47210a05bd073847ab5a",
@ -29,6 +59,16 @@
"digest": "a6636e2ee1a176161e022296bc045e79",
"sector_size": 268435456
},
"v12-zigzag-proof-of-replication-6496c180c2eab89ee0638bc73879ced01daf512486eb38b9e1c9402ba578e010.params": {
"cid": "QmUBpwbVwu5uzjTS8n1yesJ2QUaVbWJ8D9p2VaSsfjgAUr",
"digest": "e7aa73a1b06290d30f567bfac8324bf1",
"sector_size": 16777216
},
"v12-zigzag-proof-of-replication-6496c180c2eab89ee0638bc73879ced01daf512486eb38b9e1c9402ba578e010.vk": {
"cid": "Qme2uK8sQJUT3DzF291p2b91eFQzn7cKSJFiKVBsvrjTgt",
"digest": "d318cd116803c8ccbd3ac4cded9400ad",
"sector_size": 16777216
},
"v12-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.params": {
"cid": "QmTfhTnkFvbpFfw8UydFdnPCDfxgAxEcw4fRdGsELpcFnh",
"digest": "906b6c0c9dc5bb581d9641c11b54e197",
@ -39,4 +79,4 @@
"digest": "3632776cd23e376694c625390b9a73ea",
"sector_size": 1024
}
}
}

12
chain/deals/client_utils.go
View File

@ -2,9 +2,9 @@ package deals
import (
"context"
"github.com/filecoin-project/go-lotus/lib/sectorbuilder"
"runtime"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
cbor "github.com/ipfs/go-ipld-cbor"
@ -56,11 +56,11 @@ func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, int64, error)
return nil, 0, err
}
var commP [sectorbuilder.CommitmentBytesLen]byte
err = withTemp(uf, func(f string) error {
commP, err = sectorbuilder.GeneratePieceCommitment(f, uint64(size))
return err
})
commP, err := sectorbuilder.GeneratePieceCommitment(uf, uint64(size))
if err != nil {
return nil, 0, err
}
return commP[:], size, err
}

11
chain/deals/handler_states.go
View File

@ -5,6 +5,7 @@ import (
"context"
"github.com/filecoin-project/go-lotus/api"
"github.com/filecoin-project/go-lotus/build"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipfs/go-merkledag"
@ -202,13 +203,13 @@ func (h *Handler) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder
return sectorbuilder.SectorSealingStatus{}, err
}
switch status.SealStatusCode {
case 0: // sealed
case 2: // failed
switch status.State {
case sealing_state.Sealed:
case sealing_state.Failed:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg)
case 1: // pending
case sealing_state.Pending:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID)
case 3: // sealing
case sealing_state.Sealing:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID)
default:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID)

28
chain/deals/utils.go
View File

@ -1,28 +0,0 @@
package deals
import (
"io"
"io/ioutil"
"os"
)
func withTemp(r io.Reader, cb func(string) error) error {
f, err := ioutil.TempFile(os.TempDir(), "lotus-temp-")
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
err = cb(f.Name())
if err != nil {
os.Remove(f.Name())
return err
}
return os.Remove(f.Name())
}

2
cmd/lotus-storage-miner/sectors.go
View File

@ -65,7 +65,7 @@ var sectorsStatusCmd = &cli.Command{
}
fmt.Printf("SectorID:\t%d\n", status.SectorID)
fmt.Printf("SealStatusCode:\t%d\n", status.SealStatusCode)
fmt.Printf("Status:\t%s\n", status.State.String())
fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg)
fmt.Printf("CommD:\t\t%x\n", status.CommD)
fmt.Printf("CommR:\t\t%x\n", status.CommR)

65
lib/sectorbuilder/sectorbuilder.go
View File

@ -2,6 +2,9 @@ package sectorbuilder
import (
"encoding/binary"
"io"
"os"
"sync"
"unsafe"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
@ -64,8 +67,18 @@ func (sb *SectorBuilder) Destroy() {
sectorbuilder.DestroySectorBuilder(sb.handle)
}
func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, piecePath string) (uint64, error) {
return sectorbuilder.AddPiece(sb.handle, pieceKey, pieceSize, piecePath)
func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Reader) (uint64, error) {
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return 0, err
}
sectorID, err := sectorbuilder.AddPieceFromFile(sb.handle, pieceKey, pieceSize, f)
if err != nil {
return 0, err
}
return sectorID, werr()
}
// TODO: should *really really* return an io.ReadCloser
@ -118,3 +131,51 @@ func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo {
func VerifyPost(sectorSize uint64, sectorInfo SortedSectorInfo, challengeSeed [CommLen]byte, proof []byte, faults []uint64) (bool, error) {
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeed, proof, faults)
}
func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
f, werr, err := toReadableFile(piece, int64(pieceSize))
if err != nil {
return [32]byte{}, err
}
commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize)
if err != nil {
return [32]byte{}, err
}
return commP, werr()
}
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
f, ok := r.(*os.File)
if ok {
return f, func() error { return nil }, nil
}
var w *os.File
f, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
var copyWait sync.WaitGroup
copyWait.Add(1)
var werr error
go func() {
defer copyWait.Done()
_, werr = io.CopyN(w, r, n)
err := w.Close()
if werr == nil {
werr = err
}
}()
return f, func() error {
copyWait.Wait()
return nil
}, nil
}

34
storage/sector/store.go
View File

@ -2,10 +2,9 @@ package sector
import (
"context"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
"golang.org/x/xerrors"
"io"
"io/ioutil"
"os"
"sync"
"time"
@ -63,7 +62,7 @@ func (s *Store) poll() {
continue
}
if status.SealStatusCode == 0 { // constant pls, zero implies the last step?
if status.State == sealing_state.Sealed {
done = append(done, status)
}
}
@ -102,10 +101,8 @@ func (s *Store) service() {
}
func (s *Store) AddPiece(ref string, size uint64, r io.Reader) (sectorID uint64, err error) {
err = withTemp(r, func(f string) (err error) {
sectorID, err = s.sb.AddPiece(ref, size, f)
return err
})
sectorID, err = s.sb.AddPiece(ref, size, r)
if err != nil {
return 0, err
}
@ -191,26 +188,3 @@ func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte
func (s *Store) Stop() {
close(s.closeCh)
}
func withTemp(r io.Reader, cb func(string) error) error {
f, err := ioutil.TempFile(os.TempDir(), "lotus-temp-")
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
err = cb(f.Name())
if err != nil {
if err := os.Remove(f.Name()); err != nil {
log.Errorf("couldn't remove temp file '%s'", f.Name())
}
return err
}
return os.Remove(f.Name())
}