cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Cleanup after dropping sectorbuilder

Browse Source
This commit is contained in:
Łukasz Magiera 2020-03-26 20:34:38 +01:00
parent e8e90300f0
commit 617e3135c1
10 changed files with 176 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ffiwrapper/sealer.go
View File

@ -7,7 +7,7 @@ import (
var log = logging.Logger("ffiwrapper")
type SectorBuilder struct {
type Sealer struct {
sealProofType abi.RegisteredProof
postProofType abi.RegisteredProof
ssize abi.SectorSize // a function of sealProofType and postProofType
@ -24,18 +24,18 @@ func fallbackPostChallengeCount(sectors uint64, faults uint64) uint64 {
return challengeCount
}
func (sb *SectorBuilder) Stop() {
func (sb *Sealer) Stop() {
close(sb.stopping)
}
func (sb *SectorBuilder) SectorSize() abi.SectorSize {
func (sb *Sealer) SectorSize() abi.SectorSize {
return sb.ssize
}
func (sb *SectorBuilder) SealProofType() abi.RegisteredProof {
func (sb *Sealer) SealProofType() abi.RegisteredProof {
return sb.sealProofType
}
func (sb *SectorBuilder) PoStProofType() abi.RegisteredProof {
func (sb *Sealer) PoStProofType() abi.RegisteredProof {
return sb.postProofType
}

82
ffiwrapper/sealer_cgo.go
View File

@ -19,15 +19,15 @@ import (
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
)
var _ Basic = &SectorBuilder{}
var _ Storage = &Sealer{}
func New(sectors SectorProvider, cfg *Config) (*SectorBuilder, error) {
func New(sectors SectorProvider, cfg *Config) (*Sealer, error) {
sectorSize, err := sizeFromConfig(*cfg)
if err != nil {
return nil, err
}
sb := &SectorBuilder{
sb := &Sealer{
sealProofType: cfg.SealProofType,
postProofType: cfg.PoStProofType,
ssize: sectorSize,
@ -40,47 +40,13 @@ func New(sectors SectorProvider, cfg *Config) (*SectorBuilder, error) {
return sb, nil
}
func (sb *SectorBuilder) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, faults []abi.SectorNumber) (ffi.SortedPrivateSectorInfo, error) {
fmap := map[abi.SectorNumber]struct{}{}
for _, fault := range faults {
fmap[fault] = struct{}{}
}
var out []ffi.PrivateSectorInfo
for _, s := range sectorInfo {
if _, faulty := fmap[s.SectorNumber]; faulty {
continue
}
paths, done, err := sb.sectors.AcquireSector(ctx, abi.SectorID{Miner: mid, Number: s.SectorNumber}, stores.FTCache|stores.FTSealed, 0, false)
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquire sector paths: %w", err)
}
done() // TODO: This is a tiny bit suboptimal
postProofType, err := s.RegisteredProof.RegisteredPoStProof()
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquiring registered PoSt proof from sector info %+v: %w", s, err)
}
out = append(out, ffi.PrivateSectorInfo{
CacheDirPath: paths.Cache,
PoStProofType: postProofType,
SealedSectorPath: paths.Sealed,
SectorInfo: s,
})
}
return ffi.NewSortedPrivateSectorInfo(out...), nil
}
func (sb *SectorBuilder) NewSector(ctx context.Context, sector abi.SectorID) error {
func (sb *Sealer) NewSector(ctx context.Context, sector abi.SectorID) error {
// TODO: Allocate the sector here instead of in addpiece
return nil
}
func (sb *SectorBuilder) AddPiece(ctx context.Context, sector abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error) {
func (sb *Sealer) AddPiece(ctx context.Context, sector abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error) {
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return abi.PieceInfo{}, err
@ -143,7 +109,7 @@ func (sb *SectorBuilder) AddPiece(ctx context.Context, sector abi.SectorID, exis
}, werr()
}
func (sb *SectorBuilder) ReadPieceFromSealedSector(ctx context.Context, sector abi.SectorID, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealedCID cid.Cid) (io.ReadCloser, error) {
func (sb *Sealer) ReadPieceFromSealedSector(ctx context.Context, sector abi.SectorID, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealedCID cid.Cid) (io.ReadCloser, error) {
path, doneUnsealed, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTUnsealed, false)
if err != nil {
return nil, xerrors.Errorf("acquire unsealed sector path: %w", err)
@ -212,7 +178,7 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(ctx context.Context, sector a
}, nil
}
func (sb *SectorBuilder) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
func (sb *Sealer) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTSealed|stores.FTCache, true)
if err != nil {
return nil, xerrors.Errorf("acquiring sector paths: %w", err)
@ -269,7 +235,7 @@ func (sb *SectorBuilder) SealPreCommit1(ctx context.Context, sector abi.SectorID
return p1o, nil
}
func (sb *SectorBuilder) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) {
func (sb *Sealer) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, true)
if err != nil {
return storage.SectorCids{}, xerrors.Errorf("acquiring sector paths: %w", err)
@ -287,7 +253,7 @@ func (sb *SectorBuilder) SealPreCommit2(ctx context.Context, sector abi.SectorID
}, nil
}
func (sb *SectorBuilder) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
func (sb *Sealer) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, true)
if err != nil {
return nil, xerrors.Errorf("acquire sector paths: %w", err)
@ -314,37 +280,11 @@ func (sb *SectorBuilder) SealCommit1(ctx context.Context, sector abi.SectorID, t
return output, nil
}
func (sb *SectorBuilder) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (storage.Proof, error) {
func (sb *Sealer) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (storage.Proof, error) {
return ffi.SealCommitPhase2(phase1Out, sector.Number, sector.Miner)
}
func (sb *SectorBuilder) GenerateFallbackPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) (storage.FallbackPostOut, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return storage.FallbackPostOut{}, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
challengeSeed[31] = 0
candidates, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return storage.FallbackPostOut{}, err
}
winners := make([]abi.PoStCandidate, len(candidates))
for idx := range winners {
winners[idx] = candidates[idx].Candidate
}
proof, err := ffi.GeneratePoSt(miner, privsectors, challengeSeed, winners)
return storage.FallbackPostOut{
PoStInputs: ffiToStorageCandidates(candidates),
Proof: proof,
}, err
}
func (sb *SectorBuilder) FinalizeSector(ctx context.Context, sector abi.SectorID) error {
func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID) error {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, false)
if err != nil {
return xerrors.Errorf("acquiring sector cache path: %w", err)

6
ffiwrapper/sealer_test.go
View File

@ -37,7 +37,7 @@ type seal struct {
ticket abi.SealRandomness
}
func (s *seal) precommit(t *testing.T, sb *SectorBuilder, id abi.SectorID, done func()) {
func (s *seal) precommit(t *testing.T, sb *Sealer, id abi.SectorID, done func()) {
defer done()
dlen := abi.PaddedPieceSize(sectorSize).Unpadded()
@ -61,7 +61,7 @@ func (s *seal) precommit(t *testing.T, sb *SectorBuilder, id abi.SectorID, done
s.cids = cids
}
func (s *seal) commit(t *testing.T, sb *SectorBuilder, done func()) {
func (s *seal) commit(t *testing.T, sb *Sealer, done func()) {
defer done()
seed := abi.InteractiveSealRandomness{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
@ -95,7 +95,7 @@ func (s *seal) commit(t *testing.T, sb *SectorBuilder, done func()) {
}
}
func post(t *testing.T, sb *SectorBuilder, seals ...seal) time.Time {
func post(t *testing.T, sb *Sealer, seals ...seal) time.Time {
randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7}
sis := make([]abi.SectorInfo, len(seals))

9
ffiwrapper/types.go
View File

@ -3,9 +3,10 @@ package ffiwrapper
import (
"context"
"errors"
"github.com/ipfs/go-cid"
"io"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
@ -20,14 +21,14 @@ type Validator interface {
CanProve(sector stores.SectorPaths) (bool, error)
}
type Sealer interface {
type StorageSealer interface {
storage.Sealer
storage.Storage
}
type Basic interface {
type Storage interface {
storage.Prover
Sealer
StorageSealer
ReadPieceFromSealedSector(context.Context, abi.SectorID, UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error)
}

80
ffiwrapper/varifier_cgo.go
View File

@ -1,80 +0,0 @@
//+build cgo
package ffiwrapper
import (
"context"
"go.opencensus.io/trace"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
)
func (sb *SectorBuilder) ComputeElectionPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) {
challengeSeed[31] = 0
privsects, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, nil) // TODO: faults
if err != nil {
return nil, err
}
return ffi.GeneratePoSt(miner, privsects, challengeSeed, winners)
}
func (sb *SectorBuilder) GenerateEPostCandidates(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return nil, err
}
challengeSeed[31] = 0
challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
pc, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, err
}
return ffiToStorageCandidates(pc), nil
}
func ffiToStorageCandidates(pc []ffi.PoStCandidateWithTicket) []storage.PoStCandidateWithTicket {
out := make([]storage.PoStCandidateWithTicket, len(pc))
for i := range out {
out[i] = storage.PoStCandidateWithTicket{
Candidate: pc[i].Candidate,
Ticket: pc[i].Ticket,
}
}
return out
}
var _ Verifier = ProofVerifier
type proofVerifier struct{}
var ProofVerifier = proofVerifier{}
func (proofVerifier) VerifySeal(info abi.SealVerifyInfo) (bool, error) {
return ffi.VerifySeal(info)
}
func (proofVerifier) VerifyElectionPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func (proofVerifier) VerifyFallbackPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func verifyPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
info.Randomness[31] = 0
return ffi.VerifyPoSt(info)
}

142
ffiwrapper/verifier_cgo.go Normal file
View File

@ -0,0 +1,142 @@
//+build cgo
package ffiwrapper
import (
"context"
"golang.org/x/xerrors"
"go.opencensus.io/trace"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
)
func (sb *Sealer) ComputeElectionPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) {
challengeSeed[31] = 0
privsects, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, nil) // TODO: faults
if err != nil {
return nil, err
}
return ffi.GeneratePoSt(miner, privsects, challengeSeed, winners)
}
func (sb *Sealer) GenerateFallbackPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) (storage.FallbackPostOut, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return storage.FallbackPostOut{}, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
challengeSeed[31] = 0
candidates, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return storage.FallbackPostOut{}, err
}
winners := make([]abi.PoStCandidate, len(candidates))
for idx := range winners {
winners[idx] = candidates[idx].Candidate
}
proof, err := ffi.GeneratePoSt(miner, privsectors, challengeSeed, winners)
return storage.FallbackPostOut{
PoStInputs: ffiToStorageCandidates(candidates),
Proof: proof,
}, err
}
func (sb *Sealer) GenerateEPostCandidates(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return nil, err
}
challengeSeed[31] = 0
challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
pc, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, err
}
return ffiToStorageCandidates(pc), nil
}
func ffiToStorageCandidates(pc []ffi.PoStCandidateWithTicket) []storage.PoStCandidateWithTicket {
out := make([]storage.PoStCandidateWithTicket, len(pc))
for i := range out {
out[i] = storage.PoStCandidateWithTicket{
Candidate: pc[i].Candidate,
Ticket: pc[i].Ticket,
}
}
return out
}
func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, faults []abi.SectorNumber) (ffi.SortedPrivateSectorInfo, error) {
fmap := map[abi.SectorNumber]struct{}{}
for _, fault := range faults {
fmap[fault] = struct{}{}
}
var out []ffi.PrivateSectorInfo
for _, s := range sectorInfo {
if _, faulty := fmap[s.SectorNumber]; faulty {
continue
}
paths, done, err := sb.sectors.AcquireSector(ctx, abi.SectorID{Miner: mid, Number: s.SectorNumber}, stores.FTCache|stores.FTSealed, 0, false)
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquire sector paths: %w", err)
}
done() // TODO: This is a tiny bit suboptimal
postProofType, err := s.RegisteredProof.RegisteredPoStProof()
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquiring registered PoSt proof from sector info %+v: %w", s, err)
}
out = append(out, ffi.PrivateSectorInfo{
CacheDirPath: paths.Cache,
PoStProofType: postProofType,
SealedSectorPath: paths.Sealed,
SectorInfo: s,
})
}
return ffi.NewSortedPrivateSectorInfo(out...), nil
}
var _ Verifier = ProofVerifier
type proofVerifier struct{}
var ProofVerifier = proofVerifier{}
func (proofVerifier) VerifySeal(info abi.SealVerifyInfo) (bool, error) {
return ffi.VerifySeal(info)
}
func (proofVerifier) VerifyElectionPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func (proofVerifier) VerifyFallbackPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func verifyPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
info.Randomness[31] = 0
return ffi.VerifyPoSt(info)
}

6
manager.go
View File

@ -4,7 +4,6 @@ import (
"container/list"
"context"
"errors"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"net/http"
"sync"
@ -19,6 +18,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
@ -30,7 +30,7 @@ var ErrNoWorkers = errors.New("no suitable workers found")
type URLs []string
type Worker interface {
ffiwrapper.Sealer
ffiwrapper.StorageSealer
TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error)
@ -47,7 +47,7 @@ type SectorManager interface {
ReadPieceFromSealedSector(context.Context, abi.SectorID, ffiwrapper.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error)
ffiwrapper.Sealer
ffiwrapper.StorageSealer
storage.Prover
}

8
mock/mock.go
View File

@ -4,8 +4,6 @@ import (
"bytes"
"context"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"io/ioutil"
"math/big"
@ -20,6 +18,8 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
var log = logging.Logger("sbmock")
@ -124,7 +124,7 @@ func (sb *SectorMgr) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticke
ss, ok := sb.sectors[sid]
sb.lk.Unlock()
if !ok {
return nil, xerrors.Errorf("no sector with id %d in sectorbuilder", sid)
return nil, xerrors.Errorf("no sector with id %d in storage", sid)
}
ss.lk.Lock()
@ -237,7 +237,7 @@ func (sb *SectorMgr) FailSector(sid abi.SectorID) error {
defer sb.lk.Unlock()
ss, ok := sb.sectors[sid]
if !ok {
return fmt.Errorf("no such sector in sectorbuilder")
return fmt.Errorf("no such sector in storage")
}
ss.failed = true

4
worker_local.go
View File

@ -2,7 +2,6 @@ package sectorstorage
import (
"context"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"os"
@ -14,6 +13,7 @@ import (
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
@ -87,7 +87,7 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi.
}, nil
}
func (l *LocalWorker) sb() (ffiwrapper.Basic, error) {
func (l *LocalWorker) sb() (ffiwrapper.Storage, error) {
return ffiwrapper.New(&localWorkerPathProvider{w: l}, l.scfg)
}

2
zerocomm/zerocomm_test.go
View File

@ -3,7 +3,6 @@ package zerocomm_test
import (
"bytes"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
"io"
"testing"
@ -12,6 +11,7 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
)
func TestComms(t *testing.T) {