cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #1335 from filecoin-project/feat/update-sb-ffi

Browse Source 
Update FFI
This commit is contained in:
Whyrusleeping 2020-03-06 00:10:53 -08:00 committed by GitHub
commit 30d28a942e
18 changed files with 98 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
chain/gen/gen.go
View File

@ -8,12 +8,12 @@ import (
"io/ioutil"
"sync/atomic"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
commcid "github.com/filecoin-project/go-fil-commcid"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-storage/storage"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-car"
@ -514,14 +514,14 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*cr
}
type ElectionPoStProver interface {
GenerateCandidates(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]ffi.PoStCandidateWithTicket, error)
ComputeProof(context.Context, []abi.SectorInfo, []byte, []ffi.PoStCandidateWithTicket) ([]abi.PoStProof, error)
GenerateCandidates(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]storage.PoStCandidateWithTicket, error)
ComputeProof(context.Context, []abi.SectorInfo, []byte, []storage.PoStCandidateWithTicket) ([]abi.PoStProof, error)
}
type eppProvider struct{}
func (epp *eppProvider) GenerateCandidates(ctx context.Context, _ []abi.SectorInfo, eprand abi.PoStRandomness) ([]ffi.PoStCandidateWithTicket, error) {
return []ffi.PoStCandidateWithTicket{
func (epp *eppProvider) GenerateCandidates(ctx context.Context, _ []abi.SectorInfo, eprand abi.PoStRandomness) ([]storage.PoStCandidateWithTicket, error) {
return []storage.PoStCandidateWithTicket{
{
Candidate: abi.PoStCandidate{
RegisteredProof: abi.RegisteredProof_StackedDRG2KiBPoSt,
@ -534,7 +534,7 @@ func (epp *eppProvider) GenerateCandidates(ctx context.Context, _ []abi.SectorIn
}, nil
}
func (epp *eppProvider) ComputeProof(ctx context.Context, _ []abi.SectorInfo, eprand []byte, winners []ffi.PoStCandidateWithTicket) ([]abi.PoStProof, error) {
func (epp *eppProvider) ComputeProof(ctx context.Context, _ []abi.SectorInfo, eprand []byte, winners []storage.PoStCandidateWithTicket) ([]abi.PoStProof, error) {
return []abi.PoStProof{{
ProofBytes: []byte("valid proof"),
@ -544,7 +544,7 @@ func (epp *eppProvider) ComputeProof(ctx context.Context, _ []abi.SectorInfo, ep
type ProofInput struct {
sectors []abi.SectorInfo
hvrf []byte
winners []ffi.PoStCandidateWithTicket
winners []storage.PoStCandidateWithTicket
vrfout []byte
}
@ -600,7 +600,7 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round int64, miner add
return nil, xerrors.Errorf("failed to look up miners sector size: %w", err)
}
var winners []ffi.PoStCandidateWithTicket
var winners []storage.PoStCandidateWithTicket
for _, c := range candidates {
if types.IsTicketWinner(c.Candidate.PartialTicket, ssize, uint64(len(sinfos)), pow.TotalPower) {
winners = append(winners, c)

3
cmd/lotus-bench/main.go
View File

@ -180,7 +180,6 @@ func main() {
sbfs := &fs.Basic{
Miner: maddr,
NextID: 1,
Root: sbdir,
}
@ -205,7 +204,7 @@ func main() {
r := rand.New(rand.NewSource(100 + int64(i)))
pi, err := sb.AddPiece(context.TODO(), abi.PaddedPieceSize(sectorSize).Unpadded(), i, r, nil)
pi, err := sb.AddPiece(context.TODO(), i, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r)
if err != nil {
return err
}

11
cmd/lotus-seed/seed/seed.go
View File

@ -52,9 +52,10 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum
return nil, nil, err
}
next := offset
sbfs := &fs.Basic{
Miner: maddr,
NextID: offset,
Root: sbroot,
}
@ -70,12 +71,10 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum
var sealedSectors []*genesis.PreSeal
for i := 0; i < sectors; i++ {
sid, err := sbfs.AcquireSectorNumber()
if err != nil {
return nil, nil, err
}
sid := next
next++
pi, err := sb.AddPiece(context.TODO(), abi.PaddedPieceSize(ssize).Unpadded(), sid, rand.Reader, nil)
pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(ssize).Unpadded(), rand.Reader)
if err != nil {
return nil, nil, err
}

2
extern/filecoin-ffi vendored

@ -1 +1 @@
Subproject commit 4da0bccccefb48df4bce63869d058f9bbf02b686
Subproject commit 4446ff8a1bb9115703010e6fa7e94b7f7dbf5618

5
go.mod
View File

@ -11,7 +11,7 @@ require (
github.com/coreos/go-systemd/v22 v22.0.0
github.com/docker/go-units v0.4.0
github.com/filecoin-project/chain-validation v0.0.6-0.20200305212458-670d41260fd7
github.com/filecoin-project/filecoin-ffi v0.0.0-20200226231125-fc253ccb5294
github.com/filecoin-project/filecoin-ffi v0.0.0-20200304181354-4446ff8a1bb9
github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be
github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
@ -21,9 +21,10 @@ require (
github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200306043753-5cdbe369b47d
github.com/filecoin-project/go-statestore v0.1.0
github.com/filecoin-project/specs-actors v0.0.0-20200306001214-5b98d3ed4bc4
github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/google/uuid v1.1.1

7
go.sum
View File

@ -120,18 +120,21 @@ github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.m
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA=
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7 h1:pGvvPZgIfBXAda+OGc943zFcM8Wt4GG9UVymVeyfx6c=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200306043753-5cdbe369b47d h1:tRS0QB50rhUaetohnt3GwKNGk28WOhkdniHvP461HXc=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200306043753-5cdbe369b47d/go.mod h1:NcE+iL0bbYnamGmYQgCPVGbSaf8VF2/CLra/61B3I3I=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
github.com/filecoin-project/specs-actors v0.0.0-20200305205312-53bb01da9aeb h1:9Ch6f/LYCh/BoriHfSp8sRX0OaO+9gQVK6x0vGl9o0s=
github.com/filecoin-project/specs-actors v0.0.0-20200305205312-53bb01da9aeb/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
github.com/filecoin-project/specs-actors v0.0.0-20200306001214-5b98d3ed4bc4 h1:EFOQ/DA3jOu2r2/r7/vnkGRXoCoDGY5knXb/bVl/X/c=
github.com/filecoin-project/specs-actors v0.0.0-20200306001214-5b98d3ed4bc4/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513 h1:okBx3lPomwDxlPmRvyP078BwivDfdxNUlpCDhDD0ia8=
github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=

8
node/builder.go
View File

@ -5,9 +5,6 @@ import (
"errors"
"time"
"github.com/filecoin-project/specs-actors/actors/runtime"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
ci "github.com/libp2p/go-libp2p-core/crypto"
@ -26,6 +23,9 @@ import (
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/runtime"
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain"
@ -261,7 +261,7 @@ func Online() Option {
Override(new(*advmgr.Manager), advmgr.New),
Override(new(sealmgr.Manager), From(new(*advmgr.Manager))),
Override(new(sectorbuilder.Prover), From(new(sealmgr.Manager))),
Override(new(storage2.Prover), From(new(sealmgr.Manager))),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
Override(new(sealing.TicketFn), modules.SealTicketGen),

19
node/node_test.go
View File

@ -75,6 +75,11 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
err = ds.Put(datastore.NewKey("miner-address"), act.Bytes())
require.NoError(t, err)
nic := storedcounter.New(ds, datastore.NewKey("/storage/nextid"))
for i := 0; i < nPreseal; i++ {
nic.Next()
}
err = lr.Close()
require.NoError(t, err)
@ -173,7 +178,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
if err != nil {
t.Fatal(err)
}
genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, 2, tdir, []byte("make genesis mem random"), nil)
genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nPreseal, tdir, []byte("make genesis mem random"), nil)
if err != nil {
t.Fatal(err)
}
@ -266,6 +271,8 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
return fulls, storers
}
const nPreseal = 2
func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
ctx := context.Background()
mn := mocknet.New(ctx)
@ -301,7 +308,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
if err != nil {
t.Fatal(err)
}
genm, k, err := sbmock.PreSeal(2048, maddr, 2)
genm, k, err := sbmock.PreSeal(2048, maddr, nPreseal)
if err != nil {
t.Fatal(err)
}
@ -434,7 +441,7 @@ func TestAPIRPC(t *testing.T) {
func TestAPIDealFlow(t *testing.T) {
logging.SetLogLevel("miner", "ERROR")
//logging.SetLogLevel("chainstore", "ERROR")
logging.SetLogLevel("chainstore", "ERROR")
logging.SetLogLevel("chain", "ERROR")
logging.SetLogLevel("sub", "ERROR")
logging.SetLogLevel("storageminer", "ERROR")
@ -446,5 +453,11 @@ func TestAPIDealFlowReal(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
logging.SetLogLevel("miner", "ERROR")
logging.SetLogLevel("chainstore", "ERROR")
logging.SetLogLevel("chain", "ERROR")
logging.SetLogLevel("sub", "ERROR")
logging.SetLogLevel("storageminer", "ERROR")
test.TestDealFlow(t, builder, time.Second)
}

8
storage/fpost_sched.go
View File

@ -4,12 +4,12 @@ import (
"context"
"sync"
"github.com/filecoin-project/specs-actors/actors/abi"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
@ -22,7 +22,7 @@ const StartConfidence = 4 // TODO: config
type FPoStScheduler struct {
api storageMinerApi
sb sectorbuilder.Prover
sb storage.Prover
actor address.Address
worker address.Address
@ -37,7 +37,7 @@ type FPoStScheduler struct {
failLk sync.Mutex
}
func NewFPoStScheduler(api storageMinerApi, sb sectorbuilder.Prover, actor address.Address, worker address.Address) *FPoStScheduler {
func NewFPoStScheduler(api storageMinerApi, sb storage.Prover, actor address.Address, worker address.Address) *FPoStScheduler {
return &FPoStScheduler{api: api, sb: sb, actor: actor, worker: worker}
}

11
storage/miner.go
View File

@ -6,16 +6,15 @@ import (
"time"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-storage/storage"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/host"
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/storage/sealmgr"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/specs-actors/actors/crypto"
@ -121,16 +120,16 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error {
}
type SectorBuilderEpp struct {
prover sectorbuilder.Prover
prover storage.Prover
}
func NewElectionPoStProver(sb sectorbuilder.Prover) *SectorBuilderEpp {
func NewElectionPoStProver(sb storage.Prover) *SectorBuilderEpp {
return &SectorBuilderEpp{sb}
}
var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil)
func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.SectorInfo, rand abi.PoStRandomness) ([]ffi.PoStCandidateWithTicket, error) {
func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.SectorInfo, rand abi.PoStRandomness) ([]storage.PoStCandidateWithTicket, error) {
start := time.Now()
var faults []abi.SectorNumber // TODO
@ -142,7 +141,7 @@ func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.S
return cds, nil
}
func (epp *SectorBuilderEpp) ComputeProof(ctx context.Context, ssi []abi.SectorInfo, rand []byte, winners []ffi.PoStCandidateWithTicket) ([]abi.PoStProof, error) {
func (epp *SectorBuilderEpp) ComputeProof(ctx context.Context, ssi []abi.SectorInfo, rand []byte, winners []storage.PoStCandidateWithTicket) ([]abi.PoStProof, error) {
if build.InsecurePoStValidation {
log.Warn("Generating fake EPost proof! You should only see this while running tests!")
return []abi.PoStProof{{ProofBytes: []byte("valid proof")}}, nil

23
storage/sbmock/sbmock.go
View File

@ -13,13 +13,12 @@ import (
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
ffi "github.com/filecoin-project/filecoin-ffi"
)
var log = logging.Logger("sbmock")
@ -75,7 +74,7 @@ func (sb *SBMock) RateLimit() func() {
}
}
func (sb *SBMock) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, sectorId abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) {
func (sb *SBMock) AddPiece(ctx context.Context, sectorId abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) {
log.Warn("Add piece: ", sectorId, size, sb.proofType)
sb.lk.Lock()
ss, ok := sb.sectors[sectorId]
@ -115,11 +114,11 @@ func (sb *SBMock) AcquireSectorNumber() (abi.SectorNumber, error) {
return id, nil
}
func (sb *SBMock) GenerateFallbackPoSt([]abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) {
func (sb *SBMock) GenerateFallbackPoSt([]abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, []abi.PoStProof, error) {
panic("NYI")
}
func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) {
func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
sb.lk.Lock()
ss, ok := sb.sectors[sid]
sb.lk.Unlock()
@ -174,7 +173,7 @@ func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorNumber, tick
return cc, nil
}
func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorNumber, phase1Out storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
db := []byte(string(phase1Out))
db[0] ^= 'd'
@ -190,7 +189,7 @@ func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorNumber, phas
return commR, d, nil
}
func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCid cid.Cid, unsealed cid.Cid) (output []byte, err error) {
func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCid cid.Cid, unsealed cid.Cid) (output storage.Commit1Out, err error) {
sb.lk.Lock()
ss, ok := sb.sectors[sid]
sb.lk.Unlock()
@ -218,7 +217,7 @@ func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket
return out[:], nil
}
func (sb *SBMock) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) {
func (sb *SBMock) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage.Commit1Out) (proof storage.Proof, err error) {
var out [32]byte
for i := range out {
out[i] = phase1Out[i] ^ byte(sectorNum&0xff)
@ -261,7 +260,7 @@ func (sb *SBMock) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed
panic("implement me")
}
func (sb *SBMock) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) {
func (sb *SBMock) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) {
if len(faults) > 0 {
panic("todo")
}
@ -271,13 +270,13 @@ func (sb *SBMock) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challenge
n = uint64(len(sectorInfo))
}
out := make([]ffi.PoStCandidateWithTicket, n)
out := make([]storage.PoStCandidateWithTicket, n)
seed := big.NewInt(0).SetBytes(challengeSeed[:])
start := seed.Mod(seed, big.NewInt(int64(len(sectorInfo)))).Int64()
for i := range out {
out[i] = ffi.PoStCandidateWithTicket{
out[i] = storage.PoStCandidateWithTicket{
Candidate: abi.PoStCandidate{
SectorID: abi.SectorID{
Number: abi.SectorNumber((int(start) + i) % len(sectorInfo)),
@ -308,7 +307,7 @@ func (sb *SBMock) StageFakeData() (abi.SectorNumber, []abi.PieceInfo, error) {
buf := make([]byte, usize)
rand.Read(buf)
pi, err := sb.AddPiece(context.TODO(), usize, sid, bytes.NewReader(buf), nil)
pi, err := sb.AddPiece(context.TODO(), sid, nil, usize, bytes.NewReader(buf))
if err != nil {
return 0, nil, err
}

2
storage/sealing/garbage.go
View File

@ -25,7 +25,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e
out := make([]Piece, len(sizes))
for i, size := range sizes {
ppi, err := m.sealer.AddPiece(ctx, size, sectorID, m.pledgeReader(size), existingPieceSizes)
ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, m.pledgeReader(size))
if err != nil {
return nil, xerrors.Errorf("add piece: %w", err)
}

2
storage/sealing/sealing.go
View File

@ -116,7 +116,7 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector
func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, dealID abi.DealID) error {
log.Infof("Seal piece for deal %d", dealID)
ppi, err := m.sealer.AddPiece(ctx, size, sectorID, r, []abi.UnpaddedPieceSize{})
ppi, err := m.sealer.AddPiece(ctx, sectorID, []abi.UnpaddedPieceSize{}, size, r)
if err != nil {
return xerrors.Errorf("adding piece to sector: %w", err)
}

23
storage/sealmgr/advmgr/local.go
View File

@ -4,6 +4,7 @@ import (
"context"
"io"
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
@ -23,15 +24,7 @@ type localWorkerPathProvider struct {
w *localWorker
}
func (l *localWorkerPathProvider) AcquireSectorNumber() (abi.SectorNumber, error) {
return 0, xerrors.Errorf("unsupported")
}
func (l *localWorkerPathProvider) FinalizeSector(abi.SectorNumber) error {
return xerrors.Errorf("unsupported")
}
func (l *localWorkerPathProvider) AcquireSector(id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
mid, err := address.IDFromAddress(l.w.scfg.Miner)
if err != nil {
return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("get miner ID: %w", err)
@ -44,16 +37,16 @@ func (l *localWorker) sb() (sectorbuilder.Basic, error) {
return sectorbuilder.New(&localWorkerPathProvider{w: l}, l.scfg)
}
func (l *localWorker) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, epcs []abi.UnpaddedPieceSize) (abi.PieceInfo, error) {
func (l *localWorker) AddPiece(ctx context.Context, sn abi.SectorNumber, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) {
sb, err := l.sb()
if err != nil {
return abi.PieceInfo{}, err
}
return sb.AddPiece(ctx, sz, sn, r, epcs)
return sb.AddPiece(ctx, sn, epcs, sz, r)
}
func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) {
func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) {
sb, err := l.sb()
if err != nil {
return nil, err
@ -62,7 +55,7 @@ func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNu
return sb.SealPreCommit1(ctx, sectorNum, ticket, pieces)
}
func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
sb, err := l.sb()
if err != nil {
return cid.Undef, cid.Undef, err
@ -71,7 +64,7 @@ func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNu
return sb.SealPreCommit2(ctx, sectorNum, phase1Out)
}
func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) {
func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) {
sb, err := l.sb()
if err != nil {
return nil, err
@ -80,7 +73,7 @@ func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumbe
return sb.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID)
}
func (l *localWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) {
func (l *localWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) {
sb, err := l.sb()
if err != nil {
return nil, err

15
storage/sealmgr/advmgr/manager.go
View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/sealmgr"
@ -49,7 +50,7 @@ type Manager struct {
storage *storage
sectorbuilder.Prover
storage2.Prover
}
func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manager, error) {
@ -151,7 +152,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.Stor
return workers, paths
}
func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) {
func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) {
// TODO: consider multiple paths vs workers when initially allocating
var best []config.StorageMeta
@ -173,10 +174,10 @@ func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi
// TODO: select(candidateWorkers, ...)
// TODO: remove the sectorbuilder abstraction, pass path directly
return candidateWorkers[0].AddPiece(ctx, sz, sn, r, existingPieces)
return candidateWorkers[0].AddPiece(ctx, sn, existingPieces, sz, r)
}
func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) {
func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) {
// TODO: also consider where the unsealed data sits
best, err := m.storage.findBestAllocStorage(sectorbuilder.FTCache|sectorbuilder.FTSealed, true)
@ -191,7 +192,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber
return candidateWorkers[0].SealPreCommit1(ctx, sectorNum, ticket, pieces)
}
func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
// TODO: allow workers to fetch the sectors
best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed)
@ -206,7 +207,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber
return candidateWorkers[0].SealPreCommit2(ctx, sectorNum, phase1Out)
}
func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) {
func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) {
best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed)
if err != nil {
return nil, xerrors.Errorf("finding path for sector sealing: %w", err)
@ -219,7 +220,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, t
return candidateWorkers[0].SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID)
}
func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) {
func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) {
for _, worker := range m.workers {
if _, ok := worker.TaskTypes()[sealmgr.TTCommit2]; !ok {
continue

12
storage/sealmgr/advmgr/roprov.go
View File

@ -1,6 +1,8 @@
package advmgr
import (
"context"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"golang.org/x/xerrors"
@ -11,15 +13,7 @@ type readonlyProvider struct {
stor *storage
}
func (l *readonlyProvider) AcquireSectorNumber() (abi.SectorNumber, error) {
return 0, xerrors.New("read-only provider")
}
func (l *readonlyProvider) FinalizeSector(abi.SectorNumber) error {
return xerrors.New("read-only provider")
}
func (l *readonlyProvider) AcquireSector(id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
if allocate != 0 {
return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage")
}

19
storage/sealmgr/simple.go
View File

@ -12,8 +12,7 @@ import (
"github.com/filecoin-project/go-fil-markets/storedcounter"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/specs-storage/storage"
)
type LocalWorker struct {
@ -61,35 +60,35 @@ func (s *Simple) NewSector() (abi.SectorNumber, error) {
return abi.SectorNumber(n), nil
}
func (s *Simple) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sectorNum abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) {
func (s *Simple) AddPiece(ctx context.Context, sectorNum abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r storage.Data) (abi.PieceInfo, error) {
s.rateLimiter.Lock()
defer s.rateLimiter.Unlock()
return s.worker.AddPiece(ctx, sz, sectorNum, r, existingPieces)
return s.worker.AddPiece(ctx, sectorNum, existingPieces, sz, r)
}
func (s *Simple) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) {
func (s *Simple) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
s.rateLimiter.Lock()
defer s.rateLimiter.Unlock()
return s.worker.SealPreCommit1(ctx, sectorNum, ticket, pieces)
}
func (s *Simple) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
func (s *Simple) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
s.rateLimiter.Lock()
defer s.rateLimiter.Unlock()
return s.worker.SealPreCommit2(ctx, sectorNum, phase1Out)
}
func (s *Simple) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) {
func (s *Simple) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage.Commit1Out, err error) {
s.rateLimiter.Lock()
defer s.rateLimiter.Unlock()
return s.worker.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID)
}
func (s *Simple) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) {
func (s *Simple) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage.Commit1Out) (proof storage.Proof, err error) {
s.rateLimiter.Lock()
defer s.rateLimiter.Unlock()
@ -103,11 +102,11 @@ func (s *Simple) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber)
return s.worker.FinalizeSector(ctx, sectorNum)
}
func (s *Simple) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) {
func (s *Simple) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) {
return s.worker.GenerateEPostCandidates(sectorInfo, challengeSeed, faults)
}
func (s *Simple) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) {
func (s *Simple) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, []abi.PoStProof, error) {
return s.worker.GenerateFallbackPoSt(sectorInfo, challengeSeed, faults)
}

8
storage/sealmgr/types.go
View File

@ -4,14 +4,16 @@ import (
"context"
"io"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/specs-storage/storage"
)
type Worker interface {
sectorbuilder.Sealer
sectorbuilder.Prover
storage.Prover
}
type Manager interface {
@ -28,5 +30,5 @@ type Manager interface {
ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error)
sectorbuilder.Sealer
sectorbuilder.Prover
storage.Prover
}