From 6dffe9377fa0a162438980d9d25713b0ddcee42f Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 7 Apr 2020 15:34:30 -0700 Subject: [PATCH 1/3] provide deal scheduling info to FSM so that it can set pre-commit expiry --- cmd/lotus-storage-miner/init.go | 14 +++++++++++--- go.mod | 2 +- go.sum | 4 ++-- markets/storageadapter/provider.go | 8 +++++++- node/impl/storminer.go | 4 ++-- node/modules/storageminer.go | 11 ++++++++--- storage/miner.go | 4 +++- storage/sealing.go | 4 ++-- storage/sectorblocks/blocks.go | 8 +++++--- 9 files changed, 41 insertions(+), 18 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 535abf000..2ecfe8b74 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -292,9 +292,17 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, SectorNumber: sector.SectorID, Pieces: []sealing.Piece{ { - DealID: &dealID, - Size: abi.PaddedPieceSize(meta.SectorSize).Unpadded(), - CommP: sector.CommD, + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(meta.SectorSize), + PieceCID: commD, + }, + DealInfo: &sealing.DealInfo{ + DealID: dealID, + DealSchedule: sealing.DealSchedule{ + StartEpoch: sector.Deal.StartEpoch, + EndEpoch: sector.Deal.EndEpoch, + }, + }, }, }, CommD: &commD, diff --git a/go.mod b/go.mod index fef2cc410..e4bf3038a 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/filecoin-project/sector-storage v0.0.0-20200406195014-a6d093838576 github.com/filecoin-project/specs-actors v0.0.0-20200324235424-aef9b20a9fb1 github.com/filecoin-project/specs-storage v0.0.0-20200317225704-7420bc655c38 - github.com/filecoin-project/storage-fsm v0.0.0-20200407163248-25993479b13f + github.com/filecoin-project/storage-fsm v0.0.0-20200408153957-1c356922353f github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 diff --git a/go.sum b/go.sum index c64b6abf6..bd8a7f8bd 100644 --- a/go.sum +++ b/go.sum @@ -176,8 +176,8 @@ github.com/filecoin-project/specs-actors v0.0.0-20200324235424-aef9b20a9fb1 h1:I github.com/filecoin-project/specs-actors v0.0.0-20200324235424-aef9b20a9fb1/go.mod h1:5WngRgTN5Eo4+0SjCBqLzEr2l6Mj45DrP2606gBhqI0= github.com/filecoin-project/specs-storage v0.0.0-20200317225704-7420bc655c38 h1:ky+rfX3bG1TjOBLn14V674q+iwZpalyKzZxGRNzA11I= github.com/filecoin-project/specs-storage v0.0.0-20200317225704-7420bc655c38/go.mod h1:dUmzHS7izOD6HW3/JpzFrjxnptxbsHXBlO8puK2UzBk= -github.com/filecoin-project/storage-fsm v0.0.0-20200407163248-25993479b13f h1:WPA4Lj1PLBVKFe8nKs8UdJ7fj4luFKR62eRDpVNo9CU= -github.com/filecoin-project/storage-fsm v0.0.0-20200407163248-25993479b13f/go.mod h1:t3dyEJiCkdQjo0h75mir3x5a0vSPHi9woQO7UkkZpFg= +github.com/filecoin-project/storage-fsm v0.0.0-20200408153957-1c356922353f h1:ocVYJgS622P5p/LOOPzb875M+wlJHe6in2DcoGcd9J8= +github.com/filecoin-project/storage-fsm v0.0.0-20200408153957-1c356922353f/go.mod h1:9nHIzwfHk6cNXaaNnUJNWNRRytp5QYMBOA+NtcTUKJM= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 1150d92d6..eab3fd203 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -102,7 +102,13 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark } func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, pieceSize abi.UnpaddedPieceSize, pieceData io.Reader) error { - _, err := n.secb.AddPiece(ctx, abi.UnpaddedPieceSize(pieceSize), pieceData, deal.DealID) + _, err := n.secb.AddPiece(ctx, pieceSize, pieceData, sealing.DealInfo{ + DealID: deal.DealID, + DealSchedule: sealing.DealSchedule{ + StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch, + EndEpoch: deal.ClientDealProposal.Proposal.EndEpoch, + }, + }) if err != nil { return xerrors.Errorf("AddPiece failed: %s", err) } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 72b82fea1..d66585161 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -75,10 +75,10 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb deals := make([]abi.DealID, len(info.Pieces)) for i, piece := range info.Pieces { - if piece.DealID == nil { + if piece.DealInfo == nil { continue } - deals[i] = *piece.DealID + deals[i] = piece.DealInfo.DealID } log := make([]api.SectorLog, len(info.Log)) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 0b041e5ae..619af71a0 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -282,10 +282,15 @@ func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode } func SealTicketGen(fapi lapi.FullNode) sealing.TicketFn { - return func(ctx context.Context) (abi.SealRandomness, abi.ChainEpoch, error) { - ts, err := fapi.ChainHead(ctx) + return func(ctx context.Context, tok sealing.TipSetToken) (abi.SealRandomness, abi.ChainEpoch, error) { + tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { - return nil, 0, xerrors.Errorf("getting head ts for SealTicket failed: %w", err) + return nil, 0, xerrors.Errorf("could not unmarshal TipSetToken to TipSetKey: %w", err) + } + + ts, err := fapi.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, 0, xerrors.Errorf("getting TipSet for key failed: %w", err) } r, err := fapi.ChainGetRandomness(ctx, ts.Key(), crypto.DomainSeparationTag_SealRandomness, ts.Height()-build.SealRandomnessLookback, nil) diff --git a/storage/miner.go b/storage/miner.go index 6d0d6f569..9cf7a6aa7 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -99,7 +99,9 @@ func (m *Miner) Run(ctx context.Context) error { } evts := events.NewEvents(ctx, m.api) - m.sealing = sealing.New(NewSealingAPIAdapter(m.api), NewEventsAdapter(evts), m.maddr, m.worker, m.ds, m.sealer, m.sc, m.verif, m.tktFn) + adaptedAPI := NewSealingAPIAdapter(m.api) + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, 100) + m.sealing = sealing.New(adaptedAPI, NewEventsAdapter(evts), m.maddr, m.worker, m.ds, m.sealer, m.sc, m.verif, m.tktFn, &pcp) go m.sealing.Run(ctx) diff --git a/storage/sealing.go b/storage/sealing.go index 3e81bdb87..c8b485379 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -20,8 +20,8 @@ func (m *Miner) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.SectorNu return m.sealing.AllocatePiece(size) } -func (m *Miner) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, dealID abi.DealID) error { - return m.sealing.SealPiece(ctx, size, r, sectorID, dealID) +func (m *Miner) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, d sealing.DealInfo) error { + return m.sealing.SealPiece(ctx, size, r, sectorID, d) } func (m *Miner) ListSectors() ([]sealing.SectorInfo, error) { diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index b8561fffa..0f5c7c5a3 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -8,6 +8,8 @@ import ( "io" "sync" + sealing "github.com/filecoin-project/storage-fsm" + "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-datastore" @@ -96,18 +98,18 @@ func (st *SectorBlocks) writeRef(dealID abi.DealID, sectorID abi.SectorNumber, o return st.keys.Put(DealIDToDsKey(dealID), newRef) // TODO: batch somehow } -func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, dealID abi.DealID) (sectorID abi.SectorNumber, err error) { +func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d sealing.DealInfo) (sectorID abi.SectorNumber, err error) { sectorID, pieceOffset, err := st.Miner.AllocatePiece(padreader.PaddedSize(uint64(size))) if err != nil { return 0, err } - err = st.writeRef(dealID, sectorID, pieceOffset, size) + err = st.writeRef(d.DealID, sectorID, pieceOffset, size) if err != nil { return 0, err } - return sectorID, st.Miner.SealPiece(ctx, size, r, sectorID, dealID) + return sectorID, st.Miner.SealPiece(ctx, size, r, sectorID, d) } func (st *SectorBlocks) List() (map[uint64][]api.SealedRef, error) { From 4a12a984514718ec64ae4e78e63598361acf35b8 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 8 Apr 2020 08:45:21 -0700 Subject: [PATCH 2/3] set pre-commit expiration to 10000000, which was original value in FSM --- storage/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/miner.go b/storage/miner.go index 9cf7a6aa7..bd2533aea 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -100,7 +100,7 @@ func (m *Miner) Run(ctx context.Context) error { evts := events.NewEvents(ctx, m.api) adaptedAPI := NewSealingAPIAdapter(m.api) - pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, 100) + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, 10000000) m.sealing = sealing.New(adaptedAPI, NewEventsAdapter(evts), m.maddr, m.worker, m.ds, m.sealer, m.sc, m.verif, m.tktFn, &pcp) go m.sealing.Run(ctx) From 19573c03191d9c46c7973712bfabb73102673463 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 8 Apr 2020 08:46:54 -0700 Subject: [PATCH 3/3] adjust imports to conform to convention --- storage/sectorblocks/blocks.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 0f5c7c5a3..5b4b3e3c4 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -8,10 +8,6 @@ import ( "io" "sync" - sealing "github.com/filecoin-project/storage-fsm" - - "github.com/filecoin-project/go-padreader" - "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/query" @@ -19,6 +15,9 @@ import ( "golang.org/x/xerrors" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/specs-actors/actors/abi" + sealing "github.com/filecoin-project/storage-fsm" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/modules/dtypes"