lotus/extern/storage-sealing/sealing.go

510 lines
15 KiB
Go
Raw Normal View History

package sealing
import (
"context"
"errors"
"io"
2020-08-18 14:20:31 +00:00
"math"
2020-07-01 13:30:25 +00:00
"sync"
"time"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
padreader "github.com/filecoin-project/go-padreader"
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
2021-01-13 21:19:10 +00:00
"github.com/filecoin-project/go-state-types/dline"
2020-12-02 20:47:45 +00:00
"github.com/filecoin-project/go-state-types/network"
statemachine "github.com/filecoin-project/go-statemachine"
2020-12-02 20:47:45 +00:00
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
2020-09-17 15:30:15 +00:00
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
2020-08-17 13:39:33 +00:00
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
)
const SectorStorePrefix = "/sectors"
2020-09-11 11:28:09 +00:00
var ErrTooManySectorsSealing = xerrors.New("too many sectors sealing")
var log = logging.Logger("sectors")
2020-07-15 14:51:02 +00:00
type SectorLocation struct {
2020-07-15 14:53:17 +00:00
Deadline uint64
2020-07-15 14:51:02 +00:00
Partition uint64
}
var ErrSectorAllocated = errors.New("sectorNumber is allocated, but PreCommit info wasn't found on chain")
type SealingAPI interface {
StateWaitMsg(context.Context, cid.Cid) (MsgLookup, error)
StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error)
2020-06-15 13:13:35 +00:00
StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tok TipSetToken) (cid.Cid, error)
// Can return ErrSectorAllocated in case precommit info wasn't found, but the sector number is marked as allocated
2020-04-06 21:03:47 +00:00
StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*miner.SectorPreCommitOnChainInfo, error)
2020-05-28 00:10:50 +00:00
StateSectorGetInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*miner.SectorOnChainInfo, error)
2020-07-15 14:51:02 +00:00
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*SectorLocation, error)
2020-04-06 20:23:37 +00:00
StateMinerSectorSize(context.Context, address.Address, TipSetToken) (abi.SectorSize, error)
StateMinerWorkerAddress(ctx context.Context, maddr address.Address, tok TipSetToken) (address.Address, error)
StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error)
2020-06-26 15:58:29 +00:00
StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error)
StateMinerInfo(context.Context, address.Address, TipSetToken) (miner.MinerInfo, error)
2020-10-13 19:35:29 +00:00
StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, TipSetToken) (bool, error)
StateMarketStorageDeal(context.Context, abi.DealID, TipSetToken) (market.DealProposal, error)
2020-09-17 02:34:13 +00:00
StateNetworkVersion(ctx context.Context, tok TipSetToken) (network.Version, error)
2021-01-13 21:19:10 +00:00
StateMinerProvingDeadline(context.Context, address.Address, TipSetToken) (*dline.Info, error)
StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tok TipSetToken) ([]api.Partition, error)
2020-08-12 17:47:00 +00:00
SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error)
ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error)
2020-08-11 23:58:35 +00:00
ChainGetRandomnessFromBeacon(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
ChainGetRandomnessFromTickets(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
2020-04-15 18:00:29 +00:00
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
}
type SectorStateNotifee func(before, after SectorInfo)
2020-12-02 20:47:45 +00:00
type AddrSel func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error)
type Sealing struct {
api SealingAPI
2020-08-12 17:47:00 +00:00
feeCfg FeeConfig
events Events
maddr address.Address
2020-03-23 11:40:02 +00:00
sealer sectorstorage.SectorManager
sectors *statemachine.StateGroup
2020-03-17 20:19:52 +00:00
sc SectorIDCounter
verif ffiwrapper.Verifier
pcp PreCommitPolicy
unsealedInfoMap UnsealedSectorMap
2020-07-01 13:30:25 +00:00
upgradeLk sync.Mutex
toUpgrade map[abi.SectorNumber]struct{}
notifee SectorStateNotifee
2020-12-02 20:47:45 +00:00
addrSel AddrSel
stats SectorStats
2021-01-12 23:42:01 +00:00
terminator *TerminateBatcher
2020-08-18 14:20:31 +00:00
getConfig GetSealingConfigFunc
2020-06-23 21:47:53 +00:00
}
2020-08-12 17:47:00 +00:00
type FeeConfig struct {
MaxPreCommitGasFee abi.TokenAmount
MaxCommitGasFee abi.TokenAmount
2021-01-12 23:42:01 +00:00
MaxTerminateGasFee abi.TokenAmount
2020-06-23 21:47:53 +00:00
}
type UnsealedSectorMap struct {
infos map[abi.SectorNumber]UnsealedSectorInfo
2020-08-18 14:20:31 +00:00
lk sync.Mutex
}
2020-06-23 21:47:53 +00:00
type UnsealedSectorInfo struct {
2020-07-20 16:43:19 +00:00
numDeals uint64
// stored should always equal sum of pieceSizes.Padded()
2020-07-30 12:22:05 +00:00
stored abi.PaddedPieceSize
2020-06-23 21:47:53 +00:00
pieceSizes []abi.UnpaddedPieceSize
ssize abi.SectorSize
}
2020-12-02 20:47:45 +00:00
func New(api SealingAPI, fc FeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing {
s := &Sealing{
2020-01-16 02:54:57 +00:00
api: api,
2020-08-12 17:47:00 +00:00
feeCfg: fc,
events: events,
maddr: maddr,
sealer: sealer,
sc: sc,
verif: verif,
pcp: pcp,
unsealedInfoMap: UnsealedSectorMap{
infos: make(map[abi.SectorNumber]UnsealedSectorInfo),
2020-08-18 14:20:31 +00:00
lk: sync.Mutex{},
},
2020-07-01 14:33:59 +00:00
2020-08-18 14:20:31 +00:00
toUpgrade: map[abi.SectorNumber]struct{}{},
notifee: notifee,
2020-12-02 20:47:45 +00:00
addrSel: as,
2021-01-12 23:42:01 +00:00
terminator: NewTerminationBatcher(context.TODO(), maddr, api, as, fc),
2020-08-18 14:20:31 +00:00
getConfig: gc,
stats: SectorStats{
bySector: map[abi.SectorID]statSectorState{},
},
}
s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{})
s.unsealedInfoMap.lk.Lock() // released after initialized in .Run()
return s
}
func (m *Sealing) Run(ctx context.Context) error {
if err := m.restartSectors(ctx); err != nil {
log.Errorf("%+v", err)
return xerrors.Errorf("failed load sector states: %w", err)
}
return nil
}
func (m *Sealing) Stop(ctx context.Context) error {
2021-01-12 23:42:01 +00:00
if err := m.terminator.Stop(ctx); err != nil {
return err
}
if err := m.sectors.Stop(ctx); err != nil {
return err
}
return nil
}
2020-07-30 12:22:05 +00:00
func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) {
log.Infof("Adding piece for deal %d (publish msg: %s)", d.DealID, d.PublishCid)
2020-02-11 01:10:50 +00:00
if (padreader.PaddedSize(uint64(size))) != size {
return 0, 0, xerrors.Errorf("cannot allocate unpadded piece")
}
2020-06-23 19:32:22 +00:00
sp, err := m.currentSealProof(ctx)
if err != nil {
return 0, 0, xerrors.Errorf("getting current seal proof type: %w", err)
}
ssize, err := sp.SectorSize()
if err != nil {
return 0, 0, err
}
if size > abi.PaddedPieceSize(ssize).Unpadded() {
return 0, 0, xerrors.Errorf("piece cannot fit into a sector")
}
2020-08-18 14:20:31 +00:00
m.unsealedInfoMap.lk.Lock()
sid, pads, err := m.getSectorAndPadding(ctx, size)
if err != nil {
2020-08-18 14:20:31 +00:00
m.unsealedInfoMap.lk.Unlock()
return 0, 0, xerrors.Errorf("getting available sector: %w", err)
}
for _, p := range pads {
2020-08-14 14:06:53 +00:00
err = m.addPiece(ctx, sid, p.Unpadded(), NewNullReader(p.Unpadded()), nil)
if err != nil {
2020-08-18 14:20:31 +00:00
m.unsealedInfoMap.lk.Unlock()
return 0, 0, xerrors.Errorf("writing pads: %w", err)
}
}
offset := m.unsealedInfoMap.infos[sid].stored
err = m.addPiece(ctx, sid, size, r, &d)
if err != nil {
2020-08-18 14:20:31 +00:00
m.unsealedInfoMap.lk.Unlock()
return 0, 0, xerrors.Errorf("adding piece to sector: %w", err)
}
startPacking := m.unsealedInfoMap.infos[sid].numDeals >= getDealPerSectorLimit(ssize)
2020-08-18 14:20:31 +00:00
m.unsealedInfoMap.lk.Unlock()
if startPacking {
2020-07-30 12:22:05 +00:00
if err := m.StartPacking(sid); err != nil {
return 0, 0, xerrors.Errorf("start packing: %w", err)
}
2020-07-20 16:43:19 +00:00
}
return sid, offset, nil
}
2020-08-18 14:20:31 +00:00
// Caller should hold m.unsealedInfoMap.lk
func (m *Sealing) addPiece(ctx context.Context, sectorID abi.SectorNumber, size abi.UnpaddedPieceSize, r io.Reader, di *DealInfo) error {
log.Infof("Adding piece to sector %d", sectorID)
sp, err := m.currentSealProof(ctx)
if err != nil {
return xerrors.Errorf("getting current seal proof type: %w", err)
}
ssize, err := sp.SectorSize()
if err != nil {
return err
}
ppi, err := m.sealer.AddPiece(sectorstorage.WithPriority(ctx, DealSectorPriority), m.minerSector(sp, sectorID), m.unsealedInfoMap.infos[sectorID].pieceSizes, size, r)
if err != nil {
return xerrors.Errorf("writing piece: %w", err)
}
piece := Piece{
Piece: ppi,
DealInfo: di,
}
err = m.sectors.Send(uint64(sectorID), SectorAddPiece{NewPiece: piece})
2020-06-23 21:47:53 +00:00
if err != nil {
return err
}
ui := m.unsealedInfoMap.infos[sectorID]
2020-07-20 16:43:19 +00:00
num := m.unsealedInfoMap.infos[sectorID].numDeals
if di != nil {
num = num + 1
}
m.unsealedInfoMap.infos[sectorID] = UnsealedSectorInfo{
2020-07-20 16:43:19 +00:00
numDeals: num,
2020-07-30 12:22:05 +00:00
stored: ui.stored + piece.Piece.Size,
2020-06-23 21:47:53 +00:00
pieceSizes: append(ui.pieceSizes, piece.Piece.Size.Unpadded()),
ssize: ssize,
2020-06-23 21:47:53 +00:00
}
return nil
}
2020-06-23 19:32:22 +00:00
func (m *Sealing) Remove(ctx context.Context, sid abi.SectorNumber) error {
return m.sectors.Send(uint64(sid), SectorRemove{})
}
2021-01-12 23:42:01 +00:00
func (m *Sealing) Terminate(ctx context.Context, sid abi.SectorNumber) error {
return m.sectors.Send(uint64(sid), SectorTerminate{})
}
2021-01-13 22:32:04 +00:00
func (m *Sealing) TerminateFlush(ctx context.Context) (*cid.Cid, error) {
return m.terminator.Flush(ctx)
}
2021-01-14 11:37:23 +00:00
func (m *Sealing) TerminatePending(ctx context.Context) ([]abi.SectorID, error) {
return m.terminator.Pending(ctx)
}
2020-08-18 14:20:31 +00:00
// Caller should NOT hold m.unsealedInfoMap.lk
func (m *Sealing) StartPacking(sectorID abi.SectorNumber) error {
// locking here ensures that when the SectorStartPacking event is sent, the sector won't be picked up anywhere else
m.unsealedInfoMap.lk.Lock()
defer m.unsealedInfoMap.lk.Unlock()
// cannot send SectorStartPacking to sectors that have already been packed, otherwise it will cause the state machine to exit
if _, ok := m.unsealedInfoMap.infos[sectorID]; !ok {
log.Warnf("call start packing, but sector %v not in unsealedInfoMap.infos, maybe have called", sectorID)
return nil
}
log.Infof("Starting packing sector %d", sectorID)
2020-06-23 21:47:53 +00:00
err := m.sectors.Send(uint64(sectorID), SectorStartPacking{})
if err != nil {
return err
}
log.Infof("send Starting packing event success sector %d", sectorID)
2020-06-23 21:47:53 +00:00
delete(m.unsealedInfoMap.infos, sectorID)
2020-06-23 21:47:53 +00:00
return nil
}
2020-08-18 14:20:31 +00:00
// Caller should hold m.unsealedInfoMap.lk
func (m *Sealing) getSectorAndPadding(ctx context.Context, size abi.UnpaddedPieceSize) (abi.SectorNumber, []abi.PaddedPieceSize, error) {
for tries := 0; tries < 100; tries++ {
for k, v := range m.unsealedInfoMap.infos {
pads, padLength := ffiwrapper.GetRequiredPadding(v.stored, size.Padded())
if v.stored+size.Padded()+padLength <= abi.PaddedPieceSize(v.ssize) {
return k, pads, nil
}
}
if len(m.unsealedInfoMap.infos) > 0 {
log.Infow("tried to put a piece into an open sector, found none with enough space", "open", len(m.unsealedInfoMap.infos), "size", size, "tries", tries)
}
ns, ssize, err := m.newDealSector(ctx)
switch err {
case nil:
m.unsealedInfoMap.infos[ns] = UnsealedSectorInfo{
numDeals: 0,
stored: 0,
pieceSizes: nil,
ssize: ssize,
}
case errTooManySealing:
m.unsealedInfoMap.lk.Unlock()
2020-11-25 17:55:58 +00:00
select {
case <-time.After(2 * time.Second):
case <-ctx.Done():
m.unsealedInfoMap.lk.Lock()
return 0, nil, xerrors.Errorf("getting sector for piece: %w", ctx.Err())
}
m.unsealedInfoMap.lk.Lock()
continue
default:
return 0, nil, xerrors.Errorf("creating new sector: %w", err)
}
return ns, nil, nil
}
return 0, nil, xerrors.Errorf("failed to allocate piece to a sector")
}
var errTooManySealing = errors.New("too many sectors sealing")
2020-08-18 14:20:31 +00:00
// newDealSector creates a new sector for deal storage
func (m *Sealing) newDealSector(ctx context.Context) (abi.SectorNumber, abi.SectorSize, error) {
2020-08-18 14:20:31 +00:00
// First make sure we don't have too many 'open' sectors
cfg, err := m.getConfig()
if err != nil {
return 0, 0, xerrors.Errorf("getting config: %w", err)
2020-08-18 14:20:31 +00:00
}
if cfg.MaxSealingSectorsForDeals > 0 {
if m.stats.curSealing() > cfg.MaxSealingSectorsForDeals {
return 0, 0, ErrTooManySectorsSealing
}
}
2020-11-25 17:55:58 +00:00
if cfg.MaxWaitDealsSectors > 0 && uint64(len(m.unsealedInfoMap.infos)) >= cfg.MaxWaitDealsSectors {
// Too many sectors are sealing in parallel. Start sealing one, and retry
// allocating the piece to a sector (we're dropping the lock here, so in
// case other goroutines are also trying to create a sector, we retry in
// getSectorAndPadding instead of here - otherwise if we have lots of
// parallel deals in progress, we can start creating a ton of sectors
// with just a single deal in them)
2020-11-25 17:55:58 +00:00
var mostStored abi.PaddedPieceSize = math.MaxUint64
var best abi.SectorNumber = math.MaxUint64
2020-08-18 14:20:31 +00:00
2020-11-25 17:55:58 +00:00
for sn, info := range m.unsealedInfoMap.infos {
if info.stored+1 > mostStored+1 { // 18446744073709551615 + 1 = 0
best = sn
2020-08-18 14:20:31 +00:00
}
2020-11-25 17:55:58 +00:00
}
2020-08-18 14:20:31 +00:00
2020-11-25 17:55:58 +00:00
if best != math.MaxUint64 {
m.unsealedInfoMap.lk.Unlock()
err := m.StartPacking(best)
m.unsealedInfoMap.lk.Lock()
2020-08-18 14:20:31 +00:00
2020-11-25 17:55:58 +00:00
if err != nil {
log.Errorf("newDealSector StartPacking error: %+v", err)
// let's pretend this is fine
2020-08-18 14:20:31 +00:00
}
}
2020-11-25 17:55:58 +00:00
return 0, 0, errTooManySealing // will wait a bit and retry
2020-08-18 14:20:31 +00:00
}
spt, err := m.currentSealProof(ctx)
if err != nil {
return 0, 0, xerrors.Errorf("getting current seal proof type: %w", err)
}
// Now actually create a new sector
sid, err := m.sc.Next()
if err != nil {
return 0, 0, xerrors.Errorf("getting sector number: %w", err)
}
err = m.sealer.NewSector(context.TODO(), m.minerSector(spt, sid))
if err != nil {
return 0, 0, xerrors.Errorf("initializing sector: %w", err)
}
2020-06-23 19:32:22 +00:00
log.Infof("Creating sector %d", sid)
2020-06-23 21:47:53 +00:00
err = m.sectors.Send(uint64(sid), SectorStart{
2020-03-22 20:44:27 +00:00
ID: sid,
SectorType: spt,
})
2020-06-23 21:47:53 +00:00
if err != nil {
return 0, 0, xerrors.Errorf("starting the sector fsm: %w", err)
2020-06-23 21:47:53 +00:00
}
2020-08-18 14:20:31 +00:00
cf, err := m.getConfig()
if err != nil {
return 0, 0, xerrors.Errorf("getting the sealing delay: %w", err)
}
2020-08-18 14:20:31 +00:00
if cf.WaitDealsDelay > 0 {
timer := time.NewTimer(cf.WaitDealsDelay)
go func() {
<-timer.C
if err := m.StartPacking(sid); err != nil {
log.Errorf("starting sector %d: %+v", sid, err)
}
}()
}
ssize, err := spt.SectorSize()
return sid, ssize, err
}
2020-03-17 20:19:52 +00:00
2020-06-23 19:32:22 +00:00
// newSectorCC accepts a slice of pieces with no deal (junk data)
func (m *Sealing) newSectorCC(ctx context.Context, sid abi.SectorNumber, pieces []Piece) error {
spt, err := m.currentSealProof(ctx)
if err != nil {
return xerrors.Errorf("getting current seal proof type: %w", err)
}
2020-06-23 19:32:22 +00:00
log.Infof("Creating CC sector %d", sid)
return m.sectors.Send(uint64(sid), SectorStartCC{
ID: sid,
Pieces: pieces,
SectorType: spt,
2020-06-23 19:32:22 +00:00
})
2020-06-22 16:42:38 +00:00
}
func (m *Sealing) currentSealProof(ctx context.Context) (abi.RegisteredSealProof, error) {
mi, err := m.api.StateMinerInfo(ctx, m.maddr, nil)
if err != nil {
return 0, err
}
return miner.PreferredSealProofTypeFromWindowPoStType(mi.WindowPoStProofType)
}
func (m *Sealing) minerSector(spt abi.RegisteredSealProof, num abi.SectorNumber) storage.SectorRef {
return storage.SectorRef{
ID: m.minerSectorID(num),
ProofType: spt,
}
}
func (m *Sealing) minerSectorID(num abi.SectorNumber) abi.SectorID {
2020-03-17 20:19:52 +00:00
mid, err := address.IDFromAddress(m.maddr)
if err != nil {
panic(err)
}
return abi.SectorID{
Number: num,
2020-03-18 01:08:11 +00:00
Miner: abi.ActorID(mid),
2020-03-17 20:19:52 +00:00
}
}
func (m *Sealing) Address() address.Address {
return m.maddr
}
2020-07-20 16:43:19 +00:00
func getDealPerSectorLimit(size abi.SectorSize) uint64 {
if size < 64<<30 {
return 256
}
return 512
2020-07-20 16:43:19 +00:00
}