Fix widowed PoSt scheduler
This commit is contained in:
parent
f53bd6bdba
commit
af87b9aa98
@ -253,7 +253,9 @@ type FullNode interface {
|
|||||||
// StateMinerInfo returns info about the indicated miner
|
// StateMinerInfo returns info about the indicated miner
|
||||||
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error)
|
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error)
|
||||||
// StateMinerDeadlines returns all the proving deadlines for the given miner
|
// StateMinerDeadlines returns all the proving deadlines for the given miner
|
||||||
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error)
|
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error)
|
||||||
|
// StateMinerPartitions loads miner partitions for the specified miner/deadline
|
||||||
|
StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)
|
||||||
// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
|
// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
|
||||||
StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
|
StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
|
||||||
// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
|
// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
|
||||||
|
@ -131,7 +131,8 @@ type FullNodeStruct struct {
|
|||||||
StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"`
|
StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"`
|
||||||
StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"`
|
StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"`
|
||||||
StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"`
|
StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"`
|
||||||
StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error) `perm:"read"`
|
StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) `perm:"read"`
|
||||||
|
StateMinerPartitions func(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)
|
||||||
StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"`
|
StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"`
|
||||||
StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
|
StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
|
||||||
StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"`
|
StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error) `perm:"read"`
|
||||||
@ -593,10 +594,14 @@ func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Addre
|
|||||||
return c.Internal.StateMinerInfo(ctx, actor, tsk)
|
return c.Internal.StateMinerInfo(ctx, actor, tsk)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) {
|
func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) {
|
||||||
return c.Internal.StateMinerDeadlines(ctx, m, tsk)
|
return c.Internal.StateMinerDeadlines(ctx, m, tsk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *FullNodeStruct) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) {
|
||||||
|
return c.Internal.StateMinerPartitions(ctx, m, dlIdx, tsk)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
|
func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
|
||||||
return c.Internal.StateMinerFaults(ctx, actor, tsk)
|
return c.Internal.StateMinerFaults(ctx, actor, tsk)
|
||||||
}
|
}
|
||||||
|
@ -323,36 +323,6 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetMinerDeadlines(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*miner.Deadlines, error) {
|
|
||||||
var mas miner.State
|
|
||||||
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return mas.LoadDeadlines(sm.cs.Store(ctx))
|
|
||||||
}
|
|
||||||
|
|
||||||
/*func GetMinerFaults(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*abi.BitField, error) {
|
|
||||||
var mas miner.State
|
|
||||||
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("(get faults) failed to load miner actor state: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return mas.Faults, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetMinerRecoveries(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*abi.BitField, error) {
|
|
||||||
var mas miner.State
|
|
||||||
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("(get recoveries) failed to load miner actor state: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return mas.Recoveries, nil
|
|
||||||
}*/
|
|
||||||
|
|
||||||
func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) {
|
func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) {
|
||||||
var state market.State
|
var state market.State
|
||||||
if _, err := sm.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil {
|
if _, err := sm.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil {
|
||||||
|
@ -88,12 +88,30 @@ func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, ts
|
|||||||
return api.NewApiMinerInfo(mi), nil
|
return api.NewApiMinerInfo(mi), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) {
|
func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) {
|
||||||
ts, err := a.Chain.GetTipSetFromKey(tsk)
|
var out []*miner.Deadline
|
||||||
if err != nil {
|
return out, a.StateManager.WithParentStateTsk(tsk,
|
||||||
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
|
a.StateManager.WithActor(m,
|
||||||
}
|
a.StateManager.WithActorState(ctx,
|
||||||
return stmgr.GetMinerDeadlines(ctx, a.StateManager, ts, m)
|
a.StateManager.WithDeadlines(
|
||||||
|
a.StateManager.WithEachDeadline(
|
||||||
|
func(store adt.Store, idx uint64, deadline *miner.Deadline) error {
|
||||||
|
out = append(out, deadline)
|
||||||
|
return nil
|
||||||
|
})))))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) {
|
||||||
|
var out []*miner.Partition
|
||||||
|
return out, a.StateManager.WithParentStateTsk(tsk,
|
||||||
|
a.StateManager.WithActor(m,
|
||||||
|
a.StateManager.WithActorState(ctx,
|
||||||
|
a.StateManager.WithDeadlines(
|
||||||
|
a.StateManager.WithDeadline(dlIdx,
|
||||||
|
a.StateManager.WithEachPartition(func(store adt.Store, partIdx uint64, partition *miner.Partition) error {
|
||||||
|
out = append(out, partition)
|
||||||
|
return nil
|
||||||
|
}))))))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
|
func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
|
||||||
|
@ -72,7 +72,7 @@ func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr ad
|
|||||||
return mi.Worker, nil
|
return mi.Worker, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (*miner.Deadlines, error) {
|
func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]*miner.Deadline, error) {
|
||||||
tsk, err := types.TipSetKeyFromBytes(tok)
|
tsk, err := types.TipSetKeyFromBytes(tok)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err)
|
return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err)
|
||||||
@ -184,7 +184,6 @@ func (s SealingAPIAdapter) StateSectorGetInfo(ctx context.Context, maddr address
|
|||||||
return s.delegate.StateSectorGetInfo(ctx, maddr, sectorNumber, tsk)
|
return s.delegate.StateSectorGetInfo(ctx, maddr, sectorNumber, tsk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*sealing.SectorLocation, error) {
|
func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*sealing.SectorLocation, error) {
|
||||||
tsk, err := types.TipSetKeyFromBytes(tok)
|
tsk, err := types.TipSetKeyFromBytes(tok)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -48,7 +48,8 @@ type Miner struct {
|
|||||||
type storageMinerApi interface {
|
type storageMinerApi interface {
|
||||||
// Call a read only method on actors (no interaction with the chain required)
|
// Call a read only method on actors (no interaction with the chain required)
|
||||||
StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error)
|
StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error)
|
||||||
StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) (*miner.Deadlines, error)
|
StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) ([]*miner.Deadline, error)
|
||||||
|
StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)
|
||||||
StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)
|
StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)
|
||||||
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
|
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
|
||||||
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
|
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
|
||||||
|
@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-bitfield"
|
"github.com/filecoin-project/go-bitfield"
|
||||||
@ -107,66 +108,65 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check *abi.BitFi
|
|||||||
return &sbf, nil
|
return &sbf, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline uint64, deadlineSectors *abi.BitField, ts *types.TipSet) error {
|
func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) error {
|
||||||
faults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key())
|
ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries")
|
||||||
if err != nil {
|
defer span.End()
|
||||||
return xerrors.Errorf("getting on-chain faults: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fc, err := faults.Count()
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("counting faulty sectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fc == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
recov, err := s.api.StateMinerRecoveries(ctx, s.actor, ts.Key())
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("getting on-chain recoveries: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
unrecovered, err := bitfield.SubtractBitField(faults, recov)
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("subtracting recovered set from fault set: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
unrecovered, err = bitfield.IntersectBitField(unrecovered, deadlineSectors)
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("intersect unrecovered set with deadlineSectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
uc, err := unrecovered.Count()
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("counting unrecovered sectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if uc == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
sbf, err := s.checkSectors(ctx, unrecovered)
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("checking unrecovered sectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// if all sectors failed to recover, don't declare recoveries
|
|
||||||
sbfCount, err := sbf.Count()
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("counting recovered sectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if sbfCount == 0 {
|
|
||||||
log.Warnw("No recoveries to declare", "deadline", deadline, "faulty", uc)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
params := &miner.DeclareFaultsRecoveredParams{
|
params := &miner.DeclareFaultsRecoveredParams{
|
||||||
Recoveries: []miner.RecoveryDeclaration{{Deadline: deadline, Sectors: sbf}},
|
Recoveries: []miner.RecoveryDeclaration{},
|
||||||
}
|
}
|
||||||
|
|
||||||
enc, aerr := actors.SerializeParams(params)
|
faulty := uint64(0)
|
||||||
|
|
||||||
|
for partIdx, partition := range partitions {
|
||||||
|
unrecovered, err := bitfield.SubtractBitField(partition.Faults, partition.Recoveries)
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("subtracting recovered set from fault set: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
uc, err := unrecovered.Count()
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("counting unrecovered sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if uc == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
faulty += uc
|
||||||
|
|
||||||
|
recovered, err := s.checkSectors(ctx, unrecovered)
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("checking unrecovered sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// if all sectors failed to recover, don't declare recoveries
|
||||||
|
recoveredCount, err := recovered.Count()
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("counting recovered sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if recoveredCount == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
params.Recoveries = append(params.Recoveries, miner.RecoveryDeclaration{
|
||||||
|
Deadline: dlIdx,
|
||||||
|
Partition: uint64(partIdx),
|
||||||
|
Sectors: recovered,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if len(params.Recoveries) == 0 {
|
||||||
|
if faulty != 0 {
|
||||||
|
log.Warnw("No recoveries to declare", "deadline", dlIdx, "faulty", faulty)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
enc, aerr := actors.SerializeParams(params)
|
||||||
if aerr != nil {
|
if aerr != nil {
|
||||||
return xerrors.Errorf("could not serialize declare recoveries parameters: %w", aerr)
|
return xerrors.Errorf("could not serialize declare recoveries parameters: %w", aerr)
|
||||||
}
|
}
|
||||||
@ -200,51 +200,56 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, deadline
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, deadline uint64, deadlineSectors *abi.BitField, ts *types.TipSet) error {
|
func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) error {
|
||||||
dc, err := deadlineSectors.Count()
|
ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults")
|
||||||
if err != nil {
|
defer span.End()
|
||||||
return xerrors.Errorf("counting deadline sectors: %w", err)
|
|
||||||
}
|
|
||||||
if dc == 0 {
|
|
||||||
// nothing can become faulty
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
toCheck, err := s.getSectorsToProve(ctx, deadlineSectors, true, ts)
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("getting next sectors to prove: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
good, err := s.checkSectors(ctx, deadlineSectors)
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("checking sectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
faulty, err := bitfield.SubtractBitField(toCheck, good)
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("calculating faulty sector set: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c, err := faulty.Count()
|
|
||||||
if err != nil {
|
|
||||||
return xerrors.Errorf("counting faulty sectors: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if c == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Errorw("DETECTED FAULTY SECTORS, declaring faults", "count", c)
|
|
||||||
|
|
||||||
params := &miner.DeclareFaultsParams{
|
params := &miner.DeclareFaultsParams{
|
||||||
Faults: []miner.FaultDeclaration{
|
Faults: []miner.FaultDeclaration{},
|
||||||
{
|
|
||||||
Deadline: deadline,
|
|
||||||
Sectors: faulty,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bad := uint64(0)
|
||||||
|
|
||||||
|
for partIdx, partition := range partitions {
|
||||||
|
toCheck, err := partition.ActiveSectors()
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("getting active sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
good, err := s.checkSectors(ctx, toCheck)
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("checking sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
faulty, err := bitfield.SubtractBitField(toCheck, good)
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("calculating faulty sector set: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c, err := faulty.Count()
|
||||||
|
if err != nil {
|
||||||
|
return xerrors.Errorf("counting faulty sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if c == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
bad += c
|
||||||
|
|
||||||
|
params.Faults = append(params.Faults, miner.FaultDeclaration{
|
||||||
|
Deadline: dlIdx,
|
||||||
|
Partition: uint64(partIdx),
|
||||||
|
Sectors: faulty,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(params.Faults) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Errorw("DETECTED FAULTY SECTORS, declaring faults", "count", bad)
|
||||||
|
|
||||||
enc, aerr := actors.SerializeParams(params)
|
enc, aerr := actors.SerializeParams(params)
|
||||||
if aerr != nil {
|
if aerr != nil {
|
||||||
return xerrors.Errorf("could not serialize declare faults parameters: %w", aerr)
|
return xerrors.Errorf("could not serialize declare faults parameters: %w", aerr)
|
||||||
@ -279,76 +284,37 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, deadline uint
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// the input sectors must match with the miner actor
|
|
||||||
func (s *WindowPoStScheduler) getSectorsToProve(ctx context.Context, deadlineSectors *abi.BitField, ignoreRecoveries bool, ts *types.TipSet) (*abi.BitField, error) {
|
|
||||||
stateFaults, err := s.api.StateMinerFaults(ctx, s.actor, ts.Key())
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("getting on-chain faults: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
faults, err := bitfield.IntersectBitField(deadlineSectors, stateFaults)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("failed to intersect proof sectors with faults: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
recoveries, err := s.api.StateMinerRecoveries(ctx, s.actor, ts.Key())
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("getting on-chain recoveries: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !ignoreRecoveries {
|
|
||||||
expectedRecoveries, err := bitfield.IntersectBitField(faults, recoveries)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("failed to intersect recoveries with faults: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
faults, err = bitfield.SubtractBitField(faults, expectedRecoveries)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("failed to subtract recoveries from faults: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
nonFaults, err := bitfield.SubtractBitField(deadlineSectors, faults)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("failed to diff bitfields: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
empty, err := nonFaults.IsEmpty()
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("failed to check if bitfield was empty: %w", err)
|
|
||||||
}
|
|
||||||
if empty {
|
|
||||||
return nil, xerrors.Errorf("no non-faulty sectors in partitions: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nonFaults, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo, ts *types.TipSet) (*miner.SubmitWindowedPoStParams, error) {
|
func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo, ts *types.TipSet) (*miner.SubmitWindowedPoStParams, error) {
|
||||||
ctx, span := trace.StartSpan(ctx, "storage.runPost")
|
ctx, span := trace.StartSpan(ctx, "storage.runPost")
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
deadlines, err := s.api.StateMinerDeadlines(ctx, s.actor, ts.Key())
|
var declWait sync.WaitGroup
|
||||||
if err != nil {
|
defer declWait.Wait()
|
||||||
return nil, xerrors.Errorf("getting miner deadlines: %w", err)
|
declWait.Add(1)
|
||||||
}
|
|
||||||
|
go func(){
|
||||||
|
defer declWait.Done()
|
||||||
|
|
||||||
{
|
|
||||||
// check faults / recoveries for the *next* deadline. It's already too
|
// check faults / recoveries for the *next* deadline. It's already too
|
||||||
// late to declare them for this deadline
|
// late to declare them for this deadline
|
||||||
declDeadline := (di.Index + 1) % miner.WPoStPeriodDeadlines
|
declDeadline := (di.Index + 1) % miner.WPoStPeriodDeadlines
|
||||||
|
|
||||||
if err := s.checkNextRecoveries(ctx, declDeadline, deadlines.Due[declDeadline], ts); err != nil {
|
partitions, err := s.api.StateMinerPartitions(ctx, s.actor, declDeadline, ts.Key())
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("getting partitions: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.checkNextRecoveries(ctx, declDeadline, partitions); err != nil {
|
||||||
// TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse
|
// TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse
|
||||||
log.Errorf("checking sector recoveries: %v", err)
|
log.Errorf("checking sector recoveries: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.checkNextFaults(ctx, declDeadline, deadlines.Due[declDeadline], ts); err != nil {
|
if err := s.checkNextFaults(ctx, declDeadline, partitions); err != nil {
|
||||||
// TODO: This is also potentially really bad, but we try to post anyways
|
// TODO: This is also potentially really bad, but we try to post anyways
|
||||||
log.Errorf("checking sector faults: %v", err)
|
log.Errorf("checking sector faults: %v", err)
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
}
|
|
||||||
|
|
||||||
buf := new(bytes.Buffer)
|
buf := new(bytes.Buffer)
|
||||||
if err := s.actor.MarshalCBOR(buf); err != nil {
|
if err := s.actor.MarshalCBOR(buf); err != nil {
|
||||||
@ -359,64 +325,69 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
|
|||||||
return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err)
|
return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
firstPartition, _, err := miner.PartitionsForDeadline(deadlines, s.partitionSectors, di.Index)
|
partitions, err := s.api.StateMinerPartitions(ctx, s.actor, di.Index, ts.Key())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("getting partitions for deadline: %w", err)
|
return nil, xerrors.Errorf("getting partitions: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
partitionCount, _, err := miner.DeadlineCount(deadlines, s.partitionSectors, di.Index)
|
params := &miner.SubmitWindowedPoStParams{
|
||||||
if err != nil {
|
Deadline: di.Index,
|
||||||
return nil, xerrors.Errorf("getting deadline partition count: %w", err)
|
Partitions: make([]miner.PoStPartition, len(partitions)),
|
||||||
|
Proofs: nil,
|
||||||
}
|
}
|
||||||
|
|
||||||
dc, err := deadlines.Due[di.Index].Count()
|
var sinfos []abi.SectorInfo
|
||||||
if err != nil {
|
sidToPart := map[abi.SectorNumber]uint64{}
|
||||||
return nil, xerrors.Errorf("get deadline count: %w", err)
|
skipCount := uint64(0)
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("di: %+v", di)
|
for partIdx, partition := range partitions {
|
||||||
log.Infof("dc: %+v", dc)
|
// TODO: Can do this in parallel
|
||||||
log.Infof("fp: %+v", firstPartition)
|
toProve, err := partition.ActiveSectors()
|
||||||
log.Infof("pc: %+v", partitionCount)
|
if err != nil {
|
||||||
log.Infof("ts: %+v (%d)", ts.Key(), ts.Height())
|
return nil, xerrors.Errorf("getting active sectors: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
if partitionCount == 0 {
|
toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries)
|
||||||
return nil, errNoPartitions
|
if err != nil {
|
||||||
}
|
return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
partitions := make([]uint64, partitionCount)
|
good, err := s.checkSectors(ctx, toProve)
|
||||||
for i := range partitions {
|
if err != nil {
|
||||||
partitions[i] = firstPartition + uint64(i)
|
return nil, xerrors.Errorf("checking sectors to skip: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
nps, err := s.getSectorsToProve(ctx, deadlines.Due[di.Index], false, ts)
|
skipped, err := bitfield.SubtractBitField(toProve, good)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("get need prove sectors: %w", err)
|
return nil, xerrors.Errorf("toProve - good: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
good, err := s.checkSectors(ctx, nps)
|
sc, err := skipped.Count()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("checking sectors to skip: %w", err)
|
return nil, xerrors.Errorf("getting skipped sector count: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
skipped, err := bitfield.SubtractBitField(nps, good)
|
skipCount += sc
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("nps - good: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
skipCount, err := skipped.Count()
|
ssi, err := s.sectorInfo(ctx, good, ts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("getting skipped sector count: %w", err)
|
return nil, xerrors.Errorf("getting sorted sector info: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ssi, err := s.sortedSectorInfo(ctx, good, ts)
|
sinfos = append(sinfos, ssi...)
|
||||||
if err != nil {
|
for _, si := range ssi {
|
||||||
return nil, xerrors.Errorf("getting sorted sector info: %w", err)
|
sidToPart[si.SectorNumber] = uint64(partIdx)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(ssi) == 0 {
|
if len(ssi) == 0 {
|
||||||
log.Warn("attempted to run windowPost without any sectors...")
|
log.Warn("attempted to run windowPost without any sectors...")
|
||||||
return nil, xerrors.Errorf("no sectors to run windowPost on")
|
return nil, xerrors.Errorf("no sectors to run windowPost on")
|
||||||
|
}
|
||||||
|
|
||||||
|
params.Partitions[partIdx] = miner.PoStPartition{
|
||||||
|
Index: uint64(partIdx),
|
||||||
|
Skipped: skipped,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infow("running windowPost",
|
log.Infow("running windowPost",
|
||||||
@ -425,22 +396,16 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
|
|||||||
"height", ts.Height(),
|
"height", ts.Height(),
|
||||||
"skipped", skipCount)
|
"skipped", skipCount)
|
||||||
|
|
||||||
var snums []abi.SectorNumber
|
|
||||||
for _, si := range ssi {
|
|
||||||
snums = append(snums, si.SectorNumber)
|
|
||||||
}
|
|
||||||
|
|
||||||
tsStart := time.Now()
|
tsStart := time.Now()
|
||||||
|
|
||||||
log.Infow("generating windowPost",
|
log.Infow("generating windowPost", "sectors", len(sinfos))
|
||||||
"sectors", len(ssi))
|
|
||||||
|
|
||||||
mid, err := address.IDFromAddress(s.actor)
|
mid, err := address.IDFromAddress(s.actor)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
postOut, postSkipped, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), ssi, abi.PoStRandomness(rand))
|
postOut, postSkipped, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, xerrors.Errorf("running post failed: %w", err)
|
return nil, xerrors.Errorf("running post failed: %w", err)
|
||||||
}
|
}
|
||||||
@ -450,21 +415,16 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, sector := range postSkipped {
|
for _, sector := range postSkipped {
|
||||||
skipped.Set(uint64(sector.Number))
|
params.Partitions[sidToPart[sector.Number]].Skipped.Set(uint64(sector.Number))
|
||||||
}
|
}
|
||||||
|
|
||||||
elapsed := time.Since(tsStart)
|
elapsed := time.Since(tsStart)
|
||||||
log.Infow("submitting window PoSt", "elapsed", elapsed)
|
log.Infow("submitting window PoSt", "elapsed", elapsed)
|
||||||
|
|
||||||
return &miner.SubmitWindowedPoStParams{
|
return params, nil
|
||||||
Deadline: di.Index,
|
|
||||||
Partitions: partitions,
|
|
||||||
Proofs: postOut,
|
|
||||||
Skipped: *skipped,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WindowPoStScheduler) sortedSectorInfo(ctx context.Context, deadlineSectors *abi.BitField, ts *types.TipSet) ([]abi.SectorInfo, error) {
|
func (s *WindowPoStScheduler) sectorInfo(ctx context.Context, deadlineSectors *abi.BitField, ts *types.TipSet) ([]abi.SectorInfo, error) {
|
||||||
sset, err := s.api.StateMinerSectors(ctx, s.actor, deadlineSectors, false, ts.Key())
|
sset, err := s.api.StateMinerSectors(ctx, s.actor, deadlineSectors, false, ts.Key())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
Loading…
Reference in New Issue
Block a user