cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #5041 from filecoin-project/feat/5013

Browse Source 
Sector check command
This commit is contained in:
Łukasz Magiera 2020-12-01 14:59:01 +01:00 committed by GitHub
commit 614f45dcfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 173 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/api_storage.go
View File

@ -19,6 +19,7 @@ import (
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/specs-storage/storage"
)
// StorageMiner is a low-level interface to the Filecoin network storage miner node
@ -116,6 +117,8 @@ type StorageMiner interface {
// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error)
}
type SealRes struct {

6
api/apistruct/struct.go
View File

@ -363,6 +363,8 @@ type StorageMinerStruct struct {
PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"`
CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) `perm:"admin"`
}
}
@ -1514,6 +1516,10 @@ func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) err
return c.Internal.CreateBackup(ctx, fpath)
}
func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) {
return c.Internal.CheckProvable(ctx, pp, sectors)
}
// WorkerStruct
func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {

3
api/docgen/docgen.go
View File

@ -234,6 +234,9 @@ func init() {
},
})
addExample(storiface.ErrorCode(0))
addExample(map[abi.SectorNumber]string{
123: "can't acquire read lock",
})
// worker specific
addExample(storiface.AcquireMove)

105
cmd/lotus-storage-miner/proving.go
View File

@ -10,11 +10,14 @@ import (
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api/apibstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/specs-storage/storage"
)
var provingCmd = &cli.Command{
@ -25,6 +28,7 @@ var provingCmd = &cli.Command{
provingDeadlinesCmd,
provingDeadlineInfoCmd,
provingFaultsCmd,
provingCheckProvableCmd,
},
}
@ -371,3 +375,104 @@ var provingDeadlineInfoCmd = &cli.Command{
return nil
},
}
var provingCheckProvableCmd = &cli.Command{
Name: "check",
Usage: "Check sectors provable",
ArgsUsage: "<deadlineIdx>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "only-bad",
Usage: "print only bad sectors",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return xerrors.Errorf("must pass deadline index")
}
dlIdx, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse deadline index: %w", err)
}
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
sapi, scloser, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer scloser()
ctx := lcli.ReqContext(cctx)
addr, err := sapi.ActorAddress(ctx)
if err != nil {
return err
}
mid, err := address.IDFromAddress(addr)
if err != nil {
return err
}
info, err := api.StateMinerInfo(ctx, addr, types.EmptyTSK)
if err != nil {
return err
}
pf, err := info.SealProofType.RegisteredWindowPoStProof()
if err != nil {
return err
}
partitions, err := api.StateMinerPartitions(ctx, addr, dlIdx, types.EmptyTSK)
if err != nil {
return err
}
tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
_, _ = fmt.Fprintln(tw, "deadline\tpartition\tsector\tstatus")
for parIdx, par := range partitions {
sectors := make(map[abi.SectorNumber]struct{})
sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.AllSectors, types.EmptyTSK)
if err != nil {
return err
}
var tocheck []storage.SectorRef
for _, info := range sectorInfos {
sectors[info.SectorNumber] = struct{}{}
tocheck = append(tocheck, storage.SectorRef{
ProofType: info.SealProof,
ID: abi.SectorID{
Miner: abi.ActorID(mid),
Number: info.SectorNumber,
},
})
}
bad, err := sapi.CheckProvable(ctx, pf, tocheck)
if err != nil {
return err
}
for s := range sectors {
if err, exist := bad[s]; exist {
_, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\n", dlIdx, parIdx, s, color.RedString("bad")+fmt.Sprintf(" (%s)", err))
} else if !cctx.Bool("only-bad") {
_, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\n", dlIdx, parIdx, s, color.GreenString("good"))
}
}
}
return tw.Flush()
},
}

25
documentation/en/api-methods-miner.md
View File

@ -10,6 +10,8 @@
* [Auth](#Auth)
* [AuthNew](#AuthNew)
* [AuthVerify](#AuthVerify)
* [Check](#Check)
* [CheckProvable](#CheckProvable)
* [Create](#Create)
* [CreateBackup](#CreateBackup)
* [Deals](#Deals)
@ -218,6 +220,29 @@ Inputs:
Response: `null`
## Check
### CheckProvable
There are not yet any comments for this method.
Perms: admin
Inputs:
```json
[
8,
null
]
```
Response:
```json
{
"123": "can't acquire read lock"
}
```
## Create

18
extern/sector-storage/faults.go vendored
View File

@ -16,12 +16,12 @@ import (
// FaultTracker TODO: Track things more actively
type FaultTracker interface {
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error)
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error)
}
// CheckProvable returns unprovable sectors
func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) {
var bad []abi.SectorID
func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) {
var bad = make(map[abi.SectorID]string)
ssize, err := pp.SectorSize()
if err != nil {
@ -41,20 +41,20 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
if !locked {
log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
bad = append(bad, sector.ID)
bad[sector.ID] = fmt.Sprint("can't acquire read lock")
return nil
}
lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
if err != nil {
log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
bad = append(bad, sector.ID)
bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
return nil
}
if lp.Sealed == "" || lp.Cache == "" {
log.Warnw("CheckProvable Sector FAULT: cache an/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
bad = append(bad, sector.ID)
log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", lp.Cache, lp.Sealed)
return nil
}
@ -70,14 +70,14 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
st, err := os.Stat(p)
if err != nil {
log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "err", err)
bad = append(bad, sector.ID)
bad[sector.ID] = fmt.Sprintf("%s", err)
return nil
}
if sz != 0 {
if st.Size() != int64(ssize)*sz {
log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz)
bad = append(bad, sector.ID)
bad[sector.ID] = fmt.Sprintf("%s is wrong size (got %d, expect %d)", p, st.Size(), int64(ssize)*sz)
return nil
}
}

6
extern/sector-storage/mock/mock.go vendored
View File

@ -405,14 +405,14 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro
return nil
}
func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) ([]abi.SectorID, error) {
var bad []abi.SectorID
func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) (map[abi.SectorID]string, error) {
bad := map[abi.SectorID]string{}
for _, sid := range ids {
_, found := mgr.sectors[sid.ID]
if !found || mgr.sectors[sid.ID].failed {
bad = append(bad, sid.ID)
bad[sid.ID] = "mock fail"
}
}

15
node/impl/storminer.go
View File

@ -37,6 +37,7 @@ import (
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
sto "github.com/filecoin-project/specs-storage/storage"
)
type StorageMinerAPI struct {
@ -543,4 +544,18 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error
return backup(sm.DS, fpath)
}
func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef) (map[abi.SectorNumber]string, error) {
bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors)
if err != nil {
return nil, err
}
var out = make(map[abi.SectorNumber]string)
for sid, err := range bad {
out[sid.Number] = err
}
return out, nil
}
var _ api.StorageMiner = &StorageMinerAPI{}

3
storage/wdpost_run.go
View File

@ -212,14 +212,13 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B
Number: info.SectorNumber,
},
})
}
bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err)
}
for _, id := range bad {
for id := range bad {
delete(sectors, id.Number)
}

6
storage/wdpost_run_test.go
View File

@ -125,9 +125,9 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si
type mockFaultTracker struct {
}
func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) {
// Returns "bad" sectors so just return nil meaning all sectors are good
return nil, nil
func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) {
// Returns "bad" sectors so just return empty map meaning all sectors are good
return map[abi.SectorID]string{}, nil
}
// TestWDPostDoPost verifies that doPost will send the correct number of window