From 663b3f4f215365878a1a6ec6ba6cdf177e0e3803 Mon Sep 17 00:00:00 2001 From: s1m0n21 Date: Thu, 26 Nov 2020 15:02:43 +0800 Subject: [PATCH 1/4] add a sector check command --- api/api_storage.go | 3 + api/apistruct/struct.go | 6 ++ cmd/lotus-storage-miner/proving.go | 102 +++++++++++++++++++++++++++++ extern/sector-storage/faults.go | 18 ++--- node/impl/storminer.go | 15 +++++ storage/wdpost_run.go | 3 +- 6 files changed, 136 insertions(+), 11 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 738a05e09..2ddbb9d12 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/specs-storage/storage" ) // StorageMiner is a low-level interface to the Filecoin network storage miner node @@ -116,6 +117,8 @@ type StorageMiner interface { // LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that // the path specified when calling CreateBackup is within the base path CreateBackup(ctx context.Context, fpath string) error + + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) } type SealRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index ba25e2f0f..d60756bf8 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -363,6 +363,8 @@ type StorageMinerStruct struct { PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"` CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` + + CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) `perm:"admin"` } } @@ -1510,6 +1512,10 @@ func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) err return c.Internal.CreateBackup(ctx, fpath) } +func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) { + return c.Internal.CheckProvable(ctx, pp, sectors) +} + // WorkerStruct func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) { diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 377b81d32..f053af054 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -10,11 +10,14 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/specs-storage/storage" ) var provingCmd = &cli.Command{ @@ -25,6 +28,7 @@ var provingCmd = &cli.Command{ provingDeadlinesCmd, provingDeadlineInfoCmd, provingFaultsCmd, + provingCheckProvableCmd, }, } @@ -371,3 +375,101 @@ var provingDeadlineInfoCmd = &cli.Command{ return nil }, } + +var provingCheckProvableCmd = &cli.Command{ + Name: "check", + Usage: "Check sectors provable", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "only-bad", + Usage: "print only bad sectors", + Value: false, + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("must pass deadline index") + } + + dlIdx, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) + if err != nil { + return xerrors.Errorf("could not parse deadline index: %w", err) + } + + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + sapi, scloser, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer scloser() + + ctx := lcli.ReqContext(cctx) + + addr, err := sapi.ActorAddress(ctx) + if err != nil { + return err + } + + mid, err := address.IDFromAddress(addr) + if err != nil { + return err + } + + info, err := api.StateMinerInfo(ctx, addr, types.EmptyTSK) + if err != nil { + return err + } + + pf, err := info.SealProofType.RegisteredWindowPoStProof() + if err != nil { + return err + } + + partitions, err := api.StateMinerPartitions(ctx, addr, dlIdx, types.EmptyTSK) + if err != nil { + return err + } + + tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "deadline\tpartition\tsector\tstatus") + + for parIdx, par := range partitions { + sectors := make(map[abi.SectorNumber]struct{}) + + sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.AllSectors, types.EmptyTSK) + + var tocheck []storage.SectorRef + for _, info := range sectorInfos { + sectors[info.SectorNumber] = struct{}{} + tocheck = append(tocheck, storage.SectorRef{ + ProofType: info.SealProof, + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: info.SectorNumber, + }, + }) + } + + bad, err := sapi.CheckProvable(ctx, pf, tocheck) + if err != nil { + return err + } + + for s := range sectors { + if err, exist := bad[s]; exist { + _, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\n", dlIdx, parIdx, s, color.RedString("bad")+fmt.Sprintf(" (%s)", err)) + } else if !cctx.Bool("only-bad") { + _, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\n", dlIdx, parIdx, s, color.GreenString("good")) + } + } + } + + return tw.Flush() + }, +} diff --git a/extern/sector-storage/faults.go b/extern/sector-storage/faults.go index 4b98c8187..7af81f002 100644 --- a/extern/sector-storage/faults.go +++ b/extern/sector-storage/faults.go @@ -16,12 +16,12 @@ import ( // FaultTracker TODO: Track things more actively type FaultTracker interface { - CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) } // CheckProvable returns unprovable sectors -func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) { - var bad []abi.SectorID +func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { + var bad = make(map[abi.SectorID]string) ssize, err := pp.SectorSize() if err != nil { @@ -41,20 +41,20 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, if !locked { log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector) - bad = append(bad, sector.ID) + bad[sector.ID] = fmt.Sprint("can't acquire read lock") return nil } lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) if err != nil { log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err) - bad = append(bad, sector.ID) + bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err) return nil } if lp.Sealed == "" || lp.Cache == "" { - log.Warnw("CheckProvable Sector FAULT: cache an/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache) - bad = append(bad, sector.ID) + log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache) + bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", lp.Cache, lp.Sealed) return nil } @@ -70,14 +70,14 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, st, err := os.Stat(p) if err != nil { log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "err", err) - bad = append(bad, sector.ID) + bad[sector.ID] = fmt.Sprintf("%s", err) return nil } if sz != 0 { if st.Size() != int64(ssize)*sz { log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz) - bad = append(bad, sector.ID) + bad[sector.ID] = fmt.Sprintf("%s is wrong size (got %d, expect %d)", p, st.Size(), int64(ssize)*sz) return nil } } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 89c4bbb8a..488b80f89 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -37,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" + sto "github.com/filecoin-project/specs-storage/storage" ) type StorageMinerAPI struct { @@ -543,4 +544,18 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error return backup(sm.DS, fpath) } +func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef) (map[abi.SectorNumber]string, error) { + bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors) + if err != nil { + return nil, err + } + + var out = make(map[abi.SectorNumber]string) + for sid, err := range bad { + out[sid.Number] = err + } + + return out, nil +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 7cb656f30..87dd8ad15 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -212,14 +212,13 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B Number: info.SectorNumber, }, }) - } bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck) if err != nil { return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err) } - for _, id := range bad { + for id := range bad { delete(sectors, id.Number) } From dc7c7608dfea7f7bdfc246a55e3a4a96b91979a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 27 Nov 2020 16:32:12 +0100 Subject: [PATCH 2/4] docsgen --- api/docgen/docgen.go | 3 +++ documentation/en/api-methods-miner.md | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index bc29cf91d..14b1a90f3 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -234,6 +234,9 @@ func init() { }, }) addExample(storiface.ErrorCode(0)) + addExample(map[abi.SectorNumber]string{ + 123: "can't acquire read lock", + }) // worker specific addExample(storiface.AcquireMove) diff --git a/documentation/en/api-methods-miner.md b/documentation/en/api-methods-miner.md index 10429d575..2289bcd21 100644 --- a/documentation/en/api-methods-miner.md +++ b/documentation/en/api-methods-miner.md @@ -10,6 +10,8 @@ * [Auth](#Auth) * [AuthNew](#AuthNew) * [AuthVerify](#AuthVerify) +* [Check](#Check) + * [CheckProvable](#CheckProvable) * [Create](#Create) * [CreateBackup](#CreateBackup) * [Deals](#Deals) @@ -218,6 +220,29 @@ Inputs: Response: `null` +## Check + + +### CheckProvable +There are not yet any comments for this method. + +Perms: admin + +Inputs: +```json +[ + 8, + null +] +``` + +Response: +```json +{ + "123": "can't acquire read lock" +} +``` + ## Create From e8d2560581b4910f5eedc59fdbeae49a5c774c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 27 Nov 2020 16:34:41 +0100 Subject: [PATCH 3/4] Fix tests --- cmd/lotus-storage-miner/proving.go | 3 +++ extern/sector-storage/mock/mock.go | 6 +++--- storage/wdpost_run_test.go | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index f053af054..b930476e6 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -443,6 +443,9 @@ var provingCheckProvableCmd = &cli.Command{ sectors := make(map[abi.SectorNumber]struct{}) sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.AllSectors, types.EmptyTSK) + if err != nil { + return err + } var tocheck []storage.SectorRef for _, info := range sectorInfos { diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 6179f40d7..59d7d0503 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -405,14 +405,14 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro return nil } -func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) ([]abi.SectorID, error) { - var bad []abi.SectorID +func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) (map[abi.SectorID]string, error) { + bad := map[abi.SectorID]string{} for _, sid := range ids { _, found := mgr.sectors[sid.ID] if !found || mgr.sectors[sid.ID].failed { - bad = append(bad, sid.ID) + bad[sid.ID] = "mock fail" } } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index d426f5f31..5b85261c2 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -125,9 +125,9 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si type mockFaultTracker struct { } -func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) { +func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { // Returns "bad" sectors so just return nil meaning all sectors are good - return nil, nil + return map[abi.SectorID]string{}, nil } // TestWDPostDoPost verifies that doPost will send the correct number of window From 054327b72a117793073ca6e45c972bef6521d840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 1 Dec 2020 14:50:51 +0100 Subject: [PATCH 4/4] wdpost: Update comment Co-authored-by: dirkmc --- storage/wdpost_run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 5b85261c2..80b4f66af 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -126,7 +126,7 @@ type mockFaultTracker struct { } func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { - // Returns "bad" sectors so just return nil meaning all sectors are good + // Returns "bad" sectors so just return empty map meaning all sectors are good return map[abi.SectorID]string{}, nil }