From 2e95a536790113df7bcc95bc9215301fda23b65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 17 Dec 2019 23:23:43 +0100 Subject: [PATCH] Wire up faults in fPoSt --- chain/actors/actor_miner.go | 4 +- chain/types/blockheader.go | 6 +-- lib/jsonrpc/websocket.go | 2 +- lib/sectorbuilder/scrub.go | 84 ++++++++++++++++++++++++++++++ lib/sectorbuilder/sectorbuilder.go | 25 ++++++--- lib/sectorbuilder/simple.go | 6 +-- storage/fpost_run.go | 57 +++++++++++++++++++- 7 files changed, 166 insertions(+), 18 deletions(-) create mode 100644 lib/sectorbuilder/scrub.go diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 59ccf0b2a..0379fda4d 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -508,7 +508,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM } if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize, - sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID); !ok || lerr != nil { + sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, 0); !ok || lerr != nil { // TODO: FORK - set faults to len(faults) if lerr != nil { // TODO: study PoST errors return nil, aerrors.Absorb(lerr, 4, "PoST error") @@ -824,7 +824,7 @@ func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMConte self.LastFaultSubmission = vmctx.BlockHeight() nstate, aerr := vmctx.Storage().Put(self) - if err != nil { + if err != nil { // TODO: FORK: should be aerr return nil, aerr } if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index e44ca54b4..aaa41cb2f 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -176,7 +176,7 @@ const sha256bits = 256 func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow BigInt) bool { ssize := NewInt(ssizeI) - ssampled := ElectionPostChallengeCount(snum) + ssampled := ElectionPostChallengeCount(snum, 0) // TODO: faults in epost? /* Need to check that (h(vrfout) + 1) / (max(h) + 1) <= e * sectorSize / totalPower @@ -213,12 +213,12 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big return lhs.Cmp(rhs) < 0 } -func ElectionPostChallengeCount(sectors uint64) uint64 { +func ElectionPostChallengeCount(sectors uint64, faults int) uint64 { if sectors == 0 { return 0 } // ceil(sectors / build.SectorChallengeRatioDiv) - return (sectors-1)/build.SectorChallengeRatioDiv + 1 + return (sectors-uint64(faults)-1)/build.SectorChallengeRatioDiv + 1 } func (t *Ticket) Equals(ot *Ticket) bool { diff --git a/lib/jsonrpc/websocket.go b/lib/jsonrpc/websocket.go index 5ed47ce4f..23da51598 100644 --- a/lib/jsonrpc/websocket.go +++ b/lib/jsonrpc/websocket.go @@ -423,7 +423,7 @@ func (c *wsConn) handleWsConn(ctx context.Context) { if !ok { if c.incomingErr != nil { if !websocket.IsCloseError(c.incomingErr, websocket.CloseNormalClosure) { - log.Warnw("websocket error", "error", c.incomingErr) + log.Debugw("websocket error", "error", c.incomingErr) } } return // remote closed diff --git a/lib/sectorbuilder/scrub.go b/lib/sectorbuilder/scrub.go new file mode 100644 index 000000000..a6ed8ce7c --- /dev/null +++ b/lib/sectorbuilder/scrub.go @@ -0,0 +1,84 @@ +package sectorbuilder + +import ( + "io/ioutil" + "os" + "path/filepath" + + sectorbuilder "github.com/filecoin-project/filecoin-ffi" + "golang.org/x/xerrors" +) + +type Fault struct { + SectorID uint64 + + Err error +} + +func (sb *SectorBuilder) Scrub(sectorSet sectorbuilder.SortedPublicSectorInfo) []*Fault { + var faults []*Fault + + for _, sector := range sectorSet.Values() { + err := sb.checkSector(sector.SectorID) + if err != nil { + faults = append(faults, &Fault{SectorID: sector.SectorID, Err: err}) + } + } + + return faults +} + +func (sb *SectorBuilder) checkSector(sectorID uint64) error { + cache, err := sb.sectorCacheDir(sectorID) + if err != nil { + return xerrors.Errorf("getting sector cache dir: %w", err) + } + + if err := assertFile(filepath.Join(cache, "p_aux"), 96, 96); err != nil { + return err + } + if err := assertFile(filepath.Join(cache, "sc-01-data-tree-r-last.dat"), (2*sb.ssize)-32, (2*sb.ssize)-32); err != nil { + return err + } + + // TODO: better validate this + if err := assertFile(filepath.Join(cache, "t_aux"), 100, 32000); err != nil { // TODO: what should this actually be? + return err + } + + dent, err := ioutil.ReadDir(cache) + if err != nil { + return xerrors.Errorf("reading cache dir %s", cache) + } + if len(dent) != 3 { + return xerrors.Errorf("found %d files in %s, expected 3", len(dent), cache) + } + + sealed, err := sb.SealedSectorPath(sectorID) + if err != nil { + return xerrors.Errorf("getting sealed sector path: %w", err) + } + + if err := assertFile(filepath.Join(sealed), sb.ssize, sb.ssize); err != nil { + return err + } + + return nil +} + +func assertFile(path string, minSz uint64, maxSz uint64) error { + st, err := os.Stat(path) + if err != nil { + return xerrors.Errorf("stat %s: %w", path, err) + } + + if st.IsDir() { + return xerrors.Errorf("expected %s to be a regular file", path) + } + + if uint64(st.Size()) < minSz || uint64(st.Size()) > maxSz { + return xerrors.Errorf("%s wasn't within size bounds, expected %d < f < %d, got %d", minSz, maxSz, st.Size()) + } + + return nil +} diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 59f8bd176..7564f1c6b 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -623,7 +623,7 @@ func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, var cseed [CommLen]byte copy(cseed[:], challengeSeed) - privsects, err := sb.pubSectorToPriv(sectorInfo) + privsects, err := sb.pubSectorToPriv(sectorInfo, nil) // TODO: faults if err != nil { return nil, err } @@ -634,20 +634,29 @@ func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, } func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) { - privsectors, err := sb.pubSectorToPriv(sectorInfo) + privsectors, err := sb.pubSectorToPriv(sectorInfo, faults) if err != nil { return nil, err } - challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults)) proverID := addressToProverID(sb.Miner) return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors) } -func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo) (SortedPrivateSectorInfo, error) { +func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo, faults []uint64) (SortedPrivateSectorInfo, error) { + fmap := map[uint64]struct{}{} + for _, fault := range faults { + fmap[fault] = struct{}{} + } + var out []sectorbuilder.PrivateSectorInfo for _, s := range sectorInfo.Values() { + if _, faulty := fmap[s.SectorID]; faulty { + continue + } + cachePath, err := sb.sectorCacheDir(s.SectorID) if err != nil { return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err) @@ -669,12 +678,12 @@ func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo) (Sor } func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) { - privsectors, err := sb.pubSectorToPriv(sectorInfo) + privsectors, err := sb.pubSectorToPriv(sectorInfo, faults) if err != nil { return nil, nil, err } - challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults)) proverID := addressToProverID(sb.Miner) candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors) @@ -690,8 +699,8 @@ func (sb *SectorBuilder) Stop() { close(sb.stopping) } -func fallbackPostChallengeCount(sectors uint64) uint64 { - challengeCount := types.ElectionPostChallengeCount(sectors) +func fallbackPostChallengeCount(sectors uint64, faults int) uint64 { + challengeCount := types.ElectionPostChallengeCount(sectors, faults) if challengeCount > build.MaxFallbackPostChallengeCount { return build.MaxFallbackPostChallengeCount } diff --git a/lib/sectorbuilder/simple.go b/lib/sectorbuilder/simple.go index 457c4b2e1..082489b8a 100644 --- a/lib/sectorbuilder/simple.go +++ b/lib/sectorbuilder/simple.go @@ -37,12 +37,12 @@ func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedP } func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), 0) return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID) } -func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values()))) +func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address, faults int) (bool, error) { + challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), faults) return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID) } diff --git a/storage/fpost_run.go b/storage/fpost_run.go index 5f453188b..713a1c072 100644 --- a/storage/fpost_run.go +++ b/storage/fpost_run.go @@ -50,6 +50,57 @@ func (s *fpostScheduler) doPost(ctx context.Context, eps uint64, ts *types.TipSe }() } +func (s *fpostScheduler) checkFaults(ctx context.Context, ssi sectorbuilder.SortedPublicSectorInfo) ([]uint64, error) { + faults := s.sb.Scrub(ssi) + var faultIDs []uint64 + + if len(faults) > 0 { + params := &actors.DeclareFaultsParams{Faults: types.NewBitField()} + + for _, fault := range faults { + log.Warnf("fault detected: sector %d: %s", fault.SectorID, fault.Err) + faultIDs = append(faultIDs, fault.SectorID) + + // TODO: omit already declared (with finality in mind though..) + params.Faults.Set(fault.SectorID) + } + + log.Warnf("DECLARING %d FAULTS", len(faults)) + + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return nil, xerrors.Errorf("could not serialize declare faults parameters: %w", aerr) + } + + msg := &types.Message{ + To: s.actor, + From: s.worker, + Method: actors.MAMethods.DeclareFaults, + Params: enc, + Value: types.NewInt(0), + GasLimit: types.NewInt(10000000), // i dont know help + GasPrice: types.NewInt(1), + } + + sm, err := s.api.MpoolPushMessage(ctx, msg) + if err != nil { + return nil, xerrors.Errorf("pushing faults message to mpool: %w", err) + } + + rec, err := s.api.StateWaitMsg(ctx, sm.Cid()) + if err != nil { + return nil, xerrors.Errorf("waiting for declare faults: %w", err) + } + + if rec.Receipt.ExitCode != 0 { + return nil, xerrors.Errorf("declare faults exit %d", rec.Receipt.ExitCode) + } + log.Infof("Faults declared successfully") + } + + return faultIDs, nil +} + func (s *fpostScheduler) runPost(ctx context.Context, eps uint64, ts *types.TipSet) (*actors.SubmitFallbackPoStParams, error) { ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() @@ -68,8 +119,12 @@ func (s *fpostScheduler) runPost(ctx context.Context, eps uint64, ts *types.TipS log.Infow("running fPoSt", "chain-random", rand, "eps", eps, "height", ts.Height()) + faults, err := s.checkFaults(ctx, ssi) + if err != nil { + log.Errorf("Failed to declare faults: %+v", err) + } + tsStart := time.Now() - var faults []uint64 // TODO var seed [32]byte copy(seed[:], rand)