From 5e98b8a0d62b7772f932a1ec867bee9c19868498 Mon Sep 17 00:00:00 2001 From: Aayush Date: Tue, 2 Aug 2022 15:28:43 -0400 Subject: [PATCH] add a test for Post max sectors recovery --- .circleci/config.yml | 6 +- ...k_config_test.go => wdpost_config_test.go} | 114 ++++++++++++++++++ storage/wdpost/wdpost_run_faults.go | 20 +-- 3 files changed, 127 insertions(+), 13 deletions(-) rename itests/{wdpost_check_config_test.go => wdpost_config_test.go} (61%) diff --git a/.circleci/config.yml b/.circleci/config.yml index dad86860a..b4d7f87be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -976,9 +976,9 @@ workflows: target: "./itests/verifreg_test.go" - test: - name: test-itest-wdpost_check_config - suite: itest-wdpost_check_config - target: "./itests/wdpost_check_config_test.go" + name: test-itest-wdpost_config + suite: itest-wdpost_config + target: "./itests/wdpost_config_test.go" - test: name: test-itest-wdpost_dispute diff --git a/itests/wdpost_check_config_test.go b/itests/wdpost_config_test.go similarity index 61% rename from itests/wdpost_check_config_test.go rename to itests/wdpost_config_test.go index cc10af04f..61b64387d 100644 --- a/itests/wdpost_check_config_test.go +++ b/itests/wdpost_config_test.go @@ -186,3 +186,117 @@ func TestWindowPostNoPreChecks(t *testing.T) { sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged } + +func TestWindowPostMaxSectorsRecoveryConfig(t *testing.T) { + oldVal := wdpost.RecoveringSectorLimit + defer func() { + wdpost.RecoveringSectorLimit = oldVal + }() + wdpost.RecoveringSectorLimit = 1 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client, miner, ens := kit.EnsembleMinimal(t, + kit.LatestActorsAt(-1), + kit.MockProofs()) + ens.InterconnectAll().BeginMining(2 * time.Millisecond) + + nSectors := 10 + + miner.PledgeSectors(ctx, nSectors, 0, nil) + + maddr, err := miner.ActorAddress(ctx) + require.NoError(t, err) + di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + mid, err := address.IDFromAddress(maddr) + require.NoError(t, err) + + t.Log("Running one proving period") + waitUntil := di.Open + di.WPoStProvingPeriod + t.Logf("End for head.Height > %d", waitUntil) + + ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) + + p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + ssz, err := miner.ActorSectorSize(ctx, maddr) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+kit.DefaultPresealsPerBootstrapMiner))) + + t.Log("Drop some sectors") + + // Drop 2 sectors from deadline 2 partition 0 (full partition / deadline) + parts, err := client.StateMinerPartitions(ctx, maddr, 2, types.EmptyTSK) + require.NoError(t, err) + require.Greater(t, len(parts), 0) + + secs := parts[0].AllSectors + n, err := secs.Count() + require.NoError(t, err) + require.Equal(t, uint64(2), n) + + // Drop the partition + err = secs.ForEach(func(sid uint64) error { + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sid), + }, + }, true) + }) + require.NoError(t, err) + + di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + t.Log("Go through another PP, wait for sectors to become faulty") + waitUntil = di.Open + di.WPoStProvingPeriod + t.Logf("End for head.Height > %d", waitUntil) + + ts = client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) + + p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + + sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz) + require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-2, int(sectors)) // -2 just removed sectors + + t.Log("Make the sectors recoverable") + + err = secs.ForEach(func(sid uint64) error { + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sid), + }, + }, false) + }) + require.NoError(t, err) + + di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + waitUntil = di.Open + di.WPoStProvingPeriod + 200 + t.Logf("End for head.Height > %d", waitUntil) + + ts = client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) + + p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + + sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) + require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-1, int(sectors)) // -1 not recovered sector +} diff --git a/storage/wdpost/wdpost_run_faults.go b/storage/wdpost/wdpost_run_faults.go index 59cb9b441..22186b551 100644 --- a/storage/wdpost/wdpost_run_faults.go +++ b/storage/wdpost/wdpost_run_faults.go @@ -22,12 +22,12 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -var recoveringSectorLimit uint64 = 0 +var RecoveringSectorLimit uint64 = 0 func init() { if rcl := os.Getenv("LOTUS_RECOVERING_SECTOR_LIMIT"); rcl != "" { var err error - recoveringSectorLimit, err = strconv.ParseUint(rcl, 10, 64) + RecoveringSectorLimit, err = strconv.ParseUint(rcl, 10, 64) if err != nil { log.Errorw("parsing LOTUS_RECOVERING_SECTOR_LIMIT", "error", err) } @@ -94,14 +94,14 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 } // rules to follow if we have indicated that we don't want to recover more than X sectors in a deadline - if recoveringSectorLimit > 0 { + if RecoveringSectorLimit > 0 { // something weird happened, break because we can't recover any more - if recoveringSectorLimit < totalSectorsToRecover { - log.Warnf("accepted more recoveries (%d) than recoveringSectorLimit (%d)", totalSectorsToRecover, recoveringSectorLimit) + if RecoveringSectorLimit < totalSectorsToRecover { + log.Warnf("accepted more recoveries (%d) than RecoveringSectorLimit (%d)", totalSectorsToRecover, RecoveringSectorLimit) break } - maxNewRecoverable := recoveringSectorLimit - totalSectorsToRecover + maxNewRecoverable := RecoveringSectorLimit - totalSectorsToRecover // we need to trim the recover bitfield if recoveredCount > maxNewRecoverable { @@ -111,9 +111,9 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 break } - log.Warnf("only adding %d sectors to respect recoveringSectorLimit %d", maxNewRecoverable, recoveringSectorLimit) + log.Warnf("only adding %d sectors to respect RecoveringSectorLimit %d", maxNewRecoverable, RecoveringSectorLimit) - recovered = bitfield.NewFromSet(recoverySlice[0:maxNewRecoverable]) + recovered = bitfield.NewFromSet(recoverySlice[:maxNewRecoverable]) recoveredCount = maxNewRecoverable } } @@ -132,9 +132,9 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 totalSectorsToRecover += recoveredCount - if recoveringSectorLimit > 0 && totalSectorsToRecover >= recoveringSectorLimit { + if RecoveringSectorLimit > 0 && totalSectorsToRecover >= RecoveringSectorLimit { log.Errorf("reached recovering sector limit %d, only marking %d sectors for recovery now", - recoveringSectorLimit, + RecoveringSectorLimit, totalSectorsToRecover) break }