Merge pull request #2789 from filecoin-project/fix/post-fault-recover-decls

wdpost: Fix Fault/Recovery declaration timing

api/test/ccupgrade.go

@@ -54,7 +54,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	CC := abi.SectorNumber(GenesisPreseals + 1)
 	Upgraded := CC + 1
 
-	pledgeSectors(t, ctx, miner, 1, nil)
+	pledgeSectors(t, ctx, miner, 1, 0, nil)
 
 	sl, err := miner.SectorsList(ctx)
 	if err != nil {

api/test/window_post.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
 
 	"os"
 	"strings"
@@ -14,10 +12,14 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/sector-storage/mock"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	sealing "github.com/filecoin-project/storage-fsm"
 
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	bminer "github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/impl"
@@ -60,13 +62,13 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect
 		}
 	}()
 
-	pledgeSectors(t, ctx, miner, nSectors, blockNotif)
+	pledgeSectors(t, ctx, miner, nSectors, 0, blockNotif)
 
 	mine = false
 	<-done
 }
 
-func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n int, blockNotif <-chan struct{}) {
+func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, existing int, blockNotif <-chan struct{}) {
 	for i := 0; i < n; i++ {
 		err := miner.PledgeSector(ctx)
 		require.NoError(t, err)
@@ -79,7 +81,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n i
 		s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
 		require.NoError(t, err)
 		fmt.Printf("Sectors: %d\n", len(s))
-		if len(s) >= n {
+		if len(s) >= n + existing {
 			break
 		}
 
@@ -143,7 +145,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		}
 	}()
 
-	pledgeSectors(t, ctx, miner, nSectors, nil)
+	pledgeSectors(t, ctx, miner, nSectors, 0, nil)
 
 	maddr, err := miner.ActorAddress(ctx)
 	require.NoError(t, err)
@@ -151,7 +153,10 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
 	require.NoError(t, err)
 
-	fmt.Printf("Running one proving periods\n")
+	mid, err := address.IDFromAddress(maddr)
+	require.NoError(t, err)
+
+	fmt.Printf("Running one proving period\n")
 
 	for {
 		head, err := client.ChainHead(ctx)
@@ -176,6 +181,8 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	require.Equal(t, p.MinerPower, p.TotalPower)
 	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals)))
 
+	fmt.Printf("Drop some sectors\n")
+
 	// Drop 2 sectors from deadline 2 partition 0 (full partition / deadline)
 	{
 		parts, err := client.StateMinerPartitions(ctx, maddr, 2, types.EmptyTSK)
@@ -188,11 +195,16 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 
 		// Drop the partition
 		err = parts[0].Sectors.ForEach(func(sid uint64) error {
-			return miner.SectorRemove(ctx, abi.SectorNumber(sid))
+			return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(abi.SectorID{
+				Miner:  abi.ActorID(mid),
+				Number: abi.SectorNumber(sid),
+			}, true)
 		})
 		require.NoError(t, err)
 	}
 
+	var s abi.SectorID
+
 	// Drop 1 sectors from deadline 3 partition 0
 	{
 		parts, err := client.StateMinerPartitions(ctx, maddr, 3, types.EmptyTSK)
@@ -204,16 +216,23 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		require.Equal(t, uint64(2), n)
 
 		// Drop the sector
-		s, err := parts[0].Sectors.First()
+		sn, err := parts[0].Sectors.First()
 		require.NoError(t, err)
 
-		err = miner.SectorRemove(ctx, abi.SectorNumber(s))
+		s = abi.SectorID{
+			Miner:  abi.ActorID(mid),
+			Number: abi.SectorNumber(sn),
+		}
+
+		err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, true)
 		require.NoError(t, err)
 	}
 
 	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
 	require.NoError(t, err)
 
+	fmt.Printf("Go through another PP, wait for sectors to become faulty\n")
+
 	for {
 		head, err := client.ChainHead(ctx)
 		require.NoError(t, err)
@@ -236,6 +255,66 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
 	require.Equal(t, nSectors+GenesisPreseals-3, int(sectors)) // -3 just removed sectors
 
+	fmt.Printf("Recover one sector\n")
+
+	err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, false)
+	require.NoError(t, err)
+
+	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	for {
+		head, err := client.ChainHead(ctx)
+		require.NoError(t, err)
+
+		if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
+			break
+		}
+
+		if head.Height()%100 == 0 {
+			fmt.Printf("@%d\n", head.Height())
+		}
+		build.Clock.Sleep(blocktime)
+	}
+
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+
+	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
+	require.Equal(t, nSectors+GenesisPreseals-2, int(sectors)) // -2 not recovered sectors
+
+	// pledge a sector after recovery
+
+	pledgeSectors(t, ctx, miner, 1, nSectors, nil)
+
+	{
+		// wait a bit more
+
+		head, err := client.ChainHead(ctx)
+		require.NoError(t, err)
+
+		waitUntil := head.Height() + 10
+
+		for {
+			head, err := client.ChainHead(ctx)
+			require.NoError(t, err)
+
+			if head.Height() > waitUntil {
+				break
+			}
+		}
+	}
+
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+
+	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
+	require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged
+
 	mine = false
 	<-done
 }

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/sector-storage v0.0.0-20200730203805-7153e1dd05b5
+	github.com/filecoin-project/sector-storage v0.0.0-20200803184904-3cab915fd225
 	github.com/filecoin-project/specs-actors v0.8.6
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
 	github.com/filecoin-project/storage-fsm v0.0.0-20200730122205-d423ae90d8d4

go.sum

@@ -262,8 +262,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
 github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo=
 github.com/filecoin-project/sector-storage v0.0.0-20200730050024-3ee28c3b6d9a/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA=
-github.com/filecoin-project/sector-storage v0.0.0-20200730203805-7153e1dd05b5 h1:jULKRxvzn6CIKcKi74mGH0w1i54KXgD29IP7EhXBavQ=
+github.com/filecoin-project/sector-storage v0.0.0-20200803184904-3cab915fd225 h1:Or2lM5Cdsq0nDrSWp2YO70tjd8Ohg0jVWT/KGP3BX+I=
-github.com/filecoin-project/sector-storage v0.0.0-20200730203805-7153e1dd05b5/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA=
+github.com/filecoin-project/sector-storage v0.0.0-20200803184904-3cab915fd225/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA=
 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
 github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
 github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=

node/impl/storminer.go

@@ -48,6 +48,7 @@ type StorageMinerAPI struct {
 	BlockMiner        *miner.Miner
 	Full              api.FullNode
 	StorageMgr        *sectorstorage.Manager `optional:"true"`
+	IStorageMgr       sectorstorage.SectorManager
 	*stores.Index
 
 	ConsiderOnlineStorageDealsConfigFunc       dtypes.ConsiderOnlineStorageDealsConfigFunc

storage/wdpost_run.go

@@ -288,11 +288,13 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 	declWait.Add(1)
 
 	go func() {
+		// TODO: extract from runPost, run on fault cutoff boundaries
+
 		defer declWait.Done()
 
 		// check faults / recoveries for the *next* deadline. It's already too
 		// late to declare them for this deadline
-		declDeadline := (di.Index + 1) % miner.WPoStPeriodDeadlines
+		declDeadline := (di.Index + 2) % miner.WPoStPeriodDeadlines
 
 		partitions, err := s.api.StateMinerPartitions(ctx, s.actor, declDeadline, ts.Key())
 		if err != nil {