diff --git a/itests/batch_deal_test.go b/itests/batch_deal_test.go index f5c425d86..f51a75094 100644 --- a/itests/batch_deal_test.go +++ b/itests/batch_deal_test.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" @@ -14,7 +15,7 @@ import ( ) func TestBatchDealInput(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() var ( blockTime = 10 * time.Millisecond @@ -29,7 +30,7 @@ func TestBatchDealInput(t *testing.T) { ) // Set max deals per publish deals message to maxDealsPerMsg - minerDef := []StorageMiner{{ + minerDef := []kit.StorageMiner{{ Full: 0, Opts: node.Options( node.Override( @@ -49,23 +50,23 @@ func TestBatchDealInput(t *testing.T) { }, nil }), ), - Preseal: PresealGenesis, + Preseal: kit.PresealGenesis, }} // Create a connect client and miner node - n, sn := MockSbBuilder(t, OneFull, minerDef) + n, sn := kit.MockSbBuilder(t, kit.OneFull, minerDef) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] - s := connectAndStartMining(t, blockTime, client, miner) - defer s.blockMiner.Stop() + s := kit.ConnectAndStartMining(t, blockTime, client, miner) + defer s.BlockMiner.Stop() // Starts a deal and waits until it's published runDealTillSeal := func(rseed int) { - res, _, err := CreateClientFile(s.ctx, s.client, rseed) + res, _, err := kit.CreateClientFile(s.Ctx, s.Client, rseed) require.NoError(t, err) - dc := startDeal(t, s.ctx, s.miner, s.client, res.Root, false, dealStartEpoch) - waitDealSealed(t, s.ctx, s.miner, s.client, dc, false) + dc := kit.StartDeal(t, s.Ctx, s.Miner, s.Client, res.Root, false, dealStartEpoch) + kit.WaitDealSealed(t, s.Ctx, s.Miner, s.Client, dc, false) } // Run maxDealsPerMsg+1 deals in parallel @@ -83,7 +84,7 @@ func TestBatchDealInput(t *testing.T) { <-done } - sl, err := sn[0].SectorsList(s.ctx) + sl, err := sn[0].SectorsList(s.Ctx) require.NoError(t, err) require.GreaterOrEqual(t, len(sl), 4) require.LessOrEqual(t, len(sl), 5) diff --git a/itests/ccupgrade_test.go b/itests/ccupgrade_test.go index ceff0cecf..f9f024b27 100644 --- a/itests/ccupgrade_test.go +++ b/itests/ccupgrade_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" @@ -16,7 +17,7 @@ import ( ) func TestCCUpgrade(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() for _, height := range []abi.ChainEpoch{ -1, // before @@ -26,14 +27,14 @@ func TestCCUpgrade(t *testing.T) { } { height := height // make linters happy by copying t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { - runTestCCUpgrade(t, MockSbBuilder, 5*time.Millisecond, height) + runTestCCUpgrade(t, kit.MockSbBuilder, 5*time.Millisecond, height) }) } } -func runTestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { +func runTestCCUpgrade(t *testing.T, b kit.APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, []FullNodeOpts{FullNodeWithLatestActorsAt(upgradeHeight)}, OneMiner) + n, sn := b(t, []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(upgradeHeight)}, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -53,7 +54,7 @@ func runTestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgra defer close(done) for atomic.LoadInt64(&mine) == 1 { time.Sleep(blocktime) - if err := sn[0].MineOne(ctx, MineNext); err != nil { + if err := sn[0].MineOne(ctx, kit.MineNext); err != nil { t.Error(err) } } @@ -64,10 +65,10 @@ func runTestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgra t.Fatal(err) } - CC := abi.SectorNumber(GenesisPreseals + 1) + CC := abi.SectorNumber(kit.GenesisPreseals + 1) Upgraded := CC + 1 - pledgeSectors(t, ctx, miner, 1, 0, nil) + kit.PledgeSectors(t, ctx, miner, 1, 0, nil) sl, err := miner.SectorsList(ctx) if err != nil { @@ -91,7 +92,7 @@ func runTestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgra t.Fatal(err) } - MakeDeal(t, ctx, 6, client, miner, false, false, 0) + kit.MakeDeal(t, ctx, 6, client, miner, false, false, 0) // Validate upgrade diff --git a/itests/cli_test.go b/itests/cli_test.go index 864e332b1..10e2af15c 100644 --- a/itests/cli_test.go +++ b/itests/cli_test.go @@ -7,15 +7,16 @@ import ( "time" "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/itests/kit" ) // TestClient does a basic test to exercise the client CLI commands. func TestClient(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - clientNode, _ := StartOneNodeOneMiner(ctx, t, blocktime) - RunClientTest(t, cli.Commands, clientNode) + clientNode, _ := kit.StartOneNodeOneMiner(ctx, t, blocktime) + kit.RunClientTest(t, cli.Commands, clientNode) } diff --git a/itests/deadlines_test.go b/itests/deadlines_test.go index a236f1057..4822e0f19 100644 --- a/itests/deadlines_test.go +++ b/itests/deadlines_test.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/extern/sector-storage/mock" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/ipfs/go-cid" @@ -74,7 +75,7 @@ func TestDeadlineToggling(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := MockSbBuilder(t, []FullNodeOpts{FullNodeWithLatestActorsAt(upgradeH)}, OneMiner) + n, sn := kit.MockSbBuilder(t, []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(upgradeH)}, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) minerA := sn[0] @@ -103,7 +104,7 @@ func TestDeadlineToggling(t *testing.T) { defer close(done) for ctx.Err() == nil { build.Clock.Sleep(blocktime) - if err := minerA.MineOne(ctx, MineNext); err != nil { + if err := minerA.MineOne(ctx, kit.MineNext); err != nil { if ctx.Err() != nil { // context was canceled, ignore the error. return @@ -117,8 +118,8 @@ func TestDeadlineToggling(t *testing.T) { <-done }() - minerB := n[0].Stb(ctx, t, TestSpt, defaultFrom) - minerC := n[0].Stb(ctx, t, TestSpt, defaultFrom) + minerB := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) + minerC := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) maddrB, err := minerB.ActorAddress(ctx) require.NoError(t, err) @@ -130,7 +131,7 @@ func TestDeadlineToggling(t *testing.T) { // pledge sectors on C, go through a PP, check for power { - pledgeSectors(t, ctx, minerC, sectorsC, 0, nil) + kit.PledgeSectors(t, ctx, minerC, sectorsC, 0, nil) di, err := client.StateMinerProvingDeadline(ctx, maddrC, types.EmptyTSK) require.NoError(t, err) @@ -199,8 +200,8 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) require.GreaterOrEqual(t, nv, network.Version12) - minerD := n[0].Stb(ctx, t, TestSpt, defaultFrom) - minerE := n[0].Stb(ctx, t, TestSpt, defaultFrom) + minerD := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) + minerE := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) maddrD, err := minerD.ActorAddress(ctx) require.NoError(t, err) @@ -208,7 +209,7 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) // first round of miner checks - checkMiner(maddrA, types.NewInt(uint64(ssz)*GenesisPreseals), true, types.EmptyTSK) + checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.GenesisPreseals), true, types.EmptyTSK) checkMiner(maddrC, types.NewInt(uint64(ssz)*sectorsC), true, types.EmptyTSK) checkMiner(maddrB, types.NewInt(0), false, types.EmptyTSK) @@ -216,10 +217,10 @@ func TestDeadlineToggling(t *testing.T) { checkMiner(maddrE, types.NewInt(0), false, types.EmptyTSK) // pledge sectors on minerB/minerD, stop post on minerC - pledgeSectors(t, ctx, minerB, sectorsB, 0, nil) + kit.PledgeSectors(t, ctx, minerB, sectorsB, 0, nil) checkMiner(maddrB, types.NewInt(0), true, types.EmptyTSK) - pledgeSectors(t, ctx, minerD, sectorsD, 0, nil) + kit.PledgeSectors(t, ctx, minerD, sectorsD, 0, nil) checkMiner(maddrD, types.NewInt(0), true, types.EmptyTSK) minerC.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).Fail() @@ -235,7 +236,7 @@ func TestDeadlineToggling(t *testing.T) { params := &miner.SectorPreCommitInfo{ Expiration: 2880 * 300, SectorNumber: 22, - SealProof: TestSpt, + SealProof: kit.TestSpt, SealedCID: cr, SealRandEpoch: head.Height() - 200, @@ -285,7 +286,7 @@ func TestDeadlineToggling(t *testing.T) { } // second round of miner checks - checkMiner(maddrA, types.NewInt(uint64(ssz)*GenesisPreseals), true, types.EmptyTSK) + checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.GenesisPreseals), true, types.EmptyTSK) checkMiner(maddrC, types.NewInt(0), true, types.EmptyTSK) checkMiner(maddrB, types.NewInt(uint64(ssz)*sectorsB), true, types.EmptyTSK) checkMiner(maddrD, types.NewInt(uint64(ssz)*sectorsD), true, types.EmptyTSK) @@ -356,7 +357,7 @@ func TestDeadlineToggling(t *testing.T) { } // third round of miner checks - checkMiner(maddrA, types.NewInt(uint64(ssz)*GenesisPreseals), true, types.EmptyTSK) + checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.GenesisPreseals), true, types.EmptyTSK) checkMiner(maddrC, types.NewInt(0), true, types.EmptyTSK) checkMiner(maddrB, types.NewInt(0), true, types.EmptyTSK) checkMiner(maddrD, types.NewInt(0), false, types.EmptyTSK) diff --git a/itests/deals_test.go b/itests/deals_test.go index 66abcca73..b7570629f 100644 --- a/itests/deals_test.go +++ b/itests/deals_test.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" @@ -24,7 +25,7 @@ import ( ) func TestDealCycle(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() blockTime := 10 * time.Millisecond @@ -34,16 +35,19 @@ func TestDealCycle(t *testing.T) { dealStartEpoch := abi.ChainEpoch(2 << 12) t.Run("TestFullDealCycle_Single", func(t *testing.T) { - runFullDealCycles(t, 1, MockSbBuilder, blockTime, false, false, dealStartEpoch) + runFullDealCycles(t, 1, kit.MockSbBuilder, blockTime, false, false, dealStartEpoch) }) t.Run("TestFullDealCycle_Two", func(t *testing.T) { - runFullDealCycles(t, 2, MockSbBuilder, blockTime, false, false, dealStartEpoch) + runFullDealCycles(t, 2, kit.MockSbBuilder, blockTime, false, false, dealStartEpoch) }) t.Run("WithExportedCAR", func(t *testing.T) { - runFullDealCycles(t, 1, MockSbBuilder, blockTime, true, false, dealStartEpoch) + runFullDealCycles(t, 1, kit.MockSbBuilder, blockTime, true, false, dealStartEpoch) }) t.Run("TestFastRetrievalDealCycle", func(t *testing.T) { - TestFastRetrievalDealFlow(t, MockSbBuilder, blockTime, dealStartEpoch) + runFastRetrievalDealFlowT(t, kit.MockSbBuilder, blockTime, dealStartEpoch) + }) + t.Run("TestZeroPricePerByteRetrievalDealFlow", func(t *testing.T) { + runZeroPricePerByteRetrievalDealFlow(t, kit.MockSbBuilder, blockTime, dealStartEpoch) }) } @@ -52,7 +56,7 @@ func TestAPIDealFlowReal(t *testing.T) { t.Skip("skipping test in short mode") } - QuietMiningLogs() + kit.QuietMiningLogs() // TODO: just set this globally? oldDelay := policy.GetPreCommitChallengeDelay() @@ -62,22 +66,22 @@ func TestAPIDealFlowReal(t *testing.T) { }) t.Run("basic", func(t *testing.T) { - runFullDealCycles(t, 1, Builder, time.Second, false, false, 0) + runFullDealCycles(t, 1, kit.Builder, time.Second, false, false, 0) }) t.Run("fast-retrieval", func(t *testing.T) { - runFullDealCycles(t, 1, Builder, time.Second, false, true, 0) + runFullDealCycles(t, 1, kit.Builder, time.Second, false, true, 0) }) t.Run("retrieval-second", func(t *testing.T) { - runSecondDealRetrievalTest(t, Builder, time.Second) + runSecondDealRetrievalTest(t, kit.Builder, time.Second) }) } func TestPublishDealsBatching(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() - b := MockSbBuilder + b := kit.MockSbBuilder blocktime := 10 * time.Millisecond startEpoch := abi.ChainEpoch(2 << 12) @@ -85,7 +89,7 @@ func TestPublishDealsBatching(t *testing.T) { maxDealsPerMsg := uint64(2) // Set max deals per publish deals message to 2 - minerDef := []StorageMiner{{ + minerDef := []kit.StorageMiner{{ Full: 0, Opts: node.Override( new(*storageadapter.DealPublisher), @@ -93,25 +97,25 @@ func TestPublishDealsBatching(t *testing.T) { Period: publishPeriod, MaxDealsPerMsg: maxDealsPerMsg, })), - Preseal: PresealGenesis, + Preseal: kit.PresealGenesis, }} // Create a connect client and miner node - n, sn := b(t, OneFull, minerDef) + n, sn := b(t, kit.OneFull, minerDef) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] - s := connectAndStartMining(t, blocktime, client, miner) - defer s.blockMiner.Stop() + s := kit.ConnectAndStartMining(t, blocktime, client, miner) + defer s.BlockMiner.Stop() // Starts a deal and waits until it's published runDealTillPublish := func(rseed int) { - res, _, err := CreateClientFile(s.ctx, s.client, rseed) + res, _, err := kit.CreateClientFile(s.Ctx, s.Client, rseed) require.NoError(t, err) - upds, err := client.ClientGetDealUpdates(s.ctx) + upds, err := client.ClientGetDealUpdates(s.Ctx) require.NoError(t, err) - startDeal(t, s.ctx, s.miner, s.client, res.Root, false, startEpoch) + kit.StartDeal(t, s.Ctx, s.Miner, s.Client, res.Root, false, startEpoch) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) @@ -143,11 +147,11 @@ func TestPublishDealsBatching(t *testing.T) { } // Expect a single PublishStorageDeals message that includes the first two deals - msgCids, err := s.client.StateListMessages(s.ctx, &api.MessageMatch{To: market.Address}, types.EmptyTSK, 1) + msgCids, err := s.Client.StateListMessages(s.Ctx, &api.MessageMatch{To: market.Address}, types.EmptyTSK, 1) require.NoError(t, err) count := 0 for _, msgCid := range msgCids { - msg, err := s.client.ChainGetMessage(s.ctx, msgCid) + msg, err := s.Client.ChainGetMessage(s.Ctx, msgCid) require.NoError(t, err) if msg.Method == market.Methods.PublishStorageDeals { @@ -177,14 +181,14 @@ func TestDealMining(t *testing.T) { t.Skip("skipping test in short mode") } - QuietMiningLogs() + kit.QuietMiningLogs() - b := MockSbBuilder + b := kit.MockSbBuilder blocktime := 50 * time.Millisecond ctx := context.Background() - n, sn := b(t, OneFull, []StorageMiner{ - {Full: 0, Preseal: PresealGenesis}, + n, sn := b(t, kit.OneFull, []kit.StorageMiner{ + {Full: 0, Preseal: kit.PresealGenesis}, {Full: 0, Preseal: 0}, // TODO: Add support for miners on non-first full node }) client := n[0].FullNode.(*impl.FullNodeAPI) @@ -282,12 +286,12 @@ func TestDealMining(t *testing.T) { } }() - deal := startDeal(t, ctx, provider, client, fcid, false, 0) + deal := kit.StartDeal(t, ctx, provider, client, fcid, false, 0) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) - waitDealSealed(t, ctx, provider, client, deal, false) + kit.WaitDealSealed(t, ctx, provider, client, deal, false) <-minedTwo @@ -296,12 +300,98 @@ func TestDealMining(t *testing.T) { <-done } -func runFullDealCycles(t *testing.T, n int, b APIBuilder, blocktime time.Duration, carExport, fastRet bool, startEpoch abi.ChainEpoch) { - s := setupOneClientOneMiner(t, b, blocktime) - defer s.blockMiner.Stop() +func runFullDealCycles(t *testing.T, n int, b kit.APIBuilder, blocktime time.Duration, carExport, fastRet bool, startEpoch abi.ChainEpoch) { + s := kit.SetupOneClientOneMiner(t, b, blocktime) + defer s.BlockMiner.Stop() baseseed := 6 for i := 0; i < n; i++ { - MakeDeal(t, s.ctx, baseseed+i, s.client, s.miner, carExport, fastRet, startEpoch) + kit.MakeDeal(t, s.Ctx, baseseed+i, s.Client, s.Miner, carExport, fastRet, startEpoch) } } + +func runFastRetrievalDealFlowT(t *testing.T, b kit.APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) { + s := kit.SetupOneClientOneMiner(t, b, blocktime) + defer s.BlockMiner.Stop() + + data := make([]byte, 1600) + rand.New(rand.NewSource(int64(8))).Read(data) + + r := bytes.NewReader(data) + fcid, err := s.Client.ClientImportLocal(s.Ctx, r) + if err != nil { + t.Fatal(err) + } + + fmt.Println("FILE CID: ", fcid) + + deal := kit.StartDeal(t, s.Ctx, s.Miner, s.Client, fcid, true, startEpoch) + + kit.WaitDealPublished(t, s.Ctx, s.Miner, deal) + fmt.Println("deal published, retrieving") + // Retrieval + info, err := s.Client.ClientGetDealInfo(s.Ctx, *deal) + require.NoError(t, err) + + kit.TestRetrieval(t, s.Ctx, s.Client, fcid, &info.PieceCID, false, data) +} + +func runSecondDealRetrievalTest(t *testing.T, b kit.APIBuilder, blocktime time.Duration) { + s := kit.SetupOneClientOneMiner(t, b, blocktime) + defer s.BlockMiner.Stop() + + { + data1 := make([]byte, 800) + rand.New(rand.NewSource(int64(3))).Read(data1) + r := bytes.NewReader(data1) + + fcid1, err := s.Client.ClientImportLocal(s.Ctx, r) + if err != nil { + t.Fatal(err) + } + + data2 := make([]byte, 800) + rand.New(rand.NewSource(int64(9))).Read(data2) + r2 := bytes.NewReader(data2) + + fcid2, err := s.Client.ClientImportLocal(s.Ctx, r2) + if err != nil { + t.Fatal(err) + } + + deal1 := kit.StartDeal(t, s.Ctx, s.Miner, s.Client, fcid1, true, 0) + + // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(time.Second) + kit.WaitDealSealed(t, s.Ctx, s.Miner, s.Client, deal1, true) + + deal2 := kit.StartDeal(t, s.Ctx, s.Miner, s.Client, fcid2, true, 0) + + time.Sleep(time.Second) + kit.WaitDealSealed(t, s.Ctx, s.Miner, s.Client, deal2, false) + + // Retrieval + info, err := s.Client.ClientGetDealInfo(s.Ctx, *deal2) + require.NoError(t, err) + + rf, _ := s.Miner.SectorsRefs(s.Ctx) + fmt.Printf("refs: %+v\n", rf) + + kit.TestRetrieval(t, s.Ctx, s.Client, fcid2, &info.PieceCID, false, data2) + } +} + +func runZeroPricePerByteRetrievalDealFlow(t *testing.T, b kit.APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) { + s := kit.SetupOneClientOneMiner(t, b, blocktime) + defer s.BlockMiner.Stop() + + // Set price-per-byte to zero + ask, err := s.Miner.MarketGetRetrievalAsk(s.Ctx) + require.NoError(t, err) + + ask.PricePerByte = abi.NewTokenAmount(0) + err = s.Miner.MarketSetRetrievalAsk(s.Ctx, ask) + require.NoError(t, err) + + kit.MakeDeal(t, s.Ctx, 6, s.Client, s.Miner, false, false, startEpoch) +} diff --git a/itests/h_blockminer.go b/itests/kit/blockminer.go similarity index 80% rename from itests/h_blockminer.go rename to itests/kit/blockminer.go index 8418634e9..1b9774c8d 100644 --- a/itests/h_blockminer.go +++ b/itests/kit/blockminer.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "context" @@ -11,15 +11,16 @@ import ( "github.com/filecoin-project/lotus/miner" ) -// BlockMiner is a utility that makes a test miner mine blocks on a timer. +// BlockMiner is a utility that makes a test Miner Mine blocks on a timer. type BlockMiner struct { ctx context.Context t *testing.T miner TestStorageNode blocktime time.Duration - mine int64 - nulls int64 done chan struct{} + + Mine int64 + Nulls int64 } func NewBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blocktime time.Duration) *BlockMiner { @@ -28,7 +29,7 @@ func NewBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blo t: t, miner: miner, blocktime: blocktime, - mine: int64(1), + Mine: int64(1), done: make(chan struct{}), } } @@ -37,14 +38,14 @@ func (bm *BlockMiner) MineBlocks() { time.Sleep(time.Second) go func() { defer close(bm.done) - for atomic.LoadInt64(&bm.mine) == 1 { + for atomic.LoadInt64(&bm.Mine) == 1 { select { case <-bm.ctx.Done(): return case <-time.After(bm.blocktime): } - nulls := atomic.SwapInt64(&bm.nulls, 0) + nulls := atomic.SwapInt64(&bm.Nulls, 0) if err := bm.miner.MineOne(bm.ctx, miner.MineReq{ InjectNulls: abi.ChainEpoch(nulls), Done: func(bool, abi.ChainEpoch, error) {}, @@ -56,7 +57,7 @@ func (bm *BlockMiner) MineBlocks() { } func (bm *BlockMiner) Stop() { - atomic.AddInt64(&bm.mine, -1) + atomic.AddInt64(&bm.Mine, -1) fmt.Println("shutting down mining") <-bm.done } diff --git a/itests/client.go b/itests/kit/client.go similarity index 77% rename from itests/client.go rename to itests/kit/client.go index 1a9380a44..914e5396c 100644 --- a/itests/client.go +++ b/itests/kit/client.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "context" @@ -20,7 +20,7 @@ import ( lcli "github.com/urfave/cli/v2" ) -// RunClientTest exercises some of the client CLI commands +// RunClientTest exercises some of the Client CLI commands func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestNode) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -29,7 +29,7 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestNode) { mockCLI := NewMockCLI(ctx, t, cmds) clientCLI := mockCLI.Client(clientNode.ListenAddr) - // Get the miner address + // Get the Miner address addrs, err := clientNode.StateListMiners(ctx, types.EmptyTSK) require.NoError(t, err) require.Len(t, addrs, 1) @@ -37,33 +37,33 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestNode) { minerAddr := addrs[0] fmt.Println("Miner:", minerAddr) - // client query-ask - out := clientCLI.RunCmd("client", "query-ask", minerAddr.String()) + // Client query-ask + out := clientCLI.RunCmd("Client", "query-ask", minerAddr.String()) require.Regexp(t, regexp.MustCompile("Ask:"), out) // Create a deal (non-interactive) - // client deal --start-epoch= 1000000attofil + // Client deal --start-epoch= 1000000attofil res, _, err := CreateClientFile(ctx, clientNode, 1) require.NoError(t, err) startEpoch := fmt.Sprintf("--start-epoch=%d", 2<<12) dataCid := res.Root price := "1000000attofil" duration := fmt.Sprintf("%d", build.MinDealDuration) - out = clientCLI.RunCmd("client", "deal", startEpoch, dataCid.String(), minerAddr.String(), price, duration) - fmt.Println("client deal", out) + out = clientCLI.RunCmd("Client", "deal", startEpoch, dataCid.String(), minerAddr.String(), price, duration) + fmt.Println("Client deal", out) // Create a deal (interactive) - // client deal + // Client deal // // (in days) - // - // "no" (verified client) + // + // "no" (verified Client) // "yes" (confirm deal) res, _, err = CreateClientFile(ctx, clientNode, 2) require.NoError(t, err) dataCid2 := res.Root duration = fmt.Sprintf("%d", build.MinDealDuration/builtin.EpochsInDay) - cmd := []string{"client", "deal"} + cmd := []string{"Client", "deal"} interactiveCmds := []string{ dataCid2.String(), duration, @@ -72,13 +72,13 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestNode) { "yes", } out = clientCLI.RunInteractiveCmd(cmd, interactiveCmds) - fmt.Println("client deal:\n", out) + fmt.Println("Client deal:\n", out) // Wait for provider to start sealing deal dealStatus := "" for { - // client list-deals - out = clientCLI.RunCmd("client", "list-deals") + // Client list-deals + out = clientCLI.RunCmd("Client", "list-deals") fmt.Println("list-deals:\n", out) lines := strings.Split(out, "\n") @@ -97,12 +97,12 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestNode) { time.Sleep(time.Second) } - // Retrieve the first file from the miner - // client retrieve - tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-client") + // Retrieve the first file from the Miner + // Client retrieve + tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-Client") require.NoError(t, err) path := filepath.Join(tmpdir, "outfile.dat") - out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path) + out = clientCLI.RunCmd("Client", "retrieve", dataCid.String(), path) fmt.Println("retrieve:\n", out) require.Regexp(t, regexp.MustCompile("Success"), out) } diff --git a/itests/deals.go b/itests/kit/deals.go similarity index 63% rename from itests/deals.go rename to itests/kit/deals.go index 295436d22..1a38ed086 100644 --- a/itests/deals.go +++ b/itests/kit/deals.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "bytes" @@ -38,17 +38,17 @@ func MakeDeal(t *testing.T, ctx context.Context, rseed int, client api.FullNode, fcid := res.Root fmt.Println("FILE CID: ", fcid) - deal := startDeal(t, ctx, miner, client, fcid, fastRet, startEpoch) + deal := StartDeal(t, ctx, miner, client, fcid, fastRet, startEpoch) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) - waitDealSealed(t, ctx, miner, client, deal, false) + WaitDealSealed(t, ctx, miner, client, deal, false) // Retrieval info, err := client.ClientGetDealInfo(ctx, *deal) require.NoError(t, err) - testRetrieval(t, ctx, client, fcid, &info.PieceCID, carExport, data) + TestRetrieval(t, ctx, client, fcid, &info.PieceCID, carExport, data) } func CreateClientFile(ctx context.Context, client api.FullNode, rseed int) (*api.ImportRes, []byte, error) { @@ -73,93 +73,7 @@ func CreateClientFile(ctx context.Context, client api.FullNode, rseed int) (*api return res, data, nil } -func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) { - s := setupOneClientOneMiner(t, b, blocktime) - defer s.blockMiner.Stop() - - data := make([]byte, 1600) - rand.New(rand.NewSource(int64(8))).Read(data) - - r := bytes.NewReader(data) - fcid, err := s.client.ClientImportLocal(s.ctx, r) - if err != nil { - t.Fatal(err) - } - - fmt.Println("FILE CID: ", fcid) - - deal := startDeal(t, s.ctx, s.miner, s.client, fcid, true, startEpoch) - - waitDealPublished(t, s.ctx, s.miner, deal) - fmt.Println("deal published, retrieving") - // Retrieval - info, err := s.client.ClientGetDealInfo(s.ctx, *deal) - require.NoError(t, err) - - testRetrieval(t, s.ctx, s.client, fcid, &info.PieceCID, false, data) -} - -func runSecondDealRetrievalTest(t *testing.T, b APIBuilder, blocktime time.Duration) { - s := setupOneClientOneMiner(t, b, blocktime) - defer s.blockMiner.Stop() - - { - data1 := make([]byte, 800) - rand.New(rand.NewSource(int64(3))).Read(data1) - r := bytes.NewReader(data1) - - fcid1, err := s.client.ClientImportLocal(s.ctx, r) - if err != nil { - t.Fatal(err) - } - - data2 := make([]byte, 800) - rand.New(rand.NewSource(int64(9))).Read(data2) - r2 := bytes.NewReader(data2) - - fcid2, err := s.client.ClientImportLocal(s.ctx, r2) - if err != nil { - t.Fatal(err) - } - - deal1 := startDeal(t, s.ctx, s.miner, s.client, fcid1, true, 0) - - // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this - time.Sleep(time.Second) - waitDealSealed(t, s.ctx, s.miner, s.client, deal1, true) - - deal2 := startDeal(t, s.ctx, s.miner, s.client, fcid2, true, 0) - - time.Sleep(time.Second) - waitDealSealed(t, s.ctx, s.miner, s.client, deal2, false) - - // Retrieval - info, err := s.client.ClientGetDealInfo(s.ctx, *deal2) - require.NoError(t, err) - - rf, _ := s.miner.SectorsRefs(s.ctx) - fmt.Printf("refs: %+v\n", rf) - - testRetrieval(t, s.ctx, s.client, fcid2, &info.PieceCID, false, data2) - } -} - -func TestZeroPricePerByteRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) { - s := setupOneClientOneMiner(t, b, blocktime) - defer s.blockMiner.Stop() - - // Set price-per-byte to zero - ask, err := s.miner.MarketGetRetrievalAsk(s.ctx) - require.NoError(t, err) - - ask.PricePerByte = abi.NewTokenAmount(0) - err = s.miner.MarketSetRetrievalAsk(s.ctx, ask) - require.NoError(t, err) - - MakeDeal(t, s.ctx, 6, s.client, s.miner, false, false, startEpoch) -} - -func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid { +func StartDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid { maddr, err := miner.ActorAddress(ctx) if err != nil { t.Fatal(err) @@ -187,7 +101,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client return deal } -func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, deal *cid.Cid, noseal bool) { +func WaitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, deal *cid.Cid, noseal bool) { loop: for { di, err := client.ClientGetDealInfo(ctx, *deal) @@ -199,7 +113,7 @@ loop: if noseal { return } - startSealingWaiting(t, ctx, miner) + StartSealingWaiting(t, ctx, miner) case storagemarket.StorageDealProposalRejected: t.Fatal("deal rejected") case storagemarket.StorageDealFailing: @@ -215,7 +129,7 @@ loop: } } -func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) { +func WaitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) { subCtx, cancel := context.WithCancel(ctx) defer cancel() updates, err := miner.MarketGetDealUpdates(subCtx) @@ -245,7 +159,7 @@ func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, } } -func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNode) { +func StartSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNode) { snums, err := miner.SectorsList(ctx) require.NoError(t, err) @@ -260,7 +174,7 @@ func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNod } } -func testRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) { +func TestRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) { offers, err := client.ClientFindData(ctx, fcid, piece) if err != nil { t.Fatal(err) @@ -301,7 +215,7 @@ func testRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid } if carExport { - rdata = extractCarData(t, ctx, rdata, rpath) + rdata = ExtractCarData(t, ctx, rdata, rpath) } if !bytes.Equal(rdata, data) { @@ -309,7 +223,7 @@ func testRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid } } -func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath string) []byte { +func ExtractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath string) []byte { bserv := dstest.Bserv() ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata)) if err != nil { @@ -339,21 +253,21 @@ func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath strin return rdata } -type dealsScaffold struct { - ctx context.Context - client *impl.FullNodeAPI - miner TestStorageNode - blockMiner *BlockMiner +type DealsScaffold struct { + Ctx context.Context + Client *impl.FullNodeAPI + Miner TestStorageNode + BlockMiner *BlockMiner } -func setupOneClientOneMiner(t *testing.T, b APIBuilder, blocktime time.Duration) *dealsScaffold { +func SetupOneClientOneMiner(t *testing.T, b APIBuilder, blocktime time.Duration) *DealsScaffold { n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] - return connectAndStartMining(t, blocktime, client, miner) + return ConnectAndStartMining(t, blocktime, client, miner) } -func connectAndStartMining(t *testing.T, blocktime time.Duration, client *impl.FullNodeAPI, miner TestStorageNode) *dealsScaffold { +func ConnectAndStartMining(t *testing.T, blocktime time.Duration, client *impl.FullNodeAPI, miner TestStorageNode) *DealsScaffold { ctx := context.Background() addrinfo, err := client.NetAddrsListen(ctx) if err != nil { @@ -368,10 +282,10 @@ func connectAndStartMining(t *testing.T, blocktime time.Duration, client *impl.F blockMiner := NewBlockMiner(ctx, t, miner, blocktime) blockMiner.MineBlocks() - return &dealsScaffold{ - ctx: ctx, - client: client, - miner: miner, - blockMiner: blockMiner, + return &DealsScaffold{ + Ctx: ctx, + Client: client, + Miner: miner, + BlockMiner: blockMiner, } } diff --git a/itests/kit/funds.go b/itests/kit/funds.go new file mode 100644 index 000000000..64b0eed41 --- /dev/null +++ b/itests/kit/funds.go @@ -0,0 +1,37 @@ +package kit + +import ( + "context" + "testing" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/go-address" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) { + senderAddr, err := sender.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + msg := &types.Message{ + From: senderAddr, + To: addr, + Value: amount, + } + + sm, err := sender.MpoolPushMessage(ctx, msg, nil) + if err != nil { + t.Fatal(err) + } + res, err := sender.StateWaitMsg(ctx, sm.Cid(), 3, lapi.LookbackNoLimit, true) + if err != nil { + t.Fatal(err) + } + if res.Receipt.ExitCode != 0 { + t.Fatal("did not successfully send money") + } +} diff --git a/itests/log.go b/itests/kit/log.go similarity index 87% rename from itests/log.go rename to itests/kit/log.go index 47ffe481b..af7d4093d 100644 --- a/itests/log.go +++ b/itests/kit/log.go @@ -1,9 +1,9 @@ -package itests +package kit import logging "github.com/ipfs/go-log/v2" func QuietMiningLogs() { - _ = logging.SetLogLevel("miner", "ERROR") + _ = logging.SetLogLevel("Miner", "ERROR") _ = logging.SetLogLevel("chainstore", "ERROR") _ = logging.SetLogLevel("chain", "ERROR") _ = logging.SetLogLevel("sub", "ERROR") diff --git a/itests/h_util2.go b/itests/kit/mining.go similarity index 56% rename from itests/h_util2.go rename to itests/kit/mining.go index 174499ef7..c091e1384 100644 --- a/itests/h_util2.go +++ b/itests/kit/mining.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "context" @@ -6,38 +6,9 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/go-address" - lapi "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/miner" ) -func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) { - senderAddr, err := sender.WalletDefaultAddress(ctx) - if err != nil { - t.Fatal(err) - } - - msg := &types.Message{ - From: senderAddr, - To: addr, - Value: amount, - } - - sm, err := sender.MpoolPushMessage(ctx, msg, nil) - if err != nil { - t.Fatal(err) - } - res, err := sender.StateWaitMsg(ctx, sm.Cid(), 3, lapi.LookbackNoLimit, true) - if err != nil { - t.Fatal(err) - } - if res.Receipt.ExitCode != 0 { - t.Fatal("did not successfully send money") - } -} - func MineUntilBlock(ctx context.Context, t *testing.T, fn TestNode, sn TestStorageNode, cb func(abi.ChainEpoch)) { for i := 0; i < 1000; i++ { var success bool @@ -81,7 +52,7 @@ func MineUntilBlock(ctx context.Context, t *testing.T, fn TestNode, sn TestStora } return } - t.Log("did not mine block, trying again", i) + t.Log("did not Mine block, trying again", i) } - t.Fatal("failed to mine 1000 times in a row...") + t.Fatal("failed to Mine 1000 times in a row...") } diff --git a/itests/h_mockcli.go b/itests/kit/mockcli.go similarity index 99% rename from itests/h_mockcli.go rename to itests/kit/mockcli.go index fa9a89036..c0f218920 100644 --- a/itests/h_mockcli.go +++ b/itests/kit/mockcli.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "bytes" diff --git a/itests/h_net.go b/itests/kit/net.go similarity index 99% rename from itests/h_net.go rename to itests/kit/net.go index 969ed1ec5..107e84d4a 100644 --- a/itests/h_net.go +++ b/itests/kit/net.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "context" diff --git a/itests/h_node_builder.go b/itests/kit/node_builder.go similarity index 99% rename from itests/h_node_builder.go rename to itests/kit/node_builder.go index 84c2b844e..86995e2be 100644 --- a/itests/h_node_builder.go +++ b/itests/kit/node_builder.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "bytes" @@ -84,7 +84,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr ds, err := lr.Datastore(context.TODO(), "/metadata") require.NoError(t, err) - err = ds.Put(datastore.NewKey("miner-address"), act.Bytes()) + err = ds.Put(datastore.NewKey("Miner-address"), act.Bytes()) require.NoError(t, err) nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix)) @@ -139,10 +139,10 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr t.Cleanup(func() { _ = stop(context.Background()) }) /*// Bootstrap with full node - remoteAddrs, err := tnd.NetAddrsListen(ctx) + remoteAddrs, err := tnd.NetAddrsListen(Ctx) require.NoError(t, err) - err = minerapi.NetConnect(ctx, remoteAddrs) + err = minerapi.NetConnect(Ctx, remoteAddrs) require.NoError(t, err)*/ mineOne := func(ctx context.Context, req lotusminer.MineReq) error { select { diff --git a/itests/h_pledge.go b/itests/kit/pledge.go similarity index 94% rename from itests/h_pledge.go rename to itests/kit/pledge.go index 2223d585a..94a1978a8 100644 --- a/itests/h_pledge.go +++ b/itests/kit/pledge.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "context" @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, existing int, blockNotif <-chan struct{}) { +func PledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, existing int, blockNotif <-chan struct{}) { for i := 0; i < n; i++ { if i%3 == 0 && blockNotif != nil { <-blockNotif diff --git a/itests/t.go b/itests/kit/t.go similarity index 98% rename from itests/t.go rename to itests/kit/t.go index d90f398b2..a29ddbab5 100644 --- a/itests/t.go +++ b/itests/kit/t.go @@ -1,4 +1,4 @@ -package itests +package kit import ( "context" @@ -57,7 +57,7 @@ const GenesisPreseals = 2 const TestSpt = abi.RegisteredSealProof_StackedDrg2KiBV1_1 -// Options for setting up a mock storage miner +// Options for setting up a mock storage Miner type StorageMiner struct { Full int Opts node.Option diff --git a/itests/multisig_test.go b/itests/multisig_test.go index 8593d61c3..3dcafc693 100644 --- a/itests/multisig_test.go +++ b/itests/multisig_test.go @@ -18,7 +18,7 @@ import ( // TestMultisig does a basic test to exercise the multisig CLI commands func TestMultisig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - QuietMiningLogs() + harness.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() diff --git a/itests/paych_api_test.go b/itests/paych_api_test.go index 7b3d7cf3e..3535e6901 100644 --- a/itests/paych_api_test.go +++ b/itests/paych_api_test.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/itests/kit" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" @@ -27,10 +28,10 @@ import ( ) func TestPaymentChannelsAPI(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() ctx := context.Background() - n, sn := MockSbBuilder(t, TwoFull, OneMiner) + n, sn := kit.MockSbBuilder(t, kit.TwoFull, kit.OneMiner) paymentCreator := n[0] paymentReceiver := n[1] @@ -51,7 +52,7 @@ func TestPaymentChannelsAPI(t *testing.T) { } // start mining blocks - bm := NewBlockMiner(ctx, t, miner, 5*time.Millisecond) + bm := kit.NewBlockMiner(ctx, t, miner, 5*time.Millisecond) bm.MineBlocks() // send some funds to register the receiver @@ -60,7 +61,7 @@ func TestPaymentChannelsAPI(t *testing.T) { t.Fatal(err) } - SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18)) + kit.SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18)) // setup the payment channel createrAddr, err := paymentCreator.WalletDefaultAddress(ctx) @@ -267,7 +268,7 @@ func TestPaymentChannelsAPI(t *testing.T) { bm.Stop() } -func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentReceiver TestNode, receiverAddr address.Address, count int) { +func waitForBlocks(ctx context.Context, t *testing.T, bm *kit.BlockMiner, paymentReceiver kit.TestNode, receiverAddr address.Address, count int) { // We need to add null blocks in batches, if we add too many the chain can't sync batchSize := 60 for i := 0; i < count; i += batchSize { @@ -277,7 +278,7 @@ func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentRec } // Add a batch of null blocks - atomic.StoreInt64(&bm.nulls, int64(size-1)) + atomic.StoreInt64(&bm.Nulls, int64(size-1)) // Add a real block m, err := paymentReceiver.MpoolPushMessage(ctx, &types.Message{ @@ -296,7 +297,7 @@ func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentRec } } -func waitForMessage(ctx context.Context, t *testing.T, paymentCreator TestNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup { +func waitForMessage(ctx context.Context, t *testing.T, paymentCreator kit.TestNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup { ctx, cancel := context.WithTimeout(ctx, duration) defer cancel() diff --git a/itests/paych_cli_test.go b/itests/paych_cli_test.go index f7541a78e..62478e9da 100644 --- a/itests/paych_cli_test.go +++ b/itests/paych_cli_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -36,18 +37,18 @@ func init() { // commands func TestPaymentChannelsBasic(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - nodes, addrs := StartTwoNodesOneMiner(ctx, t, blocktime) + nodes, addrs := kit.StartTwoNodesOneMiner(ctx, t, blocktime) paymentCreator := nodes[0] paymentReceiver := nodes[1] creatorAddr := addrs[0] receiverAddr := addrs[1] // Create mock CLI - mockCLI := NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) receiverCLI := mockCLI.Client(paymentReceiver.ListenAddr) @@ -88,17 +89,17 @@ type voucherSpec struct { // TestPaymentChannelStatus tests the payment channel status CLI command func TestPaymentChannelStatus(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - nodes, addrs := StartTwoNodesOneMiner(ctx, t, blocktime) + nodes, addrs := kit.StartTwoNodesOneMiner(ctx, t, blocktime) paymentCreator := nodes[0] creatorAddr := addrs[0] receiverAddr := addrs[1] // Create mock CLI - mockCLI := NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) // creator: paych status-by-from-to @@ -167,18 +168,18 @@ func TestPaymentChannelStatus(t *testing.T) { // channel voucher commands func TestPaymentChannelVouchers(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - nodes, addrs := StartTwoNodesOneMiner(ctx, t, blocktime) + nodes, addrs := kit.StartTwoNodesOneMiner(ctx, t, blocktime) paymentCreator := nodes[0] paymentReceiver := nodes[1] creatorAddr := addrs[0] receiverAddr := addrs[1] // Create mock CLI - mockCLI := NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) receiverCLI := mockCLI.Client(paymentReceiver.ListenAddr) @@ -299,17 +300,17 @@ func TestPaymentChannelVouchers(t *testing.T) { // is greater than what's left in the channel, voucher create fails func TestPaymentChannelVoucherCreateShortfall(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - nodes, addrs := StartTwoNodesOneMiner(ctx, t, blocktime) + nodes, addrs := kit.StartTwoNodesOneMiner(ctx, t, blocktime) paymentCreator := nodes[0] creatorAddr := addrs[0] receiverAddr := addrs[1] // Create mock CLI - mockCLI := NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) // creator: paych add-funds @@ -377,7 +378,7 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { } // waitForHeight waits for the node to reach the given chain epoch -func waitForHeight(ctx context.Context, t *testing.T, node TestNode, height abi.ChainEpoch) { +func waitForHeight(ctx context.Context, t *testing.T, node kit.TestNode, height abi.ChainEpoch) { atHeight := make(chan struct{}) chainEvents := events.NewEvents(ctx, node) err := chainEvents.ChainAt(func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { @@ -395,7 +396,7 @@ func waitForHeight(ctx context.Context, t *testing.T, node TestNode, height abi. } // getPaychState gets the state of the payment channel with the given address -func getPaychState(ctx context.Context, t *testing.T, node TestNode, chAddr address.Address) paych.State { +func getPaychState(ctx context.Context, t *testing.T, node kit.TestNode, chAddr address.Address) paych.State { act, err := node.StateGetActor(ctx, chAddr, types.EmptyTSK) require.NoError(t, err) diff --git a/itests/sdr_upgrade_test.go b/itests/sdr_upgrade_test.go index c99ff92b8..efe5e3b03 100644 --- a/itests/sdr_upgrade_test.go +++ b/itests/sdr_upgrade_test.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/itests/kit" bminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl" "github.com/stretchr/testify/assert" @@ -18,7 +19,7 @@ import ( ) func TestSDRUpgrade(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() oldDelay := policy.GetPreCommitChallengeDelay() policy.SetPreCommitChallengeDelay(5) @@ -31,7 +32,7 @@ func TestSDRUpgrade(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := MockSbBuilder(t, []FullNodeOpts{FullNodeWithSDRAt(500, 1000)}, OneMiner) + n, sn := kit.MockSbBuilder(t, []kit.FullNodeOpts{kit.FullNodeWithSDRAt(500, 1000)}, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -88,7 +89,7 @@ func TestSDRUpgrade(t *testing.T) { }() // before. - pledgeSectors(t, ctx, miner, 9, 0, pledge) + kit.PledgeSectors(t, ctx, miner, 9, 0, pledge) s, err := miner.SectorsList(ctx) require.NoError(t, err) diff --git a/itests/sector_pledge_test.go b/itests/sector_pledge_test.go index c3de173dd..515916250 100644 --- a/itests/sector_pledge_test.go +++ b/itests/sector_pledge_test.go @@ -8,19 +8,20 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/itests/kit" bminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl" ) func TestPledgeSectors(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() t.Run("1", func(t *testing.T) { - runPledgeSectorTest(t, MockSbBuilder, 50*time.Millisecond, 1) + runPledgeSectorTest(t, kit.MockSbBuilder, 50*time.Millisecond, 1) }) t.Run("100", func(t *testing.T) { - runPledgeSectorTest(t, MockSbBuilder, 50*time.Millisecond, 100) + runPledgeSectorTest(t, kit.MockSbBuilder, 50*time.Millisecond, 100) }) t.Run("1000", func(t *testing.T) { @@ -28,15 +29,15 @@ func TestPledgeSectors(t *testing.T) { t.Skip("skipping test in short mode") } - runPledgeSectorTest(t, MockSbBuilder, 50*time.Millisecond, 1000) + runPledgeSectorTest(t, kit.MockSbBuilder, 50*time.Millisecond, 1000) }) } -func runPledgeSectorTest(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { +func runPledgeSectorTest(t *testing.T, b kit.APIBuilder, blocktime time.Duration, nSectors int) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, OneFull, OneMiner) + n, sn := b(t, kit.OneFull, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -64,7 +65,7 @@ func runPledgeSectorTest(t *testing.T, b APIBuilder, blocktime time.Duration, nS } }() - pledgeSectors(t, ctx, miner, nSectors, 0, nil) + kit.PledgeSectors(t, ctx, miner, nSectors, 0, nil) atomic.StoreInt64(&mine, 0) <-done diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index 90627be85..f265d3e97 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" "github.com/stretchr/testify/require" ) @@ -21,7 +22,7 @@ func TestTerminate(t *testing.T) { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") } - QuietMiningLogs() + kit.QuietMiningLogs() const blocktime = 2 * time.Millisecond @@ -30,9 +31,9 @@ func TestTerminate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := MockSbBuilder(t, - []FullNodeOpts{FullNodeWithLatestActorsAt(-1)}, - []StorageMiner{{Full: 0, Preseal: int(nSectors)}}, + n, sn := kit.MockSbBuilder(t, + []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1)}, + []kit.StorageMiner{{Full: 0, Preseal: int(nSectors)}}, ) client := n[0].FullNode.(*impl.FullNodeAPI) @@ -53,7 +54,7 @@ func TestTerminate(t *testing.T) { defer close(done) for ctx.Err() == nil { build.Clock.Sleep(blocktime) - if err := sn[0].MineOne(ctx, MineNext); err != nil { + if err := sn[0].MineOne(ctx, kit.MineNext); err != nil { if ctx.Err() != nil { // context was canceled, ignore the error. return @@ -80,7 +81,7 @@ func TestTerminate(t *testing.T) { fmt.Printf("Seal a sector\n") - pledgeSectors(t, ctx, miner, 1, 0, nil) + kit.PledgeSectors(t, ctx, miner, 1, 0, nil) fmt.Printf("wait for power\n") diff --git a/itests/tape_test.go b/itests/tape_test.go index 16cab8270..17bbd82e5 100644 --- a/itests/tape_test.go +++ b/itests/tape_test.go @@ -11,23 +11,24 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" "github.com/stretchr/testify/require" ) func TestTapeFix(t *testing.T) { - QuietMiningLogs() + kit.QuietMiningLogs() var blocktime = 2 * time.Millisecond // The "before" case is disabled, because we need the builder to mock 32 GiB sectors to accurately repro this case // TODO: Make the mock sector size configurable and reenable this // t.Run("before", func(t *testing.T) { testTapeFix(t, b, blocktime, false) }) - t.Run("after", func(t *testing.T) { testTapeFix(t, MockSbBuilder, blocktime, true) }) + t.Run("after", func(t *testing.T) { testTapeFix(t, kit.MockSbBuilder, blocktime, true) }) } -func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool) { +func testTapeFix(t *testing.T, b kit.APIBuilder, blocktime time.Duration, after bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -43,9 +44,9 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool }) } - n, sn := b(t, []FullNodeOpts{{Opts: func(_ []TestNode) node.Option { + n, sn := b(t, []kit.FullNodeOpts{{Opts: func(_ []kit.TestNode) node.Option { return node.Override(new(stmgr.UpgradeSchedule), upgradeSchedule) - }}}, OneMiner) + }}}, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -65,7 +66,7 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool defer close(done) for ctx.Err() == nil { build.Clock.Sleep(blocktime) - if err := sn[0].MineOne(ctx, MineNext); err != nil { + if err := sn[0].MineOne(ctx, kit.MineNext); err != nil { if ctx.Err() != nil { // context was canceled, ignore the error. return diff --git a/itests/wdpost_dispute_test.go b/itests/wdpost_dispute_test.go index a6528b3b6..923786f23 100644 --- a/itests/wdpost_dispute_test.go +++ b/itests/wdpost_dispute_test.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" minerActor "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node/impl" proof3 "github.com/filecoin-project/specs-actors/v3/actors/runtime/proof" "github.com/stretchr/testify/require" @@ -26,9 +27,9 @@ func TestWindowPostDispute(t *testing.T) { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") } - QuietMiningLogs() + kit.QuietMiningLogs() - b := MockSbBuilder + b := kit.MockSbBuilder blocktime := 2 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) @@ -38,12 +39,12 @@ func TestWindowPostDispute(t *testing.T) { // it doesn't submit proofs. // // Then we're going to manually submit bad proofs. - n, sn := b(t, []FullNodeOpts{ - FullNodeWithLatestActorsAt(-1), - }, []StorageMiner{ - {Full: 0, Preseal: PresealGenesis}, - {Full: 0}, - }) + n, sn := b(t, + []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1)}, + []kit.StorageMiner{ + {Full: 0, Preseal: kit.PresealGenesis}, + {Full: 0}, + }) client := n[0].FullNode.(*impl.FullNodeAPI) chainMiner := sn[0] @@ -75,7 +76,7 @@ func TestWindowPostDispute(t *testing.T) { defer close(done) for ctx.Err() == nil { build.Clock.Sleep(blocktime) - if err := chainMiner.MineOne(ctx, MineNext); err != nil { + if err := chainMiner.MineOne(ctx, kit.MineNext); err != nil { if ctx.Err() != nil { // context was canceled, ignore the error. return @@ -90,9 +91,9 @@ func TestWindowPostDispute(t *testing.T) { }() // Give the chain miner enough sectors to win every block. - pledgeSectors(t, ctx, chainMiner, 10, 0, nil) + kit.PledgeSectors(t, ctx, chainMiner, 10, 0, nil) // And the evil one 1 sector. No cookie for you. - pledgeSectors(t, ctx, evilMiner, 1, 0, nil) + kit.PledgeSectors(t, ctx, evilMiner, 1, 0, nil) // Let the evil miner's sectors gain power. evilMinerAddr, err := evilMiner.ActorAddress(ctx) @@ -257,15 +258,15 @@ func TestWindowPostDisputeFails(t *testing.T) { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") } - QuietMiningLogs() + kit.QuietMiningLogs() - b := MockSbBuilder + b := kit.MockSbBuilder blocktime := 2 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, []FullNodeOpts{FullNodeWithLatestActorsAt(-1)}, OneMiner) + n, sn := b(t, []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1)}, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -295,7 +296,7 @@ func TestWindowPostDisputeFails(t *testing.T) { defer close(done) for ctx.Err() == nil { build.Clock.Sleep(blocktime) - if err := miner.MineOne(ctx, MineNext); err != nil { + if err := miner.MineOne(ctx, kit.MineNext); err != nil { if ctx.Err() != nil { // context was canceled, ignore the error. return @@ -309,7 +310,7 @@ func TestWindowPostDisputeFails(t *testing.T) { <-done }() - pledgeSectors(t, ctx, miner, 10, 0, nil) + kit.PledgeSectors(t, ctx, miner, 10, 0, nil) di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) require.NoError(t, err) @@ -330,7 +331,7 @@ func TestWindowPostDisputeFails(t *testing.T) { ssz, err := miner.ActorSectorSize(ctx, maddr) require.NoError(t, err) - expectedPower := types.NewInt(uint64(ssz) * (GenesisPreseals + 10)) + expectedPower := types.NewInt(uint64(ssz) * (kit.GenesisPreseals + 10)) p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK) require.NoError(t, err) diff --git a/itests/wdpost_test.go b/itests/wdpost_test.go index c30ad812f..f8667f37f 100644 --- a/itests/wdpost_test.go +++ b/itests/wdpost_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" @@ -24,7 +25,7 @@ func TestWindowedPost(t *testing.T) { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") } - QuietMiningLogs() + kit.QuietMiningLogs() var ( blocktime = 2 * time.Millisecond @@ -38,16 +39,16 @@ func TestWindowedPost(t *testing.T) { } { height := height // copy to satisfy lints t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { - testWindowPostUpgrade(t, MockSbBuilder, blocktime, nSectors, height) + testWindowPostUpgrade(t, kit.MockSbBuilder, blocktime, nSectors, height) }) } } -func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int, upgradeHeight abi.ChainEpoch) { +func testWindowPostUpgrade(t *testing.T, b kit.APIBuilder, blocktime time.Duration, nSectors int, upgradeHeight abi.ChainEpoch) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, []FullNodeOpts{FullNodeWithLatestActorsAt(upgradeHeight)}, OneMiner) + n, sn := b(t, []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(upgradeHeight)}, kit.OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -67,7 +68,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, defer close(done) for ctx.Err() == nil { build.Clock.Sleep(blocktime) - if err := sn[0].MineOne(ctx, MineNext); err != nil { + if err := sn[0].MineOne(ctx, kit.MineNext); err != nil { if ctx.Err() != nil { // context was canceled, ignore the error. return @@ -81,7 +82,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, <-done }() - pledgeSectors(t, ctx, miner, nSectors, 0, nil) + kit.PledgeSectors(t, ctx, miner, nSectors, 0, nil) maddr, err := miner.ActorAddress(ctx) require.NoError(t, err) @@ -113,7 +114,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, require.NoError(t, err) require.Equal(t, p.MinerPower, p.TotalPower) - require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals))) + require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+kit.GenesisPreseals))) fmt.Printf("Drop some sectors\n") @@ -196,7 +197,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, require.Equal(t, p.MinerPower, p.TotalPower) sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz) - require.Equal(t, nSectors+GenesisPreseals-3, int(sectors)) // -3 just removed sectors + require.Equal(t, nSectors+kit.GenesisPreseals-3, int(sectors)) // -3 just removed sectors fmt.Printf("Recover one sector\n") @@ -226,11 +227,11 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, require.Equal(t, p.MinerPower, p.TotalPower) sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) - require.Equal(t, nSectors+GenesisPreseals-2, int(sectors)) // -2 not recovered sectors + require.Equal(t, nSectors+kit.GenesisPreseals-2, int(sectors)) // -2 not recovered sectors // pledge a sector after recovery - pledgeSectors(t, ctx, miner, 1, nSectors, nil) + kit.PledgeSectors(t, ctx, miner, 1, nSectors, nil) { // Wait until proven. @@ -257,5 +258,5 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, require.Equal(t, p.MinerPower, p.TotalPower) sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) - require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged + require.Equal(t, nSectors+kit.GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged }