testing: Node test refactoring

api/test/deals.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/ipfs/go-cid"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -36,7 +37,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 	os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, []int{0})
+	n, sn := b(t, 1, oneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -59,11 +60,6 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 		t.Fatal(err)
 	}
 
-	maddr, err := miner.ActorAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	fmt.Println("FILE CID: ", fcid)
 
 	mine := true
@@ -78,10 +74,34 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 			}
 		}
 	}()
+
+	deal := startDeal(t, miner, ctx, client, fcid)
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+
+	waitDealSealed(t, ctx, client, deal)
+
+	// Retrieval
+
+	testRetrieval(t, err, client, ctx, fcid, carExport, data)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}
+
+func startDeal(t *testing.T, miner TestStorageNode, ctx context.Context, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid {
+	maddr, err := miner.ActorAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	addr, err := client.WalletDefaultAddress(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
 		Data:              &storagemarket.DataRef{Root: fcid},
 		Wallet:            addr,
@@ -92,9 +112,10 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 	if err != nil {
 		t.Fatalf("%+v", err)
 	}
+	return deal
+}
 
-	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+func waitDealSealed(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, deal *cid.Cid) {
-	time.Sleep(time.Second)
 loop:
 	for {
 		di, err := client.ClientGetDealInfo(ctx, *deal)
@@ -115,9 +136,9 @@ loop:
 		fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
 		time.Sleep(time.Second / 2)
 	}
+}
 
-	// Retrieval
+func testRetrieval(t *testing.T, err error, client *impl.FullNodeAPI, ctx context.Context, fcid cid.Cid, carExport bool, data []byte) {
-
 	offers, err := client.ClientFindData(ctx, fcid)
 	if err != nil {
 		t.Fatal(err)
@@ -153,6 +174,15 @@ loop:
 	}
 
 	if carExport {
+		rdata = extractCarData(t, ctx, rdata, rpath)
+	}
+
+	if !bytes.Equal(rdata, data) {
+		t.Fatal("wrong data retrieved")
+	}
+}
+
+func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath string) []byte {
 	bserv := dstest.Bserv()
 	ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata))
 	if err != nil {
@@ -179,13 +209,5 @@ loop:
 	if err != nil {
 		t.Fatal(err)
 	}
-	}
+	return rdata
-
-	if !bytes.Equal(rdata, data) {
-		t.Fatal("wrong data retrieved")
-	}
-
-	mine = false
-	fmt.Println("shutting down mining")
-	<-done
 }

api/test/mining.go

@@ -10,7 +10,7 @@ import (
 
 func (ts *testSuite) testMining(t *testing.T) {
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, 1, []int{0})
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
 	h1, err := api.ChainHead(ctx)

api/test/test.go

@@ -19,12 +19,19 @@ type TestStorageNode struct {
 	MineOne func(context.Context) error
 }
 
+var PresealGenesis = -1
+
+type StorageMiner struct {
+	Full    int
+	Preseal int
+}
+
 // APIBuilder is a function which is invoked in test suite to provide
 // test nodes and networks
 //
 // storage array defines storage nodes, numbers in the array specify full node
 // index the storage node 'belongs' to
-type APIBuilder func(t *testing.T, nFull int, storage []int) ([]TestNode, []TestStorageNode)
+type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode)
 type testSuite struct {
 	makeNodes APIBuilder
 }
@@ -41,9 +48,11 @@ func TestApis(t *testing.T, b APIBuilder) {
 	t.Run("testMining", ts.testMining)
 }
 
+var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
+
 func (ts *testSuite) testVersion(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, []int{0})
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
 	v, err := api.Version(ctx)
@@ -57,7 +66,7 @@ func (ts *testSuite) testVersion(t *testing.T) {
 
 func (ts *testSuite) testID(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, []int{0})
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
 	id, err := api.ID(ctx)
@@ -69,7 +78,7 @@ func (ts *testSuite) testID(t *testing.T) {
 
 func (ts *testSuite) testConnectTwo(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 2, []int{0})
+	apis, _ := ts.makeNodes(t, 2, oneMiner)
 
 	p, err := apis[0].NetPeers(ctx)
 	if err != nil {

node/mining_test.go

@@ -0,0 +1,20 @@
+package node_test
+
+import (
+	"testing"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+
+	"github.com/filecoin-project/lotus/api/test"
+)
+
+func TestJoiningMiner(t *testing.T) {
+	logging.SetLogLevel("miner", "ERROR")
+	logging.SetLogLevel("chainstore", "ERROR")
+	logging.SetLogLevel("chain", "ERROR")
+	logging.SetLogLevel("sub", "ERROR")
+	logging.SetLogLevel("storageminer", "ERROR")
+
+	test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false)
+}

node/node_test.go

@@ -79,7 +79,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
 	require.NoError(t, err)
 
 	nic := storedcounter.New(ds, datastore.NewKey("/storage/nextid"))
-	for i := 0; i < nPreseal; i++ {
+	for i := 0; i < nGenesisPreseals; i++ {
 		nic.Next()
 	}
 	nic.Next()
@@ -146,7 +146,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
 	return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne}
 }
 
-func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
+func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
 	ctx := context.Background()
 	mn := mocknet.New(ctx)
 
@@ -182,7 +182,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 		if err != nil {
 			t.Fatal(err)
 		}
-		genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nPreseal, tdir, []byte("make genesis mem random"), nil)
+		genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nGenesisPreseals, tdir, []byte("make genesis mem random"), nil)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -238,16 +238,16 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 
 	}
 
-	for i, full := range storage {
+	for i, def := range storage {
 		// TODO: support non-bootstrap miners
 		if i != 0 {
 			t.Fatal("only one storage node supported")
 		}
-		if full != 0 {
+		if def.Full != 0 {
 			t.Fatal("storage nodes only supported on the first full node")
 		}
 
-		f := fulls[full]
+		f := fulls[def.Full]
 		if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil {
 			t.Fatal(err)
 		}
@@ -276,9 +276,9 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 	return fulls, storers
 }
 
-const nPreseal = 2
+const nGenesisPreseals = 2
 
-func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
+func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
 	ctx := context.Background()
 	mn := mocknet.New(ctx)
 
@@ -313,7 +313,13 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
 		if err != nil {
 			t.Fatal(err)
 		}
-		genm, k, err := mockstorage.PreSeal(2048, maddr, nPreseal)
+
+		preseals := storage[i].Preseal
+		if preseals == test.PresealGenesis {
+			preseals = nGenesisPreseals
+		}
+
+		genm, k, err := mockstorage.PreSeal(2048, maddr, preseals)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -369,16 +375,16 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t
 		}
 	}
 
-	for i, full := range storage {
+	for i, def := range storage {
 		// TODO: support non-bootstrap miners
 		if i != 0 {
 			t.Fatal("only one storage node supported")
 		}
-		if full != 0 {
+		if def.Full != 0 {
 			t.Fatal("storage nodes only supported on the first full node")
 		}
 
-		f := fulls[full]
+		f := fulls[def.Full]
 		if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil {
 			return nil, nil
 		}
@@ -409,7 +415,7 @@ func TestAPI(t *testing.T) {
 	test.TestApis(t, builder)
 }
 
-func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
+func rpcBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
 	fullApis, storaApis := builder(t, nFull, storage)
 	fulls := make([]test.TestNode, nFull)
 	storers := make([]test.TestStorageNode, len(storage))