From 40f56243d15fd893d429dea05710c9b1d32b1ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 23 Apr 2020 19:50:52 +0200 Subject: [PATCH] testing: Node test refactoring --- api/test/deals.go | 102 +++++++++++++++++++++++++++----------------- api/test/mining.go | 2 +- api/test/test.go | 17 ++++++-- node/mining_test.go | 20 +++++++++ node/node_test.go | 32 ++++++++------ 5 files changed, 115 insertions(+), 58 deletions(-) create mode 100644 node/mining_test.go diff --git a/api/test/deals.go b/api/test/deals.go index 8ddf8829a..e7f66ea0e 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/ipfs/go-cid" "io/ioutil" "math/rand" "os" @@ -36,7 +37,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, []int{0}) + n, sn := b(t, 1, oneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -59,11 +60,6 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport t.Fatal(err) } - maddr, err := miner.ActorAddress(ctx) - if err != nil { - t.Fatal(err) - } - fmt.Println("FILE CID: ", fcid) mine := true @@ -78,10 +74,34 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport } } }() + + deal := startDeal(t, miner, ctx, client, fcid) + + // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(time.Second) + + waitDealSealed(t, ctx, client, deal) + + // Retrieval + + testRetrieval(t, err, client, ctx, fcid, carExport, data) + + mine = false + fmt.Println("shutting down mining") + <-done +} + +func startDeal(t *testing.T, miner TestStorageNode, ctx context.Context, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid { + maddr, err := miner.ActorAddress(ctx) + if err != nil { + t.Fatal(err) + } + addr, err := client.WalletDefaultAddress(ctx) if err != nil { t.Fatal(err) } + deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{ Data: &storagemarket.DataRef{Root: fcid}, Wallet: addr, @@ -92,9 +112,10 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport if err != nil { t.Fatalf("%+v", err) } + return deal +} - // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this - time.Sleep(time.Second) +func waitDealSealed(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, deal *cid.Cid) { loop: for { di, err := client.ClientGetDealInfo(ctx, *deal) @@ -115,9 +136,9 @@ loop: fmt.Println("Deal state: ", storagemarket.DealStates[di.State]) time.Sleep(time.Second / 2) } +} - // Retrieval - +func testRetrieval(t *testing.T, err error, client *impl.FullNodeAPI, ctx context.Context, fcid cid.Cid, carExport bool, data []byte) { offers, err := client.ClientFindData(ctx, fcid) if err != nil { t.Fatal(err) @@ -153,39 +174,40 @@ loop: } if carExport { - bserv := dstest.Bserv() - ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata)) - if err != nil { - t.Fatal(err) - } - b, err := bserv.GetBlock(ctx, ch.Roots[0]) - if err != nil { - t.Fatal(err) - } - nd, err := ipld.Decode(b) - if err != nil { - t.Fatal(err) - } - dserv := dag.NewDAGService(bserv) - fil, err := unixfile.NewUnixfsFile(ctx, dserv, nd) - if err != nil { - t.Fatal(err) - } - outPath := filepath.Join(rpath, "retLoadedCAR") - if err := files.WriteTo(fil, outPath); err != nil { - t.Fatal(err) - } - rdata, err = ioutil.ReadFile(outPath) - if err != nil { - t.Fatal(err) - } + rdata = extractCarData(t, ctx, rdata, rpath) } if !bytes.Equal(rdata, data) { t.Fatal("wrong data retrieved") } - - mine = false - fmt.Println("shutting down mining") - <-done +} + +func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath string) []byte { + bserv := dstest.Bserv() + ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata)) + if err != nil { + t.Fatal(err) + } + b, err := bserv.GetBlock(ctx, ch.Roots[0]) + if err != nil { + t.Fatal(err) + } + nd, err := ipld.Decode(b) + if err != nil { + t.Fatal(err) + } + dserv := dag.NewDAGService(bserv) + fil, err := unixfile.NewUnixfsFile(ctx, dserv, nd) + if err != nil { + t.Fatal(err) + } + outPath := filepath.Join(rpath, "retLoadedCAR") + if err := files.WriteTo(fil, outPath); err != nil { + t.Fatal(err) + } + rdata, err = ioutil.ReadFile(outPath) + if err != nil { + t.Fatal(err) + } + return rdata } diff --git a/api/test/mining.go b/api/test/mining.go index 273b2bd48..94774048b 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -10,7 +10,7 @@ import ( func (ts *testSuite) testMining(t *testing.T) { ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, []int{0}) + apis, sn := ts.makeNodes(t, 1, oneMiner) api := apis[0] h1, err := api.ChainHead(ctx) diff --git a/api/test/test.go b/api/test/test.go index 7925300be..675e50171 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -19,12 +19,19 @@ type TestStorageNode struct { MineOne func(context.Context) error } +var PresealGenesis = -1 + +type StorageMiner struct { + Full int + Preseal int +} + // APIBuilder is a function which is invoked in test suite to provide // test nodes and networks // // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, nFull int, storage []int) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -41,9 +48,11 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testMining", ts.testMining) } +var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} + func (ts *testSuite) testVersion(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, []int{0}) + apis, _ := ts.makeNodes(t, 1, oneMiner) api := apis[0] v, err := api.Version(ctx) @@ -57,7 +66,7 @@ func (ts *testSuite) testVersion(t *testing.T) { func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, []int{0}) + apis, _ := ts.makeNodes(t, 1, oneMiner) api := apis[0] id, err := api.ID(ctx) @@ -69,7 +78,7 @@ func (ts *testSuite) testID(t *testing.T) { func (ts *testSuite) testConnectTwo(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 2, []int{0}) + apis, _ := ts.makeNodes(t, 2, oneMiner) p, err := apis[0].NetPeers(ctx) if err != nil { diff --git a/node/mining_test.go b/node/mining_test.go new file mode 100644 index 000000000..e949f813c --- /dev/null +++ b/node/mining_test.go @@ -0,0 +1,20 @@ +package node_test + +import ( + "testing" + "time" + + logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lotus/api/test" +) + +func TestJoiningMiner(t *testing.T) { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false) +} diff --git a/node/node_test.go b/node/node_test.go index 2bd440c18..d5fcc02f0 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -79,7 +79,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a require.NoError(t, err) nic := storedcounter.New(ds, datastore.NewKey("/storage/nextid")) - for i := 0; i < nPreseal; i++ { + for i := 0; i < nGenesisPreseals; i++ { nic.Next() } nic.Next() @@ -146,7 +146,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) { +func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { ctx := context.Background() mn := mocknet.New(ctx) @@ -182,7 +182,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te if err != nil { t.Fatal(err) } - genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nPreseal, tdir, []byte("make genesis mem random"), nil) + genm, k, err := seed.PreSeal(maddr, abi.RegisteredProof_StackedDRG2KiBPoSt, 0, nGenesisPreseals, tdir, []byte("make genesis mem random"), nil) if err != nil { t.Fatal(err) } @@ -238,16 +238,16 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te } - for i, full := range storage { + for i, def := range storage { // TODO: support non-bootstrap miners if i != 0 { t.Fatal("only one storage node supported") } - if full != 0 { + if def.Full != 0 { t.Fatal("storage nodes only supported on the first full node") } - f := fulls[full] + f := fulls[def.Full] if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil { t.Fatal(err) } @@ -276,9 +276,9 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te return fulls, storers } -const nPreseal = 2 +const nGenesisPreseals = 2 -func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) { +func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { ctx := context.Background() mn := mocknet.New(ctx) @@ -313,7 +313,13 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t if err != nil { t.Fatal(err) } - genm, k, err := mockstorage.PreSeal(2048, maddr, nPreseal) + + preseals := storage[i].Preseal + if preseals == test.PresealGenesis { + preseals = nGenesisPreseals + } + + genm, k, err := mockstorage.PreSeal(2048, maddr, preseals) if err != nil { t.Fatal(err) } @@ -369,16 +375,16 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t } } - for i, full := range storage { + for i, def := range storage { // TODO: support non-bootstrap miners if i != 0 { t.Fatal("only one storage node supported") } - if full != 0 { + if def.Full != 0 { t.Fatal("storage nodes only supported on the first full node") } - f := fulls[full] + f := fulls[def.Full] if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil { return nil, nil } @@ -409,7 +415,7 @@ func TestAPI(t *testing.T) { test.TestApis(t, builder) } -func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) { +func rpcBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { fullApis, storaApis := builder(t, nFull, storage) fulls := make([]test.TestNode, nFull) storers := make([]test.TestStorageNode, len(storage))