fixes

lotus-soup/compositions/stress-test-concurrent-natural-k8s.toml

@@ -5,7 +5,7 @@
 [global]
   plan = "lotus-soup"
   case = "deals-stress-test"
-  total_instances = 6
+  total_instances = 9
   builder = "docker:go"
   runner = "cluster:k8s"
 
@@ -22,7 +22,7 @@
   exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" }
 
 [global.run.test_params]
-  clients = "3"
+  clients = "6"
   miners = "2"
   genesis_timestamp_offset = "0"
   balance = "200000"
@@ -60,10 +60,10 @@
     memory = "4096Mi"
     cpu = "1000m"
   [groups.instances]
-    count = 3
+    count = 6
     percentage = 0.0
   [groups.run]
     [groups.run.test_params]
       role = "client"
-      deals = "10"
+      deals = "20"
       deal_mode = "concurrent"

lotus-soup/deals_stress.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -96,7 +97,7 @@ func dealStressTest(t *testkit.TestEnvironment) error {
 				time.Sleep(2 * time.Second)
 				t.RecordMessage("waiting for deal %d to be sealed", i)
 				testkit.WaitDealSealed(t, ctx, client, deal)
-				t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1)))
+				t.D().ResettingHistogram(fmt.Sprintf("deal.sealed,miner=", minerAddr.ActorAddr)).Update(int64(time.Since(t1)))
 			}(i)
 		}
 		t.RecordMessage("waiting for all deals to be sealed")
@@ -139,5 +140,7 @@ func dealStressTest(t *testkit.TestEnvironment) error {
 	t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
 	t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
 
+	time.Sleep(15 * time.Second) // wait for metrics to be emitted
+
 	return nil
 }

lotus-soup/manifest.toml

@@ -11,7 +11,7 @@ enabled = true
 [builders."docker:go"]
 enabled = true
 build_base_image = "iptestground/oni-buildbase:v4"
-runtime_image = "iptestground/oni-runtime:v2"
+runtime_image = "iptestground/oni-runtime:v2-debug"
 
 [runners."local:exec"]
 enabled = true

lotus-soup/testkit/node.go

@@ -245,27 +245,27 @@ func registerAndExportMetrics(instanceName string) {
 	view.SetReportingPeriod(5 * time.Second)
 }
 
-func collectStats(ctx context.Context, api api.FullNode) error {
-	var database string = "testground"
-	var headlag int = 3
+func collectStats(t *TestEnvironment, ctx context.Context, api api.FullNode) error {
+	t.RecordMessage("collecting blockchain stats")
 
 	influxAddr := os.Getenv("INFLUXDB_URL")
 	influxUser := ""
 	influxPass := ""
+	influxDb := "testground"
 
 	influx, err := tstats.InfluxClient(influxAddr, influxUser, influxPass)
 	if err != nil {
+		t.RecordMessage(err.Error())
 		return err
 	}
 
-	height, err := tstats.GetLastRecordedHeight(influx, database)
-	if err != nil {
-		return err
-	}
+	height := int64(0)
+	headlag := 3
 
 	go func() {
 		time.Sleep(15 * time.Second)
-		tstats.Collect(context.Background(), api, influx, database, height, headlag)
+		t.RecordMessage("calling tstats.Collect")
+		tstats.Collect(context.Background(), api, influx, influxDb, height, headlag)
 	}()
 
 	return nil

lotus-soup/testkit/role_miner.go

@@ -229,7 +229,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) {
 
 	// collect stats based on Travis' scripts
 	if t.InitContext.GroupSeq == 1 {
-		go collectStats(ctx, n.FullApi)
+		go collectStats(t, ctx, n.FullApi)
 	}
 
 	// Bootstrap with full node