Merge pull request #81 from filecoin-project/plan/deal-stress-test
Basic deal stress test
This commit is contained in:
Anton Evangelatov
GitHub
commit
e899da0063
@ -50,5 +50,4 @@ jobs:
|
||||
- setup
|
||||
- run:
|
||||
name: "build lotus-soup"
|
||||
command: pushd lotus-soup && go build .
|
||||
|
||||
command: pushd lotus-soup && go build -tags=testground .
|
||||
|
||||
@ -15,8 +15,8 @@
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": 1,
|
||||
"iteration": 1593533384941,
|
||||
"id": 16,
|
||||
"iteration": 1594055543533,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
@ -1677,6 +1677,284 @@
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "influxdb",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 53
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 23,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": true,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"dataLinks": []
|
||||
},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"alias": "deal.sealed - count",
|
||||
"groupBy": [
|
||||
{
|
||||
"params": [
|
||||
"$myinterval"
|
||||
],
|
||||
"type": "time"
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
"run"
|
||||
],
|
||||
"type": "tag"
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
"0"
|
||||
],
|
||||
"type": "fill"
|
||||
}
|
||||
],
|
||||
"measurement": "diagnostics.deal.sealed.histogram",
|
||||
"orderByTime": "ASC",
|
||||
"policy": "default",
|
||||
"refId": "A",
|
||||
"resultFormat": "time_series",
|
||||
"select": [
|
||||
[
|
||||
{
|
||||
"params": [
|
||||
"count"
|
||||
],
|
||||
"type": "field"
|
||||
},
|
||||
{
|
||||
"params": [],
|
||||
"type": "sum"
|
||||
}
|
||||
]
|
||||
],
|
||||
"tags": [
|
||||
{
|
||||
"key": "run",
|
||||
"operator": "=~",
|
||||
"value": "/^$runid$/"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "started -> sealed",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "none",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "influxdb",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 53
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 24,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": true,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"dataLinks": []
|
||||
},
|
||||
"percentage": false,
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"alias": "deal.retrieved - count",
|
||||
"groupBy": [
|
||||
{
|
||||
"params": [
|
||||
"$myinterval"
|
||||
],
|
||||
"type": "time"
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
"run"
|
||||
],
|
||||
"type": "tag"
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
"0"
|
||||
],
|
||||
"type": "fill"
|
||||
}
|
||||
],
|
||||
"measurement": "diagnostics.deal.retrieved.histogram",
|
||||
"orderByTime": "ASC",
|
||||
"policy": "default",
|
||||
"refId": "A",
|
||||
"resultFormat": "time_series",
|
||||
"select": [
|
||||
[
|
||||
{
|
||||
"params": [
|
||||
"count"
|
||||
],
|
||||
"type": "field"
|
||||
},
|
||||
{
|
||||
"params": [],
|
||||
"type": "sum"
|
||||
}
|
||||
]
|
||||
],
|
||||
"tags": [
|
||||
{
|
||||
"key": "run",
|
||||
"operator": "=~",
|
||||
"value": "/^$runid$/"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "started -> retrieved",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "none",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
@ -1744,7 +2022,7 @@
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
2748
dashboards/chain.json
Normal file
2748
dashboards/chain.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,57 @@
|
||||
[metadata]
|
||||
name = "lotus-soup"
|
||||
author = ""
|
||||
|
||||
[global]
|
||||
plan = "lotus-soup"
|
||||
case = "deals-stress-test"
|
||||
total_instances = 6
|
||||
builder = "docker:go"
|
||||
runner = "local:docker"
|
||||
|
||||
[global.build_config]
|
||||
enable_go_build_cache = true
|
||||
|
||||
[global.build]
|
||||
selectors = ["testground"]
|
||||
|
||||
[global.run_config]
|
||||
exposed_ports = ["6060", "1234", "2345"]
|
||||
|
||||
[global.run.test_params]
|
||||
clients = "3"
|
||||
miners = "2"
|
||||
genesis_timestamp_offset = "0"
|
||||
balance = "2000000000"
|
||||
sectors = "1000"
|
||||
random_beacon_type = "mock"
|
||||
|
||||
[[groups]]
|
||||
id = "bootstrapper"
|
||||
[groups.instances]
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "bootstrapper"
|
||||
|
||||
[[groups]]
|
||||
id = "miners"
|
||||
[groups.instances]
|
||||
count = 2
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "miner"
|
||||
mining_mode = "natural"
|
||||
|
||||
[[groups]]
|
||||
id = "clients"
|
||||
[groups.instances]
|
||||
count = 3
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "client"
|
||||
deals = "300"
|
||||
deal_mode = "concurrent"
|
||||
@ -0,0 +1,56 @@
|
||||
[metadata]
|
||||
name = "lotus-soup"
|
||||
author = ""
|
||||
|
||||
[global]
|
||||
plan = "lotus-soup"
|
||||
case = "deals-stress-test"
|
||||
total_instances = 6
|
||||
builder = "docker:go"
|
||||
runner = "local:docker"
|
||||
|
||||
[global.build_config]
|
||||
enable_go_build_cache = true
|
||||
|
||||
[global.build]
|
||||
selectors = ["testground"]
|
||||
|
||||
[global.run_config]
|
||||
exposed_ports = ["6060", "1234", "2345"]
|
||||
|
||||
[global.run.test_params]
|
||||
clients = "3"
|
||||
miners = "2"
|
||||
genesis_timestamp_offset = "100000"
|
||||
balance = "2000000000"
|
||||
sectors = "1000"
|
||||
random_beacon_type = "mock"
|
||||
|
||||
[[groups]]
|
||||
id = "bootstrapper"
|
||||
[groups.instances]
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "bootstrapper"
|
||||
|
||||
[[groups]]
|
||||
id = "miners"
|
||||
[groups.instances]
|
||||
count = 2
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "miner"
|
||||
|
||||
[[groups]]
|
||||
id = "clients"
|
||||
[groups.instances]
|
||||
count = 3
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "client"
|
||||
deals = "300"
|
||||
deal_mode = "concurrent"
|
||||
@ -0,0 +1,57 @@
|
||||
[metadata]
|
||||
name = "lotus-soup"
|
||||
author = ""
|
||||
|
||||
[global]
|
||||
plan = "lotus-soup"
|
||||
case = "deals-stress-test"
|
||||
total_instances = 6
|
||||
builder = "docker:go"
|
||||
runner = "local:docker"
|
||||
|
||||
[global.build_config]
|
||||
enable_go_build_cache = true
|
||||
|
||||
[global.build]
|
||||
selectors = ["testground"]
|
||||
|
||||
[global.run_config]
|
||||
exposed_ports = ["6060", "1234", "2345"]
|
||||
|
||||
[global.run.test_params]
|
||||
clients = "3"
|
||||
miners = "2"
|
||||
genesis_timestamp_offset = "0"
|
||||
balance = "2000000000"
|
||||
sectors = "1000"
|
||||
random_beacon_type = "mock"
|
||||
|
||||
[[groups]]
|
||||
id = "bootstrapper"
|
||||
[groups.instances]
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "bootstrapper"
|
||||
|
||||
[[groups]]
|
||||
id = "miners"
|
||||
[groups.instances]
|
||||
count = 2
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "miner"
|
||||
mining_mode = "natural"
|
||||
|
||||
[[groups]]
|
||||
id = "clients"
|
||||
[groups.instances]
|
||||
count = 3
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "client"
|
||||
deals = "300"
|
||||
deal_mode = "serial"
|
||||
56
lotus-soup/compositions/composition-stress-test-serial.toml
Normal file
56
lotus-soup/compositions/composition-stress-test-serial.toml
Normal file
@ -0,0 +1,56 @@
|
||||
[metadata]
|
||||
name = "lotus-soup"
|
||||
author = ""
|
||||
|
||||
[global]
|
||||
plan = "lotus-soup"
|
||||
case = "deals-stress-test"
|
||||
total_instances = 6
|
||||
builder = "docker:go"
|
||||
runner = "local:docker"
|
||||
|
||||
[global.build_config]
|
||||
enable_go_build_cache = true
|
||||
|
||||
[global.build]
|
||||
selectors = ["testground"]
|
||||
|
||||
[global.run_config]
|
||||
exposed_ports = ["6060", "1234", "2345"]
|
||||
|
||||
[global.run.test_params]
|
||||
clients = "3"
|
||||
miners = "2"
|
||||
genesis_timestamp_offset = "100000"
|
||||
balance = "2000000000"
|
||||
sectors = "1000"
|
||||
random_beacon_type = "mock"
|
||||
|
||||
[[groups]]
|
||||
id = "bootstrapper"
|
||||
[groups.instances]
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "bootstrapper"
|
||||
|
||||
[[groups]]
|
||||
id = "miners"
|
||||
[groups.instances]
|
||||
count = 2
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "miner"
|
||||
|
||||
[[groups]]
|
||||
id = "clients"
|
||||
[groups.instances]
|
||||
count = 3
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "client"
|
||||
deals = "300"
|
||||
deal_mode = "serial"
|
||||
@ -0,0 +1,69 @@
|
||||
[metadata]
|
||||
name = "lotus-soup"
|
||||
author = ""
|
||||
|
||||
[global]
|
||||
plan = "lotus-soup"
|
||||
case = "deals-stress-test"
|
||||
total_instances = 9
|
||||
builder = "docker:go"
|
||||
runner = "cluster:k8s"
|
||||
|
||||
[global.build_config]
|
||||
push_registry=true
|
||||
go_proxy_mode="remote"
|
||||
go_proxy_url="http://localhost:8081"
|
||||
registry_type="aws"
|
||||
|
||||
[global.build]
|
||||
selectors = ["testground"]
|
||||
|
||||
[global.run_config]
|
||||
exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" }
|
||||
|
||||
[global.run.test_params]
|
||||
clients = "6"
|
||||
miners = "2"
|
||||
genesis_timestamp_offset = "0"
|
||||
balance = "200000"
|
||||
sectors = "100"
|
||||
random_beacon_type = "mock"
|
||||
|
||||
[[groups]]
|
||||
id = "bootstrapper"
|
||||
[groups.resources]
|
||||
memory = "4096Mi"
|
||||
cpu = "1000m"
|
||||
[groups.instances]
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "bootstrapper"
|
||||
|
||||
[[groups]]
|
||||
id = "miners"
|
||||
[groups.resources]
|
||||
memory = "12000Mi"
|
||||
cpu = "1000m"
|
||||
[groups.instances]
|
||||
count = 2
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "miner"
|
||||
mining_mode = "natural"
|
||||
|
||||
[[groups]]
|
||||
id = "clients"
|
||||
[groups.resources]
|
||||
memory = "4096Mi"
|
||||
cpu = "1000m"
|
||||
[groups.instances]
|
||||
count = 6
|
||||
percentage = 0.0
|
||||
[groups.run]
|
||||
[groups.run.test_params]
|
||||
role = "client"
|
||||
deals = "20"
|
||||
deal_mode = "concurrent"
|
||||
110
lotus-soup/deals_e2e.go
Normal file
110
lotus-soup/deals_e2e.go
Normal file
@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
|
||||
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
||||
)
|
||||
|
||||
// This is the baseline test; Filecoin 101.
|
||||
//
|
||||
// A network with a bootstrapper, a number of miners, and a number of clients/full nodes
|
||||
// is constructed and connected through the bootstrapper.
|
||||
// Some funds are allocated to each node and a number of sectors are presealed in the genesis block.
|
||||
//
|
||||
// The test plan:
|
||||
// One or more clients store content to one or more miners, testing storage deals.
|
||||
// The plan ensures that the storage deals hit the blockchain and measure the time it took.
|
||||
// Verification: one or more clients retrieve and verify the hashes of stored content.
|
||||
// The plan ensures that all (previously) published content can be correctly retrieved
|
||||
// and measures the time it took.
|
||||
//
|
||||
// Preparation of the genesis block: this is the responsibility of the bootstrapper.
|
||||
// In order to compute the genesis block, we need to collect identities and presealed
|
||||
// sectors from each node.
|
||||
// Then we create a genesis block that allocates some funds to each node and collects
|
||||
// the presealed sectors.
|
||||
func dealsE2E(t *testkit.TestEnvironment) error {
|
||||
// Dispatch/forward non-client roles to defaults.
|
||||
if t.Role != "client" {
|
||||
return testkit.HandleDefaultRole(t)
|
||||
}
|
||||
|
||||
// This is a client role
|
||||
t.RecordMessage("running client")
|
||||
|
||||
cl, err := testkit.PrepareClient(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client := cl.FullApi
|
||||
|
||||
// select a random miner
|
||||
minerAddr := cl.MinerAddrs[rand.Intn(len(cl.MinerAddrs))]
|
||||
if err := client.NetConnect(ctx, minerAddr.PeerAddr); err != nil {
|
||||
return err
|
||||
}
|
||||
t.D().Counter(fmt.Sprintf("send-data-to,miner=%s", minerAddr.ActorAddr)).Inc(1)
|
||||
|
||||
t.RecordMessage("selected %s as the miner", minerAddr.ActorAddr)
|
||||
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// generate 1600 bytes of random data
|
||||
data := make([]byte, 1600)
|
||||
rand.New(rand.NewSource(time.Now().UnixNano())).Read(data)
|
||||
|
||||
file, err := ioutil.TempFile("/tmp", "data")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
|
||||
_, err = file.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fcid, err := client.ClientImport(ctx, api.FileRef{Path: file.Name(), IsCAR: false})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.RecordMessage("file cid: %s", fcid)
|
||||
|
||||
// start deal
|
||||
t1 := time.Now()
|
||||
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, fcid)
|
||||
t.RecordMessage("started deal: %s", deal)
|
||||
|
||||
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
t.RecordMessage("waiting for deal to be sealed")
|
||||
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||
t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1)))
|
||||
|
||||
carExport := true
|
||||
|
||||
t.RecordMessage("trying to retrieve %s", fcid)
|
||||
testkit.RetrieveData(t, ctx, client, fcid, carExport, data)
|
||||
t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1)))
|
||||
|
||||
t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
|
||||
|
||||
time.Sleep(10 * time.Second) // wait for metrics to be emitted
|
||||
|
||||
// TODO broadcast published content CIDs to other clients
|
||||
// TODO select a random piece of content published by some other client and retrieve it
|
||||
|
||||
t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
|
||||
return nil
|
||||
}
|
||||
146
lotus-soup/deals_stress.go
Normal file
146
lotus-soup/deals_stress.go
Normal file
@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
||||
)
|
||||
|
||||
func dealStressTest(t *testkit.TestEnvironment) error {
|
||||
// Dispatch/forward non-client roles to defaults.
|
||||
if t.Role != "client" {
|
||||
return testkit.HandleDefaultRole(t)
|
||||
}
|
||||
|
||||
t.RecordMessage("running client")
|
||||
|
||||
cl, err := testkit.PrepareClient(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client := cl.FullApi
|
||||
|
||||
// select a random miner
|
||||
minerAddr := cl.MinerAddrs[rand.Intn(len(cl.MinerAddrs))]
|
||||
if err := client.NetConnect(ctx, minerAddr.PeerAddr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.RecordMessage("selected %s as the miner", minerAddr.ActorAddr)
|
||||
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// prepare a number of concurrent data points
|
||||
deals := t.IntParam("deals")
|
||||
data := make([][]byte, 0, deals)
|
||||
files := make([]*os.File, 0, deals)
|
||||
cids := make([]cid.Cid, 0, deals)
|
||||
rng := rand.NewSource(time.Now().UnixNano())
|
||||
|
||||
for i := 0; i < deals; i++ {
|
||||
dealData := make([]byte, 1600)
|
||||
rand.New(rng).Read(dealData)
|
||||
|
||||
dealFile, err := ioutil.TempFile("/tmp", "data")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(dealFile.Name())
|
||||
|
||||
_, err = dealFile.Write(dealData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dealCid, err := client.ClientImport(ctx, api.FileRef{Path: dealFile.Name(), IsCAR: false})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.RecordMessage("deal %d file cid: %s", i, dealCid)
|
||||
|
||||
data = append(data, dealData)
|
||||
files = append(files, dealFile)
|
||||
cids = append(cids, dealCid)
|
||||
}
|
||||
|
||||
concurrentDeals := true
|
||||
if t.StringParam("deal_mode") == "serial" {
|
||||
concurrentDeals = false
|
||||
}
|
||||
|
||||
// this to avoid failure to get block
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
t.RecordMessage("starting storage deals")
|
||||
if concurrentDeals {
|
||||
|
||||
var wg1 sync.WaitGroup
|
||||
for i := 0; i < deals; i++ {
|
||||
wg1.Add(1)
|
||||
go func(i int) {
|
||||
defer wg1.Done()
|
||||
t1 := time.Now()
|
||||
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, cids[i])
|
||||
t.RecordMessage("started storage deal %d -> %s", i, deal)
|
||||
time.Sleep(2 * time.Second)
|
||||
t.RecordMessage("waiting for deal %d to be sealed", i)
|
||||
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||
t.D().ResettingHistogram(fmt.Sprintf("deal.sealed,miner=%s", minerAddr.ActorAddr)).Update(int64(time.Since(t1)))
|
||||
}(i)
|
||||
}
|
||||
t.RecordMessage("waiting for all deals to be sealed")
|
||||
wg1.Wait()
|
||||
t.RecordMessage("all deals sealed; starting retrieval")
|
||||
|
||||
var wg2 sync.WaitGroup
|
||||
for i := 0; i < deals; i++ {
|
||||
wg2.Add(1)
|
||||
go func(i int) {
|
||||
defer wg2.Done()
|
||||
t.RecordMessage("retrieving data for deal %d", i)
|
||||
t1 := time.Now()
|
||||
testkit.RetrieveData(t, ctx, client, cids[i], true, data[i])
|
||||
t.RecordMessage("retrieved data for deal %d", i)
|
||||
t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1)))
|
||||
}(i)
|
||||
}
|
||||
t.RecordMessage("waiting for all retrieval deals to complete")
|
||||
wg2.Wait()
|
||||
t.RecordMessage("all retrieval deals successful")
|
||||
|
||||
} else {
|
||||
|
||||
for i := 0; i < deals; i++ {
|
||||
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, cids[i])
|
||||
t.RecordMessage("started storage deal %d -> %s", i, deal)
|
||||
time.Sleep(2 * time.Second)
|
||||
t.RecordMessage("waiting for deal %d to be sealed", i)
|
||||
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||
}
|
||||
|
||||
for i := 0; i < deals; i++ {
|
||||
t.RecordMessage("retrieving data for deal %d", i)
|
||||
testkit.RetrieveData(t, ctx, client, cids[i], true, data[i])
|
||||
t.RecordMessage("retrieved data for deal %d", i)
|
||||
}
|
||||
}
|
||||
|
||||
t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
|
||||
t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
|
||||
|
||||
time.Sleep(15 * time.Second) // wait for metrics to be emitted
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -6,10 +6,10 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4
|
||||
github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200706104419-7c180fe156d4
|
||||
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
|
||||
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706172415-cf6ac44b6ec5
|
||||
github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/influxdata/influxdb v1.8.0 // indirect
|
||||
|
||||
@ -227,8 +227,9 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU
|
||||
github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo=
|
||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4 h1:VqNmKGy4/ryzo/TqevSa1kancc3hSdws7sl/NCTZzT0=
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200706104419-7c180fe156d4 h1:oI26a0ohPim5xcd/a4xW4GldplBgAGqayoeF+sgnJkQ=
|
||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200706104419-7c180fe156d4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
|
||||
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
|
||||
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
|
||||
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
|
||||
@ -246,6 +247,10 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
|
||||
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7 h1:eE3a712/0rcnml1lqwtGHYz9JrX4vLqBk9yBdYtH9Jo=
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706153752-f8b65d391143 h1:03PUHBPtjNmxdPbOL258pXLVE4Dz3xjsY86THGV1UQQ=
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706153752-f8b65d391143/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706172415-cf6ac44b6ec5 h1:wsEkRwhcWvaZowowC2Kj9ueJ2vIRDqOxOcFvqqgHdxE=
|
||||
github.com/filecoin-project/lotus v0.4.2-0.20200706172415-cf6ac44b6ec5/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
||||
github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
|
||||
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
|
||||
github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f h1:EHKqNJNIcYggqfrd5nu7SV1KR93ReZygfdSV0w/jefQ=
|
||||
|
||||
@ -1,120 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/testground/sdk-go/run"
|
||||
|
||||
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
)
|
||||
|
||||
var cases = map[string]interface{}{
|
||||
"deals-e2e": testkit.WrapTestEnvironment(dealsE2E),
|
||||
"drand-halting": testkit.WrapTestEnvironment(dealsE2E),
|
||||
"deals-e2e": testkit.WrapTestEnvironment(dealsE2E),
|
||||
"deals-stress-test": testkit.WrapTestEnvironment(dealStressTest),
|
||||
"drand-halting": testkit.WrapTestEnvironment(dealsE2E),
|
||||
}
|
||||
|
||||
func init() {
|
||||
build.BlockDelaySecs = 2
|
||||
build.PropagationDelaySecs = 4
|
||||
}
|
||||
|
||||
func main() {
|
||||
run.InvokeMap(cases)
|
||||
}
|
||||
|
||||
// This is the baseline test; Filecoin 101.
|
||||
//
|
||||
// A network with a bootstrapper, a number of miners, and a number of clients/full nodes
|
||||
// is constructed and connected through the bootstrapper.
|
||||
// Some funds are allocated to each node and a number of sectors are presealed in the genesis block.
|
||||
//
|
||||
// The test plan:
|
||||
// One or more clients store content to one or more miners, testing storage deals.
|
||||
// The plan ensures that the storage deals hit the blockchain and measure the time it took.
|
||||
// Verification: one or more clients retrieve and verify the hashes of stored content.
|
||||
// The plan ensures that all (previously) published content can be correctly retrieved
|
||||
// and measures the time it took.
|
||||
//
|
||||
// Preparation of the genesis block: this is the responsibility of the bootstrapper.
|
||||
// In order to compute the genesis block, we need to collect identities and presealed
|
||||
// sectors from each node.
|
||||
// Then we create a genesis block that allocates some funds to each node and collects
|
||||
// the presealed sectors.
|
||||
func dealsE2E(t *testkit.TestEnvironment) error {
|
||||
// Dispatch/forward non-client roles to defaults.
|
||||
if t.Role != "client" {
|
||||
return testkit.HandleDefaultRole(t)
|
||||
}
|
||||
|
||||
cl, err := testkit.PrepareClient(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// This is a client role
|
||||
t.RecordMessage("running client")
|
||||
|
||||
ctx := context.Background()
|
||||
client := cl.FullApi
|
||||
|
||||
// select a random miner
|
||||
minerAddr := cl.MinerAddrs[rand.Intn(len(cl.MinerAddrs))]
|
||||
if err := client.NetConnect(ctx, minerAddr.PeerAddr); err != nil {
|
||||
return err
|
||||
}
|
||||
t.D().Counter(fmt.Sprintf("send-data-to,miner=%s", minerAddr.ActorAddr)).Inc(1)
|
||||
|
||||
t.RecordMessage("selected %s as the miner", minerAddr.ActorAddr)
|
||||
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// generate 1600 bytes of random data
|
||||
data := make([]byte, 1600)
|
||||
rand.New(rand.NewSource(time.Now().UnixNano())).Read(data)
|
||||
|
||||
file, err := ioutil.TempFile("/tmp", "data")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
|
||||
_, err = file.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fcid, err := client.ClientImport(ctx, api.FileRef{Path: file.Name(), IsCAR: false})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.RecordMessage("file cid: %s", fcid)
|
||||
|
||||
// start deal
|
||||
t1 := time.Now()
|
||||
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, fcid)
|
||||
t.RecordMessage("started deal: %s", deal)
|
||||
|
||||
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
t.RecordMessage("waiting for deal to be sealed")
|
||||
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||
t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1)))
|
||||
|
||||
carExport := true
|
||||
|
||||
t.RecordMessage("trying to retrieve %s", fcid)
|
||||
testkit.RetrieveData(t, ctx, err, client, fcid, carExport, data)
|
||||
t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1)))
|
||||
|
||||
t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
|
||||
|
||||
time.Sleep(10 * time.Second) // wait for metrics to be emitted
|
||||
|
||||
// TODO broadcast published content CIDs to other clients
|
||||
// TODO select a random piece of content published by some other client and retrieve it
|
||||
|
||||
t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -78,3 +78,35 @@ instances = { min = 1, max = 100, default = 5 }
|
||||
# Params relevant to pubsub tracing
|
||||
enable_pubsub_tracer = { type = "bool", default = false } # Mining Mode: synchronized -vs- natural time
|
||||
mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] }
|
||||
|
||||
[[testcases]]
|
||||
name = "deals-stress-test"
|
||||
instances = { min = 1, max = 100, default = 5 }
|
||||
|
||||
[testcases.params]
|
||||
clients = { type = "int", default = 1 }
|
||||
miners = { type = "int", default = 1 }
|
||||
balance = { type = "int", default = 1 }
|
||||
sectors = { type = "int", default = 1 }
|
||||
role = { type = "string" }
|
||||
|
||||
genesis_timestamp_offset = { type = "int", default = 0 }
|
||||
|
||||
random_beacon_type = { type = "enum", default = "mock", options = ["mock", "local-drand", "external-drand"] }
|
||||
|
||||
# Params relevant to drand nodes. drand nodes should have role="drand", and must all be
|
||||
# in the same composition group. There must be at least threshold drand nodes.
|
||||
# To get lotus nodes to actually use the drand nodes, you must set random_beacon_type="local-drand"
|
||||
# for the lotus node groups.
|
||||
drand_period = { type = "duration", default="10s" }
|
||||
drand_threshold = { type = "int", default = 2 }
|
||||
drand_gossip_relay = { type = "bool", default = true }
|
||||
|
||||
# Params relevant to pubsub tracing
|
||||
enable_pubsub_tracer = { type = "bool", default = false }
|
||||
|
||||
# Mining Mode: synchronized -vs- natural time
|
||||
mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] }
|
||||
|
||||
deals = { type = "int", default = 1 }
|
||||
deal_mode = { type = "enum", default = "serial", options = ["serial", "concurrent"] }
|
||||
|
||||
@ -34,7 +34,7 @@ import (
|
||||
)
|
||||
|
||||
func init() {
|
||||
_ = logging.SetLogLevel("*", "ERROR")
|
||||
_ = logging.SetLogLevel("*", "WARN")
|
||||
|
||||
_ = os.Setenv("BELLMAN_NO_GPU", "1")
|
||||
|
||||
@ -245,27 +245,27 @@ func registerAndExportMetrics(instanceName string) {
|
||||
view.SetReportingPeriod(5 * time.Second)
|
||||
}
|
||||
|
||||
func collectStats(ctx context.Context, api api.FullNode) error {
|
||||
var database string = "testground"
|
||||
var headlag int = 3
|
||||
func collectStats(t *TestEnvironment, ctx context.Context, api api.FullNode) error {
|
||||
t.RecordMessage("collecting blockchain stats")
|
||||
|
||||
influxAddr := os.Getenv("INFLUXDB_URL")
|
||||
influxUser := ""
|
||||
influxPass := ""
|
||||
influxDb := "testground"
|
||||
|
||||
influx, err := tstats.InfluxClient(influxAddr, influxUser, influxPass)
|
||||
if err != nil {
|
||||
t.RecordMessage(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
height, err := tstats.GetLastRecordedHeight(influx, database)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
height := int64(0)
|
||||
headlag := 3
|
||||
|
||||
go func() {
|
||||
time.Sleep(15 * time.Second)
|
||||
tstats.Collect(context.Background(), api, influx, database, height, headlag)
|
||||
t.RecordMessage("calling tstats.Collect")
|
||||
tstats.Collect(context.Background(), api, influx, influxDb, height, headlag)
|
||||
}()
|
||||
|
||||
return nil
|
||||
|
||||
@ -19,7 +19,7 @@ import (
|
||||
"github.com/ipld/go-car"
|
||||
)
|
||||
|
||||
func RetrieveData(t *TestEnvironment, ctx context.Context, err error, client api.FullNode, fcid cid.Cid, carExport bool, data []byte) {
|
||||
func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, fcid cid.Cid, carExport bool, data []byte) {
|
||||
t1 := time.Now()
|
||||
offers, err := client.ClientFindData(ctx, fcid)
|
||||
if err != nil {
|
||||
|
||||
@ -229,7 +229,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) {
|
||||
|
||||
// collect stats based on Travis' scripts
|
||||
if t.InitContext.GroupSeq == 1 {
|
||||
go collectStats(ctx, n.FullApi)
|
||||
go collectStats(t, ctx, n.FullApi)
|
||||
}
|
||||
|
||||
// Bootstrap with full node
|
||||
|
||||
Loading…
Reference in New Issue
Block a user