Merge pull request #81 from filecoin-project/plan/deal-stress-test
Basic deal stress test
This commit is contained in:
Anton Evangelatov
GitHub
commit
e899da0063
@ -50,5 +50,4 @@ jobs:
|
|||||||
- setup
|
- setup
|
||||||
- run:
|
- run:
|
||||||
name: "build lotus-soup"
|
name: "build lotus-soup"
|
||||||
command: pushd lotus-soup && go build .
|
command: pushd lotus-soup && go build -tags=testground .
|
||||||
|
|
||||||
|
|||||||
@ -15,8 +15,8 @@
|
|||||||
"editable": true,
|
"editable": true,
|
||||||
"gnetId": null,
|
"gnetId": null,
|
||||||
"graphTooltip": 0,
|
"graphTooltip": 0,
|
||||||
"id": 1,
|
"id": 16,
|
||||||
"iteration": 1593533384941,
|
"iteration": 1594055543533,
|
||||||
"links": [],
|
"links": [],
|
||||||
"panels": [
|
"panels": [
|
||||||
{
|
{
|
||||||
@ -1677,6 +1677,284 @@
|
|||||||
"align": false,
|
"align": false,
|
||||||
"alignLevel": null
|
"alignLevel": null
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "influxdb",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"fill": 1,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 53
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 23,
|
||||||
|
"legend": {
|
||||||
|
"alignAsTable": true,
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"rightSide": true,
|
||||||
|
"show": true,
|
||||||
|
"total": true,
|
||||||
|
"values": true
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"options": {
|
||||||
|
"dataLinks": []
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 2,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"alias": "deal.sealed - count",
|
||||||
|
"groupBy": [
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"$myinterval"
|
||||||
|
],
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"type": "tag"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"0"
|
||||||
|
],
|
||||||
|
"type": "fill"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"measurement": "diagnostics.deal.sealed.histogram",
|
||||||
|
"orderByTime": "ASC",
|
||||||
|
"policy": "default",
|
||||||
|
"refId": "A",
|
||||||
|
"resultFormat": "time_series",
|
||||||
|
"select": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"count"
|
||||||
|
],
|
||||||
|
"type": "field"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [],
|
||||||
|
"type": "sum"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "run",
|
||||||
|
"operator": "=~",
|
||||||
|
"value": "/^$runid$/"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "started -> sealed",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "none",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "influxdb",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"fill": 1,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 53
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 24,
|
||||||
|
"legend": {
|
||||||
|
"alignAsTable": true,
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"rightSide": true,
|
||||||
|
"show": true,
|
||||||
|
"total": true,
|
||||||
|
"values": true
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"options": {
|
||||||
|
"dataLinks": []
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 2,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"alias": "deal.retrieved - count",
|
||||||
|
"groupBy": [
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"$myinterval"
|
||||||
|
],
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"type": "tag"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"0"
|
||||||
|
],
|
||||||
|
"type": "fill"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"measurement": "diagnostics.deal.retrieved.histogram",
|
||||||
|
"orderByTime": "ASC",
|
||||||
|
"policy": "default",
|
||||||
|
"refId": "A",
|
||||||
|
"resultFormat": "time_series",
|
||||||
|
"select": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"count"
|
||||||
|
],
|
||||||
|
"type": "field"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [],
|
||||||
|
"type": "sum"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "run",
|
||||||
|
"operator": "=~",
|
||||||
|
"value": "/^$runid$/"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "started -> retrieved",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "none",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"refresh": "10s",
|
"refresh": "10s",
|
||||||
@ -1744,7 +2022,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"time": {
|
"time": {
|
||||||
"from": "now-15m",
|
"from": "now-30m",
|
||||||
"to": "now"
|
"to": "now"
|
||||||
},
|
},
|
||||||
"timepicker": {
|
"timepicker": {
|
||||||
2748
dashboards/chain.json
Normal file
2748
dashboards/chain.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,57 @@
|
|||||||
|
[metadata]
|
||||||
|
name = "lotus-soup"
|
||||||
|
author = ""
|
||||||
|
|
||||||
|
[global]
|
||||||
|
plan = "lotus-soup"
|
||||||
|
case = "deals-stress-test"
|
||||||
|
total_instances = 6
|
||||||
|
builder = "docker:go"
|
||||||
|
runner = "local:docker"
|
||||||
|
|
||||||
|
[global.build_config]
|
||||||
|
enable_go_build_cache = true
|
||||||
|
|
||||||
|
[global.build]
|
||||||
|
selectors = ["testground"]
|
||||||
|
|
||||||
|
[global.run_config]
|
||||||
|
exposed_ports = ["6060", "1234", "2345"]
|
||||||
|
|
||||||
|
[global.run.test_params]
|
||||||
|
clients = "3"
|
||||||
|
miners = "2"
|
||||||
|
genesis_timestamp_offset = "0"
|
||||||
|
balance = "2000000000"
|
||||||
|
sectors = "1000"
|
||||||
|
random_beacon_type = "mock"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "bootstrapper"
|
||||||
|
[groups.instances]
|
||||||
|
count = 1
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "bootstrapper"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "miners"
|
||||||
|
[groups.instances]
|
||||||
|
count = 2
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "miner"
|
||||||
|
mining_mode = "natural"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "clients"
|
||||||
|
[groups.instances]
|
||||||
|
count = 3
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "client"
|
||||||
|
deals = "300"
|
||||||
|
deal_mode = "concurrent"
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
[metadata]
|
||||||
|
name = "lotus-soup"
|
||||||
|
author = ""
|
||||||
|
|
||||||
|
[global]
|
||||||
|
plan = "lotus-soup"
|
||||||
|
case = "deals-stress-test"
|
||||||
|
total_instances = 6
|
||||||
|
builder = "docker:go"
|
||||||
|
runner = "local:docker"
|
||||||
|
|
||||||
|
[global.build_config]
|
||||||
|
enable_go_build_cache = true
|
||||||
|
|
||||||
|
[global.build]
|
||||||
|
selectors = ["testground"]
|
||||||
|
|
||||||
|
[global.run_config]
|
||||||
|
exposed_ports = ["6060", "1234", "2345"]
|
||||||
|
|
||||||
|
[global.run.test_params]
|
||||||
|
clients = "3"
|
||||||
|
miners = "2"
|
||||||
|
genesis_timestamp_offset = "100000"
|
||||||
|
balance = "2000000000"
|
||||||
|
sectors = "1000"
|
||||||
|
random_beacon_type = "mock"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "bootstrapper"
|
||||||
|
[groups.instances]
|
||||||
|
count = 1
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "bootstrapper"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "miners"
|
||||||
|
[groups.instances]
|
||||||
|
count = 2
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "miner"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "clients"
|
||||||
|
[groups.instances]
|
||||||
|
count = 3
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "client"
|
||||||
|
deals = "300"
|
||||||
|
deal_mode = "concurrent"
|
||||||
@ -0,0 +1,57 @@
|
|||||||
|
[metadata]
|
||||||
|
name = "lotus-soup"
|
||||||
|
author = ""
|
||||||
|
|
||||||
|
[global]
|
||||||
|
plan = "lotus-soup"
|
||||||
|
case = "deals-stress-test"
|
||||||
|
total_instances = 6
|
||||||
|
builder = "docker:go"
|
||||||
|
runner = "local:docker"
|
||||||
|
|
||||||
|
[global.build_config]
|
||||||
|
enable_go_build_cache = true
|
||||||
|
|
||||||
|
[global.build]
|
||||||
|
selectors = ["testground"]
|
||||||
|
|
||||||
|
[global.run_config]
|
||||||
|
exposed_ports = ["6060", "1234", "2345"]
|
||||||
|
|
||||||
|
[global.run.test_params]
|
||||||
|
clients = "3"
|
||||||
|
miners = "2"
|
||||||
|
genesis_timestamp_offset = "0"
|
||||||
|
balance = "2000000000"
|
||||||
|
sectors = "1000"
|
||||||
|
random_beacon_type = "mock"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "bootstrapper"
|
||||||
|
[groups.instances]
|
||||||
|
count = 1
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "bootstrapper"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "miners"
|
||||||
|
[groups.instances]
|
||||||
|
count = 2
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "miner"
|
||||||
|
mining_mode = "natural"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "clients"
|
||||||
|
[groups.instances]
|
||||||
|
count = 3
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "client"
|
||||||
|
deals = "300"
|
||||||
|
deal_mode = "serial"
|
||||||
56
lotus-soup/compositions/composition-stress-test-serial.toml
Normal file
56
lotus-soup/compositions/composition-stress-test-serial.toml
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
[metadata]
|
||||||
|
name = "lotus-soup"
|
||||||
|
author = ""
|
||||||
|
|
||||||
|
[global]
|
||||||
|
plan = "lotus-soup"
|
||||||
|
case = "deals-stress-test"
|
||||||
|
total_instances = 6
|
||||||
|
builder = "docker:go"
|
||||||
|
runner = "local:docker"
|
||||||
|
|
||||||
|
[global.build_config]
|
||||||
|
enable_go_build_cache = true
|
||||||
|
|
||||||
|
[global.build]
|
||||||
|
selectors = ["testground"]
|
||||||
|
|
||||||
|
[global.run_config]
|
||||||
|
exposed_ports = ["6060", "1234", "2345"]
|
||||||
|
|
||||||
|
[global.run.test_params]
|
||||||
|
clients = "3"
|
||||||
|
miners = "2"
|
||||||
|
genesis_timestamp_offset = "100000"
|
||||||
|
balance = "2000000000"
|
||||||
|
sectors = "1000"
|
||||||
|
random_beacon_type = "mock"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "bootstrapper"
|
||||||
|
[groups.instances]
|
||||||
|
count = 1
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "bootstrapper"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "miners"
|
||||||
|
[groups.instances]
|
||||||
|
count = 2
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "miner"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "clients"
|
||||||
|
[groups.instances]
|
||||||
|
count = 3
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "client"
|
||||||
|
deals = "300"
|
||||||
|
deal_mode = "serial"
|
||||||
@ -0,0 +1,69 @@
|
|||||||
|
[metadata]
|
||||||
|
name = "lotus-soup"
|
||||||
|
author = ""
|
||||||
|
|
||||||
|
[global]
|
||||||
|
plan = "lotus-soup"
|
||||||
|
case = "deals-stress-test"
|
||||||
|
total_instances = 9
|
||||||
|
builder = "docker:go"
|
||||||
|
runner = "cluster:k8s"
|
||||||
|
|
||||||
|
[global.build_config]
|
||||||
|
push_registry=true
|
||||||
|
go_proxy_mode="remote"
|
||||||
|
go_proxy_url="http://localhost:8081"
|
||||||
|
registry_type="aws"
|
||||||
|
|
||||||
|
[global.build]
|
||||||
|
selectors = ["testground"]
|
||||||
|
|
||||||
|
[global.run_config]
|
||||||
|
exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" }
|
||||||
|
|
||||||
|
[global.run.test_params]
|
||||||
|
clients = "6"
|
||||||
|
miners = "2"
|
||||||
|
genesis_timestamp_offset = "0"
|
||||||
|
balance = "200000"
|
||||||
|
sectors = "100"
|
||||||
|
random_beacon_type = "mock"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "bootstrapper"
|
||||||
|
[groups.resources]
|
||||||
|
memory = "4096Mi"
|
||||||
|
cpu = "1000m"
|
||||||
|
[groups.instances]
|
||||||
|
count = 1
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "bootstrapper"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "miners"
|
||||||
|
[groups.resources]
|
||||||
|
memory = "12000Mi"
|
||||||
|
cpu = "1000m"
|
||||||
|
[groups.instances]
|
||||||
|
count = 2
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "miner"
|
||||||
|
mining_mode = "natural"
|
||||||
|
|
||||||
|
[[groups]]
|
||||||
|
id = "clients"
|
||||||
|
[groups.resources]
|
||||||
|
memory = "4096Mi"
|
||||||
|
cpu = "1000m"
|
||||||
|
[groups.instances]
|
||||||
|
count = 6
|
||||||
|
percentage = 0.0
|
||||||
|
[groups.run]
|
||||||
|
[groups.run.test_params]
|
||||||
|
role = "client"
|
||||||
|
deals = "20"
|
||||||
|
deal_mode = "concurrent"
|
||||||
110
lotus-soup/deals_e2e.go
Normal file
110
lotus-soup/deals_e2e.go
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/lotus/api"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This is the baseline test; Filecoin 101.
|
||||||
|
//
|
||||||
|
// A network with a bootstrapper, a number of miners, and a number of clients/full nodes
|
||||||
|
// is constructed and connected through the bootstrapper.
|
||||||
|
// Some funds are allocated to each node and a number of sectors are presealed in the genesis block.
|
||||||
|
//
|
||||||
|
// The test plan:
|
||||||
|
// One or more clients store content to one or more miners, testing storage deals.
|
||||||
|
// The plan ensures that the storage deals hit the blockchain and measure the time it took.
|
||||||
|
// Verification: one or more clients retrieve and verify the hashes of stored content.
|
||||||
|
// The plan ensures that all (previously) published content can be correctly retrieved
|
||||||
|
// and measures the time it took.
|
||||||
|
//
|
||||||
|
// Preparation of the genesis block: this is the responsibility of the bootstrapper.
|
||||||
|
// In order to compute the genesis block, we need to collect identities and presealed
|
||||||
|
// sectors from each node.
|
||||||
|
// Then we create a genesis block that allocates some funds to each node and collects
|
||||||
|
// the presealed sectors.
|
||||||
|
func dealsE2E(t *testkit.TestEnvironment) error {
|
||||||
|
// Dispatch/forward non-client roles to defaults.
|
||||||
|
if t.Role != "client" {
|
||||||
|
return testkit.HandleDefaultRole(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// This is a client role
|
||||||
|
t.RecordMessage("running client")
|
||||||
|
|
||||||
|
cl, err := testkit.PrepareClient(t)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
client := cl.FullApi
|
||||||
|
|
||||||
|
// select a random miner
|
||||||
|
minerAddr := cl.MinerAddrs[rand.Intn(len(cl.MinerAddrs))]
|
||||||
|
if err := client.NetConnect(ctx, minerAddr.PeerAddr); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.D().Counter(fmt.Sprintf("send-data-to,miner=%s", minerAddr.ActorAddr)).Inc(1)
|
||||||
|
|
||||||
|
t.RecordMessage("selected %s as the miner", minerAddr.ActorAddr)
|
||||||
|
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
|
||||||
|
// generate 1600 bytes of random data
|
||||||
|
data := make([]byte, 1600)
|
||||||
|
rand.New(rand.NewSource(time.Now().UnixNano())).Read(data)
|
||||||
|
|
||||||
|
file, err := ioutil.TempFile("/tmp", "data")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(file.Name())
|
||||||
|
|
||||||
|
_, err = file.Write(data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fcid, err := client.ClientImport(ctx, api.FileRef{Path: file.Name(), IsCAR: false})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.RecordMessage("file cid: %s", fcid)
|
||||||
|
|
||||||
|
// start deal
|
||||||
|
t1 := time.Now()
|
||||||
|
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, fcid)
|
||||||
|
t.RecordMessage("started deal: %s", deal)
|
||||||
|
|
||||||
|
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
|
||||||
|
t.RecordMessage("waiting for deal to be sealed")
|
||||||
|
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||||
|
t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1)))
|
||||||
|
|
||||||
|
carExport := true
|
||||||
|
|
||||||
|
t.RecordMessage("trying to retrieve %s", fcid)
|
||||||
|
testkit.RetrieveData(t, ctx, client, fcid, carExport, data)
|
||||||
|
t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1)))
|
||||||
|
|
||||||
|
t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
|
||||||
|
|
||||||
|
time.Sleep(10 * time.Second) // wait for metrics to be emitted
|
||||||
|
|
||||||
|
// TODO broadcast published content CIDs to other clients
|
||||||
|
// TODO select a random piece of content published by some other client and retrieve it
|
||||||
|
|
||||||
|
t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
146
lotus-soup/deals_stress.go
Normal file
146
lotus-soup/deals_stress.go
Normal file
@ -0,0 +1,146 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/lotus/api"
|
||||||
|
"github.com/ipfs/go-cid"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func dealStressTest(t *testkit.TestEnvironment) error {
|
||||||
|
// Dispatch/forward non-client roles to defaults.
|
||||||
|
if t.Role != "client" {
|
||||||
|
return testkit.HandleDefaultRole(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.RecordMessage("running client")
|
||||||
|
|
||||||
|
cl, err := testkit.PrepareClient(t)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
client := cl.FullApi
|
||||||
|
|
||||||
|
// select a random miner
|
||||||
|
minerAddr := cl.MinerAddrs[rand.Intn(len(cl.MinerAddrs))]
|
||||||
|
if err := client.NetConnect(ctx, minerAddr.PeerAddr); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
t.RecordMessage("selected %s as the miner", minerAddr.ActorAddr)
|
||||||
|
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
|
||||||
|
// prepare a number of concurrent data points
|
||||||
|
deals := t.IntParam("deals")
|
||||||
|
data := make([][]byte, 0, deals)
|
||||||
|
files := make([]*os.File, 0, deals)
|
||||||
|
cids := make([]cid.Cid, 0, deals)
|
||||||
|
rng := rand.NewSource(time.Now().UnixNano())
|
||||||
|
|
||||||
|
for i := 0; i < deals; i++ {
|
||||||
|
dealData := make([]byte, 1600)
|
||||||
|
rand.New(rng).Read(dealData)
|
||||||
|
|
||||||
|
dealFile, err := ioutil.TempFile("/tmp", "data")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(dealFile.Name())
|
||||||
|
|
||||||
|
_, err = dealFile.Write(dealData)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
dealCid, err := client.ClientImport(ctx, api.FileRef{Path: dealFile.Name(), IsCAR: false})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
t.RecordMessage("deal %d file cid: %s", i, dealCid)
|
||||||
|
|
||||||
|
data = append(data, dealData)
|
||||||
|
files = append(files, dealFile)
|
||||||
|
cids = append(cids, dealCid)
|
||||||
|
}
|
||||||
|
|
||||||
|
concurrentDeals := true
|
||||||
|
if t.StringParam("deal_mode") == "serial" {
|
||||||
|
concurrentDeals = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// this to avoid failure to get block
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
|
||||||
|
t.RecordMessage("starting storage deals")
|
||||||
|
if concurrentDeals {
|
||||||
|
|
||||||
|
var wg1 sync.WaitGroup
|
||||||
|
for i := 0; i < deals; i++ {
|
||||||
|
wg1.Add(1)
|
||||||
|
go func(i int) {
|
||||||
|
defer wg1.Done()
|
||||||
|
t1 := time.Now()
|
||||||
|
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, cids[i])
|
||||||
|
t.RecordMessage("started storage deal %d -> %s", i, deal)
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
t.RecordMessage("waiting for deal %d to be sealed", i)
|
||||||
|
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||||
|
t.D().ResettingHistogram(fmt.Sprintf("deal.sealed,miner=%s", minerAddr.ActorAddr)).Update(int64(time.Since(t1)))
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
t.RecordMessage("waiting for all deals to be sealed")
|
||||||
|
wg1.Wait()
|
||||||
|
t.RecordMessage("all deals sealed; starting retrieval")
|
||||||
|
|
||||||
|
var wg2 sync.WaitGroup
|
||||||
|
for i := 0; i < deals; i++ {
|
||||||
|
wg2.Add(1)
|
||||||
|
go func(i int) {
|
||||||
|
defer wg2.Done()
|
||||||
|
t.RecordMessage("retrieving data for deal %d", i)
|
||||||
|
t1 := time.Now()
|
||||||
|
testkit.RetrieveData(t, ctx, client, cids[i], true, data[i])
|
||||||
|
t.RecordMessage("retrieved data for deal %d", i)
|
||||||
|
t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1)))
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
t.RecordMessage("waiting for all retrieval deals to complete")
|
||||||
|
wg2.Wait()
|
||||||
|
t.RecordMessage("all retrieval deals successful")
|
||||||
|
|
||||||
|
} else {
|
||||||
|
|
||||||
|
for i := 0; i < deals; i++ {
|
||||||
|
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, cids[i])
|
||||||
|
t.RecordMessage("started storage deal %d -> %s", i, deal)
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
t.RecordMessage("waiting for deal %d to be sealed", i)
|
||||||
|
testkit.WaitDealSealed(t, ctx, client, deal)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < deals; i++ {
|
||||||
|
t.RecordMessage("retrieving data for deal %d", i)
|
||||||
|
testkit.RetrieveData(t, ctx, client, cids[i], true, data[i])
|
||||||
|
t.RecordMessage("retrieved data for deal %d", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
|
||||||
|
t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
|
||||||
|
|
||||||
|
time.Sleep(15 * time.Second) // wait for metrics to be emitted
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@ -6,10 +6,10 @@ require (
|
|||||||
github.com/davecgh/go-spew v1.1.1
|
github.com/davecgh/go-spew v1.1.1
|
||||||
github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4
|
github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4
|
||||||
github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
|
github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
|
||||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4
|
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200706104419-7c180fe156d4
|
||||||
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
|
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
|
||||||
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
|
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
|
||||||
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7
|
github.com/filecoin-project/lotus v0.4.2-0.20200706172415-cf6ac44b6ec5
|
||||||
github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf
|
github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf
|
||||||
github.com/gorilla/mux v1.7.4
|
github.com/gorilla/mux v1.7.4
|
||||||
github.com/influxdata/influxdb v1.8.0 // indirect
|
github.com/influxdata/influxdb v1.8.0 // indirect
|
||||||
|
|||||||
@ -227,8 +227,9 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU
|
|||||||
github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
|
github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
|
||||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo=
|
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo=
|
||||||
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
|
github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
|
||||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4 h1:VqNmKGy4/ryzo/TqevSa1kancc3hSdws7sl/NCTZzT0=
|
|
||||||
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
|
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
|
||||||
|
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200706104419-7c180fe156d4 h1:oI26a0ohPim5xcd/a4xW4GldplBgAGqayoeF+sgnJkQ=
|
||||||
|
github.com/filecoin-project/go-fil-markets v0.3.2-0.20200706104419-7c180fe156d4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
|
||||||
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
|
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
|
||||||
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
|
github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
|
||||||
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
|
github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
|
||||||
@ -246,6 +247,10 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
|
|||||||
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
|
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
|
||||||
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7 h1:eE3a712/0rcnml1lqwtGHYz9JrX4vLqBk9yBdYtH9Jo=
|
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7 h1:eE3a712/0rcnml1lqwtGHYz9JrX4vLqBk9yBdYtH9Jo=
|
||||||
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
github.com/filecoin-project/lotus v0.4.2-0.20200706092412-516e31d37cd7/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
||||||
|
github.com/filecoin-project/lotus v0.4.2-0.20200706153752-f8b65d391143 h1:03PUHBPtjNmxdPbOL258pXLVE4Dz3xjsY86THGV1UQQ=
|
||||||
|
github.com/filecoin-project/lotus v0.4.2-0.20200706153752-f8b65d391143/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
||||||
|
github.com/filecoin-project/lotus v0.4.2-0.20200706172415-cf6ac44b6ec5 h1:wsEkRwhcWvaZowowC2Kj9ueJ2vIRDqOxOcFvqqgHdxE=
|
||||||
|
github.com/filecoin-project/lotus v0.4.2-0.20200706172415-cf6ac44b6ec5/go.mod h1:uo3yDPhPlpHwdCKr0k41/a205WwlSclQamx+sQDKRMI=
|
||||||
github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
|
github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
|
||||||
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
|
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
|
||||||
github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f h1:EHKqNJNIcYggqfrd5nu7SV1KR93ReZygfdSV0w/jefQ=
|
github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f h1:EHKqNJNIcYggqfrd5nu7SV1KR93ReZygfdSV0w/jefQ=
|
||||||
|
|||||||
@ -1,120 +1,24 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/filecoin-project/lotus/api"
|
|
||||||
"github.com/testground/sdk-go/run"
|
"github.com/testground/sdk-go/run"
|
||||||
|
|
||||||
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
"github.com/filecoin-project/oni/lotus-soup/testkit"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/lotus/build"
|
||||||
)
|
)
|
||||||
|
|
||||||
var cases = map[string]interface{}{
|
var cases = map[string]interface{}{
|
||||||
"deals-e2e": testkit.WrapTestEnvironment(dealsE2E),
|
"deals-e2e": testkit.WrapTestEnvironment(dealsE2E),
|
||||||
|
"deals-stress-test": testkit.WrapTestEnvironment(dealStressTest),
|
||||||
"drand-halting": testkit.WrapTestEnvironment(dealsE2E),
|
"drand-halting": testkit.WrapTestEnvironment(dealsE2E),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
build.BlockDelaySecs = 2
|
||||||
|
build.PropagationDelaySecs = 4
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
run.InvokeMap(cases)
|
run.InvokeMap(cases)
|
||||||
}
|
}
|
||||||
|
|
||||||
// This is the baseline test; Filecoin 101.
|
|
||||||
//
|
|
||||||
// A network with a bootstrapper, a number of miners, and a number of clients/full nodes
|
|
||||||
// is constructed and connected through the bootstrapper.
|
|
||||||
// Some funds are allocated to each node and a number of sectors are presealed in the genesis block.
|
|
||||||
//
|
|
||||||
// The test plan:
|
|
||||||
// One or more clients store content to one or more miners, testing storage deals.
|
|
||||||
// The plan ensures that the storage deals hit the blockchain and measure the time it took.
|
|
||||||
// Verification: one or more clients retrieve and verify the hashes of stored content.
|
|
||||||
// The plan ensures that all (previously) published content can be correctly retrieved
|
|
||||||
// and measures the time it took.
|
|
||||||
//
|
|
||||||
// Preparation of the genesis block: this is the responsibility of the bootstrapper.
|
|
||||||
// In order to compute the genesis block, we need to collect identities and presealed
|
|
||||||
// sectors from each node.
|
|
||||||
// Then we create a genesis block that allocates some funds to each node and collects
|
|
||||||
// the presealed sectors.
|
|
||||||
func dealsE2E(t *testkit.TestEnvironment) error {
|
|
||||||
// Dispatch/forward non-client roles to defaults.
|
|
||||||
if t.Role != "client" {
|
|
||||||
return testkit.HandleDefaultRole(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
cl, err := testkit.PrepareClient(t)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// This is a client role
|
|
||||||
t.RecordMessage("running client")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
client := cl.FullApi
|
|
||||||
|
|
||||||
// select a random miner
|
|
||||||
minerAddr := cl.MinerAddrs[rand.Intn(len(cl.MinerAddrs))]
|
|
||||||
if err := client.NetConnect(ctx, minerAddr.PeerAddr); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
t.D().Counter(fmt.Sprintf("send-data-to,miner=%s", minerAddr.ActorAddr)).Inc(1)
|
|
||||||
|
|
||||||
t.RecordMessage("selected %s as the miner", minerAddr.ActorAddr)
|
|
||||||
|
|
||||||
time.Sleep(2 * time.Second)
|
|
||||||
|
|
||||||
// generate 1600 bytes of random data
|
|
||||||
data := make([]byte, 1600)
|
|
||||||
rand.New(rand.NewSource(time.Now().UnixNano())).Read(data)
|
|
||||||
|
|
||||||
file, err := ioutil.TempFile("/tmp", "data")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer os.Remove(file.Name())
|
|
||||||
|
|
||||||
_, err = file.Write(data)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
fcid, err := client.ClientImport(ctx, api.FileRef{Path: file.Name(), IsCAR: false})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
t.RecordMessage("file cid: %s", fcid)
|
|
||||||
|
|
||||||
// start deal
|
|
||||||
t1 := time.Now()
|
|
||||||
deal := testkit.StartDeal(ctx, minerAddr.ActorAddr, client, fcid)
|
|
||||||
t.RecordMessage("started deal: %s", deal)
|
|
||||||
|
|
||||||
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
|
|
||||||
time.Sleep(2 * time.Second)
|
|
||||||
|
|
||||||
t.RecordMessage("waiting for deal to be sealed")
|
|
||||||
testkit.WaitDealSealed(t, ctx, client, deal)
|
|
||||||
t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1)))
|
|
||||||
|
|
||||||
carExport := true
|
|
||||||
|
|
||||||
t.RecordMessage("trying to retrieve %s", fcid)
|
|
||||||
testkit.RetrieveData(t, ctx, err, client, fcid, carExport, data)
|
|
||||||
t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1)))
|
|
||||||
|
|
||||||
t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining)
|
|
||||||
|
|
||||||
time.Sleep(10 * time.Second) // wait for metrics to be emitted
|
|
||||||
|
|
||||||
// TODO broadcast published content CIDs to other clients
|
|
||||||
// TODO select a random piece of content published by some other client and retrieve it
|
|
||||||
|
|
||||||
t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@ -78,3 +78,35 @@ instances = { min = 1, max = 100, default = 5 }
|
|||||||
# Params relevant to pubsub tracing
|
# Params relevant to pubsub tracing
|
||||||
enable_pubsub_tracer = { type = "bool", default = false } # Mining Mode: synchronized -vs- natural time
|
enable_pubsub_tracer = { type = "bool", default = false } # Mining Mode: synchronized -vs- natural time
|
||||||
mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] }
|
mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] }
|
||||||
|
|
||||||
|
[[testcases]]
|
||||||
|
name = "deals-stress-test"
|
||||||
|
instances = { min = 1, max = 100, default = 5 }
|
||||||
|
|
||||||
|
[testcases.params]
|
||||||
|
clients = { type = "int", default = 1 }
|
||||||
|
miners = { type = "int", default = 1 }
|
||||||
|
balance = { type = "int", default = 1 }
|
||||||
|
sectors = { type = "int", default = 1 }
|
||||||
|
role = { type = "string" }
|
||||||
|
|
||||||
|
genesis_timestamp_offset = { type = "int", default = 0 }
|
||||||
|
|
||||||
|
random_beacon_type = { type = "enum", default = "mock", options = ["mock", "local-drand", "external-drand"] }
|
||||||
|
|
||||||
|
# Params relevant to drand nodes. drand nodes should have role="drand", and must all be
|
||||||
|
# in the same composition group. There must be at least threshold drand nodes.
|
||||||
|
# To get lotus nodes to actually use the drand nodes, you must set random_beacon_type="local-drand"
|
||||||
|
# for the lotus node groups.
|
||||||
|
drand_period = { type = "duration", default="10s" }
|
||||||
|
drand_threshold = { type = "int", default = 2 }
|
||||||
|
drand_gossip_relay = { type = "bool", default = true }
|
||||||
|
|
||||||
|
# Params relevant to pubsub tracing
|
||||||
|
enable_pubsub_tracer = { type = "bool", default = false }
|
||||||
|
|
||||||
|
# Mining Mode: synchronized -vs- natural time
|
||||||
|
mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] }
|
||||||
|
|
||||||
|
deals = { type = "int", default = 1 }
|
||||||
|
deal_mode = { type = "enum", default = "serial", options = ["serial", "concurrent"] }
|
||||||
|
|||||||
@ -34,7 +34,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
_ = logging.SetLogLevel("*", "ERROR")
|
_ = logging.SetLogLevel("*", "WARN")
|
||||||
|
|
||||||
_ = os.Setenv("BELLMAN_NO_GPU", "1")
|
_ = os.Setenv("BELLMAN_NO_GPU", "1")
|
||||||
|
|
||||||
@ -245,27 +245,27 @@ func registerAndExportMetrics(instanceName string) {
|
|||||||
view.SetReportingPeriod(5 * time.Second)
|
view.SetReportingPeriod(5 * time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectStats(ctx context.Context, api api.FullNode) error {
|
func collectStats(t *TestEnvironment, ctx context.Context, api api.FullNode) error {
|
||||||
var database string = "testground"
|
t.RecordMessage("collecting blockchain stats")
|
||||||
var headlag int = 3
|
|
||||||
|
|
||||||
influxAddr := os.Getenv("INFLUXDB_URL")
|
influxAddr := os.Getenv("INFLUXDB_URL")
|
||||||
influxUser := ""
|
influxUser := ""
|
||||||
influxPass := ""
|
influxPass := ""
|
||||||
|
influxDb := "testground"
|
||||||
|
|
||||||
influx, err := tstats.InfluxClient(influxAddr, influxUser, influxPass)
|
influx, err := tstats.InfluxClient(influxAddr, influxUser, influxPass)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
t.RecordMessage(err.Error())
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
height, err := tstats.GetLastRecordedHeight(influx, database)
|
height := int64(0)
|
||||||
if err != nil {
|
headlag := 3
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
time.Sleep(15 * time.Second)
|
time.Sleep(15 * time.Second)
|
||||||
tstats.Collect(context.Background(), api, influx, database, height, headlag)
|
t.RecordMessage("calling tstats.Collect")
|
||||||
|
tstats.Collect(context.Background(), api, influx, influxDb, height, headlag)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@ -19,7 +19,7 @@ import (
|
|||||||
"github.com/ipld/go-car"
|
"github.com/ipld/go-car"
|
||||||
)
|
)
|
||||||
|
|
||||||
func RetrieveData(t *TestEnvironment, ctx context.Context, err error, client api.FullNode, fcid cid.Cid, carExport bool, data []byte) {
|
func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, fcid cid.Cid, carExport bool, data []byte) {
|
||||||
t1 := time.Now()
|
t1 := time.Now()
|
||||||
offers, err := client.ClientFindData(ctx, fcid)
|
offers, err := client.ClientFindData(ctx, fcid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -229,7 +229,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) {
|
|||||||
|
|
||||||
// collect stats based on Travis' scripts
|
// collect stats based on Travis' scripts
|
||||||
if t.InitContext.GroupSeq == 1 {
|
if t.InitContext.GroupSeq == 1 {
|
||||||
go collectStats(ctx, n.FullApi)
|
go collectStats(t, ctx, n.FullApi)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bootstrap with full node
|
// Bootstrap with full node
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user