From 7af5ab5445f9a167685243eb31d2829baf3d218e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 27 Jul 2020 13:57:01 +0200 Subject: [PATCH] slashing/termination from failed windowed post + fast retrieval benchmarks (#95) --- Makefile | 18 +- README.md | 18 +- composer/chain-state.ipynb | 174 +++++ composer/requirements.txt | 2 + dashboards/baseline.json | 71 +- docker-images/Dockerfile.oni-runtime-debug | 6 +- docker-images/build-buildbase.sh | 26 - docker-images/build-runtime.sh | 26 - lotus-soup/_compositions/fast-k8s-3-1.toml | 68 ++ lotus-soup/_compositions/recovery-exec.toml | 80 ++ lotus-soup/_compositions/recovery-k8s.toml | 95 +++ lotus-soup/deals_e2e.go | 74 +- lotus-soup/deals_stress.go | 9 +- lotus-soup/go.mod | 11 +- lotus-soup/go.sum | 49 +- lotus-soup/init.go | 10 + lotus-soup/main.go | 10 +- lotus-soup/manifest.toml | 34 +- lotus-soup/rfwp/chain_state.go | 812 ++++++++++++++++++++ lotus-soup/rfwp/diffs.go | 295 +++++++ lotus-soup/rfwp/e2e.go | 347 +++++++++ lotus-soup/rfwp/html_chain_state.go | 66 ++ lotus-soup/testkit/deals.go | 28 +- lotus-soup/testkit/retrieval.go | 13 +- lotus-soup/testkit/role_client.go | 44 +- lotus-soup/testkit/role_miner.go | 267 +++++-- lotus-soup/testkit/sync.go | 7 + 27 files changed, 2464 insertions(+), 196 deletions(-) create mode 100644 composer/chain-state.ipynb delete mode 100755 docker-images/build-buildbase.sh delete mode 100755 docker-images/build-runtime.sh create mode 100644 lotus-soup/_compositions/fast-k8s-3-1.toml create mode 100644 lotus-soup/_compositions/recovery-exec.toml create mode 100644 lotus-soup/_compositions/recovery-k8s.toml create mode 100644 lotus-soup/rfwp/chain_state.go create mode 100644 lotus-soup/rfwp/diffs.go create mode 100644 lotus-soup/rfwp/e2e.go create mode 100644 lotus-soup/rfwp/html_chain_state.go diff --git a/Makefile b/Makefile index cc33cc2ca..06184d2d7 100644 --- a/Makefile +++ b/Makefile @@ -4,4 +4,20 @@ SHELL = /bin/bash download-proofs: go run github.com/filecoin-project/go-paramfetch/paramfetch 2048 ./docker-images/proof-parameters.json -.PHONY: download-proofs \ No newline at end of file + +build-images: + docker build -t "iptestground/oni-buildbase:v5" -f "docker-images/Dockerfile.oni-buildbase" "docker-images" + docker build -t "iptestground/oni-runtime:v2" -f "docker-images/Dockerfile.oni-runtime" "docker-images" + docker build -t "iptestground/oni-runtime:v2-debug" -f "docker-images/Dockerfile.oni-runtime-debug" "docker-images" + +push-images: + docker push iptestground/oni-buildbase:v5 + docker push iptestground/oni-runtime:v2 + docker push iptestground/oni-runtime:v2-debug + +pull-images: + docker pull iptestground/oni-buildbase:v5 + docker pull iptestground/oni-runtime:v2 + docker pull iptestground/oni-runtime:v2-debug + +.PHONY: download-proofs build-images push-images pull-images diff --git a/README.md b/README.md index 1af37b5ce..23fce81e4 100644 --- a/README.md +++ b/README.md @@ -75,16 +75,13 @@ testground daemon 3. Download required Docker images for the `lotus-soup` test plan ``` -docker pull iptestground/oni-buildbase:v5 -docker pull iptestground/oni-runtime:v2 +make pull-images ``` -Alternatively you can build them locally from the `docker-images` directory +Alternatively you can build them locally with ``` -cd docker-images -./build-buildbase.sh v5 -./build-runtime.sh v2 +make build-images ``` 4. Import the `lotus-soup` test plan into your Testground home directory @@ -166,7 +163,7 @@ Depending on the runner you want to use to run the test plan, these dependencies * `local:docker` -The Rust libraries are included in the Filecoin FFI Git submodule, which is part of the `iptestground/oni-buildbase` image. If the FFI changes on Lotus, we have to rebuild this image with the `./docker-images/build-buildbase.sh vX` command, where X is the next version (see [Docker images changelog](#docker-images-changelog) +The Rust libraries are included in the Filecoin FFI Git submodule, which is part of the `iptestground/oni-buildbase` image. If the FFI changes on Lotus, we have to rebuild this image with the `make build-images` command, where X is the next version (see [Docker images changelog](#docker-images-changelog) below). * `local:exec` @@ -179,7 +176,7 @@ The same process as for `local:docker`, however you need to make sure that the r ### proof parameters -Additional to the Filecoin FFI Git submodules, we are also bundling `proof parameters` in the `iptestground/oni-runtime` image. If these change, you will need to rebuild that image with `./docker-images/build-runtime.sh vX` command, where X is the next version. These parameters are downloaded automatically for `local:exec` if they are not present. +Additional to the Filecoin FFI Git submodules, we are also bundling `proof parameters` in the `iptestground/oni-runtime` image. If these change, you will need to rebuild that image with `make build-images` command, where X is the next version. These parameters are downloaded automatically for `local:exec` if they are not present. ## Docker images changelog @@ -196,6 +193,11 @@ Additional to the Filecoin FFI Git submodules, we are also bundling `proof param * `v1` => initial image with 2048 parameters. * `v2` => adds auxiliary tools: `net-tools netcat traceroute iputils-ping wget vim curl telnet iproute2 dnsutils`. +### oni-runtime-debug + +* `v1` => initial image +* `v2` => locking in Lotus commit e21ea53 + ## Team diff --git a/composer/chain-state.ipynb b/composer/chain-state.ipynb new file mode 100644 index 000000000..bd833dd21 --- /dev/null +++ b/composer/chain-state.ipynb @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import hvplot.pandas\n", + "import panel as pn\n", + "\n", + "STATE_FILE = './chain-state.ndjson'\n", + "\n", + "MINER_STATE_COL_RENAMES = {\n", + " 'Info.MinerAddr': 'Miner',\n", + " 'Info.MinerPower.MinerPower.RawBytePower': 'Info.MinerPowerRaw',\n", + " 'Info.MinerPower.MinerPower.QualityAdjPower': 'Info.MinerPowerQualityAdj',\n", + " 'Info.MinerPower.TotalPower.RawBytePower': 'Info.TotalPowerRaw',\n", + " 'Info.MinerPower.TotalPower.QualityAdjPower': 'Info.TotalPowerQualityAdj',\n", + "}\n", + "\n", + "MINER_NUMERIC_COLS = [\n", + " 'Info.MinerPowerRaw',\n", + " 'Info.MinerPowerQualityAdj',\n", + " 'Info.TotalPowerRaw',\n", + " 'Info.TotalPowerQualityAdj',\n", + " 'Info.Balance',\n", + " 'Info.CommittedBytes',\n", + " 'Info.ProvingBytes',\n", + " 'Info.FaultyBytes',\n", + " 'Info.FaultyPercentage',\n", + " 'Info.PreCommitDeposits',\n", + " 'Info.LockedFunds',\n", + " 'Info.AvailableFunds',\n", + " 'Info.WorkerBalance',\n", + " 'Info.MarketEscrow',\n", + " 'Info.MarketLocked',\n", + "]\n", + "\n", + "DERIVED_COLS = [\n", + " 'CommittedSectors',\n", + " 'ProvingSectors',\n", + "]\n", + "\n", + "ATTO_FIL_COLS = [\n", + " 'Info.Balance',\n", + " 'Info.PreCommitDeposits',\n", + " 'Info.LockedFunds',\n", + " 'Info.AvailableFunds',\n", + " 'Info.WorkerBalance',\n", + " 'Info.MarketEscrow',\n", + " 'Info.MarketLocked',\n", + "]\n", + "\n", + "def atto_to_fil(x):\n", + " return float(x) * pow(10, -18)\n", + "\n", + "def chain_state_to_pandas(statefile):\n", + " chain = None\n", + " \n", + " with open(statefile, 'rt') as f:\n", + " for line in f.readlines():\n", + " j = json.loads(line)\n", + " chain_height = j['Height']\n", + " \n", + " miners = j['MinerStates']\n", + " for m in miners.values():\n", + " df = pd.json_normalize(m)\n", + " df['Height'] = chain_height\n", + " df.rename(columns=MINER_STATE_COL_RENAMES, inplace=True)\n", + " if chain is None:\n", + " chain = df\n", + " else:\n", + " chain = chain.append(df, ignore_index=True)\n", + " chain.fillna(0, inplace=True)\n", + " chain.set_index('Height', inplace=True)\n", + " \n", + " for c in ATTO_FIL_COLS:\n", + " chain[c] = chain[c].apply(atto_to_fil)\n", + " \n", + " for c in MINER_NUMERIC_COLS:\n", + " chain[c] = chain[c].apply(pd.to_numeric)\n", + " \n", + " # the Sectors.* fields are lists of sector ids, but we want to plot counts, so\n", + " # we pull the length of each list into a new column\n", + " chain['CommittedSectors'] = chain['Sectors.Committed'].apply(lambda x: len(x))\n", + " chain['ProvingSectors'] = chain['Sectors.Proving'].apply(lambda x: len(x))\n", + " return chain\n", + " \n", + "cs = chain_state_to_pandas(STATE_FILE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# choose which col to plot using a widget\n", + "\n", + "cols_to_plot = MINER_NUMERIC_COLS + DERIVED_COLS\n", + "\n", + "col_selector = pn.widgets.Select(name='Field', options=cols_to_plot)\n", + "cols = ['Miner'] + cols_to_plot\n", + "plot = cs[cols].hvplot(by='Miner', y=col_selector)\n", + "pn.Column(pn.WidgetBox(col_selector), plot)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# plot all line charts in a vertical stack\n", + "\n", + "plots = []\n", + "for c in cols_to_plot:\n", + " title = c.split('.')[-1]\n", + " p = cs[['Miner', c]].hvplot(by='Miner', y=c, title=title)\n", + " plots.append(p)\n", + "pn.Column(*plots)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# miner power area chart\n", + "\n", + "mp = cs[['Miner', 'Info.MinerPowerRaw']].rename(columns={'Info.MinerPowerRaw': 'Power'})\n", + "mp = mp.pivot_table(values=['Power'], index=cs.index, columns='Miner', aggfunc='sum')\n", + "mp = mp.div(mp.sum(1), axis=0)\n", + "mp.columns = mp.columns.get_level_values(1)\n", + "mp.hvplot.area(title='Miner Power Distribution')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/composer/requirements.txt b/composer/requirements.txt index ebd5be4b7..cfdfaa463 100644 --- a/composer/requirements.txt +++ b/composer/requirements.txt @@ -4,3 +4,5 @@ jupyter panel holoviews ansi2html +matplotlib +hvplot diff --git a/dashboards/baseline.json b/dashboards/baseline.json index c6515c5d4..0678594ac 100644 --- a/dashboards/baseline.json +++ b/dashboards/baseline.json @@ -15,8 +15,8 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 16, - "iteration": 1594055543533, + "id": 15, + "iteration": 1595335476624, "links": [], "panels": [ { @@ -1588,7 +1588,7 @@ "steppedLine": false, "targets": [ { - "alias": "deal.retrieved - 95% max", + "alias": "deal.retrieved - 95% max ($tag_run)", "groupBy": [ { "params": [ @@ -1602,6 +1602,12 @@ ], "type": "tag" }, + { + "params": [ + "run" + ], + "type": "tag" + }, { "params": [ "0" @@ -1635,6 +1641,61 @@ "value": "/^$runid$/" } ] + }, + { + "alias": "deal.retrieved - min ($tag_run)", + "groupBy": [ + { + "params": [ + "$myinterval" + ], + "type": "time" + }, + { + "params": [ + "run" + ], + "type": "tag" + }, + { + "params": [ + "run" + ], + "type": "tag" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "diagnostics.deal.retrieved.histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "min" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "run", + "operator": "=~", + "value": "/^$runid$/" + } + ] } ], "thresholds": [], @@ -1957,7 +2018,7 @@ } } ], - "refresh": "10s", + "refresh": "5s", "schemaVersion": 25, "style": "dark", "tags": [], @@ -2022,7 +2083,7 @@ ] }, "time": { - "from": "now-30m", + "from": "now-15m", "to": "now" }, "timepicker": { diff --git a/docker-images/Dockerfile.oni-runtime-debug b/docker-images/Dockerfile.oni-runtime-debug index 85087adbd..e77416fad 100644 --- a/docker-images/Dockerfile.oni-runtime-debug +++ b/docker-images/Dockerfile.oni-runtime-debug @@ -12,8 +12,10 @@ RUN go get github.com/filecoin-project/go-paramfetch/paramfetch COPY /proof-parameters.json / RUN paramfetch 2048 /proof-parameters.json +ARG LOTUS_COMMIT=e21ea53 + ## for debug purposes -RUN apt update && apt install -y mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl && git clone https://github.com/filecoin-project/lotus.git && cd lotus/ && git checkout next && make clean && make all && make install +RUN apt update && apt install -y mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl && git clone https://github.com/filecoin-project/lotus.git && cd lotus/ && git checkout ${LOTUS_COMMIT} && make clean && make all && make install FROM ubuntu:18.04 @@ -22,7 +24,7 @@ COPY --from=downloader /var/tmp/filecoin-proof-parameters /var/tmp/filecoin-proo ## for debug purposes COPY --from=downloader /usr/local/bin/lotus /usr/local/bin/lll -COPY --from=downloader /usr/local/bin/lotus-storage-miner /usr/local/bin/lsm +COPY --from=downloader /usr/local/bin/lotus-miner /usr/local/bin/lm ENV FULLNODE_API_INFO="dummytoken:/ip4/127.0.0.1/tcp/1234/http" ENV STORAGE_API_INFO="dummytoken:/ip4/127.0.0.1/tcp/2345/http" diff --git a/docker-images/build-buildbase.sh b/docker-images/build-buildbase.sh deleted file mode 100755 index 1ad66b228..000000000 --- a/docker-images/build-buildbase.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -set -e -set -x - -err_report() { - echo "Error on line $1" -} - -trap 'err_report $LINENO' ERR - -TAG=$1 - -# Validate required arguments -if [ -z "$TAG" ] -then - echo -e "Please provide a tag for the build. For example: \`./build.sh v3\`" - exit 2 -fi - -dir="$(dirname "$0")" - -docker build -t "iptestground/oni-buildbase:$TAG" -f "$dir/Dockerfile.oni-buildbase" "$dir" diff --git a/docker-images/build-runtime.sh b/docker-images/build-runtime.sh deleted file mode 100755 index 8b0b2cc29..000000000 --- a/docker-images/build-runtime.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -set -e -set -x - -err_report() { - echo "Error on line $1" -} - -trap 'err_report $LINENO' ERR - -TAG=$1 - -# Validate required arguments -if [ -z "$TAG" ] -then - echo -e "Please provide a tag for the build. For example: \`./build.sh v3\`" - exit 2 -fi - -dir="$(dirname "$0")" - -docker build -t "iptestground/oni-runtime:$TAG" -f "$dir/Dockerfile.oni-runtime" "$dir" diff --git a/lotus-soup/_compositions/fast-k8s-3-1.toml b/lotus-soup/_compositions/fast-k8s-3-1.toml new file mode 100644 index 000000000..d77bfbc3e --- /dev/null +++ b/lotus-soup/_compositions/fast-k8s-3-1.toml @@ -0,0 +1,68 @@ +[metadata] + name = "lotus-soup" + author = "" + +[global] + plan = "lotus-soup" + case = "deals-e2e" + total_instances = 5 + builder = "docker:go" + runner = "cluster:k8s" + +[global.build] + selectors = ["testground"] + +[global.run_config] + exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" } + +[global.build_config] + push_registry=true + go_proxy_mode="remote" + go_proxy_url="http://localhost:8081" + registry_type="aws" + +[global.run.test_params] + clients = "3" + miners = "1" + fast_retrieval = "true" + genesis_timestamp_offset = "0" + balance = "20000000" # These balances will work for maximum 100 nodes, as TotalFilecoin is 2B + sectors = "10" + random_beacon_type = "mock" + mining_mode = "natural" + +[[groups]] + id = "bootstrapper" + [groups.resources] + memory = "512Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "bootstrapper" + +[[groups]] + id = "miners" + [groups.resources] + memory = "4096Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner" + +[[groups]] + id = "clients" + [groups.resources] + memory = "1024Mi" + cpu = "1000m" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "client" diff --git a/lotus-soup/_compositions/recovery-exec.toml b/lotus-soup/_compositions/recovery-exec.toml new file mode 100644 index 000000000..8e9ef9d6e --- /dev/null +++ b/lotus-soup/_compositions/recovery-exec.toml @@ -0,0 +1,80 @@ +[metadata] + name = "lotus-soup" + author = "" + +[global] + plan = "lotus-soup" + case = "recovery-failed-windowed-post" + total_instances = 7 + builder = "exec:go" + runner = "local:exec" + +[global.build] + selectors = ["testground"] + +[global.run_config] + exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" } + +[global.build_config] + push_registry=true + go_proxy_mode="remote" + go_proxy_url="http://localhost:8081" + registry_type="aws" + +[global.run.test_params] + clients = "3" + miners = "3" + genesis_timestamp_offset = "0" + balance = "20000000" + +[[groups]] + id = "bootstrapper" + [groups.resources] + memory = "512Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "bootstrapper" + +[[groups]] + id = "miners" + [groups.resources] + memory = "4096Mi" + cpu = "1000m" + [groups.instances] + count = 2 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner" + sectors = "10" + mining_mode = "natural" + +[[groups]] + id = "miners-biserk" + [groups.resources] + memory = "4096Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner-biserk" + sectors = "5" + mining_mode = "natural" + +[[groups]] + id = "clients" + [groups.resources] + memory = "1024Mi" + cpu = "1000m" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "client" diff --git a/lotus-soup/_compositions/recovery-k8s.toml b/lotus-soup/_compositions/recovery-k8s.toml new file mode 100644 index 000000000..5b7037e01 --- /dev/null +++ b/lotus-soup/_compositions/recovery-k8s.toml @@ -0,0 +1,95 @@ +[metadata] + name = "lotus-soup" + author = "" + +[global] + plan = "lotus-soup" + case = "recovery-failed-windowed-post" + total_instances = 9 + builder = "docker:go" + runner = "cluster:k8s" + +[global.build] + selectors = ["testground"] + +[global.run_config] + exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" } + keep_service=true + +[global.build_config] + push_registry=true + go_proxy_mode="remote" + go_proxy_url="http://localhost:8081" + registry_type="aws" + +[global.run.test_params] + clients = "4" + miners = "4" + genesis_timestamp_offset = "0" + balance = "20000000" + +[[groups]] + id = "bootstrapper" + [groups.resources] + memory = "512Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "bootstrapper" + +[[groups]] + id = "miners" + [groups.resources] + memory = "4096Mi" + cpu = "1000m" + [groups.instances] + count = 2 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner" + sectors = "10" + mining_mode = "natural" + +[[groups]] + id = "miners-full-slash" + [groups.resources] + memory = "4096Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner-full-slash" + sectors = "10" + mining_mode = "natural" + +[[groups]] + id = "miners-partial-slash" + [groups.resources] + memory = "4096Mi" + cpu = "1000m" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "miner-partial-slash" + sectors = "10" + mining_mode = "natural" + +[[groups]] + id = "clients" + [groups.resources] + memory = "1024Mi" + cpu = "1000m" + [groups.instances] + count = 4 + percentage = 0.0 + [groups.run] + [groups.run.test_params] + role = "client" diff --git a/lotus-soup/deals_e2e.go b/lotus-soup/deals_e2e.go index f6cb65dad..c0301940a 100644 --- a/lotus-soup/deals_e2e.go +++ b/lotus-soup/deals_e2e.go @@ -8,7 +8,14 @@ import ( "os" "time" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/testground/sdk-go/sync" + + mbig "math/big" + + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/oni/lotus-soup/testkit" ) @@ -38,7 +45,8 @@ func dealsE2E(t *testkit.TestEnvironment) error { } // This is a client role - t.RecordMessage("running client") + fastRetrieval := t.BooleanParam("fast_retrieval") + t.RecordMessage("running client, with fast retrieval set to: %v", fastRetrieval) cl, err := testkit.PrepareClient(t) if err != nil { @@ -57,7 +65,14 @@ func dealsE2E(t *testkit.TestEnvironment) error { t.RecordMessage("selected %s as the miner", minerAddr.MinerActorAddr) - time.Sleep(2 * time.Second) + if fastRetrieval { + err = initPaymentChannel(t, ctx, cl, minerAddr) + if err != nil { + return err + } + } + + time.Sleep(12 * time.Second) // generate 1600 bytes of random data data := make([]byte, 1600) @@ -82,7 +97,7 @@ func dealsE2E(t *testkit.TestEnvironment) error { // start deal t1 := time.Now() - deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, fcid.Root) + deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, fcid.Root, fastRetrieval) t.RecordMessage("started deal: %s", deal) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this @@ -92,10 +107,14 @@ func dealsE2E(t *testkit.TestEnvironment) error { testkit.WaitDealSealed(t, ctx, client, deal) t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1))) + // wait for all client deals to be sealed before trying to retrieve + t.SyncClient.MustSignalAndWait(ctx, sync.State("done-sealing"), t.IntParam("clients")) + carExport := true t.RecordMessage("trying to retrieve %s", fcid) - testkit.RetrieveData(t, ctx, client, fcid.Root, carExport, data) + t1 = time.Now() + _ = testkit.RetrieveData(t, ctx, client, fcid.Root, nil, carExport, data) t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1))) t.SyncClient.MustSignalEntry(ctx, testkit.StateStopMining) @@ -108,3 +127,50 @@ func dealsE2E(t *testkit.TestEnvironment) error { t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount) return nil } + +// filToAttoFil converts a fractional filecoin value into AttoFIL, rounding if necessary +func filToAttoFil(f float64) big.Int { + a := mbig.NewFloat(f) + a.Mul(a, mbig.NewFloat(float64(build.FilecoinPrecision))) + i, _ := a.Int(nil) + return big.Int{Int: i} +} + +func initPaymentChannel(t *testkit.TestEnvironment, ctx context.Context, cl *testkit.LotusClient, minerAddr testkit.MinerAddressesMsg) error { + recv := minerAddr + balance := filToAttoFil(10) + t.RecordMessage("my balance: %d", balance) + t.RecordMessage("creating payment channel; from=%s, to=%s, funds=%d", cl.Wallet.Address, recv.WalletAddr, balance) + + channel, err := cl.FullApi.PaychGet(ctx, cl.Wallet.Address, recv.WalletAddr, balance) + if err != nil { + return fmt.Errorf("failed to create payment channel: %w", err) + } + + if addr := channel.Channel; addr != address.Undef { + return fmt.Errorf("expected an Undef channel address, got: %s", addr) + } + + t.RecordMessage("payment channel created; msg_cid=%s", channel.ChannelMessage) + t.RecordMessage("waiting for payment channel message to appear on chain") + + // wait for the channel creation message to appear on chain. + _, err = cl.FullApi.StateWaitMsg(ctx, channel.ChannelMessage, 2) + if err != nil { + return fmt.Errorf("failed while waiting for payment channel creation msg to appear on chain: %w", err) + } + + // need to wait so that the channel is tracked. + // the full API waits for build.MessageConfidence (=1 in tests) before tracking the channel. + // we wait for 2 confirmations, so we have the assurance the channel is tracked. + + t.RecordMessage("reloading paych; now it should have an address") + channel, err = cl.FullApi.PaychGet(ctx, cl.Wallet.Address, recv.WalletAddr, big.Zero()) + if err != nil { + return fmt.Errorf("failed to reload payment channel: %w", err) + } + + t.RecordMessage("channel address: %s", channel.Channel) + + return nil +} diff --git a/lotus-soup/deals_stress.go b/lotus-soup/deals_stress.go index e13ff215c..f40940696 100644 --- a/lotus-soup/deals_stress.go +++ b/lotus-soup/deals_stress.go @@ -92,7 +92,7 @@ func dealsStress(t *testkit.TestEnvironment) error { go func(i int) { defer wg1.Done() t1 := time.Now() - deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, cids[i]) + deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, cids[i], false) t.RecordMessage("started storage deal %d -> %s", i, deal) time.Sleep(2 * time.Second) t.RecordMessage("waiting for deal %d to be sealed", i) @@ -111,7 +111,8 @@ func dealsStress(t *testkit.TestEnvironment) error { defer wg2.Done() t.RecordMessage("retrieving data for deal %d", i) t1 := time.Now() - testkit.RetrieveData(t, ctx, client, cids[i], true, data[i]) + _ = testkit.RetrieveData(t, ctx, client, cids[i], nil, true, data[i]) + t.RecordMessage("retrieved data for deal %d", i) t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1))) }(i) @@ -123,7 +124,7 @@ func dealsStress(t *testkit.TestEnvironment) error { } else { for i := 0; i < deals; i++ { - deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, cids[i]) + deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, cids[i], false) t.RecordMessage("started storage deal %d -> %s", i, deal) time.Sleep(2 * time.Second) t.RecordMessage("waiting for deal %d to be sealed", i) @@ -132,7 +133,7 @@ func dealsStress(t *testkit.TestEnvironment) error { for i := 0; i < deals; i++ { t.RecordMessage("retrieving data for deal %d", i) - testkit.RetrieveData(t, ctx, client, cids[i], true, data[i]) + _ = testkit.RetrieveData(t, ctx, client, cids[i], nil, true, data[i]) t.RecordMessage("retrieved data for deal %d", i) } } diff --git a/lotus-soup/go.mod b/lotus-soup/go.mod index 7e27f94e3..94a38d6b8 100644 --- a/lotus-soup/go.mod +++ b/lotus-soup/go.mod @@ -7,12 +7,16 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/drand/drand v1.0.3-0.20200714175734-29705eaf09d4 github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef - github.com/filecoin-project/go-fil-markets v0.4.1-0.20200715201050-c141144ea312 + github.com/filecoin-project/go-fil-markets v0.5.1 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/lotus v0.4.2-0.20200721105801-e21ea5355f64 - github.com/filecoin-project/specs-actors v0.8.1-0.20200720115956-cd051eabf328 + github.com/filecoin-project/lotus v0.4.3-0.20200724113535-7410c057c6b2 + github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df + github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d + github.com/filecoin-project/storage-fsm v0.0.0-20200720190000-2cfe2fe3c334 + github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 + github.com/hashicorp/go-multierror v1.1.0 github.com/influxdata/influxdb v1.8.0 // indirect github.com/ipfs/go-cid v0.0.6 github.com/ipfs/go-datastore v0.4.4 @@ -30,6 +34,7 @@ require ( github.com/multiformats/go-multiaddr-net v0.1.5 github.com/testground/sdk-go v0.2.3-0.20200706132230-6a65ddac2d8c go.opencensus.io v0.22.4 + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a ) // This will work in all build modes: docker:go, exec:go, and local go build. diff --git a/lotus-soup/go.sum b/lotus-soup/go.sum index bffdb6973..258f25847 100644 --- a/lotus-soup/go.sum +++ b/lotus-soup/go.sum @@ -222,8 +222,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/chain-validation v0.0.6-0.20200720093255-843129967fdf h1:7SkS/gSZv4ljQaQeDu4SfnF9CcvQuT9QCEf3+Hn1jp8= -github.com/filecoin-project/chain-validation v0.0.6-0.20200720093255-843129967fdf/go.mod h1:9xZvimiD8wsZbTNTUoACMPzXj4/fpIxeZBV2YjQcLhI= +github.com/filecoin-project/chain-validation v0.0.6-0.20200723211224-ffdcb7a20fe8 h1:WA2KU3u/FELAMVElQgiwEKTQe/QLUUsT52AnW4YjPjs= +github.com/filecoin-project/chain-validation v0.0.6-0.20200723211224-ffdcb7a20fe8/go.mod h1:P4FhsyLtySqsVFbOPpPVFeEShVQ4j/iA5Dzo8D2p978= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef h1:Wi5E+P1QfHP8IF27eUiTx5vYfqQZwfPxzq3oFEq8w8U= @@ -231,24 +231,27 @@ github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef/go.m github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 h1:jamfsxfK0Q9yCMHt8MPWx7Aa/O9k2Lve8eSc6FILYGQ= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.3/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 h1:xuHlrdznafh7ul5t4xEncnA4qgpQvJZEw+mr98eqHXw= github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.1.0 h1:ZDAQjvXuLzbrLnwfFruQFJP7IhImmXLuO+8i2qeAczM= +github.com/filecoin-project/go-bitfield v0.1.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.4.1-0.20200715144713-b3311844e1a5 h1:/OZ+nr0x3uMZCPrreuUbS5EUOFm9DDo4ljgdav8rp/s= -github.com/filecoin-project/go-data-transfer v0.4.1-0.20200715144713-b3311844e1a5/go.mod h1:duGDSKvsOxiKl6Dueh8DNA6ZbiM30PWUWlSKjo9ac+o= -github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= +github.com/filecoin-project/go-data-transfer v0.5.0 h1:pvWlab69BD5dwheRHjjBjFB6m7CEqEZeI+aChtVqKVk= +github.com/filecoin-project/go-data-transfer v0.5.0/go.mod h1:7yckbsPPMGuN3O1+SYNE/lowwheaUn5woGILpjN52UI= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.4.1-0.20200715201050-c141144ea312 h1:oVZggNjDWZWEjomkxPl8U3jrOLURoS4QSZA6t4YU5BY= -github.com/filecoin-project/go-fil-markets v0.4.1-0.20200715201050-c141144ea312/go.mod h1:MvrpKOiETu39e9H167gdQzdzLNcvHsUp48UkXqPSdtU= +github.com/filecoin-project/go-fil-markets v0.5.1 h1:Y69glslNCuXnygfesCmyilTVhEEjcLK7CtAohKP9SL8= +github.com/filecoin-project/go-fil-markets v0.5.1/go.mod h1:GKGigsFNMvKmx/+Mcn7093TdZTiCDLc7YGxQ7d6fq2s= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms= github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= @@ -264,22 +267,21 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/lotus v0.4.2-0.20200721105801-e21ea5355f64 h1:eGBymDewiMz2wXXrTQJyjxtVJP1jYD+HKZtebkbjU68= -github.com/filecoin-project/lotus v0.4.2-0.20200721105801-e21ea5355f64/go.mod h1:jubu0DGUMhIy/0l5HZNzKXgANucmSTLXLUAhVvVrj58= +github.com/filecoin-project/lotus v0.4.3-0.20200724113535-7410c057c6b2 h1:yGCcwE3UP40YKELdFqOz/lBvePpIyx1Y9KADgt0Fsvc= +github.com/filecoin-project/lotus v0.4.3-0.20200724113535-7410c057c6b2/go.mod h1:IQkgDgoVi+9NZEhNvYbYXW2ZCsC8fU0oB3kng1ZDdts= github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= -github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15 h1:miw6hiusb/MkV1ryoqUKKWnvHhPW00AYtyeCj0L8pqo= github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo= -github.com/filecoin-project/sector-storage v0.0.0-20200717213554-a109ef9cbeab h1:jEQtbWFyEKnCw3eAVCW3MSX/K7Nv03B3zzS/rfm2k+Q= -github.com/filecoin-project/sector-storage v0.0.0-20200717213554-a109ef9cbeab/go.mod h1:7EE+f7jM4kCy2MKHoiiwNDQGJSb+QQzZ+y+/17ugq4w= +github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df h1:VDdWrCNUNx6qeHnGU9oAy+izuGM02it9V/5+MJyhZQw= +github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df/go.mod h1:7EE+f7jM4kCy2MKHoiiwNDQGJSb+QQzZ+y+/17ugq4w= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= -github.com/filecoin-project/specs-actors v0.7.0/go.mod h1:+z0htZu/wLBDbOLcQTKKUEC2rkUTFzL2KJ/bRAVWkws= github.com/filecoin-project/specs-actors v0.7.3-0.20200716231407-60a2ae96d2e6/go.mod h1:JOMUa7EijvpOO4ofD1yeHNmqohkmmnhTvz/IpB6so4c= -github.com/filecoin-project/specs-actors v0.8.1-0.20200720061236-f4719fdd7d90/go.mod h1:JOMUa7EijvpOO4ofD1yeHNmqohkmmnhTvz/IpB6so4c= -github.com/filecoin-project/specs-actors v0.8.1-0.20200720115956-cd051eabf328 h1:jZwz1VxqzNCfINY5FDnsT+ZL03wjzLifi+JwdLkehuU= github.com/filecoin-project/specs-actors v0.8.1-0.20200720115956-cd051eabf328/go.mod h1:0+CxQ5Jeii3522irTvhKRDpr4GG1bj5Erq3p/d38DzY= +github.com/filecoin-project/specs-actors v0.8.1-0.20200723200253-a3c01bc62f99/go.mod h1:TLvIheTVl0EIuyncuKSTVXPULaj7gzhLup5CLZ/S+uM= +github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d h1:dti6ssgSFG7Tk851S3RdiDr1TNbOJ26ylc6DJ9Y2Le0= +github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d/go.mod h1:TLvIheTVl0EIuyncuKSTVXPULaj7gzhLup5CLZ/S+uM= github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= @@ -432,6 +434,8 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 h1:vQqOW42RRM5LoM/1K5dK940VipLqpH8lEVGrMz+mNjU= github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= +github.com/hannahhoward/cbor-gen-for v0.0.0-20200723175505-5892b522820a h1:wfqh5oiHXvn3Rk54xy8Cwqh+HnYihGnjMNzdNb3/ld0= +github.com/hannahhoward/cbor-gen-for v0.0.0-20200723175505-5892b522820a/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -530,7 +534,6 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9 github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3 h1:J27YvAcpuA5IvZUbeBxOcQgqnYHUPxoygc6QxxkodZ4= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= -github.com/ipfs/go-ds-badger2 v0.1.0 h1:784py6lXkwlVF+K6XSuqmdMgy5l8GI6k60ngBokb9Fg= github.com/ipfs/go-ds-badger2 v0.1.0/go.mod h1:pbR1p817OZbdId9EvLOhKBgUVTM3BMCSTan78lDDVaw= github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e h1:Xi1nil8K2lBOorBS6Ys7+hmUCzH8fr3U9ipdL/IrcEI= github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e/go.mod h1:lJnws7amT9Ehqzta0gwMrRsURU04caT0iRPr1W8AsOU= @@ -545,13 +548,12 @@ github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEP github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0= github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= -github.com/ipfs/go-graphsync v0.0.6-0.20200715142715-e2f27c4754e6 h1:+dQnaRkLV4za46Gfw6b1KNVOCcGDrdnEGZrjz3kF80k= -github.com/ipfs/go-graphsync v0.0.6-0.20200715142715-e2f27c4754e6/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= +github.com/ipfs/go-graphsync v0.0.6-0.20200715204712-ef06b3d32e83 h1:tkGDAwcZfzDFeBNyBWYOM02Qw0rGpA2UuCvq49T3K5o= +github.com/ipfs/go-graphsync v0.0.6-0.20200715204712-ef06b3d32e83/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242/go.mod h1:kq3Pi+UP3oHhAdKexE+kHHYRKMoFNuGero0R7q3hWGg= -github.com/ipfs/go-hamt-ipld v0.1.1-0.20200501020327-d53d20a7063e/go.mod h1:giiPqWYCnRBYpNTsJ/EX1ojldX5kTXrXYckSJQ7ko9M= -github.com/ipfs/go-hamt-ipld v0.1.1-0.20200605182717-0310ad2b0b1f h1:mchhWiYYUSoCuE3wDfRCo8cho5kqSoxkgnOtGcnNMZw= -github.com/ipfs/go-hamt-ipld v0.1.1-0.20200605182717-0310ad2b0b1f/go.mod h1:phOFBB7W73N9dg1glcb1fQ9HtQFDUpeyJgatW8ns0bw= +github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= +github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= @@ -1138,7 +1140,6 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= @@ -1410,13 +1411,13 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= -github.com/whyrusleeping/cbor-gen v0.0.0-20200501014322-5f9941ef88e0/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200501232601-351665a6e756/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= -github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d h1:Y25auOnuZb/GuJvqMflRSDWBz8/HBRME8fiD+H8zLfs= github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377 h1:LHFlP/ktDvOnCap7PsT87cs7Gwd0p+qv6Qm5g2ZPR+I= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200723182808-cb5de1c427f5 h1:dJgLhFKggti1Xd7GczL4DetAUyx68RhpCKCfV71ongg= +github.com/whyrusleeping/cbor-gen v0.0.0-20200723182808-cb5de1c427f5/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g= diff --git a/lotus-soup/init.go b/lotus-soup/init.go index c25d45eb7..4a5867712 100644 --- a/lotus-soup/init.go +++ b/lotus-soup/init.go @@ -31,6 +31,16 @@ func init() { // mined, e.g. payment channel creation, to be considered committed. build.MessageConfidence = 1 + // The period over which all a miner's active sectors will be challenged. + miner.WPoStProvingPeriod = abi.ChainEpoch(240) // instead of 24 hours + + // The duration of a deadline's challenge window, the period before a deadline when the challenge is available. + miner.WPoStChallengeWindow = abi.ChainEpoch(5) // instead of 30 minutes (still 48 per day) + + // Number of epochs between publishing the precommit and when the challenge for interactive PoRep is drawn + // used to ensure it is not predictable by miner. + miner.PreCommitChallengeDelay = abi.ChainEpoch(10) + power.ConsensusMinerMinPower = big.NewInt(2048) miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, diff --git a/lotus-soup/main.go b/lotus-soup/main.go index a660a63cf..8b0b58f8b 100644 --- a/lotus-soup/main.go +++ b/lotus-soup/main.go @@ -2,16 +2,18 @@ package main import ( "github.com/filecoin-project/oni/lotus-soup/paych" + "github.com/filecoin-project/oni/lotus-soup/rfwp" "github.com/filecoin-project/oni/lotus-soup/testkit" "github.com/testground/sdk-go/run" ) var cases = map[string]interface{}{ - "deals-e2e": testkit.WrapTestEnvironment(dealsE2E), - "deals-stress": testkit.WrapTestEnvironment(dealsStress), - "drand-halting": testkit.WrapTestEnvironment(dealsE2E), - "paych-stress": testkit.WrapTestEnvironment(paych.Stress), + "deals-e2e": testkit.WrapTestEnvironment(dealsE2E), + "recovery-failed-windowed-post": testkit.WrapTestEnvironment(rfwp.RecoveryFromFailedWindowedPoStE2E), + "deals-stress": testkit.WrapTestEnvironment(dealsStress), + "drand-halting": testkit.WrapTestEnvironment(dealsE2E), + "paych-stress": testkit.WrapTestEnvironment(paych.Stress), } func main() { diff --git a/lotus-soup/manifest.toml b/lotus-soup/manifest.toml index 853fdaaa9..df06bff51 100644 --- a/lotus-soup/manifest.toml +++ b/lotus-soup/manifest.toml @@ -11,7 +11,7 @@ enabled = true [builders."docker:go"] enabled = true build_base_image = "iptestground/oni-buildbase:v5" -runtime_image = "iptestground/oni-runtime:v2" +runtime_image = "iptestground/oni-runtime:v2-debug" [runners."local:exec"] enabled = true @@ -56,6 +56,9 @@ instances = { min = 1, max = 100, default = 5 } enable_pubsub_tracer = { type = "bool", default = false } mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] } + # Fast retrieval + fast_retrieval = { type = "bool", default = false } + [[testcases]] name = "drand-halting" @@ -151,3 +154,32 @@ instances = { min = 1, max = 100, default = 5 } # ********** Test-case specific ********** increments = { type = "int", default = "100", desc = "increments in which to send payment vouchers" } lane_count = { type = "int", default = "256", desc = "lanes to open; vouchers will be distributed across these lanes in round-robin fashion" } + + +[[testcases]] +name = "recovery-failed-windowed-post" +instances = { min = 1, max = 100, default = 5 } + + [testcases.params] + clients = { type = "int", default = 1 } + miners = { type = "int", default = 1 } + balance = { type = "int", default = 1 } + sectors = { type = "int", default = 1 } + role = { type = "string" } + + genesis_timestamp_offset = { type = "int", default = 0 } + + random_beacon_type = { type = "enum", default = "mock", options = ["mock", "local-drand", "external-drand"] } + + # Params relevant to drand nodes. drand nodes should have role="drand", and must all be + # in the same composition group. There must be at least threshold drand nodes. + # To get lotus nodes to actually use the drand nodes, you must set random_beacon_type="local-drand" + # for the lotus node groups. + drand_period = { type = "duration", default="10s" } + drand_threshold = { type = "int", default = 2 } + drand_gossip_relay = { type = "bool", default = true } + drand_log_level = { type = "string", default="info" } + + # Params relevant to pubsub tracing + enable_pubsub_tracer = { type = "bool", default = false } + mining_mode = { type = "enum", default = "synchronized", options = ["synchronized", "natural"] } diff --git a/lotus-soup/rfwp/chain_state.go b/lotus-soup/rfwp/chain_state.go new file mode 100644 index 000000000..3e91a25b3 --- /dev/null +++ b/lotus-soup/rfwp/chain_state.go @@ -0,0 +1,812 @@ +package rfwp + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "text/tabwriter" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/specs-actors/actors/abi/big" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/oni/lotus-soup/testkit" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + sealing "github.com/filecoin-project/storage-fsm" + + tstats "github.com/filecoin-project/lotus/tools/stats" +) + +func UpdateChainState(t *testkit.TestEnvironment, m *testkit.LotusMiner) error { + height := 0 + headlag := 3 + + ctx := context.Background() + + tipsetsCh, err := tstats.GetTips(ctx, m.FullApi, abi.ChainEpoch(height), headlag) + if err != nil { + return err + } + + jsonFilename := fmt.Sprintf("%s%cchain-state.ndjson", t.TestOutputsPath, os.PathSeparator) + jsonFile, err := os.Create(jsonFilename) + if err != nil { + return err + } + defer jsonFile.Close() + jsonEncoder := json.NewEncoder(jsonFile) + + for tipset := range tipsetsCh { + maddrs, err := m.FullApi.StateListMiners(ctx, tipset.Key()) + if err != nil { + return err + } + + snapshot := ChainSnapshot{ + Height: tipset.Height(), + MinerStates: make(map[string]*MinerStateSnapshot), + } + + err = func() error { + cs.Lock() + defer cs.Unlock() + + for _, maddr := range maddrs { + err := func() error { + filename := fmt.Sprintf("%s%cstate-%s-%d", t.TestOutputsPath, os.PathSeparator, maddr, tipset.Height()) + + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + minerInfo, err := info(t, m, maddr, w, tipset.Height()) + if err != nil { + return err + } + writeText(w, minerInfo) + + if tipset.Height()%100 == 0 { + printDiff(t, minerInfo, tipset.Height()) + } + + faultState, err := provingFaults(t, m, maddr, tipset.Height()) + if err != nil { + return err + } + writeText(w, faultState) + + provState, err := provingInfo(t, m, maddr, tipset.Height()) + if err != nil { + return err + } + writeText(w, provState) + + // record diff + recordDiff(minerInfo, provState, tipset.Height()) + + deadlines, err := provingDeadlines(t, m, maddr, tipset.Height()) + if err != nil { + return err + } + writeText(w, deadlines) + + sectorInfo, err := sectorsList(t, m, maddr, w, tipset.Height()) + if err != nil { + return err + } + writeText(w, sectorInfo) + + snapshot.MinerStates[maddr.String()] = &MinerStateSnapshot{ + Info: minerInfo, + Faults: faultState, + ProvingInfo: provState, + Deadlines: deadlines, + Sectors: sectorInfo, + } + + return jsonEncoder.Encode(snapshot) + }() + if err != nil { + return err + } + } + + cs.PrevHeight = tipset.Height() + + return nil + }() + if err != nil { + return err + } + } + + return nil +} + +type ChainSnapshot struct { + Height abi.ChainEpoch + + MinerStates map[string]*MinerStateSnapshot +} + +type MinerStateSnapshot struct { + Info *MinerInfo + Faults *ProvingFaultState + ProvingInfo *ProvingInfoState + Deadlines *ProvingDeadlines + Sectors *SectorInfo +} + +// writeText marshals m to text and writes to w, swallowing any errors along the way. +func writeText(w io.Writer, m plainTextMarshaler) { + b, err := m.MarshalPlainText() + if err != nil { + return + } + _, _ = w.Write(b) +} + +// if we make our structs `encoding.TextMarshaler`s, they all get stringified when marshaling to JSON +// instead of just using the default struct marshaler. +// so here's encoding.TextMarshaler with a different name, so that doesn't happen. +type plainTextMarshaler interface { + MarshalPlainText() ([]byte, error) +} + +type ProvingFaultState struct { + // FaultedSectors is a map of deadline indices to a list of faulted sectors for that proving window. + // If the miner has no faulty sectors, the map will be empty. + FaultedSectors map[int][]uint64 +} + +func (s *ProvingFaultState) MarshalPlainText() ([]byte, error) { + w := &bytes.Buffer{} + + if len(s.FaultedSectors) == 0 { + fmt.Fprintf(w, "no faulty sectors\n") + return w.Bytes(), nil + } + + tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) + _, _ = fmt.Fprintf(tw, "deadline\tsectors") + for deadline := 0; deadline < int(miner.WPoStPeriodDeadlines); deadline++ { + if sectors, ok := s.FaultedSectors[deadline]; ok { + for _, num := range sectors { + _, _ = fmt.Fprintf(tw, "%d\t%d\n", deadline, num) + } + } + } + + return w.Bytes(), nil +} + +func provingFaults(t *testkit.TestEnvironment, m *testkit.LotusMiner, maddr address.Address, height abi.ChainEpoch) (*ProvingFaultState, error) { + api := m.FullApi + ctx := context.Background() + + s := ProvingFaultState{FaultedSectors: make(map[int][]uint64)} + + head, err := api.ChainHead(ctx) + if err != nil { + return nil, err + } + deadlines, err := api.StateMinerDeadlines(ctx, maddr, head.Key()) + if err != nil { + return nil, err + } + for dlIdx := range deadlines { + partitions, err := api.StateMinerPartitions(ctx, maddr, uint64(dlIdx), types.EmptyTSK) + if err != nil { + return nil, err + } + + for _, partition := range partitions { + faulty, err := partition.Faults.All(10000000) + if err != nil { + return nil, err + } + + for _, num := range faulty { + s.FaultedSectors[dlIdx] = append(s.FaultedSectors[dlIdx], num) + } + } + } + + return &s, nil +} + +type ProvingInfoState struct { + CurrentEpoch abi.ChainEpoch + + ProvingPeriodStart abi.ChainEpoch + + Faults uint64 + ProvenSectors uint64 + FaultPercent float64 + Recoveries uint64 + + DeadlineIndex uint64 + DeadlineSectors uint64 + DeadlineOpen abi.ChainEpoch + DeadlineClose abi.ChainEpoch + DeadlineChallenge abi.ChainEpoch + DeadlineFaultCutoff abi.ChainEpoch + + WPoStProvingPeriod abi.ChainEpoch +} + +func (s *ProvingInfoState) MarshalPlainText() ([]byte, error) { + w := &bytes.Buffer{} + fmt.Fprintf(w, "Current Epoch: %d\n", s.CurrentEpoch) + fmt.Fprintf(w, "Chain Period: %d\n", s.CurrentEpoch/s.WPoStProvingPeriod) + fmt.Fprintf(w, "Chain Period Start: %s\n", epochTime(s.CurrentEpoch, (s.CurrentEpoch/s.WPoStProvingPeriod)*s.WPoStProvingPeriod)) + fmt.Fprintf(w, "Chain Period End: %s\n\n", epochTime(s.CurrentEpoch, (s.CurrentEpoch/s.WPoStProvingPeriod+1)*s.WPoStProvingPeriod)) + + fmt.Fprintf(w, "Proving Period Boundary: %d\n", s.ProvingPeriodStart%s.WPoStProvingPeriod) + fmt.Fprintf(w, "Proving Period Start: %s\n", epochTime(s.CurrentEpoch, s.ProvingPeriodStart)) + fmt.Fprintf(w, "Next Period Start: %s\n\n", epochTime(s.CurrentEpoch, s.ProvingPeriodStart+s.WPoStProvingPeriod)) + + fmt.Fprintf(w, "Faults: %d (%.2f%%)\n", s.Faults, s.FaultPercent) + fmt.Fprintf(w, "Recovering: %d\n", s.Recoveries) + //fmt.Fprintf(w, "New Sectors: %d\n\n", s.NewSectors) + + fmt.Fprintf(w, "Deadline Index: %d\n", s.DeadlineIndex) + fmt.Fprintf(w, "Deadline Sectors: %d\n", s.DeadlineSectors) + + fmt.Fprintf(w, "Deadline Open: %s\n", epochTime(s.CurrentEpoch, s.DeadlineOpen)) + fmt.Fprintf(w, "Deadline Close: %s\n", epochTime(s.CurrentEpoch, s.DeadlineClose)) + fmt.Fprintf(w, "Deadline Challenge: %s\n", epochTime(s.CurrentEpoch, s.DeadlineChallenge)) + fmt.Fprintf(w, "Deadline FaultCutoff: %s\n", epochTime(s.CurrentEpoch, s.DeadlineFaultCutoff)) + + return w.Bytes(), nil +} + +func provingInfo(t *testkit.TestEnvironment, m *testkit.LotusMiner, maddr address.Address, height abi.ChainEpoch) (*ProvingInfoState, error) { + api := m.FullApi + ctx := context.Background() + + head, err := api.ChainHead(ctx) + if err != nil { + return nil, err + } + + cd, err := api.StateMinerProvingDeadline(ctx, maddr, head.Key()) + if err != nil { + return nil, err + } + + deadlines, err := api.StateMinerDeadlines(ctx, maddr, head.Key()) + if err != nil { + return nil, err + } + + var mas miner.State + { + mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + rmas, err := api.ChainReadObj(ctx, mact.Head) + if err != nil { + return nil, err + } + if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { + return nil, err + } + } + + parts := map[uint64][]*miner.Partition{} + for dlIdx := range deadlines { + part, err := api.StateMinerPartitions(ctx, maddr, uint64(dlIdx), types.EmptyTSK) + if err != nil { + return nil, err + } + + parts[uint64(dlIdx)] = part + } + + proving := uint64(0) + faults := uint64(0) + recovering := uint64(0) + + for _, partitions := range parts { + for _, partition := range partitions { + sc, err := partition.Sectors.Count() + if err != nil { + return nil, err + } + proving += sc + + fc, err := partition.Faults.Count() + if err != nil { + return nil, err + } + faults += fc + + rc, err := partition.Faults.Count() + if err != nil { + return nil, err + } + recovering += rc + } + } + + var faultPerc float64 + if proving > 0 { + faultPerc = float64(faults*10000/proving) / 100 + } + + s := ProvingInfoState{ + CurrentEpoch: cd.CurrentEpoch, + ProvingPeriodStart: cd.PeriodStart, + Faults: faults, + ProvenSectors: proving, + FaultPercent: faultPerc, + Recoveries: recovering, + DeadlineIndex: cd.Index, + DeadlineOpen: cd.Open, + DeadlineClose: cd.Close, + DeadlineChallenge: cd.Challenge, + DeadlineFaultCutoff: cd.FaultCutoff, + WPoStProvingPeriod: miner.WPoStProvingPeriod, + } + + if cd.Index < miner.WPoStPeriodDeadlines { + for _, partition := range parts[cd.Index] { + sc, err := partition.Sectors.Count() + if err != nil { + return nil, err + } + s.DeadlineSectors += sc + } + } + + return &s, nil +} + +func epochTime(curr, e abi.ChainEpoch) string { + switch { + case curr > e: + return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))) + case curr == e: + return fmt.Sprintf("%d (now)", e) + case curr < e: + return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))) + } + + panic("math broke") +} + +type ProvingDeadlines struct { + Deadlines []DeadlineInfo +} + +type DeadlineInfo struct { + Sectors uint64 + Partitions int + Proven uint64 + Current bool +} + +func (d *ProvingDeadlines) MarshalPlainText() ([]byte, error) { + w := new(bytes.Buffer) + tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "deadline\tsectors\tpartitions\tproven") + + for i, di := range d.Deadlines { + var cur string + if di.Current { + cur += "\t(current)" + } + _, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%d%s\n", i, di.Sectors, di.Partitions, di.Proven, cur) + } + tw.Flush() + return w.Bytes(), nil +} + +func provingDeadlines(t *testkit.TestEnvironment, m *testkit.LotusMiner, maddr address.Address, height abi.ChainEpoch) (*ProvingDeadlines, error) { + api := m.FullApi + ctx := context.Background() + + deadlines, err := api.StateMinerDeadlines(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + + di, err := api.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + + var mas miner.State + { + mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + rmas, err := api.ChainReadObj(ctx, mact.Head) + if err != nil { + return nil, err + } + if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { + return nil, err + } + } + + infos := make([]DeadlineInfo, 0, len(deadlines)) + for dlIdx, deadline := range deadlines { + partitions, err := api.StateMinerPartitions(ctx, maddr, uint64(dlIdx), types.EmptyTSK) + if err != nil { + return nil, err + } + + provenPartitions, err := deadline.PostSubmissions.Count() + if err != nil { + return nil, err + } + + var cur string + if di.Index == uint64(dlIdx) { + cur += "\t(current)" + } + + outInfo := DeadlineInfo{ + //Sectors: c, + Partitions: len(partitions), + Proven: provenPartitions, + Current: di.Index == uint64(dlIdx), + } + infos = append(infos, outInfo) + //_, _ = fmt.Fprintf(tw, "%d\t%d\t%d%s\n", dlIdx, len(partitions), provenPartitions, cur) + } + + return &ProvingDeadlines{Deadlines: infos}, nil +} + +type SectorInfo struct { + Sectors []abi.SectorNumber + SectorStates map[abi.SectorNumber]api.SectorInfo + Committed []abi.SectorNumber + Proving []abi.SectorNumber +} + +func (i *SectorInfo) MarshalPlainText() ([]byte, error) { + provingIDs := make(map[abi.SectorNumber]struct{}, len(i.Proving)) + for _, id := range i.Proving { + provingIDs[id] = struct{}{} + } + commitedIDs := make(map[abi.SectorNumber]struct{}, len(i.Committed)) + for _, id := range i.Committed { + commitedIDs[id] = struct{}{} + } + + w := new(bytes.Buffer) + tw := tabwriter.NewWriter(w, 8, 4, 1, ' ', 0) + + for _, s := range i.Sectors { + _, inSSet := commitedIDs[s] + _, inPSet := provingIDs[s] + + st, ok := i.SectorStates[s] + if !ok { + continue + } + + fmt.Fprintf(tw, "%d: %s\tsSet: %s\tpSet: %s\ttktH: %d\tseedH: %d\tdeals: %v\n", + s, + st.State, + yesno(inSSet), + yesno(inPSet), + st.Ticket.Epoch, + st.Seed.Epoch, + st.Deals, + ) + } + + if err := tw.Flush(); err != nil { + return nil, err + } + return w.Bytes(), nil +} + +func sectorsList(t *testkit.TestEnvironment, m *testkit.LotusMiner, maddr address.Address, w io.Writer, height abi.ChainEpoch) (*SectorInfo, error) { + node := m.FullApi + ctx := context.Background() + + list, err := m.MinerApi.SectorsList(ctx) + if err != nil { + return nil, err + } + + activeSet, err := node.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + activeIDs := make(map[abi.SectorNumber]struct{}, len(activeSet)) + for _, info := range activeSet { + activeIDs[info.ID] = struct{}{} + } + + sset, err := node.StateMinerSectors(ctx, maddr, nil, true, types.EmptyTSK) + if err != nil { + return nil, err + } + commitedIDs := make(map[abi.SectorNumber]struct{}, len(activeSet)) + for _, info := range sset { + commitedIDs[info.ID] = struct{}{} + } + + sort.Slice(list, func(i, j int) bool { + return list[i] < list[j] + }) + + i := SectorInfo{Sectors: list, SectorStates: make(map[abi.SectorNumber]api.SectorInfo, len(list))} + + for _, s := range list { + st, err := m.MinerApi.SectorsStatus(ctx, s) + if err != nil { + fmt.Fprintf(w, "%d:\tError: %s\n", s, err) + continue + } + i.SectorStates[s] = st + } + return &i, nil +} + +func yesno(b bool) string { + if b { + return "YES" + } + return "NO" +} + +type MinerInfo struct { + MinerAddr address.Address + SectorSize string + + MinerPower *api.MinerPower + + CommittedBytes big.Int + ProvingBytes big.Int + FaultyBytes big.Int + FaultyPercentage float64 + + Balance big.Int + PreCommitDeposits big.Int + LockedFunds big.Int + AvailableFunds big.Int + WorkerBalance big.Int + MarketEscrow big.Int + MarketLocked big.Int + + SectorStateCounts map[sealing.SectorState]int +} + +func (i *MinerInfo) MarshalPlainText() ([]byte, error) { + w := new(bytes.Buffer) + fmt.Fprintf(w, "Miner: %s\n", i.MinerAddr) + fmt.Fprintf(w, "Sector Size: %s\n", i.SectorSize) + + pow := i.MinerPower + rpercI := types.BigDiv(types.BigMul(pow.MinerPower.RawBytePower, types.NewInt(1000000)), pow.TotalPower.RawBytePower) + qpercI := types.BigDiv(types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(1000000)), pow.TotalPower.QualityAdjPower) + + fmt.Fprintf(w, "Byte Power: %s / %s (%0.4f%%)\n", + types.SizeStr(pow.MinerPower.RawBytePower), + types.SizeStr(pow.TotalPower.RawBytePower), + float64(rpercI.Int64())/10000) + + fmt.Fprintf(w, "Actual Power: %s / %s (%0.4f%%)\n", + types.DeciStr(pow.MinerPower.QualityAdjPower), + types.DeciStr(pow.TotalPower.QualityAdjPower), + float64(qpercI.Int64())/10000) + + fmt.Fprintf(w, "\tCommitted: %s\n", types.SizeStr(i.CommittedBytes)) + + if i.FaultyBytes.Int == nil || i.FaultyBytes.IsZero() { + fmt.Fprintf(w, "\tProving: %s\n", types.SizeStr(i.ProvingBytes)) + } else { + fmt.Fprintf(w, "\tProving: %s (%s Faulty, %.2f%%)\n", + types.SizeStr(i.ProvingBytes), + types.SizeStr(i.FaultyBytes), + i.FaultyPercentage) + } + + if i.MinerPower.MinerPower.RawBytePower.LessThan(power.ConsensusMinerMinPower) { + fmt.Fprintf(w, "Below minimum power threshold, no blocks will be won\n") + } else { + expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 + if expWinChance > 0 { + if expWinChance > 1 { + expWinChance = 1 + } + winRate := time.Duration(float64(time.Second*time.Duration(build.BlockDelaySecs)) / expWinChance) + winPerDay := float64(time.Hour*24) / float64(winRate) + + fmt.Fprintln(w, "Expected block win rate: ") + fmt.Fprintf(w, "%.4f/day (every %s)\n", winPerDay, winRate.Truncate(time.Second)) + } + } + + fmt.Fprintf(w, "Miner Balance: %s\n", types.FIL(i.Balance)) + fmt.Fprintf(w, "\tPreCommit: %s\n", types.FIL(i.PreCommitDeposits)) + fmt.Fprintf(w, "\tLocked: %s\n", types.FIL(i.LockedFunds)) + fmt.Fprintf(w, "\tAvailable: %s\n", types.FIL(i.AvailableFunds)) + fmt.Fprintf(w, "Worker Balance: %s\n", types.FIL(i.WorkerBalance)) + fmt.Fprintf(w, "Market (Escrow): %s\n", types.FIL(i.MarketEscrow)) + fmt.Fprintf(w, "Market (Locked): %s\n\n", types.FIL(i.MarketLocked)) + + buckets := i.SectorStateCounts + + var sorted []stateMeta + for state, i := range buckets { + sorted = append(sorted, stateMeta{i: i, state: state}) + } + + sort.Slice(sorted, func(i, j int) bool { + return stateOrder[sorted[i].state].i < stateOrder[sorted[j].state].i + }) + + for _, s := range sorted { + _, _ = fmt.Fprintf(w, "\t%s: %d\n", s.state, s.i) + } + + return w.Bytes(), nil +} + +func info(t *testkit.TestEnvironment, m *testkit.LotusMiner, maddr address.Address, w io.Writer, height abi.ChainEpoch) (*MinerInfo, error) { + api := m.FullApi + ctx := context.Background() + + mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + var mas miner.State + { + rmas, err := api.ChainReadObj(ctx, mact.Head) + if err != nil { + return nil, err + } + if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { + return nil, err + } + } + + i := MinerInfo{MinerAddr: maddr} + + // Sector size + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + + i.SectorSize = types.SizeStr(types.NewInt(uint64(mi.SectorSize))) + + i.MinerPower, err = api.StateMinerPower(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + + secCounts, err := api.StateMinerSectorCount(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + faults, err := api.StateMinerFaults(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + + nfaults, err := faults.Count() + if err != nil { + return nil, err + } + + i.CommittedBytes = types.BigMul(types.NewInt(secCounts.Sectors), types.NewInt(uint64(mi.SectorSize))) + i.ProvingBytes = types.BigMul(types.NewInt(secCounts.Active), types.NewInt(uint64(mi.SectorSize))) + + if nfaults != 0 { + if secCounts.Sectors != 0 { + i.FaultyPercentage = float64(10000*nfaults/secCounts.Sectors) / 100. + } + i.FaultyBytes = types.BigMul(types.NewInt(nfaults), types.NewInt(uint64(mi.SectorSize))) + } + + i.Balance = mact.Balance + i.PreCommitDeposits = mas.PreCommitDeposits + i.LockedFunds = mas.LockedFunds + i.AvailableFunds = types.BigSub(mact.Balance, types.BigAdd(mas.LockedFunds, mas.PreCommitDeposits)) + + wb, err := api.WalletBalance(ctx, mi.Worker) + if err != nil { + return nil, err + } + i.WorkerBalance = wb + + mb, err := api.StateMarketBalance(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, err + } + i.MarketEscrow = mb.Escrow + i.MarketLocked = mb.Locked + + sectors, err := m.MinerApi.SectorsList(ctx) + if err != nil { + return nil, err + } + + buckets := map[sealing.SectorState]int{ + "Total": len(sectors), + } + for _, s := range sectors { + st, err := m.MinerApi.SectorsStatus(ctx, s) + if err != nil { + return nil, err + } + + buckets[sealing.SectorState(st.State)]++ + } + i.SectorStateCounts = buckets + + return &i, nil +} + +type stateMeta struct { + i int + state sealing.SectorState +} + +var stateOrder = map[sealing.SectorState]stateMeta{} +var stateList = []stateMeta{ + {state: "Total"}, + {state: sealing.Proving}, + + {state: sealing.UndefinedSectorState}, + {state: sealing.Empty}, + {state: sealing.Packing}, + {state: sealing.PreCommit1}, + {state: sealing.PreCommit2}, + {state: sealing.PreCommitting}, + {state: sealing.PreCommitWait}, + {state: sealing.WaitSeed}, + {state: sealing.Committing}, + {state: sealing.CommitWait}, + {state: sealing.FinalizeSector}, + + {state: sealing.FailedUnrecoverable}, + {state: sealing.SealPreCommit1Failed}, + {state: sealing.SealPreCommit2Failed}, + {state: sealing.PreCommitFailed}, + {state: sealing.ComputeProofFailed}, + {state: sealing.CommitFailed}, + {state: sealing.PackingFailed}, + {state: sealing.FinalizeFailed}, + {state: sealing.Faulty}, + {state: sealing.FaultReported}, + {state: sealing.FaultedFinal}, +} + +func init() { + for i, state := range stateList { + stateOrder[state.state] = stateMeta{ + i: i, + } + } +} diff --git a/lotus-soup/rfwp/diffs.go b/lotus-soup/rfwp/diffs.go new file mode 100644 index 000000000..807ff5400 --- /dev/null +++ b/lotus-soup/rfwp/diffs.go @@ -0,0 +1,295 @@ +package rfwp + +import ( + "bufio" + "fmt" + "os" + "sort" + "sync" + + "github.com/filecoin-project/oni/lotus-soup/testkit" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" +) + +type ChainState struct { + sync.Mutex + + PrevHeight abi.ChainEpoch + DiffHeight map[string]map[string]map[abi.ChainEpoch]big.Int // height -> value + DiffValue map[string]map[string]map[string][]abi.ChainEpoch // value -> []height + DiffCmp map[string]map[string]map[string][]abi.ChainEpoch // difference (height, height-1) -> []height + valueTypes []string +} + +func NewChainState() *ChainState { + cs := &ChainState{} + cs.PrevHeight = abi.ChainEpoch(-1) + cs.DiffHeight = make(map[string]map[string]map[abi.ChainEpoch]big.Int) // height -> value + cs.DiffValue = make(map[string]map[string]map[string][]abi.ChainEpoch) // value -> []height + cs.DiffCmp = make(map[string]map[string]map[string][]abi.ChainEpoch) // difference (height, height-1) -> []height + cs.valueTypes = []string{"MinerPower", "CommittedBytes", "ProvingBytes", "Balance", "PreCommitDeposits", "LockedFunds", "AvailableFunds", "WorkerBalance", "MarketEscrow", "MarketLocked", "Faults", "ProvenSectors", "Recoveries"} + return cs +} + +var ( + cs *ChainState +) + +func init() { + cs = NewChainState() +} + +func printDiff(t *testkit.TestEnvironment, mi *MinerInfo, height abi.ChainEpoch) { + maddr := mi.MinerAddr.String() + filename := fmt.Sprintf("%s%cdiff-%s-%d", t.TestOutputsPath, os.PathSeparator, maddr, height) + + f, err := os.Create(filename) + if err != nil { + panic(err) + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + keys := make([]string, 0, len(cs.DiffCmp[maddr])) + for k := range cs.DiffCmp[maddr] { + keys = append(keys, k) + } + sort.Strings(keys) + + fmt.Fprintln(w, "=====", maddr, "=====") + for i, valueName := range keys { + fmt.Fprintln(w, toCharStr(i), "=====", valueName, "=====") + if len(cs.DiffCmp[maddr][valueName]) > 0 { + fmt.Fprintf(w, "%s diff of |\n", toCharStr(i)) + } + + for difference, heights := range cs.DiffCmp[maddr][valueName] { + fmt.Fprintf(w, "%s diff of %30v at heights %v\n", toCharStr(i), difference, heights) + } + } +} + +func recordDiff(mi *MinerInfo, ps *ProvingInfoState, height abi.ChainEpoch) { + maddr := mi.MinerAddr.String() + if _, ok := cs.DiffHeight[maddr]; !ok { + cs.DiffHeight[maddr] = make(map[string]map[abi.ChainEpoch]big.Int) + cs.DiffValue[maddr] = make(map[string]map[string][]abi.ChainEpoch) + cs.DiffCmp[maddr] = make(map[string]map[string][]abi.ChainEpoch) + + for _, v := range cs.valueTypes { + cs.DiffHeight[maddr][v] = make(map[abi.ChainEpoch]big.Int) + cs.DiffValue[maddr][v] = make(map[string][]abi.ChainEpoch) + cs.DiffCmp[maddr][v] = make(map[string][]abi.ChainEpoch) + } + } + + { + value := big.Int(mi.MinerPower.MinerPower.RawBytePower) + cs.DiffHeight[maddr]["MinerPower"][height] = value + cs.DiffValue[maddr]["MinerPower"][value.String()] = append(cs.DiffValue[maddr]["MinerPower"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["MinerPower"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["MinerPower"][cmp.String()] = append(cs.DiffCmp[maddr]["MinerPower"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.CommittedBytes) + cs.DiffHeight[maddr]["CommittedBytes"][height] = value + cs.DiffValue[maddr]["CommittedBytes"][value.String()] = append(cs.DiffValue[maddr]["CommittedBytes"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["CommittedBytes"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["CommittedBytes"][cmp.String()] = append(cs.DiffCmp[maddr]["CommittedBytes"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.ProvingBytes) + cs.DiffHeight[maddr]["ProvingBytes"][height] = value + cs.DiffValue[maddr]["ProvingBytes"][value.String()] = append(cs.DiffValue[maddr]["ProvingBytes"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["ProvingBytes"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["ProvingBytes"][cmp.String()] = append(cs.DiffCmp[maddr]["ProvingBytes"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.Balance) + roundBalance(&value) + cs.DiffHeight[maddr]["Balance"][height] = value + cs.DiffValue[maddr]["Balance"][value.String()] = append(cs.DiffValue[maddr]["Balance"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["Balance"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["Balance"][cmp.String()] = append(cs.DiffCmp[maddr]["Balance"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.PreCommitDeposits) + cs.DiffHeight[maddr]["PreCommitDeposits"][height] = value + cs.DiffValue[maddr]["PreCommitDeposits"][value.String()] = append(cs.DiffValue[maddr]["PreCommitDeposits"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["PreCommitDeposits"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["PreCommitDeposits"][cmp.String()] = append(cs.DiffCmp[maddr]["PreCommitDeposits"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.LockedFunds) + roundBalance(&value) + cs.DiffHeight[maddr]["LockedFunds"][height] = value + cs.DiffValue[maddr]["LockedFunds"][value.String()] = append(cs.DiffValue[maddr]["LockedFunds"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["LockedFunds"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["LockedFunds"][cmp.String()] = append(cs.DiffCmp[maddr]["LockedFunds"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.AvailableFunds) + roundBalance(&value) + cs.DiffHeight[maddr]["AvailableFunds"][height] = value + cs.DiffValue[maddr]["AvailableFunds"][value.String()] = append(cs.DiffValue[maddr]["AvailableFunds"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["AvailableFunds"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["AvailableFunds"][cmp.String()] = append(cs.DiffCmp[maddr]["AvailableFunds"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.WorkerBalance) + cs.DiffHeight[maddr]["WorkerBalance"][height] = value + cs.DiffValue[maddr]["WorkerBalance"][value.String()] = append(cs.DiffValue[maddr]["WorkerBalance"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["WorkerBalance"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["WorkerBalance"][cmp.String()] = append(cs.DiffCmp[maddr]["WorkerBalance"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.MarketEscrow) + cs.DiffHeight[maddr]["MarketEscrow"][height] = value + cs.DiffValue[maddr]["MarketEscrow"][value.String()] = append(cs.DiffValue[maddr]["MarketEscrow"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["MarketEscrow"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["MarketEscrow"][cmp.String()] = append(cs.DiffCmp[maddr]["MarketEscrow"][cmp.String()], height) + } + } + } + + { + value := big.Int(mi.MarketLocked) + cs.DiffHeight[maddr]["MarketLocked"][height] = value + cs.DiffValue[maddr]["MarketLocked"][value.String()] = append(cs.DiffValue[maddr]["MarketLocked"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["MarketLocked"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["MarketLocked"][cmp.String()] = append(cs.DiffCmp[maddr]["MarketLocked"][cmp.String()], height) + } + } + } + + { + value := big.NewInt(int64(ps.Faults)) + cs.DiffHeight[maddr]["Faults"][height] = value + cs.DiffValue[maddr]["Faults"][value.String()] = append(cs.DiffValue[maddr]["Faults"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["Faults"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["Faults"][cmp.String()] = append(cs.DiffCmp[maddr]["Faults"][cmp.String()], height) + } + } + } + + { + value := big.NewInt(int64(ps.ProvenSectors)) + cs.DiffHeight[maddr]["ProvenSectors"][height] = value + cs.DiffValue[maddr]["ProvenSectors"][value.String()] = append(cs.DiffValue[maddr]["ProvenSectors"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["ProvenSectors"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["ProvenSectors"][cmp.String()] = append(cs.DiffCmp[maddr]["ProvenSectors"][cmp.String()], height) + } + } + } + + { + value := big.NewInt(int64(ps.Recoveries)) + cs.DiffHeight[maddr]["Recoveries"][height] = value + cs.DiffValue[maddr]["Recoveries"][value.String()] = append(cs.DiffValue[maddr]["Recoveries"][value.String()], height) + + if cs.PrevHeight != -1 { + prevValue := cs.DiffHeight[maddr]["Recoveries"][cs.PrevHeight] + cmp := big.Zero() + cmp.Sub(value.Int, prevValue.Int) // value - prevValue + if big.Cmp(cmp, big.Zero()) != 0 { + cs.DiffCmp[maddr]["Recoveries"][cmp.String()] = append(cs.DiffCmp[maddr]["Recoveries"][cmp.String()], height) + } + } + } +} + +func roundBalance(i *big.Int) { + *i = big.Div(*i, big.NewInt(1000000000000000)) + *i = big.Mul(*i, big.NewInt(1000000000000000)) +} + +func toCharStr(i int) string { + return string('a' + i) +} diff --git a/lotus-soup/rfwp/e2e.go b/lotus-soup/rfwp/e2e.go new file mode 100644 index 000000000..99794c09a --- /dev/null +++ b/lotus-soup/rfwp/e2e.go @@ -0,0 +1,347 @@ +package rfwp + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "math/rand" + "os" + "sort" + "strings" + "time" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/oni/lotus-soup/testkit" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "golang.org/x/sync/errgroup" +) + +func RecoveryFromFailedWindowedPoStE2E(t *testkit.TestEnvironment) error { + switch t.Role { + case "bootstrapper": + return testkit.HandleDefaultRole(t) + case "client": + return handleClient(t) + case "miner": + return handleMiner(t) + case "miner-full-slash": + return handleMinerFullSlash(t) + case "miner-partial-slash": + return handleMinerPartialSlash(t) + } + + return fmt.Errorf("unknown role: %s", t.Role) +} + +func handleMiner(t *testkit.TestEnvironment) error { + m, err := testkit.PrepareMiner(t) + if err != nil { + return err + } + + ctx := context.Background() + myActorAddr, err := m.MinerApi.ActorAddress(ctx) + if err != nil { + return err + } + + t.RecordMessage("running miner: %s", myActorAddr) + + if t.GroupSeq == 1 { + go FetchChainState(t, m) + } + + go UpdateChainState(t, m) + + minersToBeSlashed := 2 + ch := make(chan testkit.SlashedMinerMsg) + sub := t.SyncClient.MustSubscribe(ctx, testkit.SlashedMinerTopic, ch) + var eg errgroup.Group + + for i := 0; i < minersToBeSlashed; i++ { + select { + case slashedMiner := <-ch: + // wait for slash + eg.Go(func() error { + select { + case <-waitForSlash(t, slashedMiner): + case err = <-t.SyncClient.MustBarrier(ctx, testkit.StateAbortTest, 1).C: + if err != nil { + return err + } + return errors.New("got abort signal, exitting") + } + return nil + }) + case err := <-sub.Done(): + return fmt.Errorf("got error while waiting for slashed miners: %w", err) + case err := <-t.SyncClient.MustBarrier(ctx, testkit.StateAbortTest, 1).C: + if err != nil { + return err + } + return errors.New("got abort signal, exitting") + } + } + + errc := make(chan error) + go func() { + errc <- eg.Wait() + }() + + select { + case err := <-errc: + if err != nil { + return err + } + case err := <-t.SyncClient.MustBarrier(ctx, testkit.StateAbortTest, 1).C: + if err != nil { + return err + } + return errors.New("got abort signal, exitting") + } + + t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount) + return nil +} + +func waitForSlash(t *testkit.TestEnvironment, msg testkit.SlashedMinerMsg) chan error { + // assert that balance got reduced with that much 5 times (sector fee) + // assert that balance got reduced with that much 2 times (termination fee) + // assert that balance got increased with that much 10 times (block reward) + // assert that power got increased with that much 1 times (after sector is sealed) + // assert that power got reduced with that much 1 times (after sector is announced faulty) + slashedMiner := msg.MinerActorAddr + + errc := make(chan error) + go func() { + foundSlashConditions := false + for range time.Tick(10 * time.Second) { + if foundSlashConditions { + close(errc) + return + } + t.RecordMessage("wait for slashing, tick") + func() { + cs.Lock() + defer cs.Unlock() + + negativeAmounts := []big.Int{} + negativeDiffs := make(map[big.Int][]abi.ChainEpoch) + + for am, heights := range cs.DiffCmp[slashedMiner.String()]["LockedFunds"] { + amount, err := big.FromString(am) + if err != nil { + errc <- fmt.Errorf("cannot parse LockedFunds amount: %w:", err) + return + } + + // amount is negative => slash condition + if big.Cmp(amount, big.Zero()) < 0 { + negativeDiffs[amount] = heights + negativeAmounts = append(negativeAmounts, amount) + } + } + + t.RecordMessage("negative diffs: %d", len(negativeDiffs)) + if len(negativeDiffs) < 3 { + return + } + + sort.Slice(negativeAmounts, func(i, j int) bool { return big.Cmp(negativeAmounts[i], negativeAmounts[j]) > 0 }) + + // TODO: confirm the largest is > 18 filecoin + // TODO: confirm the next largest is > 9 filecoin + foundSlashConditions = true + }() + } + }() + + return errc +} + +func handleMinerFullSlash(t *testkit.TestEnvironment) error { + m, err := testkit.PrepareMiner(t) + if err != nil { + return err + } + + ctx := context.Background() + myActorAddr, err := m.MinerApi.ActorAddress(ctx) + if err != nil { + return err + } + + t.RecordMessage("running miner, full slash: %s", myActorAddr) + + // TODO: wait until we have sealed a deal for a client + time.Sleep(240 * time.Second) + + t.RecordMessage("shutting down miner, full slash: %s", myActorAddr) + + ctxt, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + err = m.StopFn(ctxt) + if err != nil { + //return err + t.RecordMessage("err from StopFn: %s", err.Error()) // TODO: expect this to be fixed on Lotus + } + + t.RecordMessage("shutdown miner, full slash: %s", myActorAddr) + + t.SyncClient.MustPublish(ctx, testkit.SlashedMinerTopic, testkit.SlashedMinerMsg{ + MinerActorAddr: myActorAddr, + }) + + t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount) + return nil +} + +func handleMinerPartialSlash(t *testkit.TestEnvironment) error { + m, err := testkit.PrepareMiner(t) + if err != nil { + return err + } + + ctx := context.Background() + myActorAddr, err := m.MinerApi.ActorAddress(ctx) + if err != nil { + return err + } + + t.RecordMessage("running miner, partial slash: %s", myActorAddr) + + // TODO: wait until we have sealed a deal for a client + time.Sleep(185 * time.Second) + + t.RecordMessage("shutting down miner, partial slash: %s", myActorAddr) + + ctxt, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + err = m.StopFn(ctxt) + if err != nil { + //return err + t.RecordMessage("err from StopFn: %s", err.Error()) // TODO: expect this to be fixed on Lotus + } + + t.RecordMessage("shutdown miner, partial slash: %s", myActorAddr) + + t.SyncClient.MustPublish(ctx, testkit.SlashedMinerTopic, testkit.SlashedMinerMsg{ + MinerActorAddr: myActorAddr, + }) + + time.Sleep(300 * time.Second) + + rm, err := testkit.RestoreMiner(t, m) + if err != nil { + t.RecordMessage("got err: %s", err.Error()) + return err + } + + myActorAddr, err = rm.MinerApi.ActorAddress(ctx) + if err != nil { + t.RecordMessage("got err: %s", err.Error()) + return err + } + + t.RecordMessage("running miner again, partial slash: %s", myActorAddr) + + time.Sleep(3600 * time.Second) + + //t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount) + return nil +} + +func handleClient(t *testkit.TestEnvironment) error { + cl, err := testkit.PrepareClient(t) + if err != nil { + return err + } + + // This is a client role + t.RecordMessage("running client") + + ctx := context.Background() + client := cl.FullApi + + time.Sleep(10 * time.Second) + + // select a miner based on our GroupSeq (client 1 -> miner 1 ; client 2 -> miner 2) + // this assumes that all miner instances receive the same sorted MinerAddrs slice + minerAddr := cl.MinerAddrs[t.InitContext.GroupSeq-1] + if err := client.NetConnect(ctx, minerAddr.MinerNetAddrs); err != nil { + return err + } + t.D().Counter(fmt.Sprintf("send-data-to,miner=%s", minerAddr.MinerActorAddr)).Inc(1) + + t.RecordMessage("selected %s as the miner", minerAddr.MinerActorAddr) + + time.Sleep(2 * time.Second) + + // generate 1800 bytes of random data + data := make([]byte, 1800) + rand.New(rand.NewSource(time.Now().UnixNano())).Read(data) + + file, err := ioutil.TempFile("/tmp", "data") + if err != nil { + return err + } + defer os.Remove(file.Name()) + + _, err = file.Write(data) + if err != nil { + return err + } + + fcid, err := client.ClientImport(ctx, api.FileRef{Path: file.Name(), IsCAR: false}) + if err != nil { + return err + } + t.RecordMessage("file cid: %s", fcid) + + // start deal + t1 := time.Now() + fastRetrieval := false + deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, fcid.Root, fastRetrieval) + t.RecordMessage("started deal: %s", deal) + + // this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(2 * time.Second) + + t.RecordMessage("waiting for deal to be sealed") + testkit.WaitDealSealed(t, ctx, client, deal) + t.D().ResettingHistogram("deal.sealed").Update(int64(time.Since(t1))) + + // TODO: wait to stop miner (ideally get a signal, rather than sleep) + time.Sleep(180 * time.Second) + + t.RecordMessage("trying to retrieve %s", fcid) + info, err := client.ClientGetDealInfo(ctx, *deal) + if err != nil { + return err + } + + carExport := true + err = testkit.RetrieveData(t, ctx, client, fcid.Root, &info.PieceCID, carExport, data) + if err != nil && strings.Contains(err.Error(), "cannot make retrieval deal for zero bytes") { + t.D().Counter("deal.expect-slashing").Inc(1) + } else if err != nil { + // unknown error => fail test + t.RecordFailure(err) + + // send signal to abort test + t.SyncClient.MustSignalEntry(ctx, testkit.StateAbortTest) + + t.D().ResettingHistogram("deal.retrieved.err").Update(int64(time.Since(t1))) + time.Sleep(10 * time.Second) // wait for metrics to be emitted + + return nil + } + + t.D().ResettingHistogram("deal.retrieved").Update(int64(time.Since(t1))) + time.Sleep(10 * time.Second) // wait for metrics to be emitted + + t.SyncClient.MustSignalAndWait(ctx, testkit.StateDone, t.TestInstanceCount) // TODO: not sure about this + return nil +} diff --git a/lotus-soup/rfwp/html_chain_state.go b/lotus-soup/rfwp/html_chain_state.go new file mode 100644 index 000000000..72dff8463 --- /dev/null +++ b/lotus-soup/rfwp/html_chain_state.go @@ -0,0 +1,66 @@ +package rfwp + +import ( + "context" + "fmt" + "os" + + "github.com/filecoin-project/oni/lotus-soup/testkit" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/cli" + tstats "github.com/filecoin-project/lotus/tools/stats" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/ipfs/go-cid" +) + +func FetchChainState(t *testkit.TestEnvironment, m *testkit.LotusMiner) error { + height := 0 + headlag := 3 + + ctx := context.Background() + api := m.FullApi + + tipsetsCh, err := tstats.GetTips(ctx, m.FullApi, abi.ChainEpoch(height), headlag) + if err != nil { + return err + } + + for tipset := range tipsetsCh { + err := func() error { + filename := fmt.Sprintf("%s%cchain-state-%d.html", t.TestOutputsPath, os.PathSeparator, tipset.Height()) + file, err := os.Create(filename) + defer file.Close() + if err != nil { + return err + } + + stout, err := api.StateCompute(ctx, tipset.Height(), nil, tipset.Key()) + if err != nil { + return err + } + + codeCache := map[address.Address]cid.Cid{} + getCode := func(addr address.Address) (cid.Cid, error) { + if c, found := codeCache[addr]; found { + return c, nil + } + + c, err := api.StateGetActor(ctx, addr, tipset.Key()) + if err != nil { + return cid.Cid{}, err + } + + codeCache[addr] = c.Code + return c.Code, nil + } + + return cli.ComputeStateHTMLTempl(file, tipset, stout, getCode) + }() + if err != nil { + return err + } + } + + return nil +} diff --git a/lotus-soup/testkit/deals.go b/lotus-soup/testkit/deals.go index 6e8d29a0b..a3ff85978 100644 --- a/lotus-soup/testkit/deals.go +++ b/lotus-soup/testkit/deals.go @@ -3,16 +3,18 @@ package testkit import ( "context" "fmt" - "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" + + tstats "github.com/filecoin-project/lotus/tools/stats" ) -func StartDeal(ctx context.Context, minerActorAddr address.Address, client api.FullNode, fcid cid.Cid) *cid.Cid { +func StartDeal(ctx context.Context, minerActorAddr address.Address, client api.FullNode, fcid cid.Cid, fastRetrieval bool) *cid.Cid { addr, err := client.WalletDefaultAddress(ctx) if err != nil { panic(err) @@ -24,7 +26,7 @@ func StartDeal(ctx context.Context, minerActorAddr address.Address, client api.F Miner: minerActorAddr, EpochPrice: types.NewInt(1000000), MinBlocksDuration: 1000, - FastRetrieval: false, + FastRetrieval: fastRetrieval, }) if err != nil { panic(err) @@ -33,8 +35,20 @@ func StartDeal(ctx context.Context, minerActorAddr address.Address, client api.F } func WaitDealSealed(t *TestEnvironment, ctx context.Context, client api.FullNode, deal *cid.Cid) { -loop: - for { + height := 0 + headlag := 3 + + cctx, cancel := context.WithCancel(ctx) + defer cancel() + + tipsetsCh, err := tstats.GetTips(cctx, client, abi.ChainEpoch(height), headlag) + if err != nil { + panic(err) + } + + for tipset := range tipsetsCh { + t.RecordMessage("got tipset: height %d", tipset.Height()) + di, err := client.ClientGetDealInfo(ctx, *deal) if err != nil { panic(err) @@ -48,9 +62,9 @@ loop: panic(fmt.Sprintf("deal errored %s", di.Message)) case storagemarket.StorageDealActive: t.RecordMessage("completed deal: %s", di) - break loop + return } + t.RecordMessage("deal state: %s", storagemarket.DealStates[di.State]) - time.Sleep(2 * time.Second) } } diff --git a/lotus-soup/testkit/retrieval.go b/lotus-soup/testkit/retrieval.go index c245d5cfc..de3dee6be 100644 --- a/lotus-soup/testkit/retrieval.go +++ b/lotus-soup/testkit/retrieval.go @@ -3,6 +3,7 @@ package testkit import ( "bytes" "context" + "errors" "fmt" "io/ioutil" "os" @@ -19,7 +20,7 @@ import ( "github.com/ipld/go-car" ) -func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, fcid cid.Cid, carExport bool, data []byte) { +func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, fcid cid.Cid, _ *cid.Cid, carExport bool, data []byte) error { t1 := time.Now() offers, err := client.ClientFindData(ctx, fcid, nil) if err != nil { @@ -42,7 +43,7 @@ func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, caddr, err := client.WalletDefaultAddress(ctx) if err != nil { - panic(err) + return err } ref := &api.FileRef{ @@ -52,13 +53,13 @@ func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, t1 = time.Now() err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref) if err != nil { - panic(err) + return err } t.D().ResettingHistogram("retrieve-data").Update(int64(time.Since(t1))) rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret")) if err != nil { - panic(err) + return err } if carExport { @@ -66,10 +67,12 @@ func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, } if !bytes.Equal(rdata, data) { - panic("wrong data retrieved") + return errors.New("wrong data retrieved") } t.RecordMessage("retrieved successfully") + + return nil } func ExtractCarData(ctx context.Context, rdata []byte, rpath string) []byte { diff --git a/lotus-soup/testkit/role_client.go b/lotus-soup/testkit/role_client.go index f26c20648..bc55c4a3a 100644 --- a/lotus-soup/testkit/role_client.go +++ b/lotus-soup/testkit/role_client.go @@ -15,6 +15,8 @@ import ( "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/gorilla/mux" + "github.com/hashicorp/go-multierror" ) type LotusClient struct { @@ -75,7 +77,6 @@ func PrepareClient(t *TestEnvironment) (*LotusClient, error) { if err != nil { return nil, err } - n.StopFn = stop // set the wallet err = n.setWallet(ctx, walletKey) @@ -84,11 +85,18 @@ func PrepareClient(t *TestEnvironment) (*LotusClient, error) { return nil, err } - err = startFullNodeAPIServer(t, nodeRepo, n.FullApi) + fullSrv, err := startFullNodeAPIServer(t, nodeRepo, n.FullApi) if err != nil { return nil, err } + n.StopFn = func(ctx context.Context) error { + var err *multierror.Error + err = multierror.Append(fullSrv.Shutdown(ctx)) + err = multierror.Append(stop(ctx)) + return err.ErrorOrNil() + } + registerAndExportMetrics(fmt.Sprintf("client_%d", t.GroupSeq)) t.RecordMessage("publish our address to the clients addr topic") @@ -146,40 +154,42 @@ func (c *LotusClient) RunDefault() error { return nil } -func startFullNodeAPIServer(t *TestEnvironment, repo *repo.MemRepo, api api.FullNode) error { +func startFullNodeAPIServer(t *TestEnvironment, repo repo.Repo, api api.FullNode) (*http.Server, error) { + mux := mux.NewRouter() + rpcServer := jsonrpc.NewServer() rpcServer.Register("Filecoin", api) - ah := &auth.Handler{ - Verify: func(ctx context.Context, token string) ([]auth.Permission, error) { - return apistruct.AllPermissions, nil - }, - Next: rpcServer.ServeHTTP, - } - - http.Handle("/rpc/v0", ah) + mux.Handle("/rpc/v0", rpcServer) exporter, err := prometheus.NewExporter(prometheus.Options{ Namespace: "lotus", }) if err != nil { - return err + return nil, err } - http.Handle("/debug/metrics", exporter) + mux.Handle("/debug/metrics", exporter) - srv := &http.Server{Handler: http.DefaultServeMux} + ah := &auth.Handler{ + Verify: func(ctx context.Context, token string) ([]auth.Permission, error) { + return apistruct.AllPermissions, nil + }, + Next: mux.ServeHTTP, + } + + srv := &http.Server{Handler: ah} endpoint, err := repo.APIEndpoint() if err != nil { - return fmt.Errorf("no API endpoint in repo: %w", err) + return nil, fmt.Errorf("no API endpoint in repo: %w", err) } listenAddr, err := startServer(endpoint, srv) if err != nil { - return fmt.Errorf("failed to start client API endpoint: %w", err) + return nil, fmt.Errorf("failed to start client API endpoint: %w", err) } t.RecordMessage("started node API server at %s", listenAddr) - return nil + return srv, nil } diff --git a/lotus-soup/testkit/role_miner.go b/lotus-soup/testkit/role_miner.go index f616e1b5d..cac510a54 100644 --- a/lotus-soup/testkit/role_miner.go +++ b/lotus-soup/testkit/role_miner.go @@ -3,9 +3,11 @@ package testkit import ( "context" "crypto/rand" + "encoding/json" "fmt" "io/ioutil" "net/http" + "path/filepath" "time" "contrib.go.opencensus.io/exporter/prometheus" @@ -26,11 +28,14 @@ import ( "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/sector-storage/stores" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/google/uuid" "github.com/gorilla/mux" + "github.com/hashicorp/go-multierror" "github.com/ipfs/go-datastore" libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -44,6 +49,11 @@ const ( type LotusMiner struct { *LotusNode + MinerRepo repo.Repo + NodeRepo repo.Repo + FullNetAddrs []peer.AddrInfo + GenesisMsg *GenesisMsg + t *TestEnvironment } @@ -115,52 +125,93 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { } // prepare the repo - minerRepo := repo.NewMemory(nil) - - lr, err := minerRepo.Lock(repo.StorageMiner) + minerRepoDir, err := ioutil.TempDir("", "miner-repo-dir") if err != nil { return nil, err } - ks, err := lr.KeyStore() + minerRepo, err := repo.NewFS(minerRepoDir) if err != nil { return nil, err } - kbytes, err := priv.Bytes() + err = minerRepo.Init(repo.StorageMiner) if err != nil { return nil, err } - err = ks.Put("libp2p-host", types.KeyInfo{ - Type: "libp2p-host", - PrivateKey: kbytes, - }) - if err != nil { - return nil, err - } - - ds, err := lr.Datastore("/metadata") - if err != nil { - return nil, err - } - - err = ds.Put(datastore.NewKey("miner-address"), minerAddr.Bytes()) - if err != nil { - return nil, err - } - - nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix)) - for i := 0; i < (sectors + 1); i++ { - _, err = nic.Next() + { + lr, err := minerRepo.Lock(repo.StorageMiner) if err != nil { return nil, err } - } - err = lr.Close() - if err != nil { - return nil, err + ks, err := lr.KeyStore() + if err != nil { + return nil, err + } + + kbytes, err := priv.Bytes() + if err != nil { + return nil, err + } + + err = ks.Put("libp2p-host", types.KeyInfo{ + Type: "libp2p-host", + PrivateKey: kbytes, + }) + if err != nil { + return nil, err + } + + ds, err := lr.Datastore("/metadata") + if err != nil { + return nil, err + } + + err = ds.Put(datastore.NewKey("miner-address"), minerAddr.Bytes()) + if err != nil { + return nil, err + } + + nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix)) + for i := 0; i < (sectors + 1); i++ { + _, err = nic.Next() + if err != nil { + return nil, err + } + } + + var localPaths []stores.LocalPath + + b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + ID: stores.ID(uuid.New().String()), + Weight: 10, + CanSeal: true, + CanStore: true, + }, "", " ") + if err != nil { + return nil, fmt.Errorf("marshaling storage config: %w", err) + } + + if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return nil, fmt.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) + } + + localPaths = append(localPaths, stores.LocalPath{ + Path: lr.Path(), + }) + + if err := lr.SetStorage(func(sc *stores.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, localPaths...) + }); err != nil { + return nil, err + } + + err = lr.Close() + if err != nil { + return nil, err + } } minerIP := t.NetClient.MustGetDataNetworkIP().String() @@ -169,7 +220,21 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { // we need both a full node _and_ and storage miner node n := &LotusNode{} - nodeRepo := repo.NewMemory(nil) + // prepare the repo + nodeRepoDir, err := ioutil.TempDir("", "node-repo-dir") + if err != nil { + return nil, err + } + + nodeRepo, err := repo.NewFS(nodeRepoDir) + if err != nil { + return nil, err + } + + err = nodeRepo.Init(repo.FullNode) + if err != nil { + return nil, err + } stop1, err := node.New(context.Background(), node.FullAPI(&n.FullApi), @@ -183,7 +248,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { drandOpt, ) if err != nil { - return nil, err + return nil, fmt.Errorf("node node.new error: %w", err) } // set the wallet @@ -220,22 +285,13 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { stop2, err := node.New(context.Background(), minerOpts...) if err != nil { stop1(context.TODO()) - return nil, err - } - n.StopFn = func(ctx context.Context) error { - // TODO use a multierror for this - err2 := stop2(ctx) - err1 := stop1(ctx) - if err2 != nil { - return err2 - } - return err1 + return nil, fmt.Errorf("miner node.new error: %w", err) } registerAndExportMetrics(minerAddr.String()) - // collect stats based on Travis' scripts - if t.InitContext.GroupSeq == 1 { + // collect stats based on blockchain from first instance of `miner` role + if t.InitContext.GroupSeq == 1 && t.Role == "miner" { go collectStats(t, ctx, n.FullApi) } @@ -245,11 +301,6 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { panic(err) } - // err = n.MinerApi.NetConnect(ctx, fullNodeNetAddrs) - // if err != nil { - // panic(err) - // } - // set seal delay to lower value than 1 hour err = n.MinerApi.SectorSetSealDelay(ctx, sealDelay) if err != nil { @@ -314,6 +365,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { FullNetAddrs: fullNodeNetAddrs, MinerNetAddrs: minerNetAddrs, MinerActorAddr: minerActor, + WalletAddr: walletKey.Address, }) t.RecordMessage("connecting to all other miners") @@ -323,6 +375,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { sctx, cancel := context.WithCancel(ctx) defer cancel() t.SyncClient.MustSubscribe(sctx, MinersAddrsTopic, minerCh) + var fullNetAddrs []peer.AddrInfo for i := 0; i < t.IntParam("miners"); i++ { m := <-minerCh if m.MinerActorAddr == minerActor { @@ -335,24 +388,120 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { } t.RecordMessage("connected to full node of miner %s on %v", m.MinerActorAddr, m.FullNetAddrs) + fullNetAddrs = append(fullNetAddrs, m.FullNetAddrs) } t.RecordMessage("waiting for all nodes to be ready") t.SyncClient.MustSignalAndWait(ctx, StateReady, t.TestInstanceCount) - m := &LotusMiner{n, t} - - err = startFullNodeAPIServer(t, nodeRepo, n.FullApi) + fullSrv, err := startFullNodeAPIServer(t, nodeRepo, n.FullApi) if err != nil { return nil, err } - err = startStorageMinerAPIServer(t, minerRepo, n.MinerApi) + minerSrv, err := startStorageMinerAPIServer(t, minerRepo, n.MinerApi) if err != nil { return nil, err } - return m, err + n.StopFn = func(ctx context.Context) error { + var err *multierror.Error + err = multierror.Append(fullSrv.Shutdown(ctx)) + err = multierror.Append(minerSrv.Shutdown(ctx)) + err = multierror.Append(stop2(ctx)) + err = multierror.Append(stop2(ctx)) + err = multierror.Append(stop1(ctx)) + return err.ErrorOrNil() + } + + m := &LotusMiner{n, minerRepo, nodeRepo, fullNetAddrs, genesisMsg, t} + + return m, nil +} + +func RestoreMiner(t *TestEnvironment, m *LotusMiner) (*LotusMiner, error) { + ctx, cancel := context.WithTimeout(context.Background(), PrepareNodeTimeout) + defer cancel() + + minerRepo := m.MinerRepo + nodeRepo := m.NodeRepo + fullNetAddrs := m.FullNetAddrs + genesisMsg := m.GenesisMsg + + minerIP := t.NetClient.MustGetDataNetworkIP().String() + + drandOpt, err := GetRandomBeaconOpts(ctx, t) + if err != nil { + return nil, err + } + + // create the node + // we need both a full node _and_ and storage miner node + n := &LotusNode{} + + stop1, err := node.New(context.Background(), + node.FullAPI(&n.FullApi), + node.Online(), + node.Repo(nodeRepo), + //withGenesis(genesisMsg.Genesis), + withApiEndpoint(fmt.Sprintf("/ip4/0.0.0.0/tcp/%s", t.PortNumber("node_rpc", "0"))), + withListenAddress(minerIP), + withBootstrapper(genesisMsg.Bootstrapper), + //withPubsubConfig(false, pubsubTracer), + drandOpt, + ) + if err != nil { + return nil, err + } + + minerOpts := []node.Option{ + node.StorageMiner(&n.MinerApi), + node.Online(), + node.Repo(minerRepo), + node.Override(new(api.FullNode), n.FullApi), + withApiEndpoint(fmt.Sprintf("/ip4/0.0.0.0/tcp/%s", t.PortNumber("miner_rpc", "0"))), + withMinerListenAddress(minerIP), + } + + stop2, err := node.New(context.Background(), minerOpts...) + if err != nil { + stop1(context.TODO()) + return nil, err + } + + fullSrv, err := startFullNodeAPIServer(t, nodeRepo, n.FullApi) + if err != nil { + return nil, err + } + + minerSrv, err := startStorageMinerAPIServer(t, minerRepo, n.MinerApi) + if err != nil { + return nil, err + } + + n.StopFn = func(ctx context.Context) error { + var err *multierror.Error + err = multierror.Append(fullSrv.Shutdown(ctx)) + err = multierror.Append(minerSrv.Shutdown(ctx)) + err = multierror.Append(stop2(ctx)) + err = multierror.Append(stop2(ctx)) + err = multierror.Append(stop1(ctx)) + return err.ErrorOrNil() + } + + for i := 0; i < len(fullNetAddrs); i++ { + err := n.FullApi.NetConnect(ctx, fullNetAddrs[i]) + if err != nil { + // we expect a failure since we also shutdown another miner + t.RecordMessage("failed to connect to miner %d on: %v", i, fullNetAddrs[i]) + continue + } + t.RecordMessage("connected to full node of miner %d on %v", i, fullNetAddrs[i]) + } + + pm := &LotusMiner{n, minerRepo, nodeRepo, fullNetAddrs, genesisMsg, t} + + return pm, err } func (m *LotusMiner) RunDefault() error { @@ -438,7 +587,7 @@ func (m *LotusMiner) RunDefault() error { return nil } -func startStorageMinerAPIServer(t *TestEnvironment, repo *repo.MemRepo, minerApi api.StorageMiner) error { +func startStorageMinerAPIServer(t *TestEnvironment, repo repo.Repo, minerApi api.StorageMiner) (*http.Server, error) { mux := mux.NewRouter() rpcServer := jsonrpc.NewServer() @@ -452,7 +601,7 @@ func startStorageMinerAPIServer(t *TestEnvironment, repo *repo.MemRepo, minerApi Namespace: "lotus", }) if err != nil { - return err + return nil, err } mux.Handle("/debug/metrics", exporter) @@ -466,16 +615,16 @@ func startStorageMinerAPIServer(t *TestEnvironment, repo *repo.MemRepo, minerApi endpoint, err := repo.APIEndpoint() if err != nil { - return fmt.Errorf("no API endpoint in repo: %w", err) + return nil, fmt.Errorf("no API endpoint in repo: %w", err) } srv := &http.Server{Handler: ah} listenAddr, err := startServer(endpoint, srv) if err != nil { - return fmt.Errorf("failed to start storage miner API endpoint: %w", err) + return nil, fmt.Errorf("failed to start storage miner API endpoint: %w", err) } t.RecordMessage("started storage miner API server at %s", listenAddr) - return nil + return srv, nil } diff --git a/lotus-soup/testkit/sync.go b/lotus-soup/testkit/sync.go index 51975ce29..0bd581d08 100644 --- a/lotus-soup/testkit/sync.go +++ b/lotus-soup/testkit/sync.go @@ -14,6 +14,7 @@ var ( PresealTopic = sync.NewTopic("preseal", &PresealMsg{}) ClientsAddrsTopic = sync.NewTopic("clients_addrs", &ClientAddressesMsg{}) MinersAddrsTopic = sync.NewTopic("miners_addrs", &MinerAddressesMsg{}) + SlashedMinerTopic = sync.NewTopic("slashed_miner", &SlashedMinerMsg{}) PubsubTracerTopic = sync.NewTopic("pubsub_tracer", &PubsubTracerMsg{}) DrandConfigTopic = sync.NewTopic("drand_config", &DrandRuntimeInfo{}) ) @@ -23,6 +24,7 @@ var ( StateDone = sync.State("done") StateStopMining = sync.State("stop-mining") StateMinerPickSeqNum = sync.State("miner-pick-seq-num") + StateAbortTest = sync.State("abort-test") ) type InitialBalanceMsg struct { @@ -49,6 +51,11 @@ type MinerAddressesMsg struct { FullNetAddrs peer.AddrInfo MinerNetAddrs peer.AddrInfo MinerActorAddr address.Address + WalletAddr address.Address +} + +type SlashedMinerMsg struct { + MinerActorAddr address.Address } type PubsubTracerMsg struct {