diff --git a/Makefile b/Makefile index d20343f55..e6c5f1b54 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ BUILD_DEPS+=build/.filecoin-install CLEAN+=build/.filecoin-install ffi-version-check: - @[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 2 ]] || (echo "FFI version mismatch, update submodules"; exit 1) + @[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 3 ]] || (echo "FFI version mismatch, update submodules"; exit 1) BUILD_DEPS+=ffi-version-check .PHONY: ffi-version-check diff --git a/README.md b/README.md index 761838834..0218e87e9 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). +For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme. ## Reporting a Vulnerability @@ -50,6 +50,88 @@ When implementing a change: 7. Title the PR in a meaningful way and describe the rationale and the thought process in the PR description. 8. Write clean, thoughtful, and detailed [commit messages](https://chris.beams.io/posts/git-commit/). This is even more important than the PR description, because commit messages are stored _inside_ the Git history. One good rule is: if you are happy posting the commit message as the PR description, then it's a good commit message. +## Basic Build Instructions +**System-specific Software Dependencies**: + +Building Lotus requires some system dependencies, usually provided by your distribution. + +Ubuntu/Debian: +``` +sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y +``` + +Fedora: +``` +sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev +``` + +For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos) + +#### Go + +To build Lotus, you need a working installation of [Go 1.16.4 or higher](https://golang.org/dl/): + +```bash +wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local +``` + +**TIP:** +You'll need to add `/usr/local/go/bin` to your path. For most Linux distributions you can run something like: + +```shell +echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.bashrc && source ~/.bashrc +``` + +See the [official Golang installation instructions](https://golang.org/doc/install) if you get stuck. + +### Build and install Lotus + +Once all the dependencies are installed, you can build and install the Lotus suite (`lotus`, `lotus-miner`, and `lotus-worker`). + +1. Clone the repository: + + ```sh + git clone https://github.com/filecoin-project/lotus.git + cd lotus/ + ``` + +Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the latest release[ here](https://github.com/filecoin-project/lotus/releases). + +2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases). + + If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding. + + For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below. + + ```sh + git checkout + # For example: + git checkout # tag for a release + ``` + + Currently, the latest code on the _master_ branch corresponds to mainnet. + +3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)". +4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed. + +5. Build and install Lotus: + + ```sh + make clean all #mainnet + + # Or to join a testnet or devnet: + make clean calibnet # Calibration with min 32GiB sectors + make clean nerpanet # Nerpa with min 512MiB sectors + + sudo make install + ``` + + This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`. + + `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder. + +6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain). + ## License Dual-licensed under [MIT](https://github.com/filecoin-project/lotus/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/lotus/blob/master/LICENSE-APACHE) diff --git a/build/bootstrap/calibnet.pi b/build/bootstrap/calibnet.pi index 0eb9fd2f3..20473eaaa 100644 --- a/build/bootstrap/calibnet.pi +++ b/build/bootstrap/calibnet.pi @@ -1,4 +1,4 @@ -/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWRLZAseMo9h7fRD6ojn6YYDXHsBSavX5YmjBZ9ngtAEec -/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWJFtDXgZEQMEkjJPSrbfdvh2xfjVKrXeNFG1t8ioJXAzv -/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWP1uB9Lo7yCA3S17TD4Y5wStP5Nk7Vqh53m8GsFjkyujD -/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWLrPM4WPK1YRGPCUwndWcDX8GCYgms3DiuofUmxwvhMCn +/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWJkikQQkxS58spo76BYzFt4fotaT5NpV2zngvrqm4u5ow +/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWLce5FDHR4EX4CrYavphA5xS3uDsX6aoowXh5tzDUxJav +/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWA9hFfQG9GjP6bHeuQQbMD3FDtZLdW1NayxKXUT26PQZu +/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWMHDi3LVTFG8Szqogt7RkNXvonbQYqSazxBx41A5aeuVz diff --git a/build/genesis/calibnet.car b/build/genesis/calibnet.car index d2fe7c3af..cbade953f 100644 Binary files a/build/genesis/calibnet.car and b/build/genesis/calibnet.car differ diff --git a/build/params_calibnet.go b/build/params_calibnet.go index d4cea7e07..df334a516 100644 --- a/build/params_calibnet.go +++ b/build/params_calibnet.go @@ -33,20 +33,20 @@ const UpgradeLiftoffHeight = -5 const UpgradeKumquatHeight = 90 -const UpgradeCalicoHeight = 100 +const UpgradeCalicoHeight = 120 const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1) -const UpgradeClausHeight = 250 +const UpgradeClausHeight = 270 const UpgradeOrangeHeight = 300 -const UpgradeTrustHeight = 600 -const UpgradeNorwegianHeight = 114000 +const UpgradeTrustHeight = 330 -const UpgradeTurboHeight = 193789 +const UpgradeNorwegianHeight = 360 -// 2021-06-11T14:30:00Z -const UpgradeHyperdriveHeight = 321519 +const UpgradeTurboHeight = 390 + +const UpgradeHyperdriveHeight = 420 func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30)) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index 22ec4c742..c159dc98f 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -278,13 +278,13 @@ func GetMaxSectorExpirationExtension() abi.ChainEpoch { return miner5.MaxSectorExpirationExtension } -// TODO: we'll probably need to abstract over this better in the future. -func GetMaxPoStPartitions(p abi.RegisteredPoStProof) (int, error) { +func GetMaxPoStPartitions(nv network.Version, p abi.RegisteredPoStProof) (int, error) { sectorsPerPart, err := builtin5.PoStProofWindowPoStPartitionSectors(p) if err != nil { return 0, err } - return int(miner5.AddressedSectorsMax / sectorsPerPart), nil + maxSectors := uint64(GetAddressedSectorsMax(nv)) + return int(maxSectors / sectorsPerPart), nil } func GetDefaultSectorSize() abi.SectorSize { diff --git a/chain/actors/policy/policy.go.template b/chain/actors/policy/policy.go.template index feca2370a..17b3eb0ff 100644 --- a/chain/actors/policy/policy.go.template +++ b/chain/actors/policy/policy.go.template @@ -182,13 +182,13 @@ func GetMaxSectorExpirationExtension() abi.ChainEpoch { return miner{{.latestVersion}}.MaxSectorExpirationExtension } -// TODO: we'll probably need to abstract over this better in the future. -func GetMaxPoStPartitions(p abi.RegisteredPoStProof) (int, error) { +func GetMaxPoStPartitions(nv network.Version, p abi.RegisteredPoStProof) (int, error) { sectorsPerPart, err := builtin{{.latestVersion}}.PoStProofWindowPoStPartitionSectors(p) if err != nil { return 0, err } - return int(miner{{.latestVersion}}.AddressedSectorsMax / sectorsPerPart), nil + maxSectors := uint64(GetAddressedSectorsMax(nv)) + return int(maxSectors / sectorsPerPart), nil } func GetDefaultSectorSize() abi.SectorSize { diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go index 24e47aaa0..f40250fba 100644 --- a/chain/actors/policy/policy_test.go +++ b/chain/actors/policy/policy_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" @@ -68,3 +69,12 @@ func TestPartitionSizes(t *testing.T) { require.Equal(t, sizeOld, sizeNew) } } + +func TestPoStSize(t *testing.T) { + v12PoStSize, err := GetMaxPoStPartitions(network.Version12, abi.RegisteredPoStProof_StackedDrgWindow64GiBV1) + require.Equal(t, 4, v12PoStSize) + require.NoError(t, err) + v13PoStSize, err := GetMaxPoStPartitions(network.Version13, abi.RegisteredPoStProof_StackedDrgWindow64GiBV1) + require.NoError(t, err) + require.Equal(t, 10, v13PoStSize) +} diff --git a/chain/store/store.go b/chain/store/store.go index 71fa0397a..523726863 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/filecoin-project/lotus/chain/state" @@ -404,7 +405,20 @@ func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error { // internal state as our new head, if and only if it is heavier than the current // head and does not exceed the maximum fork length. func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipSet) error { - cs.heaviestLk.Lock() + for { + cs.heaviestLk.Lock() + if len(cs.reorgCh) < reorgChBuf/2 { + break + } + cs.heaviestLk.Unlock() + log.Errorf("reorg channel is heavily backlogged, waiting a bit before trying to take process new tipsets") + select { + case <-time.After(time.Second / 2): + case <-ctx.Done(): + return ctx.Err() + } + } + defer cs.heaviestLk.Unlock() w, err := cs.Weight(ctx, ts) if err != nil { @@ -529,8 +543,10 @@ type reorg struct { new *types.TipSet } +const reorgChBuf = 32 + func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNotifee) chan<- reorg { - out := make(chan reorg, 32) + out := make(chan reorg, reorgChBuf) notifees := make([]ReorgNotifee, len(initialNotifees)) copy(notifees, initialNotifees) diff --git a/chain/sync_test.go b/chain/sync_test.go index 9f89f789b..ae4b6cc2e 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -85,6 +85,7 @@ type syncTestUtil struct { blocks []*store.FullTipSet nds []api.FullNode + us stmgr.UpgradeSchedule } func prepSyncTest(t testing.TB, h int) *syncTestUtil { @@ -104,9 +105,10 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil { mn: mocknet.New(ctx), g: g, + us: stmgr.DefaultUpgradeSchedule(), } - tu.addSourceNode(stmgr.DefaultUpgradeSchedule(), h) + tu.addSourceNode(h) //tu.checkHeight("source", source, h) @@ -119,7 +121,7 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil { func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syncTestUtil { logging.SetLogLevel("*", "INFO") - us := stmgr.UpgradeSchedule{{ + sched := stmgr.UpgradeSchedule{{ // prepare for upgrade. Network: network.Version9, Height: 1, @@ -138,7 +140,7 @@ func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syn Migration: stmgr.UpgradeActorsV5, }} - g, err := gen.NewGeneratorWithUpgradeSchedule(us) + g, err := gen.NewGeneratorWithUpgradeSchedule(sched) if err != nil { t.Fatalf("%+v", err) @@ -153,9 +155,10 @@ func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syn mn: mocknet.New(ctx), g: g, + us: sched, } - tu.addSourceNode(us, h) + tu.addSourceNode(h) //tu.checkHeight("source", source, h) // separate logs @@ -266,7 +269,7 @@ func (tu *syncTestUtil) mineNewBlock(src int, miners []int) { tu.g.CurTipset = mts } -func (tu *syncTestUtil) addSourceNode(us stmgr.UpgradeSchedule, gen int) { +func (tu *syncTestUtil) addSourceNode(gen int) { if tu.genesis != nil { tu.t.Fatal("source node already exists") } @@ -282,7 +285,7 @@ func (tu *syncTestUtil) addSourceNode(us stmgr.UpgradeSchedule, gen int) { node.Test(), node.Override(new(modules.Genesis), modules.LoadGenesis(genesis)), - node.Override(new(stmgr.UpgradeSchedule), us), + node.Override(new(stmgr.UpgradeSchedule), tu.us), ) require.NoError(tu.t, err) tu.t.Cleanup(func() { _ = stop(context.Background()) }) @@ -315,6 +318,7 @@ func (tu *syncTestUtil) addClientNode() int { node.Test(), node.Override(new(modules.Genesis), modules.LoadGenesis(tu.genesis)), + node.Override(new(stmgr.UpgradeSchedule), tu.us), ) require.NoError(tu.t, err) tu.t.Cleanup(func() { _ = stop(context.Background()) }) @@ -1032,18 +1036,24 @@ func TestDrandNull(t *testing.T) { build.UpgradeHyperdriveHeight = v5h tu := prepSyncTestWithV5Height(t, H, v5h) + p0 := tu.addClientNode() + p1 := tu.addClientNode() + + tu.loadChainToNode(p0) + tu.loadChainToNode(p1) + entropy := []byte{0, 2, 3, 4} // arbitrarily chosen pers := crypto.DomainSeparationTag_WinningPoStChallengeSeed beforeNull := tu.g.CurTipset - afterNull := tu.mineOnBlock(beforeNull, 0, nil, false, false, nil, 2) + afterNull := tu.mineOnBlock(beforeNull, p0, nil, false, false, nil, 2) nullHeight := beforeNull.TipSet().Height() + 1 if afterNull.TipSet().Height() == nullHeight { t.Fatal("didn't inject nulls as expected") } - rand, err := tu.nds[0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy) + rand, err := tu.nds[p0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy) require.NoError(t, err) // calculate the expected randomness based on the beacon BEFORE the null @@ -1054,20 +1064,20 @@ func TestDrandNull(t *testing.T) { require.Equal(t, []byte(rand), expectedRand) // zoom zoom to past the v5 upgrade by injecting many many nulls - postUpgrade := tu.mineOnBlock(afterNull, 0, nil, false, false, nil, v5h) - nv, err := tu.nds[0].StateNetworkVersion(tu.ctx, types.EmptyTSK) + postUpgrade := tu.mineOnBlock(afterNull, p0, nil, false, false, nil, v5h) + nv, err := tu.nds[p0].StateNetworkVersion(tu.ctx, postUpgrade.TipSet().Key()) require.NoError(t, err) if nv != network.Version13 { t.Fatal("expect to be v13 by now") } - afterNull = tu.mineOnBlock(postUpgrade, 0, nil, false, false, nil, 2) + afterNull = tu.mineOnBlock(postUpgrade, p0, nil, false, false, nil, 2) nullHeight = postUpgrade.TipSet().Height() + 1 if afterNull.TipSet().Height() == nullHeight { t.Fatal("didn't inject nulls as expected") } - rand, err = tu.nds[0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy) + rand0, err := tu.nds[p0].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy) require.NoError(t, err) // calculate the expected randomness based on the beacon AFTER the null @@ -1075,7 +1085,21 @@ func TestDrandNull(t *testing.T) { expectedRand, err = store.DrawRandomness(expectedBE[len(expectedBE)-1].Data, pers, nullHeight, entropy) require.NoError(t, err) - require.Equal(t, []byte(rand), expectedRand) - build.UpgradeHyperdriveHeight = ov5h + require.Equal(t, []byte(rand0), expectedRand) + // Introduce p1 to friendly p0 who has all the blocks + require.NoError(t, tu.mn.LinkAll()) + tu.connect(p0, p1) + tu.waitUntilNodeHasTs(p1, afterNull.TipSet().Key()) + p1Head := tu.getHead(p1) + + // Yes, p1 syncs well to p0's chain + require.Equal(tu.t, p1Head.Key(), afterNull.TipSet().Key()) + + // Yes, p1 sources the same randomness as p0 + rand1, err := tu.nds[p1].ChainGetRandomnessFromBeacon(tu.ctx, afterNull.TipSet().Key(), pers, nullHeight, entropy) + require.NoError(t, err) + require.Equal(t, rand0, rand1) + + build.UpgradeHyperdriveHeight = ov5h } diff --git a/chain/types/bigint.go b/chain/types/bigint.go index da4857d5b..72ef52128 100644 --- a/chain/types/bigint.go +++ b/chain/types/bigint.go @@ -47,6 +47,11 @@ func BigDiv(a, b BigInt) BigInt { return BigInt{Int: big.NewInt(0).Div(a.Int, b.Int)} } +func BigDivFloat(num, den BigInt) float64 { + res, _ := new(big.Rat).SetFrac(num.Int, den.Int).Float64() + return res +} + func BigMod(a, b BigInt) BigInt { return BigInt{Int: big.NewInt(0).Mod(a.Int, b.Int)} } diff --git a/cli/state.go b/cli/state.go index b0260f02b..6bf23d798 100644 --- a/cli/state.go +++ b/cli/state.go @@ -17,6 +17,8 @@ import ( "strings" "time" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/api/v0api" "github.com/fatih/color" @@ -183,18 +185,23 @@ var StateMinerInfo = &cli.Command{ return err } - rpercI := types.BigDiv(types.BigMul(pow.MinerPower.RawBytePower, types.NewInt(1000000)), pow.TotalPower.RawBytePower) - qpercI := types.BigDiv(types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(1000000)), pow.TotalPower.QualityAdjPower) - fmt.Printf("Byte Power: %s / %s (%0.4f%%)\n", color.BlueString(types.SizeStr(pow.MinerPower.RawBytePower)), types.SizeStr(pow.TotalPower.RawBytePower), - float64(rpercI.Int64())/10000) + types.BigDivFloat( + types.BigMul(pow.MinerPower.RawBytePower, big.NewInt(100)), + pow.TotalPower.RawBytePower, + ), + ) fmt.Printf("Actual Power: %s / %s (%0.4f%%)\n", color.GreenString(types.DeciStr(pow.MinerPower.QualityAdjPower)), types.DeciStr(pow.TotalPower.QualityAdjPower), - float64(qpercI.Int64())/10000) + types.BigDivFloat( + types.BigMul(pow.MinerPower.QualityAdjPower, big.NewInt(100)), + pow.TotalPower.QualityAdjPower, + ), + ) fmt.Println() @@ -311,8 +318,15 @@ var StatePowerCmd = &cli.Command{ tp := power.TotalPower if cctx.Args().Present() { mp := power.MinerPower - percI := types.BigDiv(types.BigMul(mp.QualityAdjPower, types.NewInt(1000000)), tp.QualityAdjPower) - fmt.Printf("%s(%s) / %s(%s) ~= %0.4f%%\n", mp.QualityAdjPower.String(), types.SizeStr(mp.QualityAdjPower), tp.QualityAdjPower.String(), types.SizeStr(tp.QualityAdjPower), float64(percI.Int64())/10000) + fmt.Printf( + "%s(%s) / %s(%s) ~= %0.4f%%\n", + mp.QualityAdjPower.String(), types.SizeStr(mp.QualityAdjPower), + tp.QualityAdjPower.String(), types.SizeStr(tp.QualityAdjPower), + types.BigDivFloat( + types.BigMul(mp.QualityAdjPower, big.NewInt(100)), + tp.QualityAdjPower, + ), + ) } else { fmt.Printf("%s(%s)\n", tp.QualityAdjPower.String(), types.SizeStr(tp.QualityAdjPower)) } diff --git a/cmd/lotus-shed/sync.go b/cmd/lotus-shed/sync.go index 65d2b6d6f..cab3bd29e 100644 --- a/cmd/lotus-shed/sync.go +++ b/cmd/lotus-shed/sync.go @@ -172,12 +172,13 @@ var syncScrapePowerCmd = &cli.Command{ return err } - qpercI := types.BigDiv(types.BigMul(totalWonPower.QualityAdjPower, types.NewInt(1000000)), totalPower.TotalPower.QualityAdjPower) - fmt.Println("Number of winning miners: ", len(miners)) fmt.Println("QAdjPower of winning miners: ", totalWonPower.QualityAdjPower) fmt.Println("QAdjPower of all miners: ", totalPower.TotalPower.QualityAdjPower) - fmt.Println("Percentage of winning QAdjPower: ", float64(qpercI.Int64())/10000) + fmt.Println("Percentage of winning QAdjPower: ", types.BigDivFloat( + types.BigMul(totalWonPower.QualityAdjPower, big.NewInt(100)), + totalPower.TotalPower.QualityAdjPower, + )) return nil }, diff --git a/cmd/lotus-storage-miner/actor_test.go b/cmd/lotus-storage-miner/actor_test.go index 7f36812bc..073a83059 100644 --- a/cmd/lotus-storage-miner/actor_test.go +++ b/cmd/lotus-storage-miner/actor_test.go @@ -10,14 +10,13 @@ import ( "testing" "time" - logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/go-state-types/network" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" @@ -32,36 +31,21 @@ func TestWorkerKeyChange(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _ = logging.SetLogLevel("*", "INFO") - - policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) - policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) - kit.QuietMiningLogs() blocktime := 1 * time.Millisecond - - clients, miners := kit.MockMinerBuilder(t, - []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1), kit.FullNodeWithLatestActorsAt(-1)}, - kit.OneMiner) - - client1 := clients[0] - client2 := clients[1] - - // Connect the nodes. - addrinfo, err := client1.NetAddrsListen(ctx) - require.NoError(t, err) - err = client2.NetConnect(ctx, addrinfo) - require.NoError(t, err) + client1, client2, miner, ens := kit.EnsembleTwoOne(t, kit.MockProofs(), + kit.ConstructorOpts(kit.InstantaneousNetworkVersion(network.Version13)), + ) + ens.InterconnectAll().BeginMining(blocktime) output := bytes.NewBuffer(nil) run := func(cmd *cli.Command, args ...string) error { app := cli.NewApp() app.Metadata = map[string]interface{}{ "repoType": repo.StorageMiner, - "testnode-full": clients[0], - "testnode-storage": miners[0], + "testnode-full": client1, + "testnode-storage": miner, } app.Writer = output api.RunningNodeType = api.NodeMiner @@ -78,9 +62,6 @@ func TestWorkerKeyChange(t *testing.T) { return cmd.Action(cctx) } - // start mining - kit.ConnectAndStartMining(t, blocktime, miners[0], client1, client2) - newKey, err := client1.WalletNew(ctx, types.KTBLS) require.NoError(t, err) @@ -105,14 +86,8 @@ func TestWorkerKeyChange(t *testing.T) { require.Error(t, run(actorConfirmChangeWorker, "--really-do-it", newKey.String())) output.Reset() - for { - head, err := client1.ChainHead(ctx) - require.NoError(t, err) - if head.Height() >= abi.ChainEpoch(targetEpoch) { - break - } - build.Clock.Sleep(10 * blocktime) - } + client1.WaitTillChain(ctx, kit.HeightAtLeast(abi.ChainEpoch(targetEpoch))) + require.NoError(t, run(actorConfirmChangeWorker, "--really-do-it", newKey.String())) output.Reset() @@ -121,23 +96,8 @@ func TestWorkerKeyChange(t *testing.T) { // Wait for finality (worker key switch). targetHeight := head.Height() + policy.ChainFinality - for { - head, err := client1.ChainHead(ctx) - require.NoError(t, err) - if head.Height() >= targetHeight { - break - } - build.Clock.Sleep(10 * blocktime) - } + client1.WaitTillChain(ctx, kit.HeightAtLeast(targetHeight)) // Make sure the other node can catch up. - for i := 0; i < 20; i++ { - head, err := client2.ChainHead(ctx) - require.NoError(t, err) - if head.Height() >= targetHeight { - return - } - build.Clock.Sleep(10 * blocktime) - } - t.Fatal("failed to reach target epoch on the second miner") + client2.WaitTillChain(ctx, kit.HeightAtLeast(targetHeight)) } diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index cbe65524e..21ae4c8ee 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -7,13 +7,9 @@ import ( "time" "github.com/filecoin-project/lotus/itests/kit" - "github.com/filecoin-project/lotus/node/impl" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/node/repo" @@ -24,12 +20,6 @@ func TestMinerAllInfo(t *testing.T) { t.Skip("skipping test in short mode") } - _ = logging.SetLogLevel("*", "INFO") - - policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) - policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) - _test = true kit.QuietMiningLogs() @@ -40,16 +30,15 @@ func TestMinerAllInfo(t *testing.T) { policy.SetPreCommitChallengeDelay(oldDelay) }) - n, sn := kit.Builder(t, kit.OneFull, kit.OneMiner) - client, miner := n[0].FullNode, sn[0] - kit.ConnectAndStartMining(t, time.Second, miner, client.(*impl.FullNodeAPI)) + client, miner, ens := kit.EnsembleMinimal(t) + ens.InterconnectAll().BeginMining(time.Second) run := func(t *testing.T) { app := cli.NewApp() app.Metadata = map[string]interface{}{ "repoType": repo.StorageMiner, - "testnode-full": n[0], - "testnode-storage": sn[0], + "testnode-full": client, + "testnode-storage": miner, } api.RunningNodeType = api.NodeMiner @@ -61,7 +50,9 @@ func TestMinerAllInfo(t *testing.T) { t.Run("pre-info-all", run) dh := kit.NewDealHarness(t, client, miner) - dh.MakeFullDeal(context.Background(), 6, false, false, 0) + deal, res, inPath := dh.MakeOnlineDeal(context.Background(), kit.MakeFullDealParams{Rseed: 6}) + outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, false) + kit.AssertFilesEqual(t, inPath, outPath) t.Run("post-info-all", run) } diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index e7c5e7904..0d8cdf07b 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "math" + corebig "math/big" "sort" "time" @@ -21,6 +23,7 @@ import ( "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -120,19 +123,23 @@ func infoCmdAct(cctx *cli.Context) error { return err } - rpercI := types.BigDiv(types.BigMul(pow.MinerPower.RawBytePower, types.NewInt(1000000)), pow.TotalPower.RawBytePower) - qpercI := types.BigDiv(types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(1000000)), pow.TotalPower.QualityAdjPower) - fmt.Printf("Power: %s / %s (%0.4f%%)\n", color.GreenString(types.DeciStr(pow.MinerPower.QualityAdjPower)), types.DeciStr(pow.TotalPower.QualityAdjPower), - float64(qpercI.Int64())/10000) + types.BigDivFloat( + types.BigMul(pow.MinerPower.QualityAdjPower, big.NewInt(100)), + pow.TotalPower.QualityAdjPower, + ), + ) fmt.Printf("\tRaw: %s / %s (%0.4f%%)\n", color.BlueString(types.SizeStr(pow.MinerPower.RawBytePower)), types.SizeStr(pow.TotalPower.RawBytePower), - float64(rpercI.Int64())/10000) - + types.BigDivFloat( + types.BigMul(pow.MinerPower.RawBytePower, big.NewInt(100)), + pow.TotalPower.RawBytePower, + ), + ) secCounts, err := api.StateMinerSectorCount(ctx, maddr, types.EmptyTSK) if err != nil { return err @@ -146,7 +153,7 @@ func infoCmdAct(cctx *cli.Context) error { } else { var faultyPercentage float64 if secCounts.Live != 0 { - faultyPercentage = float64(10000*nfaults/secCounts.Live) / 100. + faultyPercentage = float64(100*nfaults) / float64(secCounts.Live) } fmt.Printf("\tProving: %s (%s Faulty, %.2f%%)\n", types.SizeStr(types.BigMul(types.NewInt(proving), types.NewInt(uint64(mi.SectorSize)))), @@ -157,16 +164,54 @@ func infoCmdAct(cctx *cli.Context) error { if !pow.HasMinPower { fmt.Print("Below minimum power threshold, no blocks will be won") } else { - expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 - if expWinChance > 0 { - if expWinChance > 1 { - expWinChance = 1 - } - winRate := time.Duration(float64(time.Second*time.Duration(build.BlockDelaySecs)) / expWinChance) - winPerDay := float64(time.Hour*24) / float64(winRate) - fmt.Print("Expected block win rate: ") - color.Blue("%.4f/day (every %s)", winPerDay, winRate.Truncate(time.Second)) + winRatio := new(corebig.Rat).SetFrac( + types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(build.BlocksPerEpoch)).Int, + pow.TotalPower.QualityAdjPower.Int, + ) + + if winRatioFloat, _ := winRatio.Float64(); winRatioFloat > 0 { + + // if the corresponding poisson distribution isn't infinitely small then + // throw it into the mix as well, accounting for multi-wins + winRationWithPoissonFloat := -math.Expm1(-winRatioFloat) + winRationWithPoisson := new(corebig.Rat).SetFloat64(winRationWithPoissonFloat) + if winRationWithPoisson != nil { + winRatio = winRationWithPoisson + winRatioFloat = winRationWithPoissonFloat + } + + weekly, _ := new(corebig.Rat).Mul( + winRatio, + new(corebig.Rat).SetInt64(7*builtin.EpochsInDay), + ).Float64() + + avgDuration, _ := new(corebig.Rat).Mul( + new(corebig.Rat).SetInt64(builtin.EpochDurationSeconds), + new(corebig.Rat).Inv(winRatio), + ).Float64() + + fmt.Print("Projected average block win rate: ") + color.Blue( + "%.02f/week (every %s)", + weekly, + (time.Second * time.Duration(avgDuration)).Truncate(time.Second).String(), + ) + + // Geometric distribution of P(Y < k) calculated as described in https://en.wikipedia.org/wiki/Geometric_distribution#Probability_Outcomes_Examples + // https://www.wolframalpha.com/input/?i=t+%3E+0%3B+p+%3E+0%3B+p+%3C+1%3B+c+%3E+0%3B+c+%3C1%3B+1-%281-p%29%5E%28t%29%3Dc%3B+solve+t + // t == how many dice-rolls (epochs) before win + // p == winRate == ( minerPower / netPower ) + // c == target probability of win ( 99.9% in this case ) + fmt.Print("Projected block win with ") + color.Green( + "99.9%% probability every %s", + (time.Second * time.Duration( + builtin.EpochDurationSeconds*math.Log(1-0.999)/ + math.Log(1-winRatioFloat), + )).Truncate(time.Second).String(), + ) + fmt.Println("(projections DO NOT account for future network and miner growth)") } } diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 66007b63d..0e36c6508 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -171,7 +171,7 @@ var provingInfoCmd = &cli.Command{ var faultPerc float64 if proving > 0 { - faultPerc = float64(faults*10000/proving) / 100 + faultPerc = float64(faults * 100 / proving) } fmt.Printf("Current Epoch: %d\n", cd.CurrentEpoch) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index 03d397852..0411f7f13 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -235,7 +235,7 @@ var retrievalSetAskCmd = &cli.Command{ var retrievalGetAskCmd = &cli.Command{ Name: "get-ask", - Usage: "Get the provider's current retrieval ask", + Usage: "Get the provider's current retrieval ask configured by the provider in the ask-store using the set-ask CLI command", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { ctx := lcli.DaemonContext(cctx) diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index 53d88c159..e9f9d8033 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -2279,6 +2279,7 @@ Response: "ef8d99a2-6865-4189-8ffa-9fef0f806eee": { "Info": { "Hostname": "host", + "IgnoreResources": false, "Resources": { "MemPhysical": 274877906944, "MemSwap": 128849018880, diff --git a/documentation/en/api-v0-methods-worker.md b/documentation/en/api-v0-methods-worker.md index 925f8934b..c620113f4 100644 --- a/documentation/en/api-v0-methods-worker.md +++ b/documentation/en/api-v0-methods-worker.md @@ -89,6 +89,7 @@ Response: ```json { "Hostname": "string value", + "IgnoreResources": true, "Resources": { "MemPhysical": 42, "MemSwap": 42, diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index d594b9473..d0aada985 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -765,7 +765,7 @@ COMMANDS: selection Configure acceptance criteria for retrieval deal proposals list List all active retrieval deals for this miner set-ask Configure the provider's retrieval ask - get-ask Get the provider's current retrieval ask + get-ask Get the provider's current retrieval ask configured by the provider in the ask-store using the set-ask CLI command help, h Shows a list of commands or help for one command OPTIONS: @@ -868,7 +868,7 @@ OPTIONS: ### lotus-miner retrieval-deals get-ask ``` NAME: - lotus-miner retrieval-deals get-ask - Get the provider's current retrieval ask + lotus-miner retrieval-deals get-ask - Get the provider's current retrieval ask configured by the provider in the ask-store using the set-ask CLI command USAGE: lotus-miner retrieval-deals get-ask [command options] [arguments...] diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index d2e3aa7d6..d60fc680a 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit d2e3aa7d61501d69bed6e898de13d1312b021e62 +Subproject commit d60fc680aa8abeafba698f738fed5b94c9bda33d diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index 7f0dc914f..a6034cc79 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -545,6 +545,7 @@ func TestSealAndVerifyAggregate(t *testing.T) { avi.Proof, err = ProofProver.AggregateSealProofs(avi, toAggregate) require.NoError(t, err) + require.Len(t, avi.Proof, 11188) aggDone := time.Now() diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 7a73578ad..87de597fd 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -85,6 +85,20 @@ type result struct { err error } +// ResourceFilteringStrategy is an enum indicating the kinds of resource +// filtering strategies that can be configured for workers. +type ResourceFilteringStrategy string + +const ( + // ResourceFilteringHardware specifies that available hardware resources + // should be evaluated when scheduling a task against the worker. + ResourceFilteringHardware = ResourceFilteringStrategy("hardware") + + // ResourceFilteringDisabled disables resource filtering against this + // worker. The scheduler may assign any task to this worker. + ResourceFilteringDisabled = ResourceFilteringStrategy("disabled") +) + type SealerConfig struct { ParallelFetchLimit int @@ -94,6 +108,11 @@ type SealerConfig struct { AllowPreCommit2 bool AllowCommit bool AllowUnseal bool + + // ResourceFiltering instructs the system which resource filtering strategy + // to use when evaluating tasks against this worker. An empty value defaults + // to "hardware". + ResourceFiltering ResourceFilteringStrategy } type StorageAuth http.Header @@ -148,9 +167,12 @@ func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls store localTasks = append(localTasks, sealtasks.TTUnseal) } - err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{ - TaskTypes: localTasks, - }, stor, lstor, si, m, wss)) + wcfg := WorkerConfig{ + IgnoreResourceFiltering: sc.ResourceFiltering == ResourceFilteringDisabled, + TaskTypes: localTasks, + } + worker := NewLocalWorker(wcfg, stor, lstor, si, m, wss) + err = m.AddWorker(ctx, worker) if err != nil { return nil, xerrors.Errorf("adding local worker: %w", err) } diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 68c2bf3a5..273f0928e 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -123,6 +123,10 @@ func (mgr *SectorMgr) AcquireSectorNumber() (abi.SectorNumber, error) { return id, nil } +func (mgr *SectorMgr) IsUnsealed(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + return false, nil +} + func (mgr *SectorMgr) ForceState(sid storage.SectorRef, st int) error { mgr.lk.Lock() ss, ok := mgr.sectors[sid.ID] diff --git a/extern/sector-storage/piece_provider.go b/extern/sector-storage/piece_provider.go index 553dcb952..ad3a2543e 100644 --- a/extern/sector-storage/piece_provider.go +++ b/extern/sector-storage/piece_provider.go @@ -24,8 +24,11 @@ type Unsealer interface { type PieceProvider interface { // ReadPiece is used to read an Unsealed piece at the given offset and of the given size from a Sector ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) + IsUnsealed(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) } +var _ PieceProvider = &pieceProvider{} + type pieceProvider struct { storage *stores.Remote index stores.SectorIndex @@ -40,6 +43,26 @@ func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unse } } +// IsUnsealed checks if we have the unsealed piece at the given offset in an already +// existing unsealed file either locally or on any of the workers. +func (p *pieceProvider) IsUnsealed(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + if err := offset.Valid(); err != nil { + return false, xerrors.Errorf("offset is not valid: %w", err) + } + if err := size.Validate(); err != nil { + return false, xerrors.Errorf("size is not a valid piece size: %w", err) + } + + ctxLock, cancel := context.WithCancel(ctx) + defer cancel() + + if err := p.index.StorageLock(ctxLock, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil { + return false, xerrors.Errorf("acquiring read sector lock: %w", err) + } + + return p.storage.CheckIsUnsealed(ctxLock, sector, abi.PaddedPieceSize(offset.Padded()), size.Padded()) +} + // tryReadUnsealedPiece will try to read the unsealed piece from an existing unsealed sector file for the given sector from any worker that has it. // It will NOT try to schedule an Unseal of a sealed sector file for the read. // diff --git a/extern/sector-storage/piece_provider_test.go b/extern/sector-storage/piece_provider_test.go index 81dceb730..e04853d75 100644 --- a/extern/sector-storage/piece_provider_test.go +++ b/extern/sector-storage/piece_provider_test.go @@ -53,6 +53,8 @@ func TestPieceProviderSimpleNoRemoteWorker(t *testing.T) { // pre-commit 1 preCommit1 := ppt.preCommit1(t) + // check if IsUnsealed -> true + require.True(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), size)) // read piece ppt.readPiece(t, storiface.UnpaddedByteIndex(0), size, false, pieceData) @@ -60,6 +62,8 @@ func TestPieceProviderSimpleNoRemoteWorker(t *testing.T) { // pre-commit 2 ppt.preCommit2(t, preCommit1) + // check if IsUnsealed -> true + require.True(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), size)) // read piece ppt.readPiece(t, storiface.UnpaddedByteIndex(0), size, false, pieceData) @@ -67,10 +71,14 @@ func TestPieceProviderSimpleNoRemoteWorker(t *testing.T) { // finalize -> nil here will remove unsealed file ppt.finalizeSector(t, nil) + // check if IsUnsealed -> false + require.False(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), size)) // Read the piece -> will have to unseal ppt.readPiece(t, storiface.UnpaddedByteIndex(0), size, true, pieceData) + // check if IsUnsealed -> true + require.True(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), size)) // read the piece -> will not have to unseal ppt.readPiece(t, storiface.UnpaddedByteIndex(0), size, false, pieceData) @@ -118,12 +126,18 @@ func TestReadPieceRemoteWorkers(t *testing.T) { // pre-commit 1 pC1 := ppt.preCommit1(t) + + // check if IsUnsealed -> true + require.True(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), pd1size)) // Read the piece -> no need to unseal ppt.readPiece(t, storiface.UnpaddedByteIndex(0), pd1size, false, pd1) // pre-commit 2 ppt.preCommit2(t, pC1) + + // check if IsUnsealed -> true + require.True(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), pd1size)) // Read the piece -> no need to unseal ppt.readPiece(t, storiface.UnpaddedByteIndex(0), pd1size, false, pd1) @@ -133,6 +147,8 @@ func TestReadPieceRemoteWorkers(t *testing.T) { // sending nil here will remove all unsealed files after sector is finalized. ppt.finalizeSector(t, nil) + // check if IsUnsealed -> false + require.False(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), pd1size)) // Read the piece -> have to unseal since we removed the file. ppt.readPiece(t, storiface.UnpaddedByteIndex(0), pd1size, true, pd1) @@ -142,14 +158,21 @@ func TestReadPieceRemoteWorkers(t *testing.T) { // remove the unsealed file and read again -> will have to unseal. ppt.removeAllUnsealedSectorFiles(t) + // check if IsUnsealed -> false + require.False(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(0), pd1size)) ppt.readPiece(t, storiface.UnpaddedByteIndex(0), pd1size, true, pd1) + // check if IsUnsealed -> true + require.True(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(pd1size), pd2size)) // Read Piece 2 -> no unsealing as it got unsealed above. ppt.readPiece(t, storiface.UnpaddedByteIndex(pd1size), pd2size, false, pd2) // remove all unseal files -> Read Piece 2 -> will have to Unseal. ppt.removeAllUnsealedSectorFiles(t) + + // check if IsUnsealed -> false + require.False(t, ppt.isUnsealed(t, storiface.UnpaddedByteIndex(pd1size), pd2size)) ppt.readPiece(t, storiface.UnpaddedByteIndex(pd1size), pd2size, true, pd2) } @@ -306,6 +329,12 @@ func (p *pieceProviderTestHarness) preCommit2(t *testing.T, pc1 specstorage.PreC p.commD = commD } +func (p *pieceProviderTestHarness) isUnsealed(t *testing.T, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) bool { + b, err := p.pp.IsUnsealed(p.ctx, p.sector, offset, size) + require.NoError(t, err) + return b +} + func (p *pieceProviderTestHarness) readPiece(t *testing.T, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, expectedHadToUnseal bool, expectedBytes []byte) { rd, isUnsealed, err := p.pp.ReadPiece(p.ctx, p.sector, offset, size, p.ticket, p.commD) diff --git a/extern/sector-storage/sched.go b/extern/sector-storage/sched.go index 61411081a..aabf6f0ce 100644 --- a/extern/sector-storage/sched.go +++ b/extern/sector-storage/sched.go @@ -349,24 +349,24 @@ func (sh *scheduler) trySched() { defer sh.workersLk.RUnlock() windowsLen := len(sh.openWindows) - queuneLen := sh.schedQueue.Len() + queueLen := sh.schedQueue.Len() - log.Debugf("SCHED %d queued; %d open windows", queuneLen, windowsLen) + log.Debugf("SCHED %d queued; %d open windows", queueLen, windowsLen) - if windowsLen == 0 || queuneLen == 0 { + if windowsLen == 0 || queueLen == 0 { // nothing to schedule on return } windows := make([]schedWindow, windowsLen) - acceptableWindows := make([][]int, queuneLen) + acceptableWindows := make([][]int, queueLen) // Step 1 throttle := make(chan struct{}, windowsLen) var wg sync.WaitGroup - wg.Add(queuneLen) - for i := 0; i < queuneLen; i++ { + wg.Add(queueLen) + for i := 0; i < queueLen; i++ { throttle <- struct{}{} go func(sqi int) { @@ -393,7 +393,7 @@ func (sh *scheduler) trySched() { } // TODO: allow bigger windows - if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info.Resources) { + if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info) { continue } @@ -451,27 +451,27 @@ func (sh *scheduler) trySched() { // Step 2 scheduled := 0 - rmQueue := make([]int, 0, queuneLen) + rmQueue := make([]int, 0, queueLen) - for sqi := 0; sqi < queuneLen; sqi++ { + for sqi := 0; sqi < queueLen; sqi++ { task := (*sh.schedQueue)[sqi] needRes := ResourceTable[task.taskType][task.sector.ProofType] selectedWindow := -1 for _, wnd := range acceptableWindows[task.indexHeap] { wid := sh.openWindows[wnd].worker - wr := sh.workers[wid].info.Resources + info := sh.workers[wid].info log.Debugf("SCHED try assign sqi:%d sector %d to window %d", sqi, task.sector.ID.Number, wnd) // TODO: allow bigger windows - if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", wr) { + if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", info) { continue } log.Debugf("SCHED ASSIGNED sqi:%d sector %d task %s to window %d", sqi, task.sector.ID.Number, task.taskType, wnd) - windows[wnd].allocated.add(wr, needRes) + windows[wnd].allocated.add(info.Resources, needRes) // TODO: We probably want to re-sort acceptableWindows here based on new // workerHandle.utilization + windows[wnd].allocated.utilization (workerHandle.utilization is used in all // task selectors, but not in the same way, so need to figure out how to do that in a non-O(n^2 way), and diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index 3e359c121..96a1fa863 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) -func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error { +func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerInfo, r Resources, locker sync.Locker, cb func() error) error { for !a.canHandleRequest(r, id, "withResources", wr) { if a.cond == nil { a.cond = sync.NewCond(locker) @@ -14,11 +14,11 @@ func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResource a.cond.Wait() } - a.add(wr, r) + a.add(wr.Resources, r) err := cb() - a.free(wr, r) + a.free(wr.Resources, r) if a.cond != nil { a.cond.Broadcast() } @@ -44,8 +44,15 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { a.memUsedMax -= r.MaxMemory } -func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, res storiface.WorkerResources) bool { +// canHandleRequest evaluates if the worker has enough available resources to +// handle the request. +func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, info storiface.WorkerInfo) bool { + if info.IgnoreResources { + // shortcircuit; if this worker is ignoring resources, it can always handle the request. + return true + } + res := info.Resources // TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running) minNeedMem := res.MemReserved + a.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory if minNeedMem > res.MemPhysical { diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index 63f3de64d..fbc4d83ee 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -38,6 +38,20 @@ func TestWithPriority(t *testing.T) { require.Equal(t, 2222, getPriority(ctx)) } +var decentWorkerResources = storiface.WorkerResources{ + MemPhysical: 128 << 30, + MemSwap: 200 << 30, + MemReserved: 2 << 30, + CPUs: 32, + GPUs: []string{"a GPU"}, +} + +var constrainedWorkerResources = storiface.WorkerResources{ + MemPhysical: 1 << 30, + MemReserved: 2 << 30, + CPUs: 1, +} + type schedTestWorker struct { name string taskTypes map[sealtasks.TaskType]struct{} @@ -45,6 +59,9 @@ type schedTestWorker struct { closed bool session uuid.UUID + + resources storiface.WorkerResources + ignoreResources bool } func (s *schedTestWorker) SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) { @@ -107,18 +124,11 @@ func (s *schedTestWorker) Paths(ctx context.Context) ([]stores.StoragePath, erro return s.paths, nil } -var decentWorkerResources = storiface.WorkerResources{ - MemPhysical: 128 << 30, - MemSwap: 200 << 30, - MemReserved: 2 << 30, - CPUs: 32, - GPUs: []string{"a GPU"}, -} - func (s *schedTestWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) { return storiface.WorkerInfo{ - Hostname: s.name, - Resources: decentWorkerResources, + Hostname: s.name, + IgnoreResources: s.ignoreResources, + Resources: s.resources, }, nil } @@ -137,13 +147,16 @@ func (s *schedTestWorker) Close() error { var _ Worker = &schedTestWorker{} -func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}) { +func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}, resources storiface.WorkerResources, ignoreResources bool) { w := &schedTestWorker{ name: name, taskTypes: taskTypes, paths: []stores.StoragePath{{ID: "bb-8", Weight: 2, LocalPath: "food", CanSeal: true, CanStore: true}}, session: uuid.New(), + + resources: resources, + ignoreResources: ignoreResources, } for _, path := range w.paths { @@ -169,7 +182,7 @@ func TestSchedStartStop(t *testing.T) { sched := newScheduler() go sched.runSched() - addTestWorker(t, sched, stores.NewIndex(), "fred", nil) + addTestWorker(t, sched, stores.NewIndex(), "fred", nil, decentWorkerResources, false) require.NoError(t, sched.Close(context.TODO())) } @@ -183,6 +196,9 @@ func TestSched(t *testing.T) { type workerSpec struct { name string taskTypes map[sealtasks.TaskType]struct{} + + resources storiface.WorkerResources + ignoreResources bool } noopAction := func(ctx context.Context, w Worker) error { @@ -295,7 +311,7 @@ func TestSched(t *testing.T) { go sched.runSched() for _, worker := range workers { - addTestWorker(t, sched, index, worker.name, worker.taskTypes) + addTestWorker(t, sched, index, worker.name, worker.taskTypes, worker.resources, worker.ignoreResources) } rm := runMeta{ @@ -322,31 +338,42 @@ func TestSched(t *testing.T) { } } + // checks behaviour with workers with constrained resources + // the first one is not ignoring resource constraints, so we assign to the second worker, who is + t.Run("constrained-resources", testFunc([]workerSpec{ + {name: "fred1", resources: constrainedWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + {name: "fred2", resources: constrainedWorkerResources, ignoreResources: true, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + }, []task{ + sched("pc1-1", "fred2", 8, sealtasks.TTPreCommit1), + taskStarted("pc1-1"), + taskDone("pc1-1"), + })) + t.Run("one-pc1", testFunc([]workerSpec{ - {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + {name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, }, []task{ sched("pc1-1", "fred", 8, sealtasks.TTPreCommit1), taskDone("pc1-1"), })) t.Run("pc1-2workers-1", testFunc([]workerSpec{ - {name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}}, - {name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + {name: "fred2", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}}, + {name: "fred1", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, }, []task{ sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1), taskDone("pc1-1"), })) t.Run("pc1-2workers-2", testFunc([]workerSpec{ - {name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, - {name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}}, + {name: "fred1", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + {name: "fred2", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}}, }, []task{ sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1), taskDone("pc1-1"), })) t.Run("pc1-block-pc2", testFunc([]workerSpec{ - {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, + {name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, }, []task{ sched("pc1", "fred", 8, sealtasks.TTPreCommit1), taskStarted("pc1"), @@ -359,7 +386,7 @@ func TestSched(t *testing.T) { })) t.Run("pc2-block-pc1", testFunc([]workerSpec{ - {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, + {name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, }, []task{ sched("pc2", "fred", 8, sealtasks.TTPreCommit2), taskStarted("pc2"), @@ -372,7 +399,7 @@ func TestSched(t *testing.T) { })) t.Run("pc1-batching", testFunc([]workerSpec{ - {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + {name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, }, []task{ sched("t1", "fred", 8, sealtasks.TTPreCommit1), taskStarted("t1"), @@ -459,7 +486,7 @@ func TestSched(t *testing.T) { // run this one a bunch of times, it had a very annoying tendency to fail randomly for i := 0; i < 40; i++ { t.Run("pc1-pc2-prio", testFunc([]workerSpec{ - {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, + {name: "fred", resources: decentWorkerResources, taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, }, []task{ // fill queues twoPC1("w0", 0, taskStarted), diff --git a/extern/sector-storage/sched_worker.go b/extern/sector-storage/sched_worker.go index 4e18e5c6f..7bc1affc3 100644 --- a/extern/sector-storage/sched_worker.go +++ b/extern/sector-storage/sched_worker.go @@ -296,7 +296,7 @@ func (sw *schedWorker) workerCompactWindows() { for ti, todo := range window.todo { needRes := ResourceTable[todo.taskType][todo.sector.ProofType] - if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info.Resources) { + if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info) { continue } @@ -352,7 +352,7 @@ assignLoop: worker.lk.Lock() for t, todo := range firstWindow.todo { needRes := ResourceTable[todo.taskType][todo.sector.ProofType] - if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info.Resources) { + if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) { tidx = t break } @@ -424,7 +424,7 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe } // wait (if needed) for resources in the 'active' window - err = w.active.withResources(sw.wid, w.info.Resources, needRes, &sh.workersLk, func() error { + err = w.active.withResources(sw.wid, w.info, needRes, &sh.workersLk, func() error { w.lk.Lock() w.preparing.free(w.info.Resources, needRes) w.lk.Unlock() diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index e5ecbbc9b..c1b51a4a4 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -484,6 +484,87 @@ func (r *Remote) readRemote(ctx context.Context, url string, offset, size abi.Pa return resp.Body, nil } +// CheckIsUnsealed checks if we have an unsealed piece at the given offset in an already unsealed sector file for the given piece +// either locally or on any of the workers. +// Returns true if we have the unsealed piece, false otherwise. +func (r *Remote) CheckIsUnsealed(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize) (bool, error) { + ft := storiface.FTUnsealed + + paths, _, err := r.local.AcquireSector(ctx, s, ft, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) + if err != nil { + return false, xerrors.Errorf("acquire local: %w", err) + } + + path := storiface.PathByType(paths, ft) + if path != "" { + // if we have the unsealed file locally, check if it has the unsealed piece. + log.Infof("Read local %s (+%d,%d)", path, offset, size) + ssize, err := s.ProofType.SectorSize() + if err != nil { + return false, err + } + + // open the unsealed sector file for the given sector size located at the given path. + pf, err := r.pfHandler.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + if err != nil { + return false, xerrors.Errorf("opening partial file: %w", err) + } + log.Debugf("local partial file opened %s (+%d,%d)", path, offset, size) + + // even though we have an unsealed file for the given sector, we still need to determine if we have the unsealed piece + // in the unsealed sector file. That is what `HasAllocated` checks for. + has, err := r.pfHandler.HasAllocated(pf, storiface.UnpaddedByteIndex(offset.Unpadded()), size.Unpadded()) + if err != nil { + return false, xerrors.Errorf("has allocated: %w", err) + } + + // close the local unsealed file. + if err := r.pfHandler.Close(pf); err != nil { + return false, xerrors.Errorf("failed to close partial file: %s", err) + } + log.Debugf("checked if local partial file has the piece %s (+%d,%d), returning answer=%t", path, offset, size, has) + + // Sector files can technically not have a piece unsealed locally, but have it unsealed in remote storage, so we probably + // want to return only if has is true + if has { + return has, nil + } + } + + // --- We don't have the unsealed piece in an unsealed sector file locally + // Check if we have it in a remote cluster. + + si, err := r.index.StorageFindSector(ctx, s.ID, ft, 0, false) + if err != nil { + return false, xerrors.Errorf("StorageFindSector: %s", err) + } + + if len(si) == 0 { + return false, nil + } + + sort.Slice(si, func(i, j int) bool { + return si[i].Weight < si[j].Weight + }) + + for _, info := range si { + for _, url := range info.URLs { + ok, err := r.checkAllocated(ctx, url, s.ProofType, offset, size) + if err != nil { + log.Warnw("check if remote has piece", "url", url, "error", err) + continue + } + if !ok { + continue + } + + return true, nil + } + } + + return false, nil +} + // Reader returns a reader for an unsealed piece at the given offset in the given sector. // If the Miner has the unsealed piece locally, it will return a reader that reads from the local copy. // If the Miner does NOT have the unsealed piece locally, it will query all workers that have the unsealed sector file @@ -507,7 +588,7 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a if path != "" { // if we have the unsealed file locally, return a reader that can be used to read the contents of the // unsealed piece. - log.Infof("Read local %s (+%d,%d)", path, offset, size) + log.Debugf("Check local %s (+%d,%d)", path, offset, size) ssize, err := s.ProofType.SectorSize() if err != nil { return nil, err @@ -529,19 +610,18 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a } log.Debugf("check if partial file is allocated %s (+%d,%d)", path, offset, size) - if !has { - log.Debugf("miner has unsealed file but not unseal piece, %s (+%d,%d)", path, offset, size) - if err := r.pfHandler.Close(pf); err != nil { - return nil, xerrors.Errorf("close partial file: %w", err) - } - return nil, nil + if has { + log.Infof("returning piece reader for local unsealed piece sector=%+v, (offset=%d, size=%d)", s.ID, offset, size) + return r.pfHandler.Reader(pf, storiface.PaddedByteIndex(offset), size) } - log.Infof("returning piece reader for local unsealed piece sector=%+v, (offset=%d, size=%d)", s.ID, offset, size) - return r.pfHandler.Reader(pf, storiface.PaddedByteIndex(offset), size) + log.Debugf("miner has unsealed file but not unseal piece, %s (+%d,%d)", path, offset, size) + if err := r.pfHandler.Close(pf); err != nil { + return nil, xerrors.Errorf("close partial file: %w", err) + } } - // --- We don't have the unsealed sector file locally + // --- We don't have the unsealed piece in an unsealed sector file locally // if we don't have the unsealed sector file locally, we'll first lookup the Miner Sector Store Index // to determine which workers have the unsealed file and then query those workers to know diff --git a/extern/sector-storage/stores/remote_test.go b/extern/sector-storage/stores/remote_test.go index eb06a713d..b708bb68f 100644 --- a/extern/sector-storage/stores/remote_test.go +++ b/extern/sector-storage/stores/remote_test.go @@ -27,8 +27,10 @@ func TestReader(t *testing.T) { bz := []byte("Hello World") pfPath := "path" - ft := storiface.FTUnsealed emptyPartialFile := &partialfile.PartialFile{} + sectorSize := abi.SealProofInfos[1].SectorSize + + ft := storiface.FTUnsealed sectorRef := storage.SectorRef{ ID: abi.SectorID{ @@ -37,7 +39,6 @@ func TestReader(t *testing.T) { }, ProofType: 1, } - sectorSize := abi.SealProofInfos[1].SectorSize offset := abi.PaddedPieceSize(100) size := abi.PaddedPieceSize(1000) @@ -151,7 +152,7 @@ func TestReader(t *testing.T) { }, // --- nil reader when local unsealed file does NOT have unsealed piece - "nil reader when local unsealed file does not have the piece": { + "nil reader when local unsealed file does not have the unsealed piece and remote sector also dosen't have the unsealed piece": { storeFnc: func(l *mocks.MockStore) { mockSectorAcquire(l, sectorRef, pfPath, nil) }, @@ -162,7 +163,20 @@ func TestReader(t *testing.T) { false, nil) pf.EXPECT().Close(emptyPartialFile).Return(nil).Times(1) + }, + + indexFnc: func(in *mocks.MockSectorIndex, url string) { + si := stores.SectorStorageInfo{ + URLs: []string{url}, + } + + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return([]stores.SectorStorageInfo{si}, nil).Times(1) + }, + + needHttpServer: true, + getAllocatedReturnCode: 500, }, // ---- nil reader when none of the remote unsealed file has unsealed piece @@ -231,6 +245,37 @@ func TestReader(t *testing.T) { }, // --- Success for remote unsealed file + // --- Success for remote unsealed file + "successfully fetches reader from remote unsealed piece when local unsealed file does NOT have the unsealed Piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, nil) + mockCheckAllocation(pf, offset, size, emptyPartialFile, + false, nil) + + pf.EXPECT().Close(emptyPartialFile).Return(nil).Times(1) + + }, + + indexFnc: func(in *mocks.MockSectorIndex, url string) { + si := stores.SectorStorageInfo{ + URLs: []string{url}, + } + + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return([]stores.SectorStorageInfo{si}, nil).Times(1) + }, + + needHttpServer: true, + getSectorReturnCode: 200, + getAllocatedReturnCode: 200, + expectedSectorBytes: bz, + expectedNonNilReader: true, + }, + "successfully fetches reader for piece from remote unsealed piece": { storeFnc: func(l *mocks.MockStore) { mockSectorAcquire(l, sectorRef, "", nil) @@ -326,6 +371,283 @@ func TestReader(t *testing.T) { } } +func TestCheckIsUnsealed(t *testing.T) { + logging.SetAllLoggers(logging.LevelDebug) + + pfPath := "path" + ft := storiface.FTUnsealed + emptyPartialFile := &partialfile.PartialFile{} + + sectorRef := storage.SectorRef{ + ID: abi.SectorID{ + Miner: 123, + Number: 123, + }, + ProofType: 1, + } + sectorSize := abi.SealProofInfos[1].SectorSize + + offset := abi.PaddedPieceSize(100) + size := abi.PaddedPieceSize(1000) + ctx := context.Background() + + tcs := map[string]struct { + storeFnc func(s *mocks.MockStore) + pfFunc func(s *mocks.MockpartialFileHandler) + indexFnc func(s *mocks.MockSectorIndex, serverURL string) + + needHttpServer bool + + getAllocatedReturnCode int + + serverUrl string + + // expectation + errStr string + expectedIsUnealed bool + }{ + + // -------- have the unsealed file locally + "fails when error while acquiring unsealed file": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, xerrors.New("acquire error")) + }, + + errStr: "acquire error", + }, + + "fails when error while opening local partial (unsealed) file": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, xerrors.New("pf open error")) + }, + errStr: "pf open error", + }, + + "fails when error while checking if local unsealed file has piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, nil) + mockCheckAllocation(pf, offset, size, emptyPartialFile, + true, xerrors.New("piece check error")) + }, + + errStr: "piece check error", + }, + + "fails when error while closing local unsealed file": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, nil) + + mockCheckAllocation(pf, offset, size, emptyPartialFile, + false, nil) + + pf.EXPECT().Close(emptyPartialFile).Return(xerrors.New("close error")).Times(1) + }, + errStr: "close error", + }, + + // ------------------- don't have the unsealed file locally + + "fails when error while finding sector": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, "", nil) + }, + + indexFnc: func(in *mocks.MockSectorIndex, _ string) { + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return(nil, xerrors.New("find sector error")) + }, + errStr: "find sector error", + }, + + "false when no worker has unsealed file": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, "", nil) + }, + + indexFnc: func(in *mocks.MockSectorIndex, _ string) { + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return(nil, nil) + }, + }, + + // false when local unsealed file does NOT have unsealed piece + "false when local unsealed file does not have the piece and remote sector too dosen't have the piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, nil) + mockCheckAllocation(pf, offset, size, emptyPartialFile, + false, nil) + + pf.EXPECT().Close(emptyPartialFile).Return(nil).Times(1) + }, + + indexFnc: func(in *mocks.MockSectorIndex, url string) { + si := stores.SectorStorageInfo{ + URLs: []string{url}, + } + + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return([]stores.SectorStorageInfo{si}, nil).Times(1) + }, + + needHttpServer: true, + getAllocatedReturnCode: 500, + }, + + "false when none of the worker has the unsealed piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, "", nil) + }, + + indexFnc: func(in *mocks.MockSectorIndex, url string) { + si := stores.SectorStorageInfo{ + URLs: []string{url}, + } + + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return([]stores.SectorStorageInfo{si}, nil).Times(1) + }, + + needHttpServer: true, + getAllocatedReturnCode: 500, + }, + + // ---- Success for local unsealed file + "true when local unsealed file has the piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, nil) + mockCheckAllocation(pf, offset, size, emptyPartialFile, + true, nil) + pf.EXPECT().Close(emptyPartialFile).Return(nil).Times(1) + + }, + + expectedIsUnealed: true, + }, + + // --- Success for remote unsealed file + "true if we have a remote unsealed piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, "", nil) + }, + + indexFnc: func(in *mocks.MockSectorIndex, url string) { + si := stores.SectorStorageInfo{ + URLs: []string{url}, + } + + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return([]stores.SectorStorageInfo{si}, nil).Times(1) + }, + + needHttpServer: true, + getAllocatedReturnCode: 200, + expectedIsUnealed: true, + }, + + "true when local unsealed file does NOT have the unsealed Piece but remote sector has the unsealed piece": { + storeFnc: func(l *mocks.MockStore) { + mockSectorAcquire(l, sectorRef, pfPath, nil) + }, + + pfFunc: func(pf *mocks.MockpartialFileHandler) { + mockPartialFileOpen(pf, sectorSize, pfPath, nil) + mockCheckAllocation(pf, offset, size, emptyPartialFile, + false, nil) + + pf.EXPECT().Close(emptyPartialFile).Return(nil).Times(1) + }, + + indexFnc: func(in *mocks.MockSectorIndex, url string) { + si := stores.SectorStorageInfo{ + URLs: []string{url}, + } + + in.EXPECT().StorageFindSector(gomock.Any(), sectorRef.ID, storiface.FTUnsealed, gomock.Any(), + false).Return([]stores.SectorStorageInfo{si}, nil).Times(1) + }, + + needHttpServer: true, + getAllocatedReturnCode: 200, + expectedIsUnealed: true, + }, + } + + for name, tc := range tcs { + tc := tc + t.Run(name, func(t *testing.T) { + // create go mock controller here + mockCtrl := gomock.NewController(t) + // when test is done, assert expectations on all mock objects. + defer mockCtrl.Finish() + + // create them mocks + lstore := mocks.NewMockStore(mockCtrl) + pfhandler := mocks.NewMockpartialFileHandler(mockCtrl) + index := mocks.NewMockSectorIndex(mockCtrl) + + if tc.storeFnc != nil { + tc.storeFnc(lstore) + } + if tc.pfFunc != nil { + tc.pfFunc(pfhandler) + } + + if tc.needHttpServer { + // run http server + ts := httptest.NewServer(&mockHttpServer{ + expectedSectorName: storiface.SectorName(sectorRef.ID), + expectedFileType: ft.String(), + expectedOffset: fmt.Sprintf("%d", offset.Unpadded()), + expectedSize: fmt.Sprintf("%d", size.Unpadded()), + expectedSectorType: fmt.Sprintf("%d", sectorRef.ProofType), + + getAllocatedReturnCode: tc.getAllocatedReturnCode, + }) + defer ts.Close() + tc.serverUrl = fmt.Sprintf("%s/remote/%s/%s", ts.URL, ft.String(), storiface.SectorName(sectorRef.ID)) + } + if tc.indexFnc != nil { + tc.indexFnc(index, tc.serverUrl) + } + + remoteStore := stores.NewRemote(lstore, index, nil, 6000, pfhandler) + + isUnsealed, err := remoteStore.CheckIsUnsealed(ctx, sectorRef, offset, size) + + if tc.errStr != "" { + require.Error(t, err) + require.False(t, isUnsealed) + require.Contains(t, err.Error(), tc.errStr) + } else { + require.NoError(t, err) + } + + require.Equal(t, tc.expectedIsUnealed, isUnsealed) + + }) + } +} + func mockSectorAcquire(l *mocks.MockStore, sectorRef storage.SectorRef, pfPath string, err error) { l.EXPECT().AcquireSector(gomock.Any(), sectorRef, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove).Return(storiface.SectorPaths{ @@ -358,8 +680,9 @@ type mockHttpServer struct { expectedSectorType string getAllocatedReturnCode int - getSectorReturnCode int - getSectorBytes []byte + + getSectorReturnCode int + getSectorBytes []byte } func (m *mockHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index d3f4a2cd1..d1373f4c5 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -18,7 +18,12 @@ import ( type WorkerInfo struct { Hostname string - Resources WorkerResources + // IgnoreResources indicates whether the worker's available resources should + // be used ignored (true) or used (false) for the purposes of scheduling and + // task assignment. Only supported on local workers. Used for testing. + // Default should be false (zero value, i.e. resources taken into account). + IgnoreResources bool + Resources WorkerResources } type WorkerResources struct { diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index 2bb0f8300..3e63f8659 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -20,7 +20,7 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-statestore" - storage "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" @@ -33,6 +33,11 @@ var pathTypes = []storiface.SectorFileType{storiface.FTUnsealed, storiface.FTSea type WorkerConfig struct { TaskTypes []sealtasks.TaskType NoSwap bool + + // IgnoreResourceFiltering enables task distribution to happen on this + // worker regardless of its currently available resources. Used in testing + // with the local worker. + IgnoreResourceFiltering bool } // used do provide custom proofs impl (mostly used in testing) @@ -46,6 +51,9 @@ type LocalWorker struct { executor ExecutorFunc noSwap bool + // see equivalent field on WorkerConfig. + ignoreResources bool + ct *workerCallTracker acceptTasks map[sealtasks.TaskType]struct{} running sync.WaitGroup @@ -71,12 +79,12 @@ func newLocalWorker(executor ExecutorFunc, wcfg WorkerConfig, store stores.Store ct: &workerCallTracker{ st: cst, }, - acceptTasks: acceptTasks, - executor: executor, - noSwap: wcfg.NoSwap, - - session: uuid.New(), - closing: make(chan struct{}), + acceptTasks: acceptTasks, + executor: executor, + noSwap: wcfg.NoSwap, + ignoreResources: wcfg.IgnoreResourceFiltering, + session: uuid.New(), + closing: make(chan struct{}), } if w.executor == nil { @@ -501,7 +509,8 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { } return storiface.WorkerInfo{ - Hostname: hostname, + Hostname: hostname, + IgnoreResources: l.ignoreResources, Resources: storiface.WorkerResources{ MemPhysical: mem.Total, MemSwap: memSwap, diff --git a/extern/storage-sealing/fsm.go b/extern/storage-sealing/fsm.go index 359c49eb3..d3e1e9d52 100644 --- a/extern/storage-sealing/fsm.go +++ b/extern/storage-sealing/fsm.go @@ -156,7 +156,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), ), CommitFinalizeFailed: planOne( - on(SectorRetryFinalize{}, CommitFinalizeFailed), + on(SectorRetryFinalize{}, CommitFinalize), ), CommitFailed: planOne( on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), diff --git a/extern/storage-sealing/fsm_test.go b/extern/storage-sealing/fsm_test.go index f269bb96c..1d2df2784 100644 --- a/extern/storage-sealing/fsm_test.go +++ b/extern/storage-sealing/fsm_test.go @@ -154,6 +154,45 @@ func TestHappyPathFinalizeEarly(t *testing.T) { } } +func TestCommitFinalizeFailed(t *testing.T) { + var notif []struct{ before, after SectorInfo } + ma, _ := address.NewIDAddress(55151) + m := test{ + s: &Sealing{ + maddr: ma, + stats: SectorStats{ + bySector: map[abi.SectorID]statSectorState{}, + }, + notifee: func(before, after SectorInfo) { + notif = append(notif, struct{ before, after SectorInfo }{before, after}) + }, + }, + t: t, + state: &SectorInfo{State: Committing}, + } + + m.planSingle(SectorProofReady{}) + require.Equal(m.t, m.state.State, CommitFinalize) + + m.planSingle(SectorFinalizeFailed{}) + require.Equal(m.t, m.state.State, CommitFinalizeFailed) + + m.planSingle(SectorRetryFinalize{}) + require.Equal(m.t, m.state.State, CommitFinalize) + + m.planSingle(SectorFinalized{}) + require.Equal(m.t, m.state.State, SubmitCommit) + + expected := []SectorState{Committing, CommitFinalize, CommitFinalizeFailed, CommitFinalize, SubmitCommit} + for i, n := range notif { + if n.before.State != expected[i] { + t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State) + } + if n.after.State != expected[i+1] { + t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State) + } + } +} func TestSeedRevert(t *testing.T) { ma, _ := address.NewIDAddress(55151) m := test{ diff --git a/extern/storage-sealing/input.go b/extern/storage-sealing/input.go index c02d42814..1a0b7bf1e 100644 --- a/extern/storage-sealing/input.go +++ b/extern/storage-sealing/input.go @@ -396,6 +396,7 @@ func (m *Sealing) updateInput(ctx context.Context, sp abi.RegisteredSealProof) e func (m *Sealing) tryCreateDealSector(ctx context.Context, sp abi.RegisteredSealProof) error { m.startupWait.Wait() + if m.creating != nil { return nil // new sector is being created right now } @@ -449,8 +450,8 @@ func (m *Sealing) createSector(ctx context.Context, cfg sealiface.Config, sp abi func (m *Sealing) StartPacking(sid abi.SectorNumber) error { m.startupWait.Wait() - log.Infow("starting to seal deal sector", "sector", sid, "trigger", "user") + log.Infow("starting to seal deal sector", "sector", sid, "trigger", "user") return m.sectors.Send(uint64(sid), SectorStartPacking{}) } diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 3eb1970d5..0f2d7a3c2 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -132,7 +132,7 @@ type pendingPiece struct { accepted func(abi.SectorNumber, abi.UnpaddedPieceSize, error) } -func New(api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing { +func New(mctx context.Context, api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing { s := &Sealing{ api: api, feeCfg: fc, @@ -153,9 +153,9 @@ func New(api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address. notifee: notifee, addrSel: as, - terminator: NewTerminationBatcher(context.TODO(), maddr, api, as, fc, gc), - precommiter: NewPreCommitBatcher(context.TODO(), maddr, api, as, fc, gc), - commiter: NewCommitBatcher(context.TODO(), maddr, api, as, fc, gc, prov), + terminator: NewTerminationBatcher(mctx, maddr, api, as, fc, gc), + precommiter: NewPreCommitBatcher(mctx, maddr, api, as, fc, gc), + commiter: NewCommitBatcher(mctx, maddr, api, as, fc, gc, prov), getConfig: gc, dealInfo: &CurrentDealInfoManager{api}, diff --git a/go.mod b/go.mod index 137678d81..411522a36 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.6.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a - github.com/filecoin-project/go-fil-markets v1.4.0 + github.com/filecoin-project/go-fil-markets v1.5.0 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index c883dd9bb..5573587fd 100644 --- a/go.sum +++ b/go.sum @@ -283,8 +283,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.4.0 h1:J4L6o+FVOmS7ZWV6wxLPiuoDzGC7iS3S5NRFL1enEr0= -github.com/filecoin-project/go-fil-markets v1.4.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk= +github.com/filecoin-project/go-fil-markets v1.5.0 h1:3KEs01L8XFCEgujZ6ggFjr1XWjpjTQcmSSeo3I99I0k= +github.com/filecoin-project/go-fil-markets v1.5.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= diff --git a/itests/api_test.go b/itests/api_test.go index f8567bd2a..5487a2c38 100644 --- a/itests/api_test.go +++ b/itests/api_test.go @@ -7,12 +7,13 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" ) @@ -21,7 +22,7 @@ func TestAPI(t *testing.T) { runAPITest(t) }) t.Run("rpc", func(t *testing.T) { - runAPITest(t, kit2.ThroughRPC()) + runAPITest(t, kit.ThroughRPC()) }) } @@ -48,7 +49,7 @@ func (ts *apiSuite) testVersion(t *testing.T) { lapi.RunningNodeType = lapi.NodeUnknown }) - full, _, _ := kit2.EnsembleMinimal(t, ts.opts...) + full, _, _ := kit.EnsembleMinimal(t, ts.opts...) v, err := full.Version(context.Background()) require.NoError(t, err) @@ -61,7 +62,7 @@ func (ts *apiSuite) testVersion(t *testing.T) { func (ts *apiSuite) testID(t *testing.T) { ctx := context.Background() - full, _, _ := kit2.EnsembleMinimal(t, ts.opts...) + full, _, _ := kit.EnsembleMinimal(t, ts.opts...) id, err := full.ID(ctx) if err != nil { @@ -73,7 +74,7 @@ func (ts *apiSuite) testID(t *testing.T) { func (ts *apiSuite) testConnectTwo(t *testing.T) { ctx := context.Background() - one, two, _, ens := kit2.EnsembleTwoOne(t, ts.opts...) + one, two, _, ens := kit.EnsembleTwoOne(t, ts.opts...) p, err := one.NetPeers(ctx) require.NoError(t, err) @@ -97,7 +98,7 @@ func (ts *apiSuite) testConnectTwo(t *testing.T) { func (ts *apiSuite) testSearchMsg(t *testing.T) { ctx := context.Background() - full, _, ens := kit2.EnsembleMinimal(t, ts.opts...) + full, _, ens := kit.EnsembleMinimal(t, ts.opts...) senderAddr, err := full.WalletDefaultAddress(ctx) require.NoError(t, err) @@ -127,7 +128,7 @@ func (ts *apiSuite) testSearchMsg(t *testing.T) { func (ts *apiSuite) testMining(t *testing.T) { ctx := context.Background() - full, miner, _ := kit2.EnsembleMinimal(t, ts.opts...) + full, miner, _ := kit.EnsembleMinimal(t, ts.opts...) newHeads, err := full.ChainNotify(ctx) require.NoError(t, err) @@ -138,7 +139,7 @@ func (ts *apiSuite) testMining(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(h1.Height()), int64(baseHeight)) - bm := kit2.NewBlockMiner(t, miner) + bm := kit.NewBlockMiner(t, miner) bm.MineUntilBlock(ctx, full, nil) require.NoError(t, err) @@ -170,9 +171,10 @@ func (ts *apiSuite) testMiningReal(t *testing.T) { func (ts *apiSuite) testNonGenesisMiner(t *testing.T) { ctx := context.Background() - full, genesisMiner, ens := kit2.EnsembleMinimal(t, ts.opts...) + full, genesisMiner, ens := kit.EnsembleMinimal(t, append(ts.opts, kit.MockProofs())...) + ens.InterconnectAll().BeginMining(4 * time.Millisecond) - ens.BeginMining(4 * time.Millisecond) + time.Sleep(1 * time.Second) gaa, err := genesisMiner.ActorAddress(ctx) require.NoError(t, err) @@ -180,8 +182,11 @@ func (ts *apiSuite) testNonGenesisMiner(t *testing.T) { _, err = full.StateMinerInfo(ctx, gaa, types.EmptyTSK) require.NoError(t, err) - var newMiner kit2.TestMiner - ens.Miner(&newMiner, full, kit2.OwnerAddr(full.DefaultKey)).Start() + var newMiner kit.TestMiner + ens.Miner(&newMiner, full, + kit.OwnerAddr(full.DefaultKey), + kit.ProofType(abi.RegisteredSealProof_StackedDrg2KiBV1), // we're using v0 actors with old proofs. + ).Start().InterconnectAll() ta, err := newMiner.ActorAddress(ctx) require.NoError(t, err) diff --git a/itests/batch_deal_test.go b/itests/batch_deal_test.go index 6dc04c65d..300a44fa2 100644 --- a/itests/batch_deal_test.go +++ b/itests/batch_deal_test.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -18,7 +18,7 @@ import ( ) func TestBatchDealInput(t *testing.T) { - kit2.QuietMiningLogs() + kit.QuietMiningLogs() var ( blockTime = 10 * time.Millisecond @@ -37,7 +37,7 @@ func TestBatchDealInput(t *testing.T) { maxDealsPerMsg := uint64(deals) // Set max deals per publish deals message to maxDealsPerMsg - opts := kit2.ConstructorOpts(node.Options( + opts := kit.ConstructorOpts(node.Options( node.Override( new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{ @@ -56,9 +56,9 @@ func TestBatchDealInput(t *testing.T) { }, nil }), )) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), opts) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) ens.InterconnectAll().BeginMining(blockTime) - dh := kit2.NewDealHarness(t, client, miner, miner) + dh := kit.NewDealHarness(t, client, miner) err := miner.MarketSetAsk(ctx, big.Zero(), big.Zero(), 200, 128, 32<<30) require.NoError(t, err) @@ -87,7 +87,7 @@ func TestBatchDealInput(t *testing.T) { // Starts a deal and waits until it's published runDealTillSeal := func(rseed int) { - res, _, _, err := kit2.CreateImportFile(ctx, client, rseed, piece) + res, _, _, err := kit.CreateImportFile(ctx, client, rseed, piece) require.NoError(t, err) deal := dh.StartDeal(ctx, res.Root, false, dealStartEpoch) diff --git a/itests/ccupgrade_test.go b/itests/ccupgrade_test.go index 14b44291b..71bb4c065 100644 --- a/itests/ccupgrade_test.go +++ b/itests/ccupgrade_test.go @@ -6,16 +6,15 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/itests/kit2" - "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/itests/kit" + + "github.com/stretchr/testify/require" ) func TestCCUpgrade(t *testing.T) { - kit2.QuietMiningLogs() + kit.QuietMiningLogs() for _, height := range []abi.ChainEpoch{ -1, // before @@ -34,8 +33,8 @@ func runTestCCUpgrade(t *testing.T, upgradeHeight abi.ChainEpoch) { ctx := context.Background() blockTime := 5 * time.Millisecond - opts := kit2.ConstructorOpts(kit2.LatestActorsAt(upgradeHeight)) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), opts) + opts := kit.ConstructorOpts(kit.LatestActorsAt(upgradeHeight)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) ens.InterconnectAll().BeginMining(blockTime) maddr, err := miner.ActorAddress(ctx) @@ -43,7 +42,7 @@ func runTestCCUpgrade(t *testing.T, upgradeHeight abi.ChainEpoch) { t.Fatal(err) } - CC := abi.SectorNumber(kit2.DefaultPresealsPerBootstrapMiner + 1) + CC := abi.SectorNumber(kit.DefaultPresealsPerBootstrapMiner + 1) Upgraded := CC + 1 miner.PledgeSectors(ctx, 1, 0, nil) @@ -62,9 +61,10 @@ func runTestCCUpgrade(t *testing.T, upgradeHeight abi.ChainEpoch) { err = miner.SectorMarkForUpgrade(ctx, sl[0]) require.NoError(t, err) - dh := kit2.NewDealHarness(t, client, miner, miner) - - dh.MakeOnlineDeal(context.Background(), 6, false, 0) + dh := kit.NewDealHarness(t, client, miner) + deal, res, inPath := dh.MakeOnlineDeal(ctx, kit.MakeFullDealParams{Rseed: 6}) + outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, false) + kit.AssertFilesEqual(t, inPath, outPath) // Validate upgrade diff --git a/itests/cli_test.go b/itests/cli_test.go index 8436f189e..0bd1ec3b4 100644 --- a/itests/cli_test.go +++ b/itests/cli_test.go @@ -6,16 +6,16 @@ import ( "time" "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" ) // TestClient does a basic test to exercise the client CLI commands. func TestClient(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - kit2.QuietMiningLogs() + kit.QuietMiningLogs() blockTime := 5 * time.Millisecond - client, _, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), kit2.ThroughRPC()) + client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) ens.InterconnectAll().BeginMining(blockTime) - kit2.RunClientTest(t, cli.Commands, *client) + kit.RunClientTest(t, cli.Commands, client) } diff --git a/itests/deadlines_test.go b/itests/deadlines_test.go index 9551465a5..00c737b72 100644 --- a/itests/deadlines_test.go +++ b/itests/deadlines_test.go @@ -3,7 +3,6 @@ package itests import ( "bytes" "context" - "fmt" "os" "testing" "time" @@ -27,7 +26,6 @@ import ( miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" ) @@ -58,11 +56,8 @@ func TestDeadlineToggling(t *testing.T) { if os.Getenv("LOTUS_TEST_DEADLINE_TOGGLING") != "1" { t.Skip("this takes a few minutes, set LOTUS_TEST_DEADLINE_TOGGLING=1 to run") } - _ = logging.SetLogLevel("miner", "ERROR") - _ = logging.SetLogLevel("chainstore", "ERROR") - _ = logging.SetLogLevel("chain", "ERROR") - _ = logging.SetLogLevel("sub", "ERROR") - _ = logging.SetLogLevel("storageminer", "FATAL") + + kit.QuietMiningLogs() const sectorsC, sectorsD, sectorsB = 10, 9, 8 @@ -75,21 +70,26 @@ func TestDeadlineToggling(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := kit.MockMinerBuilder(t, []kit.FullNodeOpts{kit.FullNodeWithNetworkUpgradeAt(network.Version12, upgradeH)}, kit.OneMiner) + var ( + client kit.TestFullNode + minerA kit.TestMiner + minerB kit.TestMiner + minerC kit.TestMiner + minerD kit.TestMiner + minerE kit.TestMiner + ) + opts := []kit.NodeOpt{kit.ConstructorOpts(kit.NetworkUpgradeAt(network.Version12, upgradeH))} + ens := kit.NewEnsemble(t, kit.MockProofs()). + FullNode(&client, opts...). + Miner(&minerA, &client, opts...). + Start(). + InterconnectAll() + ens.BeginMining(blocktime) - client := n[0].FullNode.(*impl.FullNodeAPI) - minerA := sn[0] - - { - addrinfo, err := client.NetAddrsListen(ctx) - if err != nil { - t.Fatal(err) - } - - if err := minerA.NetConnect(ctx, addrinfo); err != nil { - t.Fatal(err) - } - } + opts = append(opts, kit.OwnerAddr(client.DefaultKey)) + ens.Miner(&minerB, &client, opts...). + Miner(&minerC, &client, opts...). + Start() defaultFrom, err := client.WalletDefaultAddress(ctx) require.NoError(t, err) @@ -99,28 +99,6 @@ func TestDeadlineToggling(t *testing.T) { build.Clock.Sleep(time.Second) - done := make(chan struct{}) - go func() { - defer close(done) - for ctx.Err() == nil { - build.Clock.Sleep(blocktime) - if err := minerA.MineOne(ctx, kit.MineNext); err != nil { - if ctx.Err() != nil { - // context was canceled, ignore the error. - return - } - t.Error(err) - } - } - }() - defer func() { - cancel() - <-done - }() - - minerB := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) - minerC := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) - maddrB, err := minerB.ActorAddress(ctx) require.NoError(t, err) maddrC, err := minerC.ActorAddress(ctx) @@ -131,20 +109,20 @@ func TestDeadlineToggling(t *testing.T) { // pledge sectors on C, go through a PP, check for power { - kit.PledgeSectors(t, ctx, minerC, sectorsC, 0, nil) + minerC.PledgeSectors(ctx, sectorsC, 0, nil) di, err := client.StateMinerProvingDeadline(ctx, maddrC, types.EmptyTSK) require.NoError(t, err) - fmt.Printf("Running one proving period (miner C)\n") - fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod*2) + t.Log("Running one proving period (miner C)") + t.Logf("End for head.Height > %d", di.PeriodStart+di.WPoStProvingPeriod*2) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+provingPeriod*2 { - fmt.Printf("Now head.Height = %d\n", head.Height()) + t.Logf("Now head.Height = %d", head.Height()) break } build.Clock.Sleep(blocktime) @@ -165,7 +143,7 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) if head.Height() > upgradeH+provingPeriod { - fmt.Printf("Now head.Height = %d\n", head.Height()) + t.Logf("Now head.Height = %d", head.Height()) break } build.Clock.Sleep(blocktime) @@ -216,8 +194,9 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) require.GreaterOrEqual(t, nv, network.Version12) - minerD := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) - minerE := n[0].Stb(ctx, t, kit.TestSpt, defaultFrom) + ens.Miner(&minerD, &client, opts...). + Miner(&minerE, &client, opts...). + Start() maddrD, err := minerD.ActorAddress(ctx) require.NoError(t, err) @@ -225,7 +204,7 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) // first round of miner checks - checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.GenesisPreseals), true, true, types.EmptyTSK) + checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.DefaultPresealsPerBootstrapMiner), true, true, types.EmptyTSK) checkMiner(maddrC, types.NewInt(uint64(ssz)*sectorsC), true, true, types.EmptyTSK) checkMiner(maddrB, types.NewInt(0), false, false, types.EmptyTSK) @@ -233,10 +212,10 @@ func TestDeadlineToggling(t *testing.T) { checkMiner(maddrE, types.NewInt(0), false, false, types.EmptyTSK) // pledge sectors on minerB/minerD, stop post on minerC - kit.PledgeSectors(t, ctx, minerB, sectorsB, 0, nil) + minerB.PledgeSectors(ctx, sectorsB, 0, nil) checkMiner(maddrB, types.NewInt(0), true, true, types.EmptyTSK) - kit.PledgeSectors(t, ctx, minerD, sectorsD, 0, nil) + minerD.PledgeSectors(ctx, sectorsD, 0, nil) checkMiner(maddrD, types.NewInt(0), true, true, types.EmptyTSK) minerC.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).Fail() @@ -281,7 +260,7 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) if head.Height() > upgradeH+provingPeriod+(provingPeriod/2) { - fmt.Printf("Now head.Height = %d\n", head.Height()) + t.Logf("Now head.Height = %d", head.Height()) break } build.Clock.Sleep(blocktime) @@ -295,14 +274,14 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) if head.Height() > upgradeH+(provingPeriod*3) { - fmt.Printf("Now head.Height = %d\n", head.Height()) + t.Logf("Now head.Height = %d", head.Height()) break } build.Clock.Sleep(blocktime) } // second round of miner checks - checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.GenesisPreseals), true, true, types.EmptyTSK) + checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.DefaultPresealsPerBootstrapMiner), true, true, types.EmptyTSK) checkMiner(maddrC, types.NewInt(0), true, true, types.EmptyTSK) checkMiner(maddrB, types.NewInt(uint64(ssz)*sectorsB), true, true, types.EmptyTSK) checkMiner(maddrD, types.NewInt(uint64(ssz)*sectorsD), true, true, types.EmptyTSK) @@ -351,7 +330,7 @@ func TestDeadlineToggling(t *testing.T) { }, nil) require.NoError(t, err) - fmt.Println("sent termination message:", smsg.Cid()) + t.Log("sent termination message:", smsg.Cid()) r, err := client.StateWaitMsg(ctx, smsg.Cid(), 2, api.LookbackNoLimit, true) require.NoError(t, err) @@ -367,13 +346,13 @@ func TestDeadlineToggling(t *testing.T) { require.NoError(t, err) if head.Height() > upgradeH+(provingPeriod*5) { - fmt.Printf("Now head.Height = %d\n", head.Height()) + t.Logf("Now head.Height = %d", head.Height()) break } build.Clock.Sleep(blocktime) } - checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.GenesisPreseals), true, true, types.EmptyTSK) + checkMiner(maddrA, types.NewInt(uint64(ssz)*kit.DefaultPresealsPerBootstrapMiner), true, true, types.EmptyTSK) checkMiner(maddrC, types.NewInt(0), true, true, types.EmptyTSK) checkMiner(maddrB, types.NewInt(0), true, true, types.EmptyTSK) checkMiner(maddrD, types.NewInt(0), false, false, types.EmptyTSK) diff --git a/itests/deals_test.go b/itests/deals_test.go index ded4ec136..0c56d1073 100644 --- a/itests/deals_test.go +++ b/itests/deals_test.go @@ -14,7 +14,8 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" @@ -27,7 +28,7 @@ func TestDealCyclesConcurrent(t *testing.T) { t.Skip("skipping test in short mode") } - kit2.QuietMiningLogs() + kit.QuietMiningLogs() //blockTime := 10 * time.Millisecond @@ -37,11 +38,15 @@ func TestDealCyclesConcurrent(t *testing.T) { startEpoch := abi.ChainEpoch(2 << 12) runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) { - api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me + //api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me - client, main, market, _ := kit2.EnsembleWithMarket(t, kit2.MockProofs(), kit2.ThroughRPC()) + ////TODO: add miner + //client, main, _, _ := kit2.EnsembleWithMarket(t, kit2.MockProofs(), kit2.ThroughRPC()) - dh := kit2.NewDealHarness(t, client, main, market) + //dh := kit.NewDealHarness(t, client, main) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) + ens.InterconnectAll().BeginMining(blockTime) + dh := kit.NewDealHarness(t, client, miner) runConcurrentDeals(t, dh, fullDealCyclesOpts{ n: n, @@ -51,7 +56,7 @@ func TestDealCyclesConcurrent(t *testing.T) { }) } - //cycles := []int{1, 2, 4, 8} + // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# cycles := []int{1} for _, n := range cycles { n := n @@ -70,7 +75,7 @@ type fullDealCyclesOpts struct { startEpoch abi.ChainEpoch } -func runConcurrentDeals(t *testing.T, dh *kit2.DealHarness, opts fullDealCyclesOpts) { +func runConcurrentDeals(t *testing.T, dh *kit.DealHarness, opts fullDealCyclesOpts) { errgrp, _ := errgroup.WithContext(context.Background()) for i := 0; i < opts.n; i++ { i := i @@ -82,9 +87,13 @@ func runConcurrentDeals(t *testing.T, dh *kit2.DealHarness, opts fullDealCyclesO err = fmt.Errorf("deal failed: %s", r) } }() - deal, res, inPath := dh.MakeOnlineDeal(context.Background(), 5+i, opts.fastRetrieval, opts.startEpoch) + deal, res, inPath := dh.MakeOnlineDeal(context.Background(), kit.MakeFullDealParams{ + Rseed: 5 + i, + FastRet: opts.fastRetrieval, + StartEpoch: opts.startEpoch, + }) outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, opts.carExport) - kit2.AssertFilesEqual(t, inPath, outPath) + kit.AssertFilesEqual(t, inPath, outPath) return nil }) } @@ -96,13 +105,13 @@ func TestDealsWithSealingAndRPC(t *testing.T) { t.Skip("skipping test in short mode") } - kit2.QuietMiningLogs() + kit.QuietMiningLogs() var blockTime = 1 * time.Second - client, miner, ens := kit2.EnsembleMinimal(t, kit2.ThroughRPC()) // no mock proofs. + client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC()) // no mock proofs. ens.InterconnectAll().BeginMining(blockTime) - dh := kit2.NewDealHarness(t, client, miner, miner) + dh := kit.NewDealHarness(t, client, miner) t.Run("stdretrieval", func(t *testing.T) { runConcurrentDeals(t, dh, fullDealCyclesOpts{n: 1}) @@ -118,6 +127,95 @@ func TestDealsWithSealingAndRPC(t *testing.T) { }) } +func TestQuotePriceForUnsealedRetrieval(t *testing.T) { + var ( + ctx = context.Background() + blocktime = 10 * time.Millisecond + ) + + kit.QuietMiningLogs() + + client, miner, ens := kit.EnsembleMinimal(t) + ens.InterconnectAll().BeginMining(blocktime) + + var ( + ppb = int64(1) + unsealPrice = int64(77) + ) + + // Set unsealed price to non-zero + ask, err := miner.MarketGetRetrievalAsk(ctx) + require.NoError(t, err) + ask.PricePerByte = abi.NewTokenAmount(ppb) + ask.UnsealPrice = abi.NewTokenAmount(unsealPrice) + err = miner.MarketSetRetrievalAsk(ctx, ask) + require.NoError(t, err) + + dh := kit.NewDealHarness(t, client, miner) + + deal1, res1, _ := dh.MakeOnlineDeal(ctx, kit.MakeFullDealParams{Rseed: 6}) + + // one more storage deal for the same data + _, res2, _ := dh.MakeOnlineDeal(ctx, kit.MakeFullDealParams{Rseed: 6}) + require.Equal(t, res1.Root, res2.Root) + + // Retrieval + dealInfo, err := client.ClientGetDealInfo(ctx, *deal1) + require.NoError(t, err) + + // fetch quote -> zero for unsealed price since unsealed file already exists. + offers, err := client.ClientFindData(ctx, res1.Root, &dealInfo.PieceCID) + require.NoError(t, err) + require.Len(t, offers, 2) + require.Equal(t, offers[0], offers[1]) + require.Equal(t, uint64(0), offers[0].UnsealPrice.Uint64()) + require.Equal(t, dealInfo.Size*uint64(ppb), offers[0].MinPrice.Uint64()) + + // remove ONLY one unsealed file + ss, err := miner.StorageList(context.Background()) + require.NoError(t, err) + _, err = miner.SectorsList(ctx) + require.NoError(t, err) + +iLoop: + for storeID, sd := range ss { + for _, sector := range sd { + err := miner.StorageDropSector(ctx, storeID, sector.SectorID, storiface.FTUnsealed) + require.NoError(t, err) + break iLoop // remove ONLY one + } + } + + // get retrieval quote -> zero for unsealed price as unsealed file exists. + offers, err = client.ClientFindData(ctx, res1.Root, &dealInfo.PieceCID) + require.NoError(t, err) + require.Len(t, offers, 2) + require.Equal(t, offers[0], offers[1]) + require.Equal(t, uint64(0), offers[0].UnsealPrice.Uint64()) + require.Equal(t, dealInfo.Size*uint64(ppb), offers[0].MinPrice.Uint64()) + + // remove the other unsealed file as well + ss, err = miner.StorageList(context.Background()) + require.NoError(t, err) + _, err = miner.SectorsList(ctx) + require.NoError(t, err) + for storeID, sd := range ss { + for _, sector := range sd { + require.NoError(t, miner.StorageDropSector(ctx, storeID, sector.SectorID, storiface.FTUnsealed)) + } + } + + // fetch quote -> non-zero for unseal price as we no more unsealed files. + offers, err = client.ClientFindData(ctx, res1.Root, &dealInfo.PieceCID) + require.NoError(t, err) + require.Len(t, offers, 2) + require.Equal(t, offers[0], offers[1]) + require.Equal(t, uint64(unsealPrice), offers[0].UnsealPrice.Uint64()) + total := (dealInfo.Size * uint64(ppb)) + uint64(unsealPrice) + require.Equal(t, total, offers[0].MinPrice.Uint64()) + +} + func TestPublishDealsBatching(t *testing.T) { var ( ctx = context.Background() @@ -126,7 +224,7 @@ func TestPublishDealsBatching(t *testing.T) { startEpoch = abi.ChainEpoch(2 << 12) ) - kit2.QuietMiningLogs() + kit.QuietMiningLogs() opts := node.Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{ @@ -135,10 +233,10 @@ func TestPublishDealsBatching(t *testing.T) { }), ) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), kit2.ConstructorOpts(opts)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ConstructorOpts(opts)) ens.InterconnectAll().BeginMining(10 * time.Millisecond) - dh := kit2.NewDealHarness(t, client, miner, miner) + dh := kit.NewDealHarness(t, client, miner) // Starts a deal and waits until it's published runDealTillPublish := func(rseed int) { @@ -213,23 +311,23 @@ func TestFirstDealEnablesMining(t *testing.T) { t.Skip("skipping test in short mode") } - kit2.QuietMiningLogs() + kit.QuietMiningLogs() var ( - client kit2.TestFullNode - genMiner kit2.TestMiner // bootstrap - provider kit2.TestMiner // no sectors, will need to create one + client kit.TestFullNode + genMiner kit.TestMiner // bootstrap + provider kit.TestMiner // no sectors, will need to create one ) - ens := kit2.NewEnsemble(t, kit2.MockProofs()) + ens := kit.NewEnsemble(t, kit.MockProofs()) ens.FullNode(&client) ens.Miner(&genMiner, &client) - ens.Miner(&provider, &client, kit2.PresealSectors(0)) + ens.Miner(&provider, &client, kit.PresealSectors(0)) ens.Start().InterconnectAll().BeginMining(50 * time.Millisecond) ctx := context.Background() - dh := kit2.NewDealHarness(t, &client, &provider, &provider) + dh := kit.NewDealHarness(t, &client, &provider) ref, _ := client.CreateImportFile(ctx, 5, 0) @@ -244,7 +342,7 @@ func TestFirstDealEnablesMining(t *testing.T) { providerMined := make(chan struct{}) go func() { - _ = client.WaitTillChain(ctx, kit2.BlockMinedBy(provider.ActorAddr)) + _ = client.WaitTillChain(ctx, kit.BlockMinedBy(provider.ActorAddr)) close(providerMined) }() @@ -269,10 +367,10 @@ func TestOfflineDealFlow(t *testing.T) { runTest := func(t *testing.T, fastRet bool) { ctx := context.Background() - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs()) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) ens.InterconnectAll().BeginMining(blocktime) - dh := kit2.NewDealHarness(t, client, miner, miner) + dh := kit.NewDealHarness(t, client, miner) // Create a random file and import on the client. res, inFile := client.CreateImportFile(ctx, 1, 0) @@ -336,7 +434,8 @@ func TestOfflineDealFlow(t *testing.T) { // Retrieve the deal outFile := dh.PerformRetrieval(ctx, proposalCid, rootCid, false) - kit2.AssertFilesEqual(t, inFile, outFile) + kit.AssertFilesEqual(t, inFile, outFile) + } t.Run("stdretrieval", func(t *testing.T) { runTest(t, false) }) @@ -348,14 +447,14 @@ func TestZeroPricePerByteRetrieval(t *testing.T) { t.Skip("skipping test in short mode") } - kit2.QuietMiningLogs() + kit.QuietMiningLogs() var ( blockTime = 10 * time.Millisecond startEpoch = abi.ChainEpoch(2 << 12) ) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs()) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) ens.InterconnectAll().BeginMining(blockTime) ctx := context.Background() @@ -367,7 +466,7 @@ func TestZeroPricePerByteRetrieval(t *testing.T) { err = miner.MarketSetRetrievalAsk(ctx, ask) require.NoError(t, err) - dh := kit2.NewDealHarness(t, client, miner, miner) + dh := kit.NewDealHarness(t, client, miner) runConcurrentDeals(t, dh, fullDealCyclesOpts{ n: 1, startEpoch: startEpoch, diff --git a/itests/gateway_test.go b/itests/gateway_test.go index 7f1b70f2d..e2329bf77 100644 --- a/itests/gateway_test.go +++ b/itests/gateway_test.go @@ -5,11 +5,10 @@ import ( "context" "fmt" "math" - "os" "testing" "time" - "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" "golang.org/x/xerrors" @@ -21,10 +20,10 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/gateway" - "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" @@ -45,7 +44,6 @@ func init() { // TestGatewayWalletMsig tests that API calls to wallet and msig can be made on a lite // node that is connected through a gateway to a full API node func TestGatewayWalletMsig(t *testing.T) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") kit.QuietMiningLogs() blocktime := 5 * time.Millisecond @@ -111,7 +109,6 @@ func TestGatewayWalletMsig(t *testing.T) { if err != nil { return cid.Undef, err } - return lite.MpoolPush(ctx, sm) } @@ -179,7 +176,6 @@ func TestGatewayWalletMsig(t *testing.T) { // TestGatewayMsigCLI tests that msig CLI calls can be made // on a lite node that is connected through a gateway to a full API node func TestGatewayMsigCLI(t *testing.T) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") kit.QuietMiningLogs() blocktime := 5 * time.Millisecond @@ -192,7 +188,6 @@ func TestGatewayMsigCLI(t *testing.T) { } func TestGatewayDealFlow(t *testing.T) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") kit.QuietMiningLogs() blocktime := 5 * time.Millisecond @@ -200,17 +195,22 @@ func TestGatewayDealFlow(t *testing.T) { nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) defer nodes.closer() + time.Sleep(5 * time.Second) + // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time dealStartEpoch := abi.ChainEpoch(2 << 12) dh := kit.NewDealHarness(t, nodes.lite, nodes.miner) - dh.MakeFullDeal(ctx, 6, false, false, dealStartEpoch) + dealCid, res, _ := dh.MakeOnlineDeal(context.Background(), kit.MakeFullDealParams{ + Rseed: 6, + StartEpoch: dealStartEpoch, + }) + dh.PerformRetrieval(ctx, dealCid, res.Root, false) } func TestGatewayCLIDealFlow(t *testing.T) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") kit.QuietMiningLogs() blocktime := 5 * time.Millisecond @@ -222,9 +222,9 @@ func TestGatewayCLIDealFlow(t *testing.T) { } type testNodes struct { - lite kit.TestFullNode - full kit.TestFullNode - miner kit.TestMiner + lite *kit.TestFullNode + full *kit.TestFullNode + miner *kit.TestMiner closer jsonrpc.ClientCloser } @@ -241,8 +241,8 @@ func startNodesWithFunds( fullWalletAddr, err := nodes.full.WalletDefaultAddress(ctx) require.NoError(t, err) - // Create a wallet on the lite node - liteWalletAddr, err := nodes.lite.WalletNew(ctx, types.KTSecp256k1) + // Get the lite node default wallet address. + liteWalletAddr, err := nodes.lite.WalletDefaultAddress(ctx) require.NoError(t, err) // Send some funds from the full node to the lite node @@ -261,66 +261,46 @@ func startNodes( ) *testNodes { var closer jsonrpc.ClientCloser - // Create one miner and two full nodes. + var ( + full *kit.TestFullNode + miner *kit.TestMiner + lite kit.TestFullNode + ) + + // - Create one full node and one lite node // - Put a gateway server in front of full node 1 // - Start full node 2 in lite mode // - Connect lite node -> gateway server -> full node - opts := append( - // Full node - kit.OneFull, - // Lite node - kit.FullNodeOpts{ - Lite: true, - Opts: func(nodes []kit.TestFullNode) node.Option { - fullNode := nodes[0] - // Create a gateway server in front of the full node - gwapi := gateway.NewNode(fullNode, lookbackCap, stateWaitLookbackLimit) - handler, err := gateway.Handler(gwapi) - require.NoError(t, err) + // create the full node and the miner. + var ens *kit.Ensemble + full, miner, ens = kit.EnsembleMinimal(t, kit.MockProofs()) + ens.InterconnectAll().BeginMining(blocktime) - srv, _ := kit.CreateRPCServer(t, handler) - - // Create a gateway client API that connects to the gateway server - var gapi api.Gateway - gapi, closer, err = client.NewGatewayRPCV1(ctx, "ws://"+srv.Listener.Addr().String()+"/rpc/v1", nil) - require.NoError(t, err) - - // Provide the gateway API to dependency injection - return node.Override(new(api.Gateway), gapi) - }, - }, - ) - n, sn := kit.RPCMockMinerBuilder(t, opts, kit.OneMiner) - - full := n[0] - lite := n[1] - miner := sn[0] - - // Get the listener address for the full node - fullAddr, err := full.NetAddrsListen(ctx) + // Create a gateway server in front of the full node + gwapi := gateway.NewNode(full, lookbackCap, stateWaitLookbackLimit) + handler, err := gateway.Handler(gwapi) require.NoError(t, err) - // Connect the miner and the full node - err = miner.NetConnect(ctx, fullAddr) + srv, _ := kit.CreateRPCServer(t, handler) + + // Create a gateway client API that connects to the gateway server + var gapi api.Gateway + gapi, closer, err = client.NewGatewayRPCV1(ctx, "ws://"+srv.Listener.Addr().String()+"/rpc/v1", nil) require.NoError(t, err) - // Connect the miner and the lite node (so that the lite node can send - // data to the miner) - liteAddr, err := lite.NetAddrsListen(ctx) - require.NoError(t, err) - err = miner.NetConnect(ctx, liteAddr) - require.NoError(t, err) + ens.FullNode(&lite, + kit.LiteNode(), + kit.ThroughRPC(), + kit.ConstructorOpts( + node.Override(new(api.Gateway), gapi), + ), + ).Start().InterconnectAll() - // Start mining blocks - bm := kit.NewBlockMiner(t, miner) - bm.MineBlocks(ctx, blocktime) - t.Cleanup(bm.Stop) - - return &testNodes{lite: lite, full: full, miner: miner, closer: closer} + return &testNodes{lite: &lite, full: full, miner: miner, closer: closer} } -func sendFunds(ctx context.Context, fromNode kit.TestFullNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { +func sendFunds(ctx context.Context, fromNode *kit.TestFullNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { msg := &types.Message{ From: fromAddr, To: toAddr, diff --git a/itests/kit/blockminer.go b/itests/kit/blockminer.go index 3b1f1fedf..2c9bd47c6 100644 --- a/itests/kit/blockminer.go +++ b/itests/kit/blockminer.go @@ -15,14 +15,14 @@ import ( // BlockMiner is a utility that makes a test miner Mine blocks on a timer. type BlockMiner struct { t *testing.T - miner TestMiner + miner *TestMiner nextNulls int64 wg sync.WaitGroup cancel context.CancelFunc } -func NewBlockMiner(t *testing.T, miner TestMiner) *BlockMiner { +func NewBlockMiner(t *testing.T, miner *TestMiner) *BlockMiner { return &BlockMiner{ t: t, miner: miner, @@ -69,7 +69,7 @@ func (bm *BlockMiner) InjectNulls(rounds abi.ChainEpoch) { atomic.AddInt64(&bm.nextNulls, int64(rounds)) } -func (bm *BlockMiner) MineUntilBlock(ctx context.Context, fn TestFullNode, cb func(abi.ChainEpoch)) { +func (bm *BlockMiner) MineUntilBlock(ctx context.Context, fn *TestFullNode, cb func(abi.ChainEpoch)) { for i := 0; i < 1000; i++ { var ( success bool @@ -93,7 +93,7 @@ func (bm *BlockMiner) MineUntilBlock(ctx context.Context, fn TestFullNode, cb fu if success { // Wait until it shows up on the given full nodes ChainHead - nloops := 50 + nloops := 200 for i := 0; i < nloops; i++ { ts, err := fn.ChainHead(ctx) require.NoError(bm.t, err) diff --git a/itests/kit/client.go b/itests/kit/client.go index 6b7d46265..bd81e0c04 100644 --- a/itests/kit/client.go +++ b/itests/kit/client.go @@ -21,7 +21,7 @@ import ( ) // RunClientTest exercises some of the Client CLI commands -func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestFullNode) { +func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode *TestFullNode) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -42,7 +42,7 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestFullNode) require.Regexp(t, regexp.MustCompile("Ask:"), out) // Create a deal (non-interactive) - // client deal --start-epoch= 1000000attofil + // client deal --start-epoch= 1000000attofil res, _, _, err := CreateImportFile(ctx, clientNode, 1, 0) require.NoError(t, err) diff --git a/itests/kit/deals.go b/itests/kit/deals.go index c768eb87f..bf986a5e7 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -1,12 +1,9 @@ package kit import ( - "bytes" "context" - "fmt" "io/ioutil" "os" - "path/filepath" "testing" "time" @@ -21,7 +18,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/node/impl" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" dstest "github.com/ipfs/go-merkledag/test" @@ -30,12 +26,18 @@ import ( type DealHarness struct { t *testing.T - client api.FullNode - miner TestMiner + client *TestFullNode + miner *TestMiner +} + +type MakeFullDealParams struct { + Rseed int + FastRet bool + StartEpoch abi.ChainEpoch } // NewDealHarness creates a test harness that contains testing utilities for deals. -func NewDealHarness(t *testing.T, client api.FullNode, miner TestMiner) *DealHarness { +func NewDealHarness(t *testing.T, client *TestFullNode, miner *TestMiner) *DealHarness { return &DealHarness{ t: t, client: client, @@ -43,38 +45,33 @@ func NewDealHarness(t *testing.T, client api.FullNode, miner TestMiner) *DealHar } } -func (dh *DealHarness) MakeFullDeal(ctx context.Context, rseed int, carExport, fastRet bool, startEpoch abi.ChainEpoch) { - res, _, data, err := CreateImportFile(ctx, dh.client, rseed, 0) - if err != nil { - dh.t.Fatal(err) - } +// MakeOnlineDeal makes an online deal, generating a random file with the +// supplied seed, and setting the specified fast retrieval flag and start epoch +// on the storage deal. It returns when the deal is sealed. +// +// TODO: convert input parameters to struct, and add size as an input param. +func (dh *DealHarness) MakeOnlineDeal(ctx context.Context, params MakeFullDealParams) (deal *cid.Cid, res *api.ImportRes, path string) { + res, path = dh.client.CreateImportFile(ctx, params.Rseed, 0) - fcid := res.Root - fmt.Println("FILE CID: ", fcid) + dh.t.Logf("FILE CID: %s", res.Root) - deal := dh.StartDeal(ctx, fcid, fastRet, startEpoch) + deal = dh.StartDeal(ctx, res.Root, params.FastRet, params.StartEpoch) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) dh.WaitDealSealed(ctx, deal, false, false, nil) - // Retrieval - info, err := dh.client.ClientGetDealInfo(ctx, *deal) - require.NoError(dh.t, err) - - dh.TestRetrieval(ctx, fcid, &info.PieceCID, carExport, data) + return deal, res, path } +// StartDeal starts a storage deal between the client and the miner. func (dh *DealHarness) StartDeal(ctx context.Context, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid { maddr, err := dh.miner.ActorAddress(ctx) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) addr, err := dh.client.WalletDefaultAddress(ctx) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) + deal, err := dh.client.ClientStartDeal(ctx, &api.StartDealParams{ Data: &storagemarket.DataRef{ TransferType: storagemarket.TTGraphsync, @@ -87,12 +84,12 @@ func (dh *DealHarness) StartDeal(ctx context.Context, fcid cid.Cid, fastRet bool MinBlocksDuration: uint64(build.MinDealDuration), FastRetrieval: fastRet, }) - if err != nil { - dh.t.Fatalf("%+v", err) - } + require.NoError(dh.t, err) + return deal } +// WaitDealSealed waits until the deal is sealed. func (dh *DealHarness) WaitDealSealed(ctx context.Context, deal *cid.Cid, noseal, noSealStart bool, cb func()) { loop: for { @@ -114,7 +111,7 @@ loop: case storagemarket.StorageDealError: dh.t.Fatal("deal errored", di.Message) case storagemarket.StorageDealActive: - fmt.Println("COMPLETE", di) + dh.t.Log("COMPLETE", di) break loop } @@ -129,7 +126,7 @@ loop: } } - fmt.Printf("Deal %d state: client:%s provider:%s\n", di.DealID, storagemarket.DealStates[di.State], storagemarket.DealStates[minerState]) + dh.t.Logf("Deal %d state: client:%s provider:%s\n", di.DealID, storagemarket.DealStates[di.State], storagemarket.DealStates[minerState]) time.Sleep(time.Second / 2) if cb != nil { cb() @@ -137,13 +134,14 @@ loop: } } +// WaitDealPublished waits until the deal is published. func (dh *DealHarness) WaitDealPublished(ctx context.Context, deal *cid.Cid) { subCtx, cancel := context.WithCancel(ctx) defer cancel() + updates, err := dh.miner.MarketGetDealUpdates(subCtx) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) + for { select { case <-ctx.Done(): @@ -158,10 +156,10 @@ func (dh *DealHarness) WaitDealPublished(ctx context.Context, deal *cid.Cid) { case storagemarket.StorageDealError: dh.t.Fatal("deal errored", di.Message) case storagemarket.StorageDealFinalizing, storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing, storagemarket.StorageDealActive: - fmt.Println("COMPLETE", di) + dh.t.Log("COMPLETE", di) return } - fmt.Println("Deal state: ", storagemarket.DealStates[di.State]) + dh.t.Log("Deal state: ", storagemarket.DealStates[di.State]) } } } @@ -180,133 +178,71 @@ func (dh *DealHarness) StartSealingWaiting(ctx context.Context) { require.NoError(dh.t, dh.miner.SectorStartSealing(ctx, snum)) } - flushSealingBatches(dh.t, ctx, dh.miner) + dh.miner.FlushSealingBatches(ctx) } } -func (dh *DealHarness) TestRetrieval(ctx context.Context, fcid cid.Cid, piece *cid.Cid, carExport bool, expect []byte) { - offers, err := dh.client.ClientFindData(ctx, fcid, piece) - if err != nil { - dh.t.Fatal(err) - } +func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool) (path string) { + // perform retrieval. + info, err := dh.client.ClientGetDealInfo(ctx, *deal) + require.NoError(dh.t, err) - if len(offers) < 1 { - dh.t.Fatal("no offers") - } + offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID) + require.NoError(dh.t, err) + require.NotEmpty(dh.t, offers, "no offers") - rpath, err := ioutil.TempDir("", "lotus-retrieve-test-") - if err != nil { - dh.t.Fatal(err) - } - defer os.RemoveAll(rpath) //nolint:errcheck + carFile, err := ioutil.TempFile(dh.t.TempDir(), "ret-car") + require.NoError(dh.t, err) + + defer carFile.Close() //nolint:errcheck caddr, err := dh.client.WalletDefaultAddress(ctx) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) ref := &api.FileRef{ - Path: filepath.Join(rpath, "ret"), + Path: carFile.Name(), IsCAR: carExport, } + updates, err := dh.client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) + for update := range updates { - if update.Err != "" { - dh.t.Fatalf("retrieval failed: %s", update.Err) - } - } - - rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret")) - if err != nil { - dh.t.Fatal(err) + require.Emptyf(dh.t, update.Err, "retrieval failed: %s", update.Err) } + ret := carFile.Name() if carExport { - rdata = dh.ExtractCarData(ctx, rdata, rpath) + actualFile := dh.ExtractFileFromCAR(ctx, carFile) + ret = actualFile.Name() + _ = actualFile.Close() //nolint:errcheck } - if !bytes.Equal(rdata, expect) { - dh.t.Fatal("wrong expect retrieved") - } + return ret } -func (dh *DealHarness) ExtractCarData(ctx context.Context, rdata []byte, rpath string) []byte { +func (dh *DealHarness) ExtractFileFromCAR(ctx context.Context, file *os.File) (out *os.File) { bserv := dstest.Bserv() - ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata)) - if err != nil { - dh.t.Fatal(err) - } + ch, err := car.LoadCar(bserv.Blockstore(), file) + require.NoError(dh.t, err) + b, err := bserv.GetBlock(ctx, ch.Roots[0]) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) + nd, err := ipld.Decode(b) - if err != nil { - dh.t.Fatal(err) - } + require.NoError(dh.t, err) + dserv := dag.NewDAGService(bserv) fil, err := unixfile.NewUnixfsFile(ctx, dserv, nd) - if err != nil { - dh.t.Fatal(err) - } - outPath := filepath.Join(rpath, "retLoadedCAR") - if err := files.WriteTo(fil, outPath); err != nil { - dh.t.Fatal(err) - } - rdata, err = ioutil.ReadFile(outPath) - if err != nil { - dh.t.Fatal(err) - } - return rdata -} - -type DealsScaffold struct { - Ctx context.Context - Client *impl.FullNodeAPI - Miner TestMiner - BlockMiner *BlockMiner -} - -func ConnectAndStartMining(t *testing.T, blocktime time.Duration, miner TestMiner, clients ...api.FullNode) *BlockMiner { - ctx := context.Background() - - for _, c := range clients { - addrinfo, err := c.NetAddrsListen(ctx) - if err != nil { - t.Fatal(err) - } - if err := miner.NetConnect(ctx, addrinfo); err != nil { - t.Fatal(err) - } - } - - time.Sleep(time.Second) - - blockMiner := NewBlockMiner(t, miner) - blockMiner.MineBlocks(ctx, blocktime) - t.Cleanup(blockMiner.Stop) - return blockMiner -} - -type TestDealState int - -const ( - TestDealStateFailed = TestDealState(-1) - TestDealStateInProgress = TestDealState(0) - TestDealStateComplete = TestDealState(1) -) - -// CategorizeDealState categorizes deal states into one of three states: -// Complete, InProgress, Failed. -func CategorizeDealState(dealStatus string) TestDealState { - switch dealStatus { - case "StorageDealFailing", "StorageDealError": - return TestDealStateFailed - case "StorageDealStaged", "StorageDealAwaitingPreCommit", "StorageDealSealing", "StorageDealActive", "StorageDealExpired", "StorageDealSlashed": - return TestDealStateComplete - } - return TestDealStateInProgress + require.NoError(dh.t, err) + + tmpfile, err := ioutil.TempFile(dh.t.TempDir(), "file-in-car") + require.NoError(dh.t, err) + + defer tmpfile.Close() //nolint:errcheck + + err = files.WriteTo(fil, tmpfile.Name()) + require.NoError(dh.t, err) + + return tmpfile } diff --git a/itests/kit2/deals_state.go b/itests/kit/deals_state.go similarity index 98% rename from itests/kit2/deals_state.go rename to itests/kit/deals_state.go index be3a9e4db..617a6d28e 100644 --- a/itests/kit2/deals_state.go +++ b/itests/kit/deals_state.go @@ -1,4 +1,4 @@ -package kit2 +package kit type TestDealState int diff --git a/itests/kit2/ensemble.go b/itests/kit/ensemble.go similarity index 95% rename from itests/kit2/ensemble.go rename to itests/kit/ensemble.go index 6e31aa742..cb4046235 100644 --- a/itests/kit2/ensemble.go +++ b/itests/kit/ensemble.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "bytes" @@ -147,13 +147,13 @@ func (n *Ensemble) FullNode(full *TestFullNode, opts ...NodeOpt) *Ensemble { require.NoError(n.t, err) } - var key *wallet.Key - if !n.bootstrapped && !options.balance.IsZero() { - // create a key+address, and assign it some FIL; this will be set as the default wallet. - var err error - key, err = wallet.GenerateKey(types.KTBLS) - require.NoError(n.t, err) + key, err := wallet.GenerateKey(types.KTBLS) + require.NoError(n.t, err) + if !n.bootstrapped && !options.balance.IsZero() { + // if we still haven't forged genesis, create a key+address, and assign + // it some FIL; this will be set as the default wallet when the node is + // started. genacc := genesis.Actor{ Type: genesis.TAccount, Balance: options.balance, @@ -300,10 +300,14 @@ func (n *Ensemble) Start() *Ensemble { ) } + // Call option builders, passing active nodes as the parameter + for _, bopt := range full.options.optBuilders { + opts = append(opts, bopt(n.active.fullnodes)) + } + // Construct the full node. stop, err := node.New(ctx, opts...) - // fullOpts[i].Opts(fulls), require.NoError(n.t, err) addr, err := full.WalletImport(context.Background(), &full.DefaultKey.KeyInfo) @@ -344,7 +348,7 @@ func (n *Ensemble) Start() *Ensemble { params, aerr := actors.SerializeParams(&power2.CreateMinerParams{ Owner: m.OwnerKey.Address, Worker: m.OwnerKey.Address, - SealProofType: n.options.proofType, + SealProofType: m.options.proofType, Peer: abi.PeerID(m.Libp2p.PeerID), }) require.NoError(n.t, aerr) @@ -493,8 +497,16 @@ func (n *Ensemble) Start() *Ensemble { node.MockHost(n.mn), - node.Override(new(v1api.FullNode), m.FullNode), + node.Override(new(v1api.FullNode), m.FullNode.FullNode), node.Override(new(*lotusminer.Miner), lotusminer.NewTestMiner(mineBlock, m.ActorAddr)), + + // disable resource filtering so that local worker gets assigned tasks + // regardless of system pressure. + node.Override(new(sectorstorage.SealerConfig), func() sectorstorage.SealerConfig { + scfg := config.DefaultStorageMiner() + scfg.Storage.ResourceFiltering = sectorstorage.ResourceFilteringDisabled + return scfg.Storage + }), } // append any node builder options. diff --git a/itests/kit2/ensemble_opts.go b/itests/kit/ensemble_opts.go similarity index 84% rename from itests/kit2/ensemble_opts.go rename to itests/kit/ensemble_opts.go index f8bc81535..440362ed1 100644 --- a/itests/kit2/ensemble_opts.go +++ b/itests/kit/ensemble_opts.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "time" @@ -16,7 +16,6 @@ type genesisAccount struct { type ensembleOpts struct { pastOffset time.Duration - proofType abi.RegisteredSealProof verifiedRoot genesisAccount accounts []genesisAccount mockProofs bool @@ -24,14 +23,6 @@ type ensembleOpts struct { var DefaultEnsembleOpts = ensembleOpts{ pastOffset: 10000000 * time.Second, // time sufficiently in the past to trigger catch-up mining. - proofType: abi.RegisteredSealProof_StackedDrg2KiBV1, -} - -func ProofType(proofType abi.RegisteredSealProof) EnsembleOpt { - return func(opts *ensembleOpts) error { - opts.proofType = proofType - return nil - } } // MockProofs activates mock proofs for the entire ensemble. diff --git a/itests/kit2/ensemble_presets.go b/itests/kit/ensemble_presets.go similarity index 99% rename from itests/kit2/ensemble_presets.go rename to itests/kit/ensemble_presets.go index 910374878..73c9e1b02 100644 --- a/itests/kit2/ensemble_presets.go +++ b/itests/kit/ensemble_presets.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "testing" diff --git a/itests/kit2/files.go b/itests/kit/files.go similarity index 98% rename from itests/kit2/files.go rename to itests/kit/files.go index 1e1509858..48592b518 100644 --- a/itests/kit2/files.go +++ b/itests/kit/files.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "bytes" diff --git a/itests/kit/funds.go b/itests/kit/funds.go index 4c739dc62..417cf9ce1 100644 --- a/itests/kit/funds.go +++ b/itests/kit/funds.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/filecoin-project/go-state-types/abi" + "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" @@ -13,11 +14,9 @@ import ( // SendFunds sends funds from the default wallet of the specified sender node // to the recipient address. -func SendFunds(ctx context.Context, t *testing.T, sender TestFullNode, recipient address.Address, amount abi.TokenAmount) { +func SendFunds(ctx context.Context, t *testing.T, sender *TestFullNode, recipient address.Address, amount abi.TokenAmount) { senderAddr, err := sender.WalletDefaultAddress(ctx) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) msg := &types.Message{ From: senderAddr, @@ -26,14 +25,10 @@ func SendFunds(ctx context.Context, t *testing.T, sender TestFullNode, recipient } sm, err := sender.MpoolPushMessage(ctx, msg, nil) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + res, err := sender.StateWaitMsg(ctx, sm.Cid(), 3, api.LookbackNoLimit, true) - if err != nil { - t.Fatal(err) - } - if res.Receipt.ExitCode != 0 { - t.Fatal("did not successfully send money") - } + require.NoError(t, err) + + require.EqualValues(t, 0, res.Receipt.ExitCode, "did not successfully send funds") } diff --git a/itests/kit/init.go b/itests/kit/init.go index 2f40ca0f0..8df4922b8 100644 --- a/itests/kit/init.go +++ b/itests/kit/init.go @@ -3,7 +3,6 @@ package kit import ( "fmt" "os" - "strings" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/build" @@ -12,21 +11,19 @@ import ( ) func init() { - bin := os.Args[0] - if !strings.HasSuffix(bin, ".test") { - panic("package itests/kit must only be imported from tests") - } - _ = logging.SetLogLevel("*", "INFO") policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) - err := os.Setenv("BELLMAN_NO_GPU", "1") - if err != nil { - panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err)) - } build.InsecurePoStValidation = true + if err := os.Setenv("BELLMAN_NO_GPU", "1"); err != nil { + panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err)) + } + + if err := os.Setenv("LOTUS_DISABLE_WATCHDOG", "1"); err != nil { + panic(fmt.Sprintf("failed to set LOTUS_DISABLE_WATCHDOG env variable: %s", err)) + } } diff --git a/itests/kit/log.go b/itests/kit/log.go index 638e768d8..3dce3af9d 100644 --- a/itests/kit/log.go +++ b/itests/kit/log.go @@ -8,7 +8,7 @@ import ( func QuietMiningLogs() { lotuslog.SetupLogLevels() - _ = logging.SetLogLevel("miner", "ERROR") + _ = logging.SetLogLevel("miner", "ERROR") // set this to INFO to watch mining happen. _ = logging.SetLogLevel("chainstore", "ERROR") _ = logging.SetLogLevel("chain", "ERROR") _ = logging.SetLogLevel("sub", "ERROR") diff --git a/itests/kit/net.go b/itests/kit/net.go deleted file mode 100644 index 54c72443f..000000000 --- a/itests/kit/net.go +++ /dev/null @@ -1,87 +0,0 @@ -package kit - -import ( - "context" - "testing" - "time" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/go-address" -) - -func StartOneNodeOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) (TestFullNode, address.Address) { - n, sn := RPCMockMinerBuilder(t, OneFull, OneMiner) - - full := n[0] - miner := sn[0] - - // Get everyone connected - addrs, err := full.NetAddrsListen(ctx) - if err != nil { - t.Fatal(err) - } - - if err := miner.NetConnect(ctx, addrs); err != nil { - t.Fatal(err) - } - - // Start mining blocks - bm := NewBlockMiner(t, miner) - bm.MineBlocks(ctx, blocktime) - t.Cleanup(bm.Stop) - - // Get the full node's wallet address - fullAddr, err := full.WalletDefaultAddress(ctx) - if err != nil { - t.Fatal(err) - } - - // Create mock CLI - return full, fullAddr -} - -func StartTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) ([]TestFullNode, []address.Address) { - n, sn := RPCMockMinerBuilder(t, TwoFull, OneMiner) - - fullNode1 := n[0] - fullNode2 := n[1] - miner := sn[0] - - // Get everyone connected - addrs, err := fullNode1.NetAddrsListen(ctx) - if err != nil { - t.Fatal(err) - } - - if err := fullNode2.NetConnect(ctx, addrs); err != nil { - t.Fatal(err) - } - - if err := miner.NetConnect(ctx, addrs); err != nil { - t.Fatal(err) - } - - // Start mining blocks - bm := NewBlockMiner(t, miner) - bm.MineBlocks(ctx, blocktime) - t.Cleanup(bm.Stop) - - // Send some funds to register the second node - fullNodeAddr2, err := fullNode2.WalletNew(ctx, types.KTSecp256k1) - if err != nil { - t.Fatal(err) - } - - SendFunds(ctx, t, fullNode1, fullNodeAddr2, abi.NewTokenAmount(1e18)) - - // Get the first node's address - fullNodeAddr1, err := fullNode1.WalletDefaultAddress(ctx) - if err != nil { - t.Fatal(err) - } - - // Create mock CLI - return n, []address.Address{fullNodeAddr1, fullNodeAddr2} -} diff --git a/itests/kit/node_builder.go b/itests/kit/node_builder.go deleted file mode 100644 index 8e615002c..000000000 --- a/itests/kit/node_builder.go +++ /dev/null @@ -1,660 +0,0 @@ -package kit - -import ( - "bytes" - "context" - "crypto/rand" - "io/ioutil" - "net/http" - "net/http/httptest" - "sync" - "testing" - "time" - - "github.com/filecoin-project/go-state-types/network" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/go-storedcounter" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/client" - "github.com/filecoin-project/lotus/api/v1api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/actors/builtin/power" - "github.com/filecoin-project/lotus/chain/gen" - genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/lotus/extern/sector-storage/mock" - "github.com/filecoin-project/lotus/genesis" - lotusminer "github.com/filecoin-project/lotus/miner" - "github.com/filecoin-project/lotus/node" - "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/node/modules/dtypes" - testing2 "github.com/filecoin-project/lotus/node/modules/testing" - "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/mockstorage" - miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - "github.com/ipfs/go-datastore" - "github.com/libp2p/go-libp2p-core/crypto" - "github.com/libp2p/go-libp2p-core/peer" - mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" - "github.com/stretchr/testify/require" -) - -func init() { - chain.BootstrapPeerThreshold = 1 - messagepool.HeadChangeCoalesceMinDelay = time.Microsecond - messagepool.HeadChangeCoalesceMaxDelay = 2 * time.Microsecond - messagepool.HeadChangeCoalesceMergeInterval = 100 * time.Nanosecond -} - -func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd TestFullNode, mn mocknet.Mocknet, opts node.Option) TestMiner { - r := repo.NewMemory(nil) - - lr, err := r.Lock(repo.StorageMiner) - require.NoError(t, err) - - ks, err := lr.KeyStore() - require.NoError(t, err) - - kbytes, err := pk.Bytes() - require.NoError(t, err) - - err = ks.Put("libp2p-host", types.KeyInfo{ - Type: "libp2p-host", - PrivateKey: kbytes, - }) - require.NoError(t, err) - - ds, err := lr.Datastore(context.TODO(), "/metadata") - require.NoError(t, err) - err = ds.Put(datastore.NewKey("miner-address"), act.Bytes()) - require.NoError(t, err) - - nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix)) - for i := 0; i < GenesisPreseals; i++ { - _, err := nic.Next() - require.NoError(t, err) - } - _, err = nic.Next() - require.NoError(t, err) - - err = lr.Close() - require.NoError(t, err) - - peerid, err := peer.IDFromPrivateKey(pk) - require.NoError(t, err) - - enc, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) - require.NoError(t, err) - - msg := &types.Message{ - To: act, - From: waddr, - Method: miner.Methods.ChangePeerID, - Params: enc, - Value: types.NewInt(0), - } - - _, err = tnd.MpoolPushMessage(ctx, msg, nil) - require.NoError(t, err) - - // start node - var minerapi api.StorageMiner - - mineBlock := make(chan lotusminer.MineReq) - stop, err := node.New(ctx, - node.StorageMiner(&minerapi), - node.Online(), - node.Repo(r), - node.Test(), - - node.MockHost(mn), - - node.Override(new(v1api.FullNode), tnd), - node.Override(new(*lotusminer.Miner), lotusminer.NewTestMiner(mineBlock, act)), - - opts, - ) - if err != nil { - t.Fatalf("failed to construct node: %v", err) - } - - t.Cleanup(func() { _ = stop(context.Background()) }) - - /*// Bootstrap with full node - remoteAddrs, err := tnd.NetAddrsListen(Ctx) - require.NoError(t, err) - - err = minerapi.NetConnect(Ctx, remoteAddrs) - require.NoError(t, err)*/ - mineOne := func(ctx context.Context, req lotusminer.MineReq) error { - select { - case mineBlock <- req: - return nil - case <-ctx.Done(): - return ctx.Err() - } - } - - return TestMiner{StorageMiner: minerapi, MineOne: mineOne, Stop: stop} -} - -func storageBuilder(parentNode TestFullNode, mn mocknet.Mocknet, opts node.Option) MinerBuilder { - return func(ctx context.Context, t *testing.T, spt abi.RegisteredSealProof, owner address.Address) TestMiner { - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - - minerPid, err := peer.IDFromPrivateKey(pk) - require.NoError(t, err) - - params, serr := actors.SerializeParams(&power2.CreateMinerParams{ - Owner: owner, - Worker: owner, - SealProofType: spt, - Peer: abi.PeerID(minerPid), - }) - require.NoError(t, serr) - - createStorageMinerMsg := &types.Message{ - To: power.Address, - From: owner, - Value: big.Zero(), - - Method: power.Methods.CreateMiner, - Params: params, - - GasLimit: 0, - GasPremium: big.NewInt(5252), - } - - signed, err := parentNode.MpoolPushMessage(ctx, createStorageMinerMsg, nil) - require.NoError(t, err) - - mw, err := parentNode.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, api.LookbackNoLimit, true) - require.NoError(t, err) - require.Equal(t, exitcode.Ok, mw.Receipt.ExitCode) - - var retval power2.CreateMinerReturn - err = retval.UnmarshalCBOR(bytes.NewReader(mw.Receipt.Return)) - require.NoError(t, err) - - return CreateTestStorageNode(ctx, t, owner, retval.IDAddress, pk, parentNode, mn, opts) - } -} - -func Builder(t *testing.T, fullOpts []FullNodeOpts, storage []StorageMiner) ([]TestFullNode, []TestMiner) { - return mockBuilderOpts(t, fullOpts, storage, false) -} - -func RPCBuilder(t *testing.T, fullOpts []FullNodeOpts, storage []StorageMiner) ([]TestFullNode, []TestMiner) { - return mockBuilderOpts(t, fullOpts, storage, true) -} - -func MockMinerBuilder(t *testing.T, fullOpts []FullNodeOpts, storage []StorageMiner) ([]TestFullNode, []TestMiner) { - return mockMinerBuilderOpts(t, fullOpts, storage, false) -} - -func RPCMockMinerBuilder(t *testing.T, fullOpts []FullNodeOpts, storage []StorageMiner) ([]TestFullNode, []TestMiner) { - return mockMinerBuilderOpts(t, fullOpts, storage, true) -} - -func mockBuilderOpts(t *testing.T, fullOpts []FullNodeOpts, storage []StorageMiner, rpc bool) ([]TestFullNode, []TestMiner) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - mn := mocknet.New(ctx) - - fulls := make([]TestFullNode, len(fullOpts)) - miners := make([]TestMiner, len(storage)) - - // ***** - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - - minerPid, err := peer.IDFromPrivateKey(pk) - require.NoError(t, err) - - var genbuf bytes.Buffer - - if len(storage) > 1 { - panic("need more peer IDs") - } - // ***** - - // PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE - // TODO: would be great if there was a better way to fake the preseals - - var ( - genms []genesis.Miner - maddrs []address.Address - genaccs []genesis.Actor - keys []*wallet.Key - ) - - var presealDirs []string - for i := 0; i < len(storage); i++ { - maddr, err := address.NewIDAddress(genesis2.MinerStart + uint64(i)) - if err != nil { - t.Fatal(err) - } - tdir, err := ioutil.TempDir("", "preseal-memgen") - if err != nil { - t.Fatal(err) - } - genm, k, err := seed.PreSeal(maddr, abi.RegisteredSealProof_StackedDrg2KiBV1, 0, GenesisPreseals, tdir, []byte("make genesis mem random"), nil, true) - if err != nil { - t.Fatal(err) - } - genm.PeerId = minerPid - - wk, err := wallet.NewKey(*k) - if err != nil { - return nil, nil - } - - genaccs = append(genaccs, genesis.Actor{ - Type: genesis.TAccount, - Balance: big.Mul(big.NewInt(400000000), types.NewInt(build.FilecoinPrecision)), - Meta: (&genesis.AccountMeta{Owner: wk.Address}).ActorMeta(), - }) - - keys = append(keys, wk) - presealDirs = append(presealDirs, tdir) - maddrs = append(maddrs, maddr) - genms = append(genms, *genm) - } - - rkhKey, err := wallet.GenerateKey(types.KTSecp256k1) - if err != nil { - return nil, nil - } - - vrk := genesis.Actor{ - Type: genesis.TAccount, - Balance: big.Mul(big.Div(big.NewInt(int64(build.FilBase)), big.NewInt(100)), big.NewInt(int64(build.FilecoinPrecision))), - Meta: (&genesis.AccountMeta{Owner: rkhKey.Address}).ActorMeta(), - } - keys = append(keys, rkhKey) - - templ := &genesis.Template{ - NetworkVersion: network.Version0, - Accounts: genaccs, - Miners: genms, - NetworkName: "test", - Timestamp: uint64(time.Now().Unix() - 10000), // some time sufficiently far in the past - VerifregRootKey: vrk, - RemainderAccount: gen.DefaultRemainderAccountActor, - } - - // END PRESEAL SECTION - - for i := 0; i < len(fullOpts); i++ { - var genesis node.Option - if i == 0 { - genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) - } else { - genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) - } - - stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), - node.Online(), - node.Repo(repo.NewMemory(nil)), - node.MockHost(mn), - node.Test(), - - genesis, - - fullOpts[i].Opts(fulls), - ) - - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { _ = stop(context.Background()) }) - - if rpc { - fulls[i] = fullRpc(t, fulls[i]) - } - - fulls[i].Stb = storageBuilder(fulls[i], mn, node.Options()) - } - - if _, err := fulls[0].FullNode.WalletImport(ctx, &rkhKey.KeyInfo); err != nil { - t.Fatal(err) - } - - for i, def := range storage { - // TODO: support non-bootstrap miners - if i != 0 { - t.Fatal("only one storage node supported") - } - if def.Full != 0 { - t.Fatal("storage nodes only supported on the first full node") - } - - f := fulls[def.Full] - if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil { - t.Fatal(err) - } - if err := f.FullNode.WalletSetDefault(ctx, keys[i].Address); err != nil { - t.Fatal(err) - } - - genMiner := maddrs[i] - wa := genms[i].Worker - - opts := def.Opts - if opts == nil { - opts = node.Options() - } - miners[i] = CreateTestStorageNode(ctx, t, wa, genMiner, pk, f, mn, opts) - if err := miners[i].StorageAddLocal(ctx, presealDirs[i]); err != nil { - t.Fatalf("%+v", err) - } - /* - sma := miners[i].StorageMiner.(*impl.StorageMinerAPI) - - psd := presealDirs[i] - */ - if rpc { - miners[i] = storerRpc(t, miners[i]) - } - } - - if err := mn.LinkAll(); err != nil { - t.Fatal(err) - } - - if len(miners) > 0 { - // Mine 2 blocks to setup some CE stuff in some actors - var wait sync.Mutex - wait.Lock() - - bm := NewBlockMiner(t, miners[0]) - t.Cleanup(bm.Stop) - - bm.MineUntilBlock(ctx, fulls[0], func(epoch abi.ChainEpoch) { - wait.Unlock() - }) - - wait.Lock() - bm.MineUntilBlock(ctx, fulls[0], func(epoch abi.ChainEpoch) { - wait.Unlock() - }) - wait.Lock() - } - - return fulls, miners -} - -func mockMinerBuilderOpts(t *testing.T, fullOpts []FullNodeOpts, storage []StorageMiner, rpc bool) ([]TestFullNode, []TestMiner) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - mn := mocknet.New(ctx) - - fulls := make([]TestFullNode, len(fullOpts)) - miners := make([]TestMiner, len(storage)) - - var genbuf bytes.Buffer - - // PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE - // TODO: would be great if there was a better way to fake the preseals - - var ( - genms []genesis.Miner - genaccs []genesis.Actor - maddrs []address.Address - keys []*wallet.Key - pidKeys []crypto.PrivKey - ) - for i := 0; i < len(storage); i++ { - maddr, err := address.NewIDAddress(genesis2.MinerStart + uint64(i)) - if err != nil { - t.Fatal(err) - } - - preseals := storage[i].Preseal - if preseals == PresealGenesis { - preseals = GenesisPreseals - } - - genm, k, err := mockstorage.PreSeal(abi.RegisteredSealProof_StackedDrg2KiBV1, maddr, preseals) - if err != nil { - t.Fatal(err) - } - - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - - minerPid, err := peer.IDFromPrivateKey(pk) - require.NoError(t, err) - - genm.PeerId = minerPid - - wk, err := wallet.NewKey(*k) - if err != nil { - return nil, nil - } - - genaccs = append(genaccs, genesis.Actor{ - Type: genesis.TAccount, - Balance: big.Mul(big.NewInt(400000000), types.NewInt(build.FilecoinPrecision)), - Meta: (&genesis.AccountMeta{Owner: wk.Address}).ActorMeta(), - }) - - keys = append(keys, wk) - pidKeys = append(pidKeys, pk) - maddrs = append(maddrs, maddr) - genms = append(genms, *genm) - } - - rkhKey, err := wallet.GenerateKey(types.KTSecp256k1) - if err != nil { - return nil, nil - } - - vrk := genesis.Actor{ - Type: genesis.TAccount, - Balance: big.Mul(big.Div(big.NewInt(int64(build.FilBase)), big.NewInt(100)), big.NewInt(int64(build.FilecoinPrecision))), - Meta: (&genesis.AccountMeta{Owner: rkhKey.Address}).ActorMeta(), - } - keys = append(keys, rkhKey) - - templ := &genesis.Template{ - NetworkVersion: network.Version0, - Accounts: genaccs, - Miners: genms, - NetworkName: "test", - Timestamp: uint64(time.Now().Unix()) - (build.BlockDelaySecs * 20000), - VerifregRootKey: vrk, - RemainderAccount: gen.DefaultRemainderAccountActor, - } - - // END PRESEAL SECTION - - for i := 0; i < len(fullOpts); i++ { - var genesis node.Option - if i == 0 { - genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) - } else { - genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) - } - - r := repo.NewMemory(nil) - - stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), - node.Online(), - node.Repo(r), - node.MockHost(mn), - node.Test(), - - node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), - node.Override(new(ffiwrapper.Prover), mock.MockProver), - - // so that we subscribe to pubsub topics immediately - node.Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(true)), - - genesis, - - fullOpts[i].Opts(fulls), - ) - if err != nil { - t.Fatalf("%+v", err) - } - - t.Cleanup(func() { _ = stop(context.Background()) }) - - if rpc { - fulls[i] = fullRpc(t, fulls[i]) - } - - fulls[i].Stb = storageBuilder(fulls[i], mn, node.Options( - node.Override(new(*mock.SectorMgr), func() (*mock.SectorMgr, error) { - return mock.NewMockSectorMgr(nil), nil - }), - - node.Override(new(sectorstorage.SectorManager), node.From(new(*mock.SectorMgr))), - node.Override(new(sectorstorage.Unsealer), node.From(new(*mock.SectorMgr))), - node.Override(new(sectorstorage.PieceProvider), node.From(new(*mock.SectorMgr))), - - node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), - node.Override(new(ffiwrapper.Prover), mock.MockProver), - node.Unset(new(*sectorstorage.Manager)), - )) - } - - if _, err := fulls[0].FullNode.WalletImport(ctx, &rkhKey.KeyInfo); err != nil { - t.Fatal(err) - } - - for i, def := range storage { - // TODO: support non-bootstrap miners - - minerID := abi.ActorID(genesis2.MinerStart + uint64(i)) - - if def.Full != 0 { - t.Fatal("storage nodes only supported on the first full node") - } - - f := fulls[def.Full] - if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil { - return nil, nil - } - if err := f.FullNode.WalletSetDefault(ctx, keys[i].Address); err != nil { - return nil, nil - } - - sectors := make([]abi.SectorID, len(genms[i].Sectors)) - for i, sector := range genms[i].Sectors { - sectors[i] = abi.SectorID{ - Miner: minerID, - Number: sector.SectorID, - } - } - - opts := def.Opts - if opts == nil { - opts = node.Options() - } - miners[i] = CreateTestStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options( - node.Override(new(*mock.SectorMgr), func() (*mock.SectorMgr, error) { - return mock.NewMockSectorMgr(sectors), nil - }), - - node.Override(new(sectorstorage.SectorManager), node.From(new(*mock.SectorMgr))), - node.Override(new(sectorstorage.Unsealer), node.From(new(*mock.SectorMgr))), - node.Override(new(sectorstorage.PieceProvider), node.From(new(*mock.SectorMgr))), - - node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), - node.Override(new(ffiwrapper.Prover), mock.MockProver), - node.Unset(new(*sectorstorage.Manager)), - opts, - )) - - if rpc { - miners[i] = storerRpc(t, miners[i]) - } - } - - if err := mn.LinkAll(); err != nil { - t.Fatal(err) - } - - bm := NewBlockMiner(t, miners[0]) - - if len(miners) > 0 { - // Mine 2 blocks to setup some CE stuff in some actors - var wait sync.Mutex - wait.Lock() - - bm.MineUntilBlock(ctx, fulls[0], func(abi.ChainEpoch) { - wait.Unlock() - }) - wait.Lock() - bm.MineUntilBlock(ctx, fulls[0], func(abi.ChainEpoch) { - wait.Unlock() - }) - wait.Lock() - } - - return fulls, miners -} - -func CreateRPCServer(t *testing.T, handler http.Handler) (*httptest.Server, multiaddr.Multiaddr) { - testServ := httptest.NewServer(handler) - t.Cleanup(testServ.Close) - t.Cleanup(testServ.CloseClientConnections) - - addr := testServ.Listener.Addr() - maddr, err := manet.FromNetAddr(addr) - require.NoError(t, err) - return testServ, maddr -} - -func fullRpc(t *testing.T, nd TestFullNode) TestFullNode { - handler, err := node.FullNodeHandler(nd.FullNode, false) - require.NoError(t, err) - - srv, maddr := CreateRPCServer(t, handler) - - var ret TestFullNode - cl, stop, err := client.NewFullNodeRPCV1(context.Background(), "ws://"+srv.Listener.Addr().String()+"/rpc/v1", nil) - require.NoError(t, err) - t.Cleanup(stop) - ret.ListenAddr, ret.FullNode = maddr, cl - - return ret -} - -func storerRpc(t *testing.T, nd TestMiner) TestMiner { - handler, err := node.MinerHandler(nd.StorageMiner, false) - require.NoError(t, err) - - srv, maddr := CreateRPCServer(t, handler) - - var ret TestMiner - cl, stop, err := client.NewStorageMinerRPCV0(context.Background(), "ws://"+srv.Listener.Addr().String()+"/rpc/v0", nil) - require.NoError(t, err) - t.Cleanup(stop) - - ret.ListenAddr, ret.StorageMiner, ret.MineOne = maddr, cl, nd.MineOne - return ret -} diff --git a/itests/kit2/node_full.go b/itests/kit/node_full.go similarity index 99% rename from itests/kit2/node_full.go rename to itests/kit/node_full.go index 3dadb4d8d..83586e188 100644 --- a/itests/kit2/node_full.go +++ b/itests/kit/node_full.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "context" diff --git a/itests/kit2/node_miner.go b/itests/kit/node_miner.go similarity index 99% rename from itests/kit2/node_miner.go rename to itests/kit/node_miner.go index c4812711d..ee144afee 100644 --- a/itests/kit2/node_miner.go +++ b/itests/kit/node_miner.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "context" diff --git a/itests/kit2/node_opts.go b/itests/kit/node_opts.go similarity index 75% rename from itests/kit2/node_opts.go rename to itests/kit/node_opts.go index c165d9517..0aff634d9 100644 --- a/itests/kit2/node_opts.go +++ b/itests/kit/node_opts.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "github.com/filecoin-project/go-state-types/abi" @@ -14,6 +14,8 @@ import ( // PresealSectors option. const DefaultPresealsPerBootstrapMiner = 2 +const TestSpt = abi.RegisteredSealProof_StackedDrg2KiBV1_1 + // nodeOpts is an options accumulating struct, where functional options are // merged into. type nodeOpts struct { @@ -24,16 +26,23 @@ type nodeOpts struct { ownerKey *wallet.Key extraNodeOpts []node.Option - subsystems MinerSubsystem - mainMiner *TestMiner + subsystems MinerSubsystem + mainMiner *TestMiner + optBuilders []OptBuilder + proofType abi.RegisteredSealProof } // DefaultNodeOpts are the default options that will be applied to test nodes. var DefaultNodeOpts = nodeOpts{ - balance: big.Mul(big.NewInt(100000000), types.NewInt(build.FilecoinPrecision)), - sectors: DefaultPresealsPerBootstrapMiner, + balance: big.Mul(big.NewInt(100000000), types.NewInt(build.FilecoinPrecision)), + sectors: DefaultPresealsPerBootstrapMiner, + proofType: abi.RegisteredSealProof_StackedDrg2KiBV1_1, // default _concrete_ proof type for non-genesis miners (notice the _1) for new actors versions. } +// OptBuilder is used to create an option after some other node is already +// active. Takes all active nodes as a parameter. +type OptBuilder func(activeNodes []*TestFullNode) node.Option + // NodeOpt is a functional option for test nodes. type NodeOpt func(opts *nodeOpts) error @@ -115,3 +124,12 @@ func ConstructorOpts(extra ...node.Option) NodeOpt { return nil } } + +// ProofType sets the proof type for this node. If you're using new actor +// versions, this should be a _1 proof type. +func ProofType(proofType abi.RegisteredSealProof) NodeOpt { + return func(opts *nodeOpts) error { + opts.proofType = proofType + return nil + } +} diff --git a/itests/kit2/node_opts_nv.go b/itests/kit/node_opts_nv.go similarity index 99% rename from itests/kit2/node_opts_nv.go rename to itests/kit/node_opts_nv.go index 39de9d9e2..d4c84b4f1 100644 --- a/itests/kit2/node_opts_nv.go +++ b/itests/kit/node_opts_nv.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "context" @@ -87,5 +87,4 @@ func SDRUpgradeAt(calico, persian abi.ChainEpoch) node.Option { Network: network.Version8, Height: persian, }}) - } diff --git a/itests/kit/nodes.go b/itests/kit/nodes.go deleted file mode 100644 index d9b04166a..000000000 --- a/itests/kit/nodes.go +++ /dev/null @@ -1,153 +0,0 @@ -package kit - -import ( - "context" - "testing" - - "github.com/multiformats/go-multiaddr" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/network" - - lapi "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/v1api" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/miner" - "github.com/filecoin-project/lotus/node" -) - -type MinerBuilder func(context.Context, *testing.T, abi.RegisteredSealProof, address.Address) TestMiner - -type TestFullNode struct { - v1api.FullNode - // ListenAddr is the address on which an API server is listening, if an - // API server is created for this Node - ListenAddr multiaddr.Multiaddr - - Stb MinerBuilder -} - -type TestMiner struct { - lapi.StorageMiner - // ListenAddr is the address on which an API server is listening, if an - // API server is created for this Node - ListenAddr multiaddr.Multiaddr - - MineOne func(context.Context, miner.MineReq) error - Stop func(context.Context) error -} - -var PresealGenesis = -1 - -const GenesisPreseals = 2 - -const TestSpt = abi.RegisteredSealProof_StackedDrg2KiBV1_1 - -// Options for setting up a mock storage Miner -type StorageMiner struct { - Full int - Opts node.Option - Preseal int -} - -type OptionGenerator func([]TestFullNode) node.Option - -// Options for setting up a mock full node -type FullNodeOpts struct { - Lite bool // run node in "lite" mode - Opts OptionGenerator // generate dependency injection options -} - -// APIBuilder is a function which is invoked in test suite to provide -// test nodes and networks -// -// fullOpts array defines options for each full node -// storage array defines storage nodes, numbers in the array specify full node -// index the storage node 'belongs' to -type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner) ([]TestFullNode, []TestMiner) - -func DefaultFullOpts(nFull int) []FullNodeOpts { - full := make([]FullNodeOpts, nFull) - for i := range full { - full[i] = FullNodeOpts{ - Opts: func(nodes []TestFullNode) node.Option { - return node.Options() - }, - } - } - return full -} - -var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} -var OneFull = DefaultFullOpts(1) -var TwoFull = DefaultFullOpts(2) - -var FullNodeWithLatestActorsAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { - // Attention: Update this when introducing new actor versions or your tests will be sad - return FullNodeWithNetworkUpgradeAt(network.Version13, upgradeHeight) -} - -var FullNodeWithNetworkUpgradeAt = func(version network.Version, upgradeHeight abi.ChainEpoch) FullNodeOpts { - fullSchedule := stmgr.UpgradeSchedule{{ - // prepare for upgrade. - Network: network.Version9, - Height: 1, - Migration: stmgr.UpgradeActorsV2, - }, { - Network: network.Version10, - Height: 2, - Migration: stmgr.UpgradeActorsV3, - }, { - Network: network.Version12, - Height: 3, - Migration: stmgr.UpgradeActorsV4, - }, { - Network: network.Version13, - Height: 4, - Migration: stmgr.UpgradeActorsV5, - }} - - schedule := stmgr.UpgradeSchedule{} - for _, upgrade := range fullSchedule { - if upgrade.Network > version { - break - } - - schedule = append(schedule, upgrade) - } - - if upgradeHeight > 0 { - schedule[len(schedule)-1].Height = upgradeHeight - } - - return FullNodeOpts{ - Opts: func(nodes []TestFullNode) node.Option { - return node.Override(new(stmgr.UpgradeSchedule), schedule) - }, - } -} - -var FullNodeWithSDRAt = func(calico, persian abi.ChainEpoch) FullNodeOpts { - return FullNodeOpts{ - Opts: func(nodes []TestFullNode) node.Option { - return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: network.Version6, - Height: 1, - Migration: stmgr.UpgradeActorsV2, - }, { - Network: network.Version7, - Height: calico, - Migration: stmgr.UpgradeCalico, - }, { - Network: network.Version8, - Height: persian, - }}) - }, - } -} - -var MineNext = miner.MineReq{ - InjectNulls: 0, - Done: func(bool, abi.ChainEpoch, error) {}, -} diff --git a/itests/kit/pledge.go b/itests/kit/pledge.go deleted file mode 100644 index 254f87bac..000000000 --- a/itests/kit/pledge.go +++ /dev/null @@ -1,88 +0,0 @@ -package kit - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/stretchr/testify/require" -) - -func PledgeSectors(t *testing.T, ctx context.Context, miner TestMiner, n, existing int, blockNotif <-chan struct{}) { //nolint:golint - toCheck := StartPledge(t, ctx, miner, n, existing, blockNotif) - - for len(toCheck) > 0 { - flushSealingBatches(t, ctx, miner) - - states := map[api.SectorState]int{} - for n := range toCheck { - st, err := miner.SectorsStatus(ctx, n, false) - require.NoError(t, err) - states[st.State]++ - if st.State == api.SectorState(sealing.Proving) { - delete(toCheck, n) - } - if strings.Contains(string(st.State), "Fail") { - t.Fatal("sector in a failed state", st.State) - } - } - - build.Clock.Sleep(100 * time.Millisecond) - fmt.Printf("WaitSeal: %d %+v\n", len(toCheck), states) - } -} - -func flushSealingBatches(t *testing.T, ctx context.Context, miner TestMiner) { //nolint:golint - pcb, err := miner.SectorPreCommitFlush(ctx) - require.NoError(t, err) - if pcb != nil { - fmt.Printf("PRECOMMIT BATCH: %+v\n", pcb) - } - - cb, err := miner.SectorCommitFlush(ctx) - require.NoError(t, err) - if cb != nil { - fmt.Printf("COMMIT BATCH: %+v\n", cb) - } -} - -func StartPledge(t *testing.T, ctx context.Context, miner TestMiner, n, existing int, blockNotif <-chan struct{}) map[abi.SectorNumber]struct{} { //nolint:golint - for i := 0; i < n; i++ { - if i%3 == 0 && blockNotif != nil { - <-blockNotif - t.Log("WAIT") - } - t.Logf("PLEDGING %d", i) - _, err := miner.PledgeSector(ctx) - require.NoError(t, err) - } - - for { - s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM - require.NoError(t, err) - fmt.Printf("Sectors: %d\n", len(s)) - if len(s) >= n+existing { - break - } - - build.Clock.Sleep(100 * time.Millisecond) - } - - fmt.Printf("All sectors is fsm\n") - - s, err := miner.SectorsList(ctx) - require.NoError(t, err) - - toCheck := map[abi.SectorNumber]struct{}{} - for _, number := range s { - toCheck[number] = struct{}{} - } - - return toCheck -} diff --git a/itests/kit2/rpc.go b/itests/kit/rpc.go similarity index 99% rename from itests/kit2/rpc.go rename to itests/kit/rpc.go index 873b64257..dab45df07 100644 --- a/itests/kit2/rpc.go +++ b/itests/kit/rpc.go @@ -1,4 +1,4 @@ -package kit2 +package kit import ( "context" diff --git a/itests/kit2/blockminer.go b/itests/kit2/blockminer.go deleted file mode 100644 index 04d425dd6..000000000 --- a/itests/kit2/blockminer.go +++ /dev/null @@ -1,124 +0,0 @@ -package kit2 - -import ( - "context" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/miner" - "github.com/stretchr/testify/require" -) - -// BlockMiner is a utility that makes a test miner Mine blocks on a timer. -type BlockMiner struct { - t *testing.T - miner *TestMiner - - nextNulls int64 - wg sync.WaitGroup - cancel context.CancelFunc -} - -func NewBlockMiner(t *testing.T, miner *TestMiner) *BlockMiner { - return &BlockMiner{ - t: t, - miner: miner, - cancel: func() {}, - } -} - -func (bm *BlockMiner) MineBlocks(ctx context.Context, blocktime time.Duration) { - time.Sleep(time.Second) - - // wrap context in a cancellable context. - ctx, bm.cancel = context.WithCancel(ctx) - - bm.wg.Add(1) - go func() { - defer bm.wg.Done() - - for { - select { - case <-time.After(blocktime): - case <-ctx.Done(): - return - } - - nulls := atomic.SwapInt64(&bm.nextNulls, 0) - err := bm.miner.MineOne(ctx, miner.MineReq{ - InjectNulls: abi.ChainEpoch(nulls), - Done: func(bool, abi.ChainEpoch, error) {}, - }) - switch { - case err == nil: // wrap around - case ctx.Err() != nil: // context fired. - return - default: // log error - bm.t.Error(err) - } - } - }() -} - -// InjectNulls injects the specified amount of null rounds in the next -// mining rounds. -func (bm *BlockMiner) InjectNulls(rounds abi.ChainEpoch) { - atomic.AddInt64(&bm.nextNulls, int64(rounds)) -} - -func (bm *BlockMiner) MineUntilBlock(ctx context.Context, fn *TestFullNode, cb func(abi.ChainEpoch)) { - for i := 0; i < 1000; i++ { - var ( - success bool - err error - epoch abi.ChainEpoch - wait = make(chan struct{}) - ) - - doneFn := func(win bool, ep abi.ChainEpoch, e error) { - success = win - err = e - epoch = ep - wait <- struct{}{} - } - - mineErr := bm.miner.MineOne(ctx, miner.MineReq{Done: doneFn}) - require.NoError(bm.t, mineErr) - <-wait - - require.NoError(bm.t, err) - - if success { - // Wait until it shows up on the given full nodes ChainHead - nloops := 200 - for i := 0; i < nloops; i++ { - ts, err := fn.ChainHead(ctx) - require.NoError(bm.t, err) - - if ts.Height() == epoch { - break - } - - require.NotEqual(bm.t, i, nloops-1, "block never managed to sync to node") - time.Sleep(time.Millisecond * 10) - } - - if cb != nil { - cb(epoch) - } - return - } - bm.t.Log("did not Mine block, trying again", i) - } - bm.t.Fatal("failed to Mine 1000 times in a row...") -} - -// Stop stops the block miner. -func (bm *BlockMiner) Stop() { - bm.t.Log("shutting down mining") - bm.cancel() - bm.wg.Wait() -} diff --git a/itests/kit2/client.go b/itests/kit2/client.go deleted file mode 100644 index 247d20836..000000000 --- a/itests/kit2/client.go +++ /dev/null @@ -1,146 +0,0 @@ -package kit2 - -import ( - "context" - "fmt" - "io/ioutil" - "math/rand" - "os" - "path/filepath" - "regexp" - "strings" - "testing" - "time" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v2/actors/builtin" - "github.com/stretchr/testify/require" - lcli "github.com/urfave/cli/v2" -) - -// RunClientTest exercises some of the Client CLI commands -func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode TestFullNode) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - // Create mock CLI - mockCLI := NewMockCLI(ctx, t, cmds) - clientCLI := mockCLI.Client(clientNode.ListenAddr) - - // Get the Miner address - addrs, err := clientNode.StateListMiners(ctx, types.EmptyTSK) - require.NoError(t, err) - require.Len(t, addrs, 1) - - minerAddr := addrs[0] - fmt.Println("Miner:", minerAddr) - - // client query-ask - out := clientCLI.RunCmd("client", "query-ask", minerAddr.String()) - require.Regexp(t, regexp.MustCompile("Ask:"), out) - - // Create a deal (non-interactive) - // client deal --start-epoch= 1000000attofil - res, _, _, err := CreateImportFile(ctx, clientNode, 1, 0) - - require.NoError(t, err) - startEpoch := fmt.Sprintf("--start-epoch=%d", 2<<12) - dataCid := res.Root - price := "1000000attofil" - duration := fmt.Sprintf("%d", build.MinDealDuration) - out = clientCLI.RunCmd("client", "deal", startEpoch, dataCid.String(), minerAddr.String(), price, duration) - fmt.Println("client deal", out) - - // Create a deal (interactive) - // client deal - // - // (in days) - // - // "no" (verified Client) - // "yes" (confirm deal) - res, _, _, err = CreateImportFile(ctx, clientNode, 2, 0) - require.NoError(t, err) - dataCid2 := res.Root - duration = fmt.Sprintf("%d", build.MinDealDuration/builtin.EpochsInDay) - cmd := []string{"client", "deal"} - interactiveCmds := []string{ - dataCid2.String(), - duration, - minerAddr.String(), - "no", - "yes", - } - out = clientCLI.RunInteractiveCmd(cmd, interactiveCmds) - fmt.Println("client deal:\n", out) - - // Wait for provider to start sealing deal - dealStatus := "" - for { - // client list-deals - out = clientCLI.RunCmd("client", "list-deals") - fmt.Println("list-deals:\n", out) - - lines := strings.Split(out, "\n") - require.GreaterOrEqual(t, len(lines), 2) - re := regexp.MustCompile(`\s+`) - parts := re.Split(lines[1], -1) - if len(parts) < 4 { - require.Fail(t, "bad list-deals output format") - } - dealStatus = parts[3] - fmt.Println(" Deal status:", dealStatus) - - st := CategorizeDealState(dealStatus) - require.NotEqual(t, TestDealStateFailed, st) - if st == TestDealStateComplete { - break - } - - time.Sleep(time.Second) - } - - // Retrieve the first file from the Miner - // client retrieve - tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-Client") - require.NoError(t, err) - path := filepath.Join(tmpdir, "outfile.dat") - out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path) - fmt.Println("retrieve:\n", out) - require.Regexp(t, regexp.MustCompile("Success"), out) -} - -func CreateImportFile(ctx context.Context, client api.FullNode, rseed int, size int) (res *api.ImportRes, path string, data []byte, err error) { - data, path, err = createRandomFile(rseed, size) - if err != nil { - return nil, "", nil, err - } - - res, err = client.ClientImport(ctx, api.FileRef{Path: path}) - if err != nil { - return nil, "", nil, err - } - return res, path, data, nil -} - -func createRandomFile(rseed, size int) ([]byte, string, error) { - if size == 0 { - size = 1600 - } - data := make([]byte, size) - rand.New(rand.NewSource(int64(rseed))).Read(data) - - dir, err := ioutil.TempDir(os.TempDir(), "test-make-deal-") - if err != nil { - return nil, "", err - } - - path := filepath.Join(dir, "sourcefile.dat") - err = ioutil.WriteFile(path, data, 0644) - if err != nil { - return nil, "", err - } - - return data, path, nil -} diff --git a/itests/kit2/deals.go b/itests/kit2/deals.go deleted file mode 100644 index 87fcca28a..000000000 --- a/itests/kit2/deals.go +++ /dev/null @@ -1,244 +0,0 @@ -package kit2 - -import ( - "context" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" - "github.com/ipld/go-car" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - ipld "github.com/ipfs/go-ipld-format" - dag "github.com/ipfs/go-merkledag" - dstest "github.com/ipfs/go-merkledag/test" - unixfile "github.com/ipfs/go-unixfs/file" -) - -type DealHarness struct { - t *testing.T - client *TestFullNode - storageMiner *TestMiner - marketMiner *TestMiner -} - -// NewDealHarness creates a test harness that contains testing utilities for deals. -func NewDealHarness(t *testing.T, client *TestFullNode, storageMiner *TestMiner, marketMiner *TestMiner) *DealHarness { - return &DealHarness{ - t: t, - client: client, - storageMiner: storageMiner, - marketMiner: marketMiner, - } -} - -// MakeOnlineDeal makes an online deal, generating a random file with the -// supplied seed, and setting the specified fast retrieval flag and start epoch -// on the storage deal. It returns when the deal is sealed. -// -// TODO: convert input parameters to struct, and add size as an input param. -func (dh *DealHarness) MakeOnlineDeal(ctx context.Context, rseed int, fastRet bool, startEpoch abi.ChainEpoch) (deal *cid.Cid, res *api.ImportRes, path string) { - res, path = dh.client.CreateImportFile(ctx, rseed, 0) - - dh.t.Logf("FILE CID: %s", res.Root) - - deal = dh.StartDeal(ctx, res.Root, fastRet, startEpoch) - - // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this - time.Sleep(time.Second) - dh.WaitDealSealed(ctx, deal, false, false, nil) - - return deal, res, path -} - -// StartDeal starts a storage deal between the client and the miner. -func (dh *DealHarness) StartDeal(ctx context.Context, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid { - maddr, err := dh.storageMiner.ActorAddress(ctx) - require.NoError(dh.t, err) - - addr, err := dh.client.WalletDefaultAddress(ctx) - require.NoError(dh.t, err) - - deal, err := dh.client.ClientStartDeal(ctx, &api.StartDealParams{ - Data: &storagemarket.DataRef{ - TransferType: storagemarket.TTGraphsync, - Root: fcid, - }, - Wallet: addr, - Miner: maddr, - EpochPrice: types.NewInt(1000000), - DealStartEpoch: startEpoch, - MinBlocksDuration: uint64(build.MinDealDuration), - FastRetrieval: fastRet, - }) - require.NoError(dh.t, err) - - return deal -} - -// WaitDealSealed waits until the deal is sealed. -func (dh *DealHarness) WaitDealSealed(ctx context.Context, deal *cid.Cid, noseal, noSealStart bool, cb func()) { -loop: - for { - di, err := dh.client.ClientGetDealInfo(ctx, *deal) - require.NoError(dh.t, err) - - switch di.State { - case storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing: - if noseal { - return - } - if !noSealStart { - dh.StartSealingWaiting(ctx) - } - case storagemarket.StorageDealProposalRejected: - dh.t.Fatal("deal rejected") - case storagemarket.StorageDealFailing: - dh.t.Fatal("deal failed") - case storagemarket.StorageDealError: - dh.t.Fatal("deal errored", di.Message) - case storagemarket.StorageDealActive: - dh.t.Log("COMPLETE", di) - break loop - } - - mds, err := dh.marketMiner.MarketListIncompleteDeals(ctx) - require.NoError(dh.t, err) - - var minerState storagemarket.StorageDealStatus - for _, md := range mds { - if md.DealID == di.DealID { - minerState = md.State - break - } - } - - dh.t.Logf("Deal %d state: client:%s provider:%s\n", di.DealID, storagemarket.DealStates[di.State], storagemarket.DealStates[minerState]) - time.Sleep(time.Second / 2) - if cb != nil { - cb() - } - } -} - -// WaitDealPublished waits until the deal is published. -func (dh *DealHarness) WaitDealPublished(ctx context.Context, deal *cid.Cid) { - subCtx, cancel := context.WithCancel(ctx) - defer cancel() - - updates, err := dh.storageMiner.MarketGetDealUpdates(subCtx) - require.NoError(dh.t, err) - - for { - select { - case <-ctx.Done(): - dh.t.Fatal("context timeout") - case di := <-updates: - if deal.Equals(di.ProposalCid) { - switch di.State { - case storagemarket.StorageDealProposalRejected: - dh.t.Fatal("deal rejected") - case storagemarket.StorageDealFailing: - dh.t.Fatal("deal failed") - case storagemarket.StorageDealError: - dh.t.Fatal("deal errored", di.Message) - case storagemarket.StorageDealFinalizing, storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing, storagemarket.StorageDealActive: - dh.t.Log("COMPLETE", di) - return - } - dh.t.Log("Deal state: ", storagemarket.DealStates[di.State]) - } - } - } -} - -func (dh *DealHarness) StartSealingWaiting(ctx context.Context) { - snums, err := dh.storageMiner.SectorsList(ctx) - require.NoError(dh.t, err) - - for _, snum := range snums { - si, err := dh.storageMiner.SectorsStatus(ctx, snum, false) - require.NoError(dh.t, err) - - dh.t.Logf("Sector state: %s", si.State) - if si.State == api.SectorState(sealing.WaitDeals) { - require.NoError(dh.t, dh.storageMiner.SectorStartSealing(ctx, snum)) - } - - dh.storageMiner.FlushSealingBatches(ctx) - } -} - -func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool) (path string) { - // perform retrieval. - info, err := dh.client.ClientGetDealInfo(ctx, *deal) - require.NoError(dh.t, err) - - offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID) - require.NoError(dh.t, err) - require.NotEmpty(dh.t, offers, "no offers") - - carFile, err := ioutil.TempFile(dh.t.TempDir(), "ret-car") - require.NoError(dh.t, err) - - defer carFile.Close() //nolint:errcheck - - caddr, err := dh.client.WalletDefaultAddress(ctx) - require.NoError(dh.t, err) - - ref := &api.FileRef{ - Path: carFile.Name(), - IsCAR: carExport, - } - - updates, err := dh.client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref) - require.NoError(dh.t, err) - - for update := range updates { - require.Emptyf(dh.t, update.Err, "retrieval failed: %s", update.Err) - } - - ret := carFile.Name() - if carExport { - actualFile := dh.ExtractFileFromCAR(ctx, carFile) - ret = actualFile.Name() - _ = actualFile.Close() //nolint:errcheck - } - - return ret -} - -func (dh *DealHarness) ExtractFileFromCAR(ctx context.Context, file *os.File) (out *os.File) { - bserv := dstest.Bserv() - ch, err := car.LoadCar(bserv.Blockstore(), file) - require.NoError(dh.t, err) - - b, err := bserv.GetBlock(ctx, ch.Roots[0]) - require.NoError(dh.t, err) - - nd, err := ipld.Decode(b) - require.NoError(dh.t, err) - - dserv := dag.NewDAGService(bserv) - fil, err := unixfile.NewUnixfsFile(ctx, dserv, nd) - require.NoError(dh.t, err) - - tmpfile, err := ioutil.TempFile(dh.t.TempDir(), "file-in-car") - require.NoError(dh.t, err) - - defer tmpfile.Close() //nolint:errcheck - - err = files.WriteTo(fil, tmpfile.Name()) - require.NoError(dh.t, err) - - return tmpfile -} diff --git a/itests/kit2/funds.go b/itests/kit2/funds.go deleted file mode 100644 index fc4a6c294..000000000 --- a/itests/kit2/funds.go +++ /dev/null @@ -1,34 +0,0 @@ -package kit2 - -import ( - "context" - "testing" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" -) - -// SendFunds sends funds from the default wallet of the specified sender node -// to the recipient address. -func SendFunds(ctx context.Context, t *testing.T, sender TestFullNode, recipient address.Address, amount abi.TokenAmount) { - senderAddr, err := sender.WalletDefaultAddress(ctx) - require.NoError(t, err) - - msg := &types.Message{ - From: senderAddr, - To: recipient, - Value: amount, - } - - sm, err := sender.MpoolPushMessage(ctx, msg, nil) - require.NoError(t, err) - - res, err := sender.StateWaitMsg(ctx, sm.Cid(), 3, api.LookbackNoLimit, true) - require.NoError(t, err) - - require.EqualValues(t, 0, res.Receipt.ExitCode, "did not successfully send funds") -} diff --git a/itests/kit2/init.go b/itests/kit2/init.go deleted file mode 100644 index dfc5a13f2..000000000 --- a/itests/kit2/init.go +++ /dev/null @@ -1,29 +0,0 @@ -package kit2 - -import ( - "fmt" - "os" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors/policy" - logging "github.com/ipfs/go-log/v2" -) - -func init() { - _ = logging.SetLogLevel("*", "INFO") - - policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) - policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) - policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) - - build.InsecurePoStValidation = true - - if err := os.Setenv("BELLMAN_NO_GPU", "1"); err != nil { - panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err)) - } - - if err := os.Setenv("LOTUS_DISABLE_WATCHDOG", "1"); err != nil { - panic(fmt.Sprintf("failed to set LOTUS_DISABLE_WATCHDOG env variable: %s", err)) - } -} diff --git a/itests/kit2/log.go b/itests/kit2/log.go deleted file mode 100644 index f255d0639..000000000 --- a/itests/kit2/log.go +++ /dev/null @@ -1,19 +0,0 @@ -package kit2 - -import ( - "github.com/filecoin-project/lotus/lib/lotuslog" - logging "github.com/ipfs/go-log/v2" -) - -func QuietMiningLogs() { - lotuslog.SetupLogLevels() - - _ = logging.SetLogLevel("miner", "ERROR") // set this to INFO to watch mining happen. - _ = logging.SetLogLevel("chainstore", "ERROR") - _ = logging.SetLogLevel("chain", "ERROR") - _ = logging.SetLogLevel("sub", "ERROR") - _ = logging.SetLogLevel("storageminer", "ERROR") - _ = logging.SetLogLevel("pubsub", "ERROR") - _ = logging.SetLogLevel("gen", "ERROR") - _ = logging.SetLogLevel("dht/RtRefreshManager", "ERROR") -} diff --git a/itests/kit2/mockcli.go b/itests/kit2/mockcli.go deleted file mode 100644 index 592c97333..000000000 --- a/itests/kit2/mockcli.go +++ /dev/null @@ -1,141 +0,0 @@ -package kit2 - -import ( - "bytes" - "context" - "flag" - "strings" - "testing" - - "github.com/multiformats/go-multiaddr" - "github.com/stretchr/testify/require" - lcli "github.com/urfave/cli/v2" -) - -type MockCLI struct { - t *testing.T - cmds []*lcli.Command - cctx *lcli.Context - out *bytes.Buffer -} - -func NewMockCLI(ctx context.Context, t *testing.T, cmds []*lcli.Command) *MockCLI { - // Create a CLI App with an --api-url flag so that we can specify which node - // the command should be executed against - app := &lcli.App{ - Flags: []lcli.Flag{ - &lcli.StringFlag{ - Name: "api-url", - Hidden: true, - }, - }, - Commands: cmds, - } - - var out bytes.Buffer - app.Writer = &out - app.Setup() - - cctx := lcli.NewContext(app, &flag.FlagSet{}, nil) - cctx.Context = ctx - return &MockCLI{t: t, cmds: cmds, cctx: cctx, out: &out} -} - -func (c *MockCLI) Client(addr multiaddr.Multiaddr) *MockCLIClient { - return &MockCLIClient{t: c.t, cmds: c.cmds, addr: addr, cctx: c.cctx, out: c.out} -} - -// MockCLIClient runs commands against a particular node -type MockCLIClient struct { - t *testing.T - cmds []*lcli.Command - addr multiaddr.Multiaddr - cctx *lcli.Context - out *bytes.Buffer -} - -func (c *MockCLIClient) RunCmd(input ...string) string { - out, err := c.RunCmdRaw(input...) - require.NoError(c.t, err, "output:\n%s", out) - - return out -} - -// Given an input, find the corresponding command or sub-command. -// eg "paych add-funds" -func (c *MockCLIClient) cmdByNameSub(input []string) (*lcli.Command, []string) { - name := input[0] - for _, cmd := range c.cmds { - if cmd.Name == name { - return c.findSubcommand(cmd, input[1:]) - } - } - return nil, []string{} -} - -func (c *MockCLIClient) findSubcommand(cmd *lcli.Command, input []string) (*lcli.Command, []string) { - // If there are no sub-commands, return the current command - if len(cmd.Subcommands) == 0 { - return cmd, input - } - - // Check each sub-command for a match against the name - subName := input[0] - for _, subCmd := range cmd.Subcommands { - if subCmd.Name == subName { - // Found a match, recursively search for sub-commands - return c.findSubcommand(subCmd, input[1:]) - } - } - return nil, []string{} -} - -func (c *MockCLIClient) RunCmdRaw(input ...string) (string, error) { - cmd, input := c.cmdByNameSub(input) - if cmd == nil { - panic("Could not find command " + input[0] + " " + input[1]) - } - - // prepend --api-url= - apiFlag := "--api-url=" + c.addr.String() - input = append([]string{apiFlag}, input...) - - fs := c.flagSet(cmd) - err := fs.Parse(input) - require.NoError(c.t, err) - - err = cmd.Action(lcli.NewContext(c.cctx.App, fs, c.cctx)) - - // Get the output - str := strings.TrimSpace(c.out.String()) - c.out.Reset() - return str, err -} - -func (c *MockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet { - // Apply app level flags (so we can process --api-url flag) - fs := &flag.FlagSet{} - for _, f := range c.cctx.App.Flags { - err := f.Apply(fs) - if err != nil { - c.t.Fatal(err) - } - } - // Apply command level flags - for _, f := range cmd.Flags { - err := f.Apply(fs) - if err != nil { - c.t.Fatal(err) - } - } - return fs -} - -func (c *MockCLIClient) RunInteractiveCmd(cmd []string, interactive []string) string { - c.toStdin(strings.Join(interactive, "\n") + "\n") - return c.RunCmd(cmd...) -} - -func (c *MockCLIClient) toStdin(s string) { - c.cctx.App.Metadata["stdin"] = bytes.NewBufferString(s) -} diff --git a/itests/multisig_test.go b/itests/multisig_test.go index 4c513640d..0afdf5f0a 100644 --- a/itests/multisig_test.go +++ b/itests/multisig_test.go @@ -3,7 +3,6 @@ package itests import ( "context" "fmt" - "os" "regexp" "strings" "testing" @@ -18,17 +17,16 @@ import ( // TestMultisig does a basic test to exercise the multisig CLI commands func TestMultisig(t *testing.T) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") kit.QuietMiningLogs() - blocktime := 5 * time.Millisecond - ctx := context.Background() - clientNode, _ := kit.StartOneNodeOneMiner(ctx, t, blocktime) + blockTime := 5 * time.Millisecond + client, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ThroughRPC()) + ens.InterconnectAll().BeginMining(blockTime) - runMultisigTests(t, clientNode) + runMultisigTests(t, client) } -func runMultisigTests(t *testing.T, clientNode kit.TestFullNode) { +func runMultisigTests(t *testing.T, clientNode *kit.TestFullNode) { // Create mock CLI ctx := context.Background() mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) diff --git a/itests/paych_api_test.go b/itests/paych_api_test.go index 8fb4cde0c..668eb14aa 100644 --- a/itests/paych_api_test.go +++ b/itests/paych_api_test.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/itests/kit" "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" @@ -23,22 +24,21 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/itests/kit2" ) func TestPaymentChannelsAPI(t *testing.T) { - kit2.QuietMiningLogs() + kit.QuietMiningLogs() ctx := context.Background() blockTime := 5 * time.Millisecond var ( - paymentCreator kit2.TestFullNode - paymentReceiver kit2.TestFullNode - miner kit2.TestMiner + paymentCreator kit.TestFullNode + paymentReceiver kit.TestFullNode + miner kit.TestMiner ) - ens := kit2.NewEnsemble(t, kit2.MockProofs()). + ens := kit.NewEnsemble(t, kit.MockProofs()). FullNode(&paymentCreator). FullNode(&paymentReceiver). Miner(&miner, &paymentCreator). @@ -51,7 +51,7 @@ func TestPaymentChannelsAPI(t *testing.T) { receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - kit2.SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18)) + kit.SendFunds(ctx, t, &paymentCreator, receiverAddr, abi.NewTokenAmount(1e18)) // setup the payment channel createrAddr, err := paymentCreator.WalletDefaultAddress(ctx) @@ -200,7 +200,7 @@ func TestPaymentChannelsAPI(t *testing.T) { require.EqualValues(t, abi.NewTokenAmount(expectedRefund), delta, "did not send correct funds from creator: expected %d, got %d", expectedRefund, delta) } -func waitForBlocks(ctx context.Context, t *testing.T, bm *kit2.BlockMiner, paymentReceiver kit2.TestFullNode, receiverAddr address.Address, count int) { +func waitForBlocks(ctx context.Context, t *testing.T, bm *kit.BlockMiner, paymentReceiver kit.TestFullNode, receiverAddr address.Address, count int) { // We need to add null blocks in batches, if we add too many the chain can't sync batchSize := 60 for i := 0; i < count; i += batchSize { @@ -225,7 +225,7 @@ func waitForBlocks(ctx context.Context, t *testing.T, bm *kit2.BlockMiner, payme } } -func waitForMessage(ctx context.Context, t *testing.T, paymentCreator kit2.TestFullNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup { +func waitForMessage(ctx context.Context, t *testing.T, paymentCreator kit.TestFullNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup { ctx, cancel := context.WithTimeout(ctx, duration) defer cancel() diff --git a/itests/paych_cli_test.go b/itests/paych_cli_test.go index 2450828d3..8a0690449 100644 --- a/itests/paych_cli_test.go +++ b/itests/paych_cli_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -37,19 +37,19 @@ func init() { // commands func TestPaymentChannelsBasic(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - kit2.QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() var ( - paymentCreator kit2.TestFullNode - paymentReceiver kit2.TestFullNode + paymentCreator kit.TestFullNode + paymentReceiver kit.TestFullNode ) creatorAddr, receiverAddr := startPaychCreatorReceiverMiner(ctx, t, &paymentCreator, &paymentReceiver, blocktime) // Create mock CLI - mockCLI := kit2.NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) receiverCLI := mockCLI.Client(paymentReceiver.ListenAddr) @@ -94,18 +94,18 @@ type voucherSpec struct { // TestPaymentChannelStatus tests the payment channel status CLI command func TestPaymentChannelStatus(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - kit2.QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() var ( - paymentCreator kit2.TestFullNode - paymentReceiver kit2.TestFullNode + paymentCreator kit.TestFullNode + paymentReceiver kit.TestFullNode ) creatorAddr, receiverAddr := startPaychCreatorReceiverMiner(ctx, t, &paymentCreator, &paymentReceiver, blocktime) // Create mock CLI - mockCLI := kit2.NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) // creator: paych status-by-from-to @@ -174,18 +174,18 @@ func TestPaymentChannelStatus(t *testing.T) { // channel voucher commands func TestPaymentChannelVouchers(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - kit2.QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() var ( - paymentCreator kit2.TestFullNode - paymentReceiver kit2.TestFullNode + paymentCreator kit.TestFullNode + paymentReceiver kit.TestFullNode ) creatorAddr, receiverAddr := startPaychCreatorReceiverMiner(ctx, t, &paymentCreator, &paymentReceiver, blocktime) // Create mock CLI - mockCLI := kit2.NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) receiverCLI := mockCLI.Client(paymentReceiver.ListenAddr) @@ -306,18 +306,18 @@ func TestPaymentChannelVouchers(t *testing.T) { // is greater than what's left in the channel, voucher create fails func TestPaymentChannelVoucherCreateShortfall(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - kit2.QuietMiningLogs() + kit.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() var ( - paymentCreator kit2.TestFullNode - paymentReceiver kit2.TestFullNode + paymentCreator kit.TestFullNode + paymentReceiver kit.TestFullNode ) creatorAddr, receiverAddr := startPaychCreatorReceiverMiner(ctx, t, &paymentCreator, &paymentReceiver, blocktime) // Create mock CLI - mockCLI := kit2.NewMockCLI(ctx, t, cli.Commands) + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) creatorCLI := mockCLI.Client(paymentCreator.ListenAddr) // creator: paych add-funds @@ -385,7 +385,7 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { } // waitForHeight waits for the node to reach the given chain epoch -func waitForHeight(ctx context.Context, t *testing.T, node kit2.TestFullNode, height abi.ChainEpoch) { +func waitForHeight(ctx context.Context, t *testing.T, node kit.TestFullNode, height abi.ChainEpoch) { atHeight := make(chan struct{}) chainEvents := events.NewEvents(ctx, node) err := chainEvents.ChainAt(func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { @@ -403,7 +403,7 @@ func waitForHeight(ctx context.Context, t *testing.T, node kit2.TestFullNode, he } // getPaychState gets the state of the payment channel with the given address -func getPaychState(ctx context.Context, t *testing.T, node kit2.TestFullNode, chAddr address.Address) paych.State { +func getPaychState(ctx context.Context, t *testing.T, node kit.TestFullNode, chAddr address.Address) paych.State { act, err := node.StateGetActor(ctx, chAddr, types.EmptyTSK) require.NoError(t, err) @@ -414,10 +414,10 @@ func getPaychState(ctx context.Context, t *testing.T, node kit2.TestFullNode, ch return chState } -func startPaychCreatorReceiverMiner(ctx context.Context, t *testing.T, paymentCreator *kit2.TestFullNode, paymentReceiver *kit2.TestFullNode, blocktime time.Duration) (address.Address, address.Address) { - var miner kit2.TestMiner - opts := kit2.ThroughRPC() - kit2.NewEnsemble(t, kit2.MockProofs()). +func startPaychCreatorReceiverMiner(ctx context.Context, t *testing.T, paymentCreator *kit.TestFullNode, paymentReceiver *kit.TestFullNode, blocktime time.Duration) (address.Address, address.Address) { + var miner kit.TestMiner + opts := kit.ThroughRPC() + kit.NewEnsemble(t, kit.MockProofs()). FullNode(paymentCreator, opts). FullNode(paymentReceiver, opts). Miner(&miner, paymentCreator). @@ -428,7 +428,7 @@ func startPaychCreatorReceiverMiner(ctx context.Context, t *testing.T, paymentCr // Send some funds to the second node receiverAddr, err := paymentReceiver.WalletDefaultAddress(ctx) require.NoError(t, err) - kit2.SendFunds(ctx, t, *paymentCreator, receiverAddr, abi.NewTokenAmount(1e18)) + kit.SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18)) // Get the first node's address creatorAddr, err := paymentCreator.WalletDefaultAddress(ctx) diff --git a/itests/sdr_upgrade_test.go b/itests/sdr_upgrade_test.go index 008c2ce61..3aa685b09 100644 --- a/itests/sdr_upgrade_test.go +++ b/itests/sdr_upgrade_test.go @@ -10,14 +10,14 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" bminer "github.com/filecoin-project/lotus/miner" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSDRUpgrade(t *testing.T) { - kit2.QuietMiningLogs() + kit.QuietMiningLogs() // oldDelay := policy.GetPreCommitChallengeDelay() // policy.SetPreCommitChallengeDelay(5) @@ -30,8 +30,8 @@ func TestSDRUpgrade(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - opts := kit2.ConstructorOpts(kit2.SDRUpgradeAt(500, 1000)) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), opts) + opts := kit.ConstructorOpts(kit.SDRUpgradeAt(500, 1000)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) ens.InterconnectAll() build.Clock.Sleep(time.Second) diff --git a/itests/sector_pledge_test.go b/itests/sector_pledge_test.go index 8e87f2658..d911dcb68 100644 --- a/itests/sector_pledge_test.go +++ b/itests/sector_pledge_test.go @@ -7,16 +7,16 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/itests/kit2" ) func TestPledgeSectors(t *testing.T) { - kit2.QuietMiningLogs() + kit.QuietMiningLogs() blockTime := 50 * time.Millisecond @@ -24,7 +24,7 @@ func TestPledgeSectors(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs()) + _, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) ens.InterconnectAll().BeginMining(blockTime) miner.PledgeSectors(ctx, nSectors, 0, nil) @@ -54,11 +54,11 @@ func TestPledgeBatching(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - opts := kit2.ConstructorOpts(kit2.LatestActorsAt(-1)) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), opts) + opts := kit.ConstructorOpts(kit.LatestActorsAt(-1)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) ens.InterconnectAll().BeginMining(blockTime) - client.WaitTillChain(ctx, kit2.HeightAtLeast(10)) + client.WaitTillChain(ctx, kit.HeightAtLeast(10)) toCheck := miner.StartPledge(ctx, nSectors, 0, nil) @@ -111,11 +111,11 @@ func TestPledgeBeforeNv13(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - opts := kit2.ConstructorOpts(kit2.LatestActorsAt(1000000000)) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), opts) + opts := kit.ConstructorOpts(kit.LatestActorsAt(1000000000)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) ens.InterconnectAll().BeginMining(blocktime) - client.WaitTillChain(ctx, kit2.HeightAtLeast(10)) + client.WaitTillChain(ctx, kit.HeightAtLeast(10)) toCheck := miner.StartPledge(ctx, nSectors, 0, nil) diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index faf12228c..94d438437 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" ) @@ -19,7 +19,7 @@ func TestTerminate(t *testing.T) { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") } - kit2.QuietMiningLogs() + kit.QuietMiningLogs() const blocktime = 2 * time.Millisecond @@ -28,8 +28,8 @@ func TestTerminate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - opts := kit2.ConstructorOpts(kit2.LatestActorsAt(-1)) - client, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), opts) + opts := kit.ConstructorOpts(kit.LatestActorsAt(-1)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) ens.InterconnectAll().BeginMining(blocktime) maddr, err := miner.ActorAddress(ctx) @@ -57,7 +57,7 @@ func TestTerminate(t *testing.T) { waitUntil := di.PeriodStart + di.WPoStProvingPeriod + 2 t.Logf("End for head.Height > %d", waitUntil) - ts := client.WaitTillChain(ctx, kit2.HeightAtLeast(waitUntil)) + ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) t.Logf("Now head.Height = %d", ts.Height()) } @@ -140,7 +140,7 @@ loop: waitUntil := di.PeriodStart + di.WPoStProvingPeriod + 2 t.Logf("End for head.Height > %d", waitUntil) - ts := client.WaitTillChain(ctx, kit2.HeightAtLeast(waitUntil)) + ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) t.Logf("Now head.Height = %d", ts.Height()) p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK) diff --git a/itests/tape_test.go b/itests/tape_test.go index 6fb3def15..08970152f 100644 --- a/itests/tape_test.go +++ b/itests/tape_test.go @@ -10,13 +10,13 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/itests/kit2" + "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/stretchr/testify/require" ) func TestTapeFix(t *testing.T) { - kit2.QuietMiningLogs() + kit.QuietMiningLogs() var blocktime = 2 * time.Millisecond @@ -42,8 +42,8 @@ func testTapeFix(t *testing.T, blocktime time.Duration, after bool) { }) } - nopts := kit2.ConstructorOpts(node.Override(new(stmgr.UpgradeSchedule), upgradeSchedule)) - _, miner, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), nopts) + nopts := kit.ConstructorOpts(node.Override(new(stmgr.UpgradeSchedule), upgradeSchedule)) + _, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), nopts) ens.InterconnectAll().BeginMining(blocktime) sid, err := miner.PledgeSector(ctx) diff --git a/itests/verifreg_test.go b/itests/verifreg_test.go index 108da6ecf..28a72263e 100644 --- a/itests/verifreg_test.go +++ b/itests/verifreg_test.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/itests/kit" verifreg4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg" "github.com/stretchr/testify/require" @@ -18,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/itests/kit2" "github.com/filecoin-project/lotus/node/impl" ) @@ -39,10 +39,10 @@ func TestVerifiedClientTopUp(t *testing.T) { bal, err := types.ParseFIL("100fil") require.NoError(t, err) - node, _, ens := kit2.EnsembleMinimal(t, kit2.MockProofs(), - kit2.RootVerifier(rootKey, abi.NewTokenAmount(bal.Int64())), - kit2.Account(verifierKey, abi.NewTokenAmount(bal.Int64())), // assign some balance to the verifier so they can send an AddClient message. - kit2.ConstructorOpts(kit2.InstantaneousNetworkVersion(nv))) + node, _, ens := kit.EnsembleMinimal(t, kit.MockProofs(), + kit.RootVerifier(rootKey, abi.NewTokenAmount(bal.Int64())), + kit.Account(verifierKey, abi.NewTokenAmount(bal.Int64())), // assign some balance to the verifier so they can send an AddClient message. + kit.ConstructorOpts(kit.InstantaneousNetworkVersion(nv))) ens.InterconnectAll().BeginMining(blockTime) diff --git a/itests/wdpost_dispute_test.go b/itests/wdpost_dispute_test.go index 6c7302af3..bf1a01e60 100644 --- a/itests/wdpost_dispute_test.go +++ b/itests/wdpost_dispute_test.go @@ -2,7 +2,6 @@ package itests import ( "context" - "fmt" "os" "testing" "time" @@ -17,7 +16,6 @@ import ( minerActor "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" - "github.com/filecoin-project/lotus/node/impl" proof3 "github.com/filecoin-project/specs-actors/v3/actors/runtime/proof" "github.com/stretchr/testify/require" ) @@ -29,26 +27,27 @@ func TestWindowPostDispute(t *testing.T) { kit.QuietMiningLogs() - b := kit.MockMinerBuilder blocktime := 2 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) defer cancel() + var ( + client kit.TestFullNode + chainMiner kit.TestMiner + evilMiner kit.TestMiner + ) + // First, we configure two miners. After sealing, we're going to turn off the first miner so // it doesn't submit proofs. // // Then we're going to manually submit bad proofs. - n, sn := b(t, - []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1)}, - []kit.StorageMiner{ - {Full: 0, Preseal: kit.PresealGenesis}, - {Full: 0}, - }) - - client := n[0].FullNode.(*impl.FullNodeAPI) - chainMiner := sn[0] - evilMiner := sn[1] + opts := kit.ConstructorOpts(kit.LatestActorsAt(-1)) + ens := kit.NewEnsemble(t, kit.MockProofs()). + FullNode(&client, opts). + Miner(&chainMiner, &client, opts). + Miner(&evilMiner, &client, opts, kit.PresealSectors(0)). + Start() { addrinfo, err := client.NetAddrsListen(ctx) @@ -68,32 +67,13 @@ func TestWindowPostDispute(t *testing.T) { defaultFrom, err := client.WalletDefaultAddress(ctx) require.NoError(t, err) - build.Clock.Sleep(time.Second) - // Mine with the _second_ node (the good one). - done := make(chan struct{}) - go func() { - defer close(done) - for ctx.Err() == nil { - build.Clock.Sleep(blocktime) - if err := chainMiner.MineOne(ctx, kit.MineNext); err != nil { - if ctx.Err() != nil { - // context was canceled, ignore the error. - return - } - t.Error(err) - } - } - }() - defer func() { - cancel() - <-done - }() + ens.BeginMining(blocktime, &chainMiner) // Give the chain miner enough sectors to win every block. - kit.PledgeSectors(t, ctx, chainMiner, 10, 0, nil) + chainMiner.PledgeSectors(ctx, 10, 0, nil) // And the evil one 1 sector. No cookie for you. - kit.PledgeSectors(t, ctx, evilMiner, 1, 0, nil) + evilMiner.PledgeSectors(ctx, 1, 0, nil) // Let the evil miner's sectors gain power. evilMinerAddr, err := evilMiner.ActorAddress(ctx) @@ -102,19 +82,13 @@ func TestWindowPostDispute(t *testing.T) { di, err := client.StateMinerProvingDeadline(ctx, evilMinerAddr, types.EmptyTSK) require.NoError(t, err) - fmt.Printf("Running one proving period\n") - fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod*2) + t.Logf("Running one proving period\n") - for { - head, err := client.ChainHead(ctx) - require.NoError(t, err) + waitUntil := di.PeriodStart + di.WPoStProvingPeriod*2 + t.Logf("End for head.Height > %d", waitUntil) - if head.Height() > di.PeriodStart+di.WPoStProvingPeriod*2 { - fmt.Printf("Now head.Height = %d\n", head.Height()) - break - } - build.Clock.Sleep(blocktime) - } + ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) p, err := client.StateMinerPower(ctx, evilMinerAddr, types.EmptyTSK) require.NoError(t, err) @@ -131,12 +105,12 @@ func TestWindowPostDispute(t *testing.T) { evilSectorLoc, err := client.StateSectorPartition(ctx, evilMinerAddr, evilSectorNo, types.EmptyTSK) require.NoError(t, err) - fmt.Println("evil miner stopping") + t.Log("evil miner stopping") // Now stop the evil miner, and start manually submitting bad proofs. require.NoError(t, evilMiner.Stop(ctx)) - fmt.Println("evil miner stopped") + t.Log("evil miner stopped") // Wait until we need to prove our sector. for { @@ -148,7 +122,7 @@ func TestWindowPostDispute(t *testing.T) { build.Clock.Sleep(blocktime) } - err = submitBadProof(ctx, client, evilMinerAddr, di, evilSectorLoc.Deadline, evilSectorLoc.Partition) + err = submitBadProof(ctx, client, evilMiner.OwnerKey.Address, evilMinerAddr, di, evilSectorLoc.Deadline, evilSectorLoc.Partition) require.NoError(t, err, "evil proof not accepted") // Wait until after the proving period. @@ -161,7 +135,7 @@ func TestWindowPostDispute(t *testing.T) { build.Clock.Sleep(blocktime) } - fmt.Println("accepted evil proof") + t.Log("accepted evil proof") // Make sure the evil node didn't lose any power. p, err = client.StateMinerPower(ctx, evilMinerAddr, types.EmptyTSK) @@ -188,7 +162,7 @@ func TestWindowPostDispute(t *testing.T) { sm, err := client.MpoolPushMessage(ctx, msg, nil) require.NoError(t, err) - fmt.Println("waiting dispute") + t.Log("waiting dispute") rec, err := client.StateWaitMsg(ctx, sm.Cid(), build.MessageConfidence, api.LookbackNoLimit, true) require.NoError(t, err) require.Zero(t, rec.Receipt.ExitCode, "dispute not accepted: %s", rec.Receipt.ExitCode.Error()) @@ -246,7 +220,7 @@ func TestWindowPostDispute(t *testing.T) { } // Now try to be evil again - err = submitBadProof(ctx, client, evilMinerAddr, di, evilSectorLoc.Deadline, evilSectorLoc.Partition) + err = submitBadProof(ctx, client, evilMiner.OwnerKey.Address, evilMinerAddr, di, evilSectorLoc.Deadline, evilSectorLoc.Partition) require.Error(t, err) require.Contains(t, err.Error(), "message execution failed: exit 16, reason: window post failed: invalid PoSt") @@ -260,27 +234,14 @@ func TestWindowPostDisputeFails(t *testing.T) { kit.QuietMiningLogs() - b := kit.MockMinerBuilder blocktime := 2 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, []kit.FullNodeOpts{kit.FullNodeWithLatestActorsAt(-1)}, kit.OneMiner) - - client := n[0].FullNode.(*impl.FullNodeAPI) - miner := sn[0] - - { - addrinfo, err := client.NetAddrsListen(ctx) - if err != nil { - t.Fatal(err) - } - - if err := miner.NetConnect(ctx, addrinfo); err != nil { - t.Fatal(err) - } - } + opts := kit.ConstructorOpts(kit.LatestActorsAt(-1)) + client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts) + ens.InterconnectAll().BeginMining(blocktime) defaultFrom, err := client.WalletDefaultAddress(ctx) require.NoError(t, err) @@ -290,48 +251,21 @@ func TestWindowPostDisputeFails(t *testing.T) { build.Clock.Sleep(time.Second) - // Mine with the _second_ node (the good one). - done := make(chan struct{}) - go func() { - defer close(done) - for ctx.Err() == nil { - build.Clock.Sleep(blocktime) - if err := miner.MineOne(ctx, kit.MineNext); err != nil { - if ctx.Err() != nil { - // context was canceled, ignore the error. - return - } - t.Error(err) - } - } - }() - defer func() { - cancel() - <-done - }() - - kit.PledgeSectors(t, ctx, miner, 10, 0, nil) + miner.PledgeSectors(ctx, 10, 0, nil) di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) require.NoError(t, err) - fmt.Printf("Running one proving period\n") - fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod*2) + t.Log("Running one proving period") + waitUntil := di.PeriodStart + di.WPoStProvingPeriod*2 + t.Logf("End for head.Height > %d", waitUntil) - for { - head, err := client.ChainHead(ctx) - require.NoError(t, err) - - if head.Height() > di.PeriodStart+di.WPoStProvingPeriod*2 { - fmt.Printf("Now head.Height = %d\n", head.Height()) - break - } - build.Clock.Sleep(blocktime) - } + ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) ssz, err := miner.ActorSectorSize(ctx, maddr) require.NoError(t, err) - expectedPower := types.NewInt(uint64(ssz) * (kit.GenesisPreseals + 10)) + expectedPower := types.NewInt(uint64(ssz) * (kit.DefaultPresealsPerBootstrapMiner + 10)) p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK) require.NoError(t, err) @@ -393,7 +327,7 @@ waitForProof: func submitBadProof( ctx context.Context, - client api.FullNode, maddr address.Address, + client api.FullNode, owner address.Address, maddr address.Address, di *dline.Info, dlIdx, partIdx uint64, ) error { head, err := client.ChainHead(ctx) @@ -401,11 +335,6 @@ func submitBadProof( return err } - from, err := client.WalletDefaultAddress(ctx) - if err != nil { - return err - } - minerInfo, err := client.StateMinerInfo(ctx, maddr, head.Key()) if err != nil { return err @@ -440,7 +369,7 @@ func submitBadProof( Method: minerActor.Methods.SubmitWindowedPoSt, Params: enc, Value: types.NewInt(0), - From: from, + From: owner, } sm, err := client.MpoolPushMessage(ctx, msg, nil) if err != nil { diff --git a/itests/wdpost_test.god b/itests/wdpost_test.go similarity index 100% rename from itests/wdpost_test.god rename to itests/wdpost_test.go diff --git a/markets/pricing/cli.go b/markets/pricing/cli.go new file mode 100644 index 000000000..3c2a5f248 --- /dev/null +++ b/markets/pricing/cli.go @@ -0,0 +1,48 @@ +package pricing + +import ( + "bytes" + "context" + "encoding/json" + "os/exec" + + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "golang.org/x/xerrors" +) + +func ExternalRetrievalPricingFunc(cmd string) dtypes.RetrievalPricingFunc { + return func(ctx context.Context, pricingInput retrievalmarket.PricingInput) (retrievalmarket.Ask, error) { + return runPricingFunc(ctx, cmd, pricingInput) + } +} + +func runPricingFunc(_ context.Context, cmd string, params interface{}) (retrievalmarket.Ask, error) { + j, err := json.Marshal(params) + if err != nil { + return retrievalmarket.Ask{}, err + } + + var out bytes.Buffer + var errb bytes.Buffer + + c := exec.Command("sh", "-c", cmd) + c.Stdin = bytes.NewReader(j) + c.Stdout = &out + c.Stderr = &errb + + switch err := c.Run().(type) { + case nil: + bz := out.Bytes() + resp := retrievalmarket.Ask{} + + if err := json.Unmarshal(bz, &resp); err != nil { + return resp, xerrors.Errorf("failed to parse pricing output %s, err=%w", string(bz), err) + } + return resp, nil + case *exec.ExitError: + return retrievalmarket.Ask{}, xerrors.Errorf("pricing func exited with error: %s", errb.String()) + default: + return retrievalmarket.Ask{}, xerrors.Errorf("pricing func cmd run error: %w", err) + } +} diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 39640c03f..6b74bde07 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -101,3 +101,67 @@ func (rpn *retrievalProviderNode) GetChainHead(ctx context.Context) (shared.TipS return head.Key().Bytes(), head.Height(), nil } + +func (rpn *retrievalProviderNode) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) { + si, err := rpn.miner.GetSectorInfo(sectorID) + if err != nil { + return false, xerrors.Errorf("failed to get sectorinfo, err=%s", err) + } + + mid, err := address.IDFromAddress(rpn.miner.Address()) + if err != nil { + return false, err + } + + ref := specstorage.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: sectorID, + }, + ProofType: si.SectorType, + } + + log.Debugf("will call IsUnsealed now sector=%+v, offset=%d, size=%d", sectorID, offset, length) + return rpn.pp.IsUnsealed(ctx, ref, storiface.UnpaddedByteIndex(offset), length) +} + +// GetRetrievalPricingInput takes a set of candidate storage deals that can serve a retrieval request, +// and returns an minimally populated PricingInput. This PricingInput should be enhanced +// with more data, and passed to the pricing function to determine the final quoted price. +func (rpn *retrievalProviderNode) GetRetrievalPricingInput(ctx context.Context, pieceCID cid.Cid, storageDeals []abi.DealID) (retrievalmarket.PricingInput, error) { + resp := retrievalmarket.PricingInput{} + + head, err := rpn.full.ChainHead(ctx) + if err != nil { + return resp, xerrors.Errorf("failed to get chain head: %w", err) + } + tsk := head.Key() + + for _, dealID := range storageDeals { + ds, err := rpn.full.StateMarketStorageDeal(ctx, dealID, tsk) + if err != nil { + return resp, xerrors.Errorf("failed to look up deal %d on chain: err=%w", dealID, err) + } + if ds.Proposal.VerifiedDeal { + resp.VerifiedDeal = true + } + + if ds.Proposal.PieceCID.Equals(pieceCID) { + resp.PieceSize = ds.Proposal.PieceSize.Unpadded() + } + + // If we've discovered a verified deal with the required PieceCID, we don't need + // to lookup more deals and we're done. + if resp.VerifiedDeal && resp.PieceSize != 0 { + break + } + } + + // Note: The piece size can never actually be zero. We only use it to here + // to assert that we didn't find a matching piece. + if resp.PieceSize == 0 { + return resp, xerrors.New("failed to find matching piece") + } + + return resp, nil +} diff --git a/markets/retrievaladapter/provider_test.go b/markets/retrievaladapter/provider_test.go new file mode 100644 index 000000000..5cdf5d060 --- /dev/null +++ b/markets/retrievaladapter/provider_test.go @@ -0,0 +1,151 @@ +package retrievaladapter + +import ( + "context" + "testing" + + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + testnet "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/mocks" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/types" + "github.com/golang/mock/gomock" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" +) + +func TestGetPricingInput(t *testing.T) { + ctx := context.Background() + tsk := &types.TipSet{} + key := tsk.Key() + + pcid := testnet.GenerateCids(1)[0] + deals := []abi.DealID{1, 2} + paddedSize := abi.PaddedPieceSize(128) + unpaddedSize := paddedSize.Unpadded() + + tcs := map[string]struct { + pieceCid cid.Cid + deals []abi.DealID + fFnc func(node *mocks.MockFullNode) + + expectedErrorStr string + expectedVerified bool + expectedPieceSize abi.UnpaddedPieceSize + }{ + "error when fails to fetch chain head": { + fFnc: func(n *mocks.MockFullNode) { + n.EXPECT().ChainHead(gomock.Any()).Return(tsk, xerrors.New("chain head error")).Times(1) + }, + expectedErrorStr: "chain head error", + }, + + "error when no piece matches": { + fFnc: func(n *mocks.MockFullNode) { + out1 := &api.MarketDeal{ + Proposal: market.DealProposal{ + PieceCID: testnet.GenerateCids(1)[0], + }, + } + out2 := &api.MarketDeal{ + Proposal: market.DealProposal{ + PieceCID: testnet.GenerateCids(1)[0], + }, + } + + n.EXPECT().ChainHead(gomock.Any()).Return(tsk, nil).Times(1) + gomock.InOrder( + n.EXPECT().StateMarketStorageDeal(gomock.Any(), deals[0], key).Return(out1, nil), + n.EXPECT().StateMarketStorageDeal(gomock.Any(), deals[1], key).Return(out2, nil), + ) + + }, + expectedErrorStr: "failed to find matching piece", + }, + + "verified is true even if one deal is verified and we get the correct piecesize": { + fFnc: func(n *mocks.MockFullNode) { + out1 := &api.MarketDeal{ + Proposal: market.DealProposal{ + PieceCID: pcid, + PieceSize: paddedSize, + }, + } + out2 := &api.MarketDeal{ + Proposal: market.DealProposal{ + PieceCID: testnet.GenerateCids(1)[0], + VerifiedDeal: true, + }, + } + + n.EXPECT().ChainHead(gomock.Any()).Return(tsk, nil).Times(1) + gomock.InOrder( + n.EXPECT().StateMarketStorageDeal(gomock.Any(), deals[0], key).Return(out1, nil), + n.EXPECT().StateMarketStorageDeal(gomock.Any(), deals[1], key).Return(out2, nil), + ) + + }, + expectedPieceSize: unpaddedSize, + expectedVerified: true, + }, + + "verified is false if both deals are unverified and we get the correct piece size": { + fFnc: func(n *mocks.MockFullNode) { + out1 := &api.MarketDeal{ + Proposal: market.DealProposal{ + PieceCID: pcid, + PieceSize: paddedSize, + VerifiedDeal: false, + }, + } + out2 := &api.MarketDeal{ + Proposal: market.DealProposal{ + PieceCID: testnet.GenerateCids(1)[0], + VerifiedDeal: false, + }, + } + + n.EXPECT().ChainHead(gomock.Any()).Return(tsk, nil).Times(1) + gomock.InOrder( + n.EXPECT().StateMarketStorageDeal(gomock.Any(), deals[0], key).Return(out1, nil), + n.EXPECT().StateMarketStorageDeal(gomock.Any(), deals[1], key).Return(out2, nil), + ) + + }, + expectedPieceSize: unpaddedSize, + expectedVerified: false, + }, + } + + for name, tc := range tcs { + tc := tc + t.Run(name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + // when test is done, assert expectations on all mock objects. + defer mockCtrl.Finish() + + mockFull := mocks.NewMockFullNode(mockCtrl) + rpn := &retrievalProviderNode{ + full: mockFull, + } + if tc.fFnc != nil { + tc.fFnc(mockFull) + } + + resp, err := rpn.GetRetrievalPricingInput(ctx, pcid, deals) + + if tc.expectedErrorStr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedErrorStr) + require.Equal(t, retrievalmarket.PricingInput{}, resp) + } else { + require.NoError(t, err) + require.Equal(t, tc.expectedPieceSize, resp.PieceSize) + require.Equal(t, tc.expectedVerified, resp.VerifiedDeal) + } + }) + } +} diff --git a/node/builder_miner.go b/node/builder_miner.go index 3165da0e2..06bd8ae12 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -63,6 +63,7 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(*stores.Local), modules.LocalStorage), Override(new(*stores.Remote), modules.RemoteStorage), + Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)), If(!cfg.Subsystems.EnableMining, If(cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))), @@ -125,6 +126,12 @@ func ConfigStorageMiner(c interface{}) Option { // Markets (retrieval deps) Override(new(sectorstorage.PieceProvider), sectorstorage.NewPieceProvider), + Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(config.DealmakingConfig{ + RetrievalPricing: &config.RetrievalPricing{ + Strategy: config.RetrievalPricingDefaultMode, + Default: &config.RetrievalPricingDefault{}, + }, + })), // Markets (retrieval) Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode), diff --git a/node/config/def.go b/node/config/def.go index d19d9ffaa..415f9e65e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -14,6 +14,14 @@ import ( sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" ) +const ( + // RetrievalPricingDefault configures the node to use the default retrieval pricing policy. + RetrievalPricingDefaultMode = "default" + // RetrievalPricingExternal configures the node to use the external retrieval pricing script + // configured by the user. + RetrievalPricingExternalMode = "external" +) + // Common is common config between full node and miner type Common struct { API API @@ -81,6 +89,29 @@ type DealmakingConfig struct { Filter string RetrievalFilter string + + RetrievalPricing *RetrievalPricing +} + +type RetrievalPricing struct { + Strategy string // possible values: "default", "external" + + Default *RetrievalPricingDefault + External *RetrievalPricingExternal +} + +type RetrievalPricingExternal struct { + // Path of the external script that will be run to price a retrieval deal. + // This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external". + Path string +} + +type RetrievalPricingDefault struct { + // VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal + // of a payloadCid that belongs to a verified storage deal. + // This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default". + // default value is true + VerifiedDealsFreeTransfer bool } type SealingConfig struct { @@ -322,6 +353,9 @@ func DefaultStorageMiner() *StorageMiner { // Default to 10 - tcp should still be able to figure this out, and // it's the ratio between 10gbit / 1gbit ParallelFetchLimit: 10, + + // By default use the hardware resource filtering strategy. + ResourceFiltering: sectorstorage.ResourceFilteringHardware, }, Dealmaking: DealmakingConfig{ @@ -337,6 +371,16 @@ func DefaultStorageMiner() *StorageMiner { PublishMsgPeriod: Duration(time.Hour), MaxDealsPerPublishMsg: 8, MaxProviderCollateralMultiplier: 2, + + RetrievalPricing: &RetrievalPricing{ + Strategy: RetrievalPricingDefaultMode, + Default: &RetrievalPricingDefault{ + VerifiedDealsFreeTransfer: true, + }, + External: &RetrievalPricingExternal{ + Path: "", + }, + }, }, Subsystems: MinerSubsystemConfig{ diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index b7a1be2e1..3c0c14383 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -92,3 +92,5 @@ type GetExpectedSealDurationFunc func() (time.Duration, error) type StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) + +type RetrievalPricingFunc func(ctx context.Context, dealPricingParams retrievalmarket.PricingInput) (retrievalmarket.Ask, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index ecc672400..7d36670af 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/filecoin-project/lotus/markets/pricing" "go.uber.org/fx" "go.uber.org/multierr" "golang.org/x/xerrors" @@ -638,6 +639,20 @@ func RetrievalNetwork(h host.Host) rmnet.RetrievalMarketNetwork { return rmnet.NewFromLibp2pHost(h) } +// RetrievalPricingFunc configures the pricing function to use for retrieval deals. +func RetrievalPricingFunc(cfg config.DealmakingConfig) func(_ dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + _ dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalPricingFunc { + + return func(_ dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + _ dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalPricingFunc { + if cfg.RetrievalPricing.Strategy == config.RetrievalPricingExternalMode { + return pricing.ExternalRetrievalPricingFunc(cfg.RetrievalPricing.External.Path) + } + + return retrievalimpl.DefaultPricingFunc(cfg.RetrievalPricing.Default.VerifiedDealsFreeTransfer) + } +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore func RetrievalProvider( maddr dtypes.MinerAddress, @@ -648,10 +663,12 @@ func RetrievalProvider( mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, pieceProvider sectorstorage.PieceProvider, + pricingFnc dtypes.RetrievalPricingFunc, userFilter dtypes.RetrievalDealFilter, ) (retrievalmarket.RetrievalProvider, error) { opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) - return retrievalimpl.NewProvider(address.Address(maddr), adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) + return retrievalimpl.NewProvider(address.Address(maddr), adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), + retrievalimpl.RetrievalPricingFunc(pricingFnc), opt) } var WorkerCallsPrefix = datastore.NewKey("/worker/calls") diff --git a/storage/miner.go b/storage/miner.go index b4c590ae3..3d29e0ef1 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -199,7 +199,7 @@ func (m *Miner) Run(ctx context.Context) error { ) // Instantiate the sealing FSM. - m.sealing = sealing.New(adaptedAPI, m.feeCfg, evtsAdapter, m.maddr, m.ds, m.sealer, m.sc, m.verif, m.prover, &pcp, cfg, m.handleSealingNotifications, as) + m.sealing = sealing.New(ctx, adaptedAPI, m.feeCfg, evtsAdapter, m.maddr, m.ds, m.sealer, m.sc, m.verif, m.prover, &pcp, cfg, m.handleSealingNotifications, as) // Run the sealing FSM. go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index d62b5e851..0fe7ea851 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -732,7 +732,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition, nv net // sectors per partition 3: ooo // partitions per message 2: oooOOO // <1><2> (3rd doesn't fit) - partitionsPerMsg, err := policy.GetMaxPoStPartitions(s.proofType) + partitionsPerMsg, err := policy.GetMaxPoStPartitions(nv, s.proofType) if err != nil { return nil, xerrors.Errorf("getting sectors per partition: %w", err) } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index b878ff97e..61f2a324b 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -31,6 +31,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/journal" @@ -185,8 +186,8 @@ func TestWDPostDoPost(t *testing.T) { // Work out the number of partitions that can be included in a message // without exceeding the message sector limit + partitionsPerMsg, err := policy.GetMaxPoStPartitions(network.Version13, proofType) require.NoError(t, err) - partitionsPerMsg := int(miner5.AddressedSectorsMax / sectorsPerPartition) if partitionsPerMsg > miner5.AddressedPartitionsMax { partitionsPerMsg = miner5.AddressedPartitionsMax } diff --git a/testplans/lotus-soup/go.mod b/testplans/lotus-soup/go.mod index 0c8e92a1b..4253a49fb 100644 --- a/testplans/lotus-soup/go.mod +++ b/testplans/lotus-soup/go.mod @@ -13,9 +13,8 @@ require ( github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/lotus v1.9.1-0.20210602131226-e1dc7ad6eb9e + github.com/filecoin-project/lotus v1.10.0-rc3.0.20210616215353-9c7db6d305e3 github.com/filecoin-project/specs-actors v0.9.14 - github.com/filecoin-project/specs-actors/v5 v5.0.0-20210602024058-0c296bb386bf // indirect github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-multierror v1.1.0 @@ -25,19 +24,19 @@ require ( github.com/ipfs/go-graphsync v0.6.2-0.20210428121800-88edb5462e17 // indirect github.com/ipfs/go-ipfs-files v0.0.8 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-log/v2 v2.1.2 + github.com/ipfs/go-log/v2 v2.1.3 github.com/ipfs/go-merkledag v0.3.2 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d github.com/kpacha/opencensus-influxdb v0.0.0-20181102202715-663e2683a27c - github.com/libp2p/go-libp2p v0.12.0 - github.com/libp2p/go-libp2p-core v0.7.0 + github.com/libp2p/go-libp2p v0.14.2 + github.com/libp2p/go-libp2p-core v0.8.5 github.com/libp2p/go-libp2p-pubsub-tracer v0.0.0-20200626141350-e730b32bf1e6 github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-net v0.2.0 github.com/testground/sdk-go v0.2.6 - go.opencensus.io v0.22.5 - golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + go.opencensus.io v0.23.0 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) // This will work in all build modes: docker:go, exec:go, and local go build. diff --git a/testplans/lotus-soup/go.sum b/testplans/lotus-soup/go.sum index 926f625cf..e1690a213 100644 --- a/testplans/lotus-soup/go.sum +++ b/testplans/lotus-soup/go.sum @@ -114,14 +114,18 @@ github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dm github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= +github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= @@ -195,8 +199,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= -github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -280,12 +286,16 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= github.com/filecoin-project/go-data-transfer v1.6.0 h1:DHIzEc23ydRCCBwtFet3MfgO8gMpZEnw60Y+s71oX6o= github.com/filecoin-project/go-data-transfer v1.6.0/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc= +github.com/filecoin-project/go-data-transfer v1.4.3 h1:ECEw69NOfmEZ7XN1NSBvj3KTbbH2mIczQs+Z2w4bD7c= +github.com/filecoin-project/go-data-transfer v1.4.3/go.mod h1:n8kbDQXWrY1c4UgfMa9KERxNCWbOTDwdNhf2MpN9dpo= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= +github.com/filecoin-project/go-fil-markets v1.2.5 h1:bQgtXbwxKyPxSEQoUI5EaTHJ0qfzyd5NosspuADCm6Y= +github.com/filecoin-project/go-fil-markets v1.2.5/go.mod h1:7JIqNBmFvOyBzk/EiPYnweVdQnWhshixb5B9b1653Ag= github.com/filecoin-project/go-fil-markets v1.4.0 h1:J4L6o+FVOmS7ZWV6wxLPiuoDzGC7iS3S5NRFL1enEr0= github.com/filecoin-project/go-fil-markets v1.4.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= @@ -301,8 +311,8 @@ github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0 github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= -github.com/filecoin-project/go-paramfetch v0.0.2-0.20210330140417-936748d3f5ec h1:gExwWUiT1TcARkxGneS4nvp9C+wBsKU0bFdg7qFpNco= -github.com/filecoin-project/go-paramfetch v0.0.2-0.20210330140417-936748d3f5ec/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 h1:G10ezOvpH1CLXQ19EA9VWNwyL0mg536ujSayjV0yg0k= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= @@ -317,8 +327,10 @@ github.com/filecoin-project/go-statestore v0.1.1 h1:ufMFq00VqnT2CAuDpcGnwLnCX1I/ github.com/filecoin-project/go-statestore v0.1.1/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/lotus v1.9.1-0.20210602131226-e1dc7ad6eb9e h1:JvtYGk30nM7K0TD4sTOUKYUePcSzZNj5ZD6g5vdrqMI= -github.com/filecoin-project/lotus v1.9.1-0.20210602131226-e1dc7ad6eb9e/go.mod h1:/ZeMXR8jPxJslaHSIW3ZxO9YPIaxcnsP+niEoBatzo8= +github.com/filecoin-project/lotus v1.10.0-rc3.0.20210616215353-9c7db6d305e3 h1:oeVa5wjoNx888oIs83L+LqAG75yqa5DCj94I2dRK+Ms= +github.com/filecoin-project/lotus v1.10.0-rc3.0.20210616215353-9c7db6d305e3/go.mod h1:a4kSO7IY58nxXhc29lpZwgZksbdTQFQ4nhBscFYPAjw= +github.com/filecoin-project/lotus v1.9.0 h1:TDKDLbmgYTL8M0mlfd9HmJVEYRlSSOQnakg4+9rfyWM= +github.com/filecoin-project/lotus v1.9.0/go.mod h1:4YC/8rizrrp2wKOYvHQEjCxZbziXi68BhrzvI+FCye0= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= @@ -326,25 +338,28 @@ github.com/filecoin-project/specs-actors v0.9.14 h1:68PVstg2UB3ZsMLF+DKFTAs/YKsq github.com/filecoin-project/specs-actors v0.9.14/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-actors/v2 v2.3.2/go.mod h1:UuJQLoTx/HPvvWeqlIFmC/ywlOLHNe8SNQ3OunFbu2Y= +github.com/filecoin-project/specs-actors/v2 v2.3.5-0.20210114162132-5b58b773f4fb h1:orr/sMzrDZUPAveRE+paBdu1kScIUO5zm+HYeh+VlhA= github.com/filecoin-project/specs-actors/v2 v2.3.5-0.20210114162132-5b58b773f4fb/go.mod h1:LljnY2Mn2homxZsmokJZCpRuhOPxfXhvcek5gWkmqAc= github.com/filecoin-project/specs-actors/v2 v2.3.5 h1:PbT4tPlSXZ8sRgajhb4D8AOEmiaaZ+jg6tc6BBv8VQc= github.com/filecoin-project/specs-actors/v2 v2.3.5/go.mod h1:LljnY2Mn2homxZsmokJZCpRuhOPxfXhvcek5gWkmqAc= +github.com/filecoin-project/specs-actors/v3 v3.1.0 h1:s4qiPw8pgypqBGAy853u/zdZJ7K9cTZdM1rTiSonHrg= github.com/filecoin-project/specs-actors/v3 v3.1.0/go.mod h1:mpynccOLlIRy0QnR008BwYBwT9fen+sPR13MA1VmMww= github.com/filecoin-project/specs-actors/v3 v3.1.1 h1:BE8fsns1GnEOxt1DTE5LxBK2FThXtWmCChgcJoHTg0E= github.com/filecoin-project/specs-actors/v3 v3.1.1/go.mod h1:mpynccOLlIRy0QnR008BwYBwT9fen+sPR13MA1VmMww= +github.com/filecoin-project/specs-actors/v4 v4.0.0 h1:vMALksY5G3J5rj3q9rbcyB+f4Tk1xrLqSgdB3jOok4s= github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= github.com/filecoin-project/specs-actors/v4 v4.0.1 h1:AiWrtvJZ63MHGe6rn7tPu4nSUY8bA1KDNszqJaD5+Fg= github.com/filecoin-project/specs-actors/v4 v4.0.1/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= github.com/filecoin-project/specs-actors/v5 v5.0.0-20210512015452-4fe3889fff57/go.mod h1:283yBMMUSDB2abcjP/hhrwTkhb9h3sfM6KGrep/ZlBI= -github.com/filecoin-project/specs-actors/v5 v5.0.0-20210528202914-a9f9f95f5e93/go.mod h1:kSDmoQuO8jlhMVzKNoesbhka1e6gHKcLQjKm9mE9Qhw= -github.com/filecoin-project/specs-actors/v5 v5.0.0-20210602024058-0c296bb386bf h1:xt9A1omyhSDbQvpVk7Na1J15a/n8y0y4GQDLeiWLpFs= -github.com/filecoin-project/specs-actors/v5 v5.0.0-20210602024058-0c296bb386bf/go.mod h1:b/btpRl84Q9SeDKlyIoORBQwe2OTmq14POrYrVvBWCM= +github.com/filecoin-project/specs-actors/v5 v5.0.0-20210609212542-73e0409ac77c h1:GnDJ6q3QEm2ytTKjPFQSvczAltgCSb3j9F1FeynwvPA= +github.com/filecoin-project/specs-actors/v5 v5.0.0-20210609212542-73e0409ac77c/go.mod h1:b/btpRl84Q9SeDKlyIoORBQwe2OTmq14POrYrVvBWCM= github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 h1:Ur/l2+6qN+lQiqjozWWc5p9UDaAMDZKTlDS98oRnlIw= github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= +github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= +github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= @@ -411,8 +426,9 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc= github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= @@ -431,8 +447,9 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -444,8 +461,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= @@ -458,14 +476,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= -github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -639,6 +659,8 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= +github.com/ipfs/go-graphsync v0.6.0 h1:x6UvDUGA7wjaKNqx5Vbo7FGT8aJ5ryYA0dMQ5jN3dF0= +github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= github.com/ipfs/go-graphsync v0.6.2-0.20210428121800-88edb5462e17 h1:rOoF88dVuDGbIx7idSdimN7JvXriyOIT96WD3eX9sHA= github.com/ipfs/go-graphsync v0.6.2-0.20210428121800-88edb5462e17/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= @@ -716,8 +738,9 @@ github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscw github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= -github.com/ipfs/go-log/v2 v2.1.2 h1:a0dRiL098zY23vay1h3dimx6y94XchEUyt5h0l4VvQU= github.com/ipfs/go-log/v2 v2.1.2/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= +github.com/ipfs/go-log/v2 v2.1.3 h1:1iS3IU7aXRlbgUpN8yTTpJ53NXYjAe37vcI5+5nYrzk= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= @@ -821,6 +844,7 @@ github.com/kilic/bls12-381 v0.0.0-20200820230200-6b2c19996391 h1:51kHw7l/dUDdOdW github.com/kilic/bls12-381 v0.0.0-20200820230200-6b2c19996391/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -837,6 +861,7 @@ github.com/kpacha/opencensus-influxdb v0.0.0-20181102202715-663e2683a27c/go.mod github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -856,8 +881,9 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= -github.com/libp2p/go-conn-security-multistream v0.2.0 h1:uNiDjS58vrvJTg9jO6bySd1rMKejieG7v45ekqHbZ1M= github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= +github.com/libp2p/go-conn-security-multistream v0.2.1 h1:ft6/POSK7F+vl/2qzegnHDaXFU0iWB4yVTYrioC6Zy0= +github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-eventbus v0.2.1 h1:VanAdErQnpTioN2TowqNcOijf6YwhuODe4pPKSDpxGc= @@ -880,8 +906,9 @@ github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qD github.com/libp2p/go-libp2p v0.8.3/go.mod h1:EsH1A+8yoWK+L4iKcbPYu6MPluZ+CHWI9El8cTaefiM= github.com/libp2p/go-libp2p v0.9.2/go.mod h1:cunHNLDVus66Ct9iXXcjKRLdmHdFdHVe1TAnbubJQqQ= github.com/libp2p/go-libp2p v0.10.0/go.mod h1:yBJNpb+mGJdgrwbKAKrhPU0u3ogyNFTfjJ6bdM+Q/G8= -github.com/libp2p/go-libp2p v0.12.0 h1:+xai9RQnQ9l5elFOKvp5wRyjyWisSwEx+6nU2+onpUA= github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0= +github.com/libp2p/go-libp2p v0.14.2 h1:qs0ABtjjNjS+RIXT1uM7sMJEvIc0pq2nKR0VQxFXhHI= +github.com/libp2p/go-libp2p v0.14.2/go.mod h1:0PQMADQEjCM2l8cSMYDpTgsb8gr6Zq7i4LUgq1mlW2E= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 h1:BM7aaOF7RpmNn9+9g6uTjGJ0cTzWr5j9i9IKeun2M8U= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= @@ -892,8 +919,9 @@ github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQ github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= github.com/libp2p/go-libp2p-autonat v0.2.3/go.mod h1:2U6bNWCNsAG9LEbwccBDQbjzQ8Krdjge1jLTE9rdoMM= -github.com/libp2p/go-libp2p-autonat v0.4.0 h1:3y8XQbpr+ssX8QfZUHekjHCYK64sj6/4hnf/awD4+Ug= github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= +github.com/libp2p/go-libp2p-autonat v0.4.2 h1:YMp7StMi2dof+baaxkbxaizXjY1RPvU71CXfxExzcUU= +github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= github.com/libp2p/go-libp2p-autonat-svc v0.1.0/go.mod h1:fqi8Obl/z3R4PFVLm8xFtZ6PBL9MlV/xumymRFkKq5A= github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= @@ -940,8 +968,12 @@ github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= -github.com/libp2p/go-libp2p-core v0.7.0 h1:4a0TMjrWNTZlNvcqxZmrMRDi/NQWrhwO2pkTuLSQ/IQ= github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.8.2/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.8.5 h1:aEgbIcPGsKy6zYcC+5AJivYFedhYa4sW7mIpWpUaLKw= +github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= @@ -976,8 +1008,10 @@ github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3 github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo= github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= -github.com/libp2p/go-libp2p-mplex v0.3.0 h1:CZyqqKP0BSGQyPLvpRQougbfXaaaJZdGgzhCpJNuNSk= github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs= +github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= +github.com/libp2p/go-libp2p-mplex v0.4.1 h1:/pyhkP1nLwjG3OM+VuaNJkQT/Pqq73WzB3aDN3Fx1sc= +github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= github.com/libp2p/go-libp2p-nat v0.0.2/go.mod h1:QrjXQSD5Dj4IJOdEcjHRkWTSomyxRo6HnUkf/TfQpLQ= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= @@ -989,8 +1023,8 @@ github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFx github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= -github.com/libp2p/go-libp2p-noise v0.1.2 h1:IH9GRihQJTx56obm+GnpdPX4KeVIlvpXrP6xnJ0wxWk= -github.com/libp2p/go-libp2p-noise v0.1.2/go.mod h1:9B10b7ueo7TIxZHHcjcDCo5Hd6kfKT2m77by82SFRfE= +github.com/libp2p/go-libp2p-noise v0.2.0 h1:wmk5nhB9a2w2RxMOyvsoKjizgJOEaJdfAakr0jN8gds= +github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q= github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= @@ -1005,8 +1039,9 @@ github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw= github.com/libp2p/go-libp2p-peerstore v0.2.4/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.2.6 h1:2ACefBX23iMdJU9Ke+dcXt3w86MIryes9v7In4+Qq3U= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= +github.com/libp2p/go-libp2p-peerstore v0.2.7 h1:83JoLxyR9OYTnNfB5vvFqvMUv/xDNa6NoPHnENhBsGw= +github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= @@ -1020,8 +1055,8 @@ github.com/libp2p/go-libp2p-pubsub-tracer v0.0.0-20200626141350-e730b32bf1e6 h1: github.com/libp2p/go-libp2p-pubsub-tracer v0.0.0-20200626141350-e730b32bf1e6/go.mod h1:8ZodgKS4qRLayfw9FDKDd9DX4C16/GMofDxSldG8QPI= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= -github.com/libp2p/go-libp2p-quic-transport v0.9.0 h1:WPuq5nV/chmIZIzvrkC2ulSdAQ0P0BDvgvAhZFOZ59E= -github.com/libp2p/go-libp2p-quic-transport v0.9.0/go.mod h1:xyY+IgxL0qsW7Kiutab0+NlxM0/p9yRtrGTYsuMWf70= +github.com/libp2p/go-libp2p-quic-transport v0.10.0 h1:koDCbWD9CCHwcHZL3/WEvP2A+e/o5/W5L3QS/2SPMA0= +github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= @@ -1048,8 +1083,9 @@ github.com/libp2p/go-libp2p-swarm v0.2.4/go.mod h1:/xIpHFPPh3wmSthtxdGbkHZ0OET1h github.com/libp2p/go-libp2p-swarm v0.2.7/go.mod h1:ZSJ0Q+oq/B1JgfPHJAT2HTall+xYRNYp1xs4S2FBWKA= github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= -github.com/libp2p/go-libp2p-swarm v0.3.1 h1:UTobu+oQHGdXTOGpZ4RefuVqYoJXcT0EBtSR74m2LkI= github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= +github.com/libp2p/go-libp2p-swarm v0.5.0 h1:HIK0z3Eqoo8ugmN8YqWAhD2RORgR+3iNXYG4U2PFd1E= +github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -1057,8 +1093,9 @@ github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MB github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= -github.com/libp2p/go-libp2p-testing v0.3.0 h1:ZiBYstPamsi7y6NJZebRudUzsYmVkt998hltyLqf8+g= github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= +github.com/libp2p/go-libp2p-testing v0.4.0 h1:PrwHRi0IGqOwVQWR3xzgigSlhlLfxgfXgkHxr77EghQ= +github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= @@ -1068,8 +1105,9 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2kn/m1w6YXxcIAYJYeI90h6BGgUc= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= -github.com/libp2p/go-libp2p-transport-upgrader v0.3.0 h1:q3ULhsknEQ34eVDhv4YwKS8iet69ffs9+Fir6a7weN4= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.2 h1:4JsnbfJzgZeRS9AWN7B9dPqn/LY/HoQTlO9gtdJTIYM= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= github.com/libp2p/go-libp2p-yamux v0.1.2/go.mod h1:xUoV/RmYkg6BW/qGxA9XJyg+HzXFYkeXbnhjmnYzKp8= github.com/libp2p/go-libp2p-yamux v0.1.3/go.mod h1:VGSQVrqkh6y4nm0189qqxMtvyBft44MOYYPpYKXiVt4= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= @@ -1079,8 +1117,9 @@ github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhLEn0XhIoZ5viCwU= github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4= github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= -github.com/libp2p/go-libp2p-yamux v0.4.1 h1:TJxRVPY9SjH7TNrNC80l1OJMBiWhs1qpKmeB+1Ug3xU= -github.com/libp2p/go-libp2p-yamux v0.4.1/go.mod h1:FA/NjRYRVNjqOzpGuGqcruH7jAU2mYIjtKBicVOL3dc= +github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= +github.com/libp2p/go-libp2p-yamux v0.5.4 h1:/UOPtT/6DHPtr3TtKXBHa6g0Le0szYuI33Xc/Xpd7fQ= +github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE= github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= @@ -1092,8 +1131,9 @@ github.com/libp2p/go-mplex v0.0.4/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTW github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= -github.com/libp2p/go-mplex v0.2.0 h1:Ov/D+8oBlbRkjBs1R1Iua8hJ8cUfbdiW8EOdZuxcgaI= github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= +github.com/libp2p/go-mplex v0.3.0 h1:U1T+vmCYJaEoDJPV1aq31N56hS+lJgb397GsylNSgrU= +github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= @@ -1105,8 +1145,9 @@ github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/ github.com/libp2p/go-nat v0.0.5 h1:qxnwkco8RLKqVh1NmjQ+tJ8p8khNLFxuElYG/TwqW4Q= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= -github.com/libp2p/go-netroute v0.1.3 h1:1ngWRx61us/EpaKkdqkMjKk/ufr/JlIFYQAxV2XX8Ig= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= +github.com/libp2p/go-netroute v0.1.6 h1:ruPJStbYyXVYGQ81uzEDzuvbYRLKRrLvTYd33yomC38= +github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -1122,8 +1163,9 @@ github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2 github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAoGVf8h9txtt/1uM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-sockaddr v0.1.0 h1:Y4s3/jNoryVRKEBrkJ576F17CPOaMIzUeCsg7dlTDj0= github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= +github.com/libp2p/go-sockaddr v0.1.1 h1:yD80l2ZOdGksnOyHrhxDdTDFrf7Oy+v3FMVArIRgZxQ= +github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= @@ -1145,8 +1187,9 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y= github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= -github.com/libp2p/go-ws-transport v0.3.1 h1:ZX5rWB8nhRRJVaPO6tmkGI/Xx8XNboYX20PW5hXIscw= github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= +github.com/libp2p/go-ws-transport v0.4.0 h1:9tvtQ9xbws6cA5LvqdE6Ne3vcmGB4f1z9SByggk4s0k= +github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= github.com/libp2p/go-yamux v1.2.1/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -1158,12 +1201,14 @@ github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/h github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= +github.com/libp2p/go-yamux/v2 v2.2.0 h1:RwtpYZ2/wVviZ5+3pjC8qdQ4TKnrak0/E01N1UWoAFU= +github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= -github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= -github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= +github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= +github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= @@ -1177,13 +1222,13 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= -github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk= github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg= -github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= +github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -1220,6 +1265,8 @@ github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nr github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -1267,8 +1314,9 @@ github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/94 github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.1.0/go.mod h1:01k2RAqtoXIuPa3DCavAE9/6jc6nM0H3EgZyfUhN2oY= -github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= +github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= +github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= @@ -1298,8 +1346,10 @@ github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wS github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= -github.com/multiformats/go-multistream v0.2.0 h1:6AuNmQVKUkRnddw2YiDjt5Elit40SFxMJkVnhmETXtU= github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= +github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= +github.com/multiformats/go-multistream v0.2.2 h1:TCYu1BHTDr1F/Qm75qwYISQdzGcRdC21nFgQW7l7GBo= +github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -1316,8 +1366,6 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= @@ -1336,6 +1384,7 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1635,6 +1684,7 @@ github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8= @@ -1663,8 +1713,8 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1718,16 +1768,19 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1754,8 +1807,9 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1766,6 +1820,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1801,6 +1856,7 @@ golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1808,8 +1864,11 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201022231255-08b38378de70 h1:Z6x4N9mAi4oF0TbHweCsH618MO6OI6UFgV0FP5n0wBY= golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 h1:0PC75Fz/kyMGhL0e1QnypqK2kQMqKt9csD1GnMJR+Zk= +golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1827,8 +1886,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1898,16 +1957,21 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426080607-c94f62235c83 h1:kHSDPqCtsHZOg0nVylfTo20DDhE9gG4Y0jn7hKQ0QAM= +golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1953,10 +2017,12 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201112185108-eeaa07dd7696 h1:Bfazo+enXJET5SbHeh95NtxabJF6fJ9r/jpfRJgd3j4= golang.org/x/tools v0.0.0-20201112185108-eeaa07dd7696/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2031,8 +2097,9 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2048,8 +2115,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= diff --git a/testplans/lotus-soup/rfwp/chain_state.go b/testplans/lotus-soup/rfwp/chain_state.go index 90159e924..d91acdff9 100644 --- a/testplans/lotus-soup/rfwp/chain_state.go +++ b/testplans/lotus-soup/rfwp/chain_state.go @@ -7,6 +7,8 @@ import ( "encoding/json" "fmt" "io" + "math" + corebig "math/big" "os" "sort" "text/tabwriter" @@ -27,6 +29,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" tstats "github.com/filecoin-project/lotus/tools/stats" ) @@ -581,18 +584,24 @@ func (i *MinerInfo) MarshalPlainText() ([]byte, error) { fmt.Fprintf(w, "Sector Size: %s\n", i.SectorSize) pow := i.MinerPower - rpercI := types.BigDiv(types.BigMul(pow.MinerPower.RawBytePower, types.NewInt(1000000)), pow.TotalPower.RawBytePower) - qpercI := types.BigDiv(types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(1000000)), pow.TotalPower.QualityAdjPower) fmt.Fprintf(w, "Byte Power: %s / %s (%0.4f%%)\n", types.SizeStr(pow.MinerPower.RawBytePower), types.SizeStr(pow.TotalPower.RawBytePower), - float64(rpercI.Int64())/10000) + types.BigDivFloat( + types.BigMul(pow.MinerPower.RawBytePower, big.NewInt(100)), + pow.TotalPower.RawBytePower, + ), + ) fmt.Fprintf(w, "Actual Power: %s / %s (%0.4f%%)\n", types.DeciStr(pow.MinerPower.QualityAdjPower), types.DeciStr(pow.TotalPower.QualityAdjPower), - float64(qpercI.Int64())/10000) + types.BigDivFloat( + types.BigMul(pow.MinerPower.QualityAdjPower, big.NewInt(100)), + pow.TotalPower.QualityAdjPower, + ), + ) fmt.Fprintf(w, "\tCommitted: %s\n", types.SizeStr(i.CommittedBytes)) @@ -608,16 +617,50 @@ func (i *MinerInfo) MarshalPlainText() ([]byte, error) { if !i.MinerPower.HasMinPower { fmt.Fprintf(w, "Below minimum power threshold, no blocks will be won\n") } else { - expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 - if expWinChance > 0 { - if expWinChance > 1 { - expWinChance = 1 - } - winRate := time.Duration(float64(time.Second*time.Duration(build.BlockDelaySecs)) / expWinChance) - winPerDay := float64(time.Hour*24) / float64(winRate) - fmt.Fprintln(w, "Expected block win rate: ") - fmt.Fprintf(w, "%.4f/day (every %s)\n", winPerDay, winRate.Truncate(time.Second)) + winRatio := new(corebig.Rat).SetFrac( + types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(build.BlocksPerEpoch)).Int, + pow.TotalPower.QualityAdjPower.Int, + ) + + if winRatioFloat, _ := winRatio.Float64(); winRatioFloat > 0 { + + // if the corresponding poisson distribution isn't infinitely small then + // throw it into the mix as well, accounting for multi-wins + winRationWithPoissonFloat := -math.Expm1(-winRatioFloat) + winRationWithPoisson := new(corebig.Rat).SetFloat64(winRationWithPoissonFloat) + if winRationWithPoisson != nil { + winRatio = winRationWithPoisson + winRatioFloat = winRationWithPoissonFloat + } + + weekly, _ := new(corebig.Rat).Mul( + winRatio, + new(corebig.Rat).SetInt64(7*builtin.EpochsInDay), + ).Float64() + + avgDuration, _ := new(corebig.Rat).Mul( + new(corebig.Rat).SetInt64(builtin.EpochDurationSeconds), + new(corebig.Rat).Inv(winRatio), + ).Float64() + + fmt.Fprintf(w, "Projected average block win rate: %.02f/week (every %s)\n", + weekly, + (time.Second * time.Duration(avgDuration)).Truncate(time.Second).String(), + ) + + // Geometric distribution of P(Y < k) calculated as described in https://en.wikipedia.org/wiki/Geometric_distribution#Probability_Outcomes_Examples + // https://www.wolframalpha.com/input/?i=t+%3E+0%3B+p+%3E+0%3B+p+%3C+1%3B+c+%3E+0%3B+c+%3C1%3B+1-%281-p%29%5E%28t%29%3Dc%3B+solve+t + // t == how many dice-rolls (epochs) before win + // p == winRate == ( minerPower / netPower ) + // c == target probability of win ( 99.9% in this case ) + fmt.Fprintf(w, "Projected block win with 99.9%% probability every %s\n", + (time.Second * time.Duration( + builtin.EpochDurationSeconds*math.Log(1-0.999)/ + math.Log(1-winRatioFloat), + )).Truncate(time.Second).String(), + ) + fmt.Fprintln(w, "(projections DO NOT account for future network and miner growth)") } }