From 2dde022a00d7b725415e5cee07d8861ebe4d1000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 20 Aug 2021 15:53:24 +0100 Subject: [PATCH 1/3] itests: support larger sector sizes; add large deal test. --- .circleci/config.yml | 5 +++++ itests/api_test.go | 6 ++--- itests/deals_512mb_test.go | 44 +++++++++++++++++++++++++++++++++++++ itests/kit/ensemble.go | 32 +++++++++++++++++---------- itests/kit/ensemble_opts.go | 5 +++++ itests/kit/init.go | 4 ++-- itests/kit/node_opts.go | 19 +++++++++------- itests/kit/sectors.go | 16 ++++++++++++++ 8 files changed, 107 insertions(+), 24 deletions(-) create mode 100644 itests/deals_512mb_test.go create mode 100644 itests/kit/sectors.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d24e25b1..d570e303c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -810,6 +810,11 @@ workflows: suite: itest-deadlines target: "./itests/deadlines_test.go" + - test: + name: test-itest-deals_512mb + suite: itest-deals_512mb + target: "./itests/deals_512mb_test.go" + - test: name: test-itest-deals_concurrent suite: itest-deals_concurrent diff --git a/itests/api_test.go b/itests/api_test.go index 6c25ef181..01e006fed 100644 --- a/itests/api_test.go +++ b/itests/api_test.go @@ -7,14 +7,14 @@ import ( "time" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/stretchr/testify/require" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" - "github.com/stretchr/testify/require" ) func TestAPI(t *testing.T) { @@ -186,7 +186,7 @@ func (ts *apiSuite) testNonGenesisMiner(t *testing.T) { var newMiner kit.TestMiner ens.Miner(&newMiner, full, kit.OwnerAddr(full.DefaultKey), - kit.ProofType(abi.RegisteredSealProof_StackedDrg2KiBV1_1), + kit.SectorSize(2<<10), kit.WithAllSubsystems(), ).Start().InterconnectAll() diff --git a/itests/deals_512mb_test.go b/itests/deals_512mb_test.go new file mode 100644 index 000000000..eac598ab9 --- /dev/null +++ b/itests/deals_512mb_test.go @@ -0,0 +1,44 @@ +package itests + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/itests/kit" +) + +func TestStorageDealMissingBlock(t *testing.T) { + ctx := context.Background() + + // enable 512MiB proofs so we can conduct larger transfers. + kit.EnableLargeSectors() + kit.QuietMiningLogs() + + client, miner, ens := kit.EnsembleMinimal(t, + kit.MockProofs(), + kit.SectorSize(512<<20), // 512MiB sectors. + ) + ens.InterconnectAll().BeginMining(50 * time.Millisecond) + + dh := kit.NewDealHarness(t, client, miner, miner) + + client.WaitTillChain(ctx, kit.HeightAtLeast(5)) + + res, _ := client.CreateImportFile(ctx, 0, 64<<20) // 64MiB file. + list, err := client.ClientListImports(ctx) + require.NoError(t, err) + require.Len(t, list, 1) + require.Equal(t, res.Root, *list[0].Root) + + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.FastRetrieval = true + dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price. + deal := dh.StartDeal(ctx, dp) + + dh.WaitDealSealed(ctx, deal, false, false, nil) +} diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 672efd6be..f876fa6f8 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -17,6 +17,14 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-storedcounter" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + "github.com/ipfs/go-datastore" + libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/peer" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" "github.com/filecoin-project/lotus/build" @@ -43,13 +51,6 @@ import ( testing2 "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/mockstorage" - miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - "github.com/ipfs/go-datastore" - libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto" - "github.com/libp2p/go-libp2p-core/peer" - mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" - "github.com/stretchr/testify/require" ) func init() { @@ -216,11 +217,11 @@ func (n *Ensemble) Miner(minerNode *TestMiner, full *TestFullNode, opts ...NodeO genm *genesis.Miner ) - // Default 2KiB sector for the network version - proofType, err := miner.SealProofTypeFromSectorSize(2<<10, n.genesis.version) + // Will use 2KiB sectors by default (default value of sectorSize). + proofType, err := miner.SealProofTypeFromSectorSize(options.sectorSize, n.genesis.version) require.NoError(n.t, err) - // create the preseal commitment. + // Create the preseal commitment. if n.options.mockProofs { genm, k, err = mockstorage.PreSeal(proofType, actorAddr, sectors) } else { @@ -363,10 +364,19 @@ func (n *Ensemble) Start() *Ensemble { if m.options.mainMiner == nil { // this is a miner created after genesis, so it won't have a preseal. // we need to create it on chain. + + // we get the proof type for the requested sector size, for + // the current network version. + nv, err := m.FullNode.FullNode.StateNetworkVersion(ctx, types.EmptyTSK) + require.NoError(n.t, err) + + proofType, err := miner.SealProofTypeFromSectorSize(m.options.sectorSize, nv) + require.NoError(n.t, err) + params, aerr := actors.SerializeParams(&power2.CreateMinerParams{ Owner: m.OwnerKey.Address, Worker: m.OwnerKey.Address, - SealProofType: m.options.proofType, + SealProofType: proofType, Peer: abi.PeerID(m.Libp2p.PeerID), }) require.NoError(n.t, aerr) diff --git a/itests/kit/ensemble_opts.go b/itests/kit/ensemble_opts.go index 9db64d3bc..327da076e 100644 --- a/itests/kit/ensemble_opts.go +++ b/itests/kit/ensemble_opts.go @@ -4,8 +4,10 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/wallet" ) @@ -37,6 +39,9 @@ var DefaultEnsembleOpts = ensembleOpts{ func MockProofs() EnsembleOpt { return func(opts *ensembleOpts) error { opts.mockProofs = true + // since we're using mock proofs, we don't need to download + // proof parameters + build.DisableBuiltinAssets = true return nil } } diff --git a/itests/kit/init.go b/itests/kit/init.go index dc8463cb4..c455cf8c6 100644 --- a/itests/kit/init.go +++ b/itests/kit/init.go @@ -6,16 +6,16 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" - logging "github.com/ipfs/go-log/v2" ) func init() { _ = logging.SetLogLevel("*", "INFO") policy.SetProviderCollateralSupplyTarget(big.Zero(), big.NewInt(1)) - policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) diff --git a/itests/kit/node_opts.go b/itests/kit/node_opts.go index 87707aa16..f56c2fb13 100644 --- a/itests/kit/node_opts.go +++ b/itests/kit/node_opts.go @@ -3,6 +3,7 @@ package kit import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -30,14 +31,14 @@ type nodeOpts struct { mainMiner *TestMiner disableLibp2p bool optBuilders []OptBuilder - proofType abi.RegisteredSealProof + sectorSize abi.SectorSize } // DefaultNodeOpts are the default options that will be applied to test nodes. var DefaultNodeOpts = nodeOpts{ - balance: big.Mul(big.NewInt(100000000), types.NewInt(build.FilecoinPrecision)), - sectors: DefaultPresealsPerBootstrapMiner, - proofType: abi.RegisteredSealProof_StackedDrg2KiBV1_1, // default _concrete_ proof type for non-genesis miners (notice the _1) for new actors versions. + balance: big.Mul(big.NewInt(100000000), types.NewInt(build.FilecoinPrecision)), + sectors: DefaultPresealsPerBootstrapMiner, + sectorSize: abi.SectorSize(2 << 10), // 2KiB. } // OptBuilder is used to create an option after some other node is already @@ -135,11 +136,13 @@ func ConstructorOpts(extra ...node.Option) NodeOpt { } } -// ProofType sets the proof type for this node. If you're using new actor -// versions, this should be a _1 proof type. -func ProofType(proofType abi.RegisteredSealProof) NodeOpt { +// SectorSize sets the sector size for this miner. Start() will populate the +// corresponding proof type depending on the network version (genesis network +// version if the Ensemble is unstarted, or the current network version +// if started). +func SectorSize(sectorSize abi.SectorSize) NodeOpt { return func(opts *nodeOpts) error { - opts.proofType = proofType + opts.sectorSize = sectorSize return nil } } diff --git a/itests/kit/sectors.go b/itests/kit/sectors.go new file mode 100644 index 000000000..5f82055c0 --- /dev/null +++ b/itests/kit/sectors.go @@ -0,0 +1,16 @@ +package kit + +import ( + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/chain/actors/policy" +) + +// EnableLargeSectors enables 512MiB sectors. This is useful in combination with +// mock proofs, for testing larger transfers. +func EnableLargeSectors() { + policy.SetSupportedProofTypes( + abi.RegisteredSealProof_StackedDrg2KiBV1, + abi.RegisteredSealProof_StackedDrg512MiBV1, // <== + ) +} From 0030e208a0a7920c7831e1d49d41843046667074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 20 Aug 2021 15:59:19 +0100 Subject: [PATCH 2/3] avoid reordering imports. --- itests/kit/ensemble.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index f876fa6f8..9e5333794 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -17,13 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-storedcounter" - miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - "github.com/ipfs/go-datastore" - libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto" - "github.com/libp2p/go-libp2p-core/peer" - mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" - "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" @@ -51,6 +44,13 @@ import ( testing2 "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/mockstorage" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + "github.com/ipfs/go-datastore" + libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/peer" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" ) func init() { From 342134f8535cea4f9f2978463520e1ce48b2ed67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 20 Aug 2021 18:07:25 +0100 Subject: [PATCH 3/3] EnableLargeSectors: restore previous supported proof types. --- itests/deals_512mb_test.go | 2 +- itests/kit/sectors.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/itests/deals_512mb_test.go b/itests/deals_512mb_test.go index eac598ab9..766d83835 100644 --- a/itests/deals_512mb_test.go +++ b/itests/deals_512mb_test.go @@ -15,7 +15,7 @@ func TestStorageDealMissingBlock(t *testing.T) { ctx := context.Background() // enable 512MiB proofs so we can conduct larger transfers. - kit.EnableLargeSectors() + kit.EnableLargeSectors(t) kit.QuietMiningLogs() client, miner, ens := kit.EnsembleMinimal(t, diff --git a/itests/kit/sectors.go b/itests/kit/sectors.go index 5f82055c0..d9c52b9ca 100644 --- a/itests/kit/sectors.go +++ b/itests/kit/sectors.go @@ -1,6 +1,8 @@ package kit import ( + "testing" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -8,9 +10,14 @@ import ( // EnableLargeSectors enables 512MiB sectors. This is useful in combination with // mock proofs, for testing larger transfers. -func EnableLargeSectors() { +func EnableLargeSectors(t *testing.T) { policy.SetSupportedProofTypes( abi.RegisteredSealProof_StackedDrg2KiBV1, - abi.RegisteredSealProof_StackedDrg512MiBV1, // <== + abi.RegisteredSealProof_StackedDrg512MiBV1, // <== here ) + t.Cleanup(func() { // reset when done. + policy.SetSupportedProofTypes( + abi.RegisteredSealProof_StackedDrg2KiBV1, + ) + }) }