itests: support larger sector sizes; add large deal test.

.circleci/config.yml

@@ -810,6 +810,11 @@ workflows:
           suite: itest-deadlines
           target: "./itests/deadlines_test.go"
       
+      - test:
+          name: test-itest-deals_512mb
+          suite: itest-deals_512mb
+          target: "./itests/deals_512mb_test.go"
+      
       - test:
           name: test-itest-deals_concurrent
           suite: itest-deals_concurrent

itests/api_test.go

@@ -7,14 +7,14 @@ import (
 	"time"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/stretchr/testify/require"
+
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/itests/kit"
-	"github.com/stretchr/testify/require"
 )
 
 func TestAPI(t *testing.T) {
@@ -186,7 +186,7 @@ func (ts *apiSuite) testNonGenesisMiner(t *testing.T) {
 	var newMiner kit.TestMiner
 	ens.Miner(&newMiner, full,
 		kit.OwnerAddr(full.DefaultKey),
-		kit.ProofType(abi.RegisteredSealProof_StackedDrg2KiBV1_1),
+		kit.SectorSize(2<<10),
 		kit.WithAllSubsystems(),
 	).Start().InterconnectAll()
 

itests/deals_512mb_test.go

@@ -0,0 +1,44 @@
+package itests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/itests/kit"
+)
+
+func TestStorageDealMissingBlock(t *testing.T) {
+	ctx := context.Background()
+
+	// enable 512MiB proofs so we can conduct larger transfers.
+	kit.EnableLargeSectors()
+	kit.QuietMiningLogs()
+
+	client, miner, ens := kit.EnsembleMinimal(t,
+		kit.MockProofs(),
+		kit.SectorSize(512<<20), // 512MiB sectors.
+	)
+	ens.InterconnectAll().BeginMining(50 * time.Millisecond)
+
+	dh := kit.NewDealHarness(t, client, miner, miner)
+
+	client.WaitTillChain(ctx, kit.HeightAtLeast(5))
+
+	res, _ := client.CreateImportFile(ctx, 0, 64<<20) // 64MiB file.
+	list, err := client.ClientListImports(ctx)
+	require.NoError(t, err)
+	require.Len(t, list, 1)
+	require.Equal(t, res.Root, *list[0].Root)
+
+	dp := dh.DefaultStartDealParams()
+	dp.Data.Root = res.Root
+	dp.FastRetrieval = true
+	dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price.
+	deal := dh.StartDeal(ctx, dp)
+
+	dh.WaitDealSealed(ctx, deal, false, false, nil)
+}

itests/kit/ensemble.go

@@ -17,6 +17,14 @@ import (
 	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/go-storedcounter"
+	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
+	power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power"
+	"github.com/ipfs/go-datastore"
+	libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto"
+	"github.com/libp2p/go-libp2p-core/peer"
+	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
+	"github.com/stretchr/testify/require"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
 	"github.com/filecoin-project/lotus/build"
@@ -43,13 +51,6 @@ import (
 	testing2 "github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/storage/mockstorage"
-	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
-	power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power"
-	"github.com/ipfs/go-datastore"
-	libp2pcrypto "github.com/libp2p/go-libp2p-core/crypto"
-	"github.com/libp2p/go-libp2p-core/peer"
-	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
-	"github.com/stretchr/testify/require"
 )
 
 func init() {
@@ -216,11 +217,11 @@ func (n *Ensemble) Miner(minerNode *TestMiner, full *TestFullNode, opts ...NodeO
 			genm    *genesis.Miner
 		)
 
-		// Default 2KiB sector for the network version
-		proofType, err := miner.SealProofTypeFromSectorSize(2<<10, n.genesis.version)
+		// Will use 2KiB sectors by default (default value of sectorSize).
+		proofType, err := miner.SealProofTypeFromSectorSize(options.sectorSize, n.genesis.version)
 		require.NoError(n.t, err)
 
-		// create the preseal commitment.
+		// Create the preseal commitment.
 		if n.options.mockProofs {
 			genm, k, err = mockstorage.PreSeal(proofType, actorAddr, sectors)
 		} else {
@@ -363,10 +364,19 @@ func (n *Ensemble) Start() *Ensemble {
 			if m.options.mainMiner == nil {
 				// this is a miner created after genesis, so it won't have a preseal.
 				// we need to create it on chain.
+
+				// we get the proof type for the requested sector size, for
+				// the current network version.
+				nv, err := m.FullNode.FullNode.StateNetworkVersion(ctx, types.EmptyTSK)
+				require.NoError(n.t, err)
+
+				proofType, err := miner.SealProofTypeFromSectorSize(m.options.sectorSize, nv)
+				require.NoError(n.t, err)
+
 				params, aerr := actors.SerializeParams(&power2.CreateMinerParams{
 					Owner:         m.OwnerKey.Address,
 					Worker:        m.OwnerKey.Address,
-					SealProofType: m.options.proofType,
+					SealProofType: proofType,
 					Peer:          abi.PeerID(m.Libp2p.PeerID),
 				})
 				require.NoError(n.t, aerr)

itests/kit/ensemble_opts.go

@@ -4,8 +4,10 @@ import (
 	"time"
 
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/stmgr"
+
 	"github.com/filecoin-project/lotus/chain/wallet"
 )
 
@@ -37,6 +39,9 @@ var DefaultEnsembleOpts = ensembleOpts{
 func MockProofs() EnsembleOpt {
 	return func(opts *ensembleOpts) error {
 		opts.mockProofs = true
+		// since we're using mock proofs, we don't need to download
+		// proof parameters
+		build.DisableBuiltinAssets = true
 		return nil
 	}
 }

itests/kit/init.go

@@ -6,16 +6,16 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+	logging "github.com/ipfs/go-log/v2"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
-	logging "github.com/ipfs/go-log/v2"
 )
 
 func init() {
 	_ = logging.SetLogLevel("*", "INFO")
 
 	policy.SetProviderCollateralSupplyTarget(big.Zero(), big.NewInt(1))
-
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
 	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
 	policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))

itests/kit/node_opts.go

@@ -3,6 +3,7 @@ package kit
 import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
@@ -30,14 +31,14 @@ type nodeOpts struct {
 	mainMiner     *TestMiner
 	disableLibp2p bool
 	optBuilders   []OptBuilder
-	proofType     abi.RegisteredSealProof
+	sectorSize    abi.SectorSize
 }
 
 // DefaultNodeOpts are the default options that will be applied to test nodes.
 var DefaultNodeOpts = nodeOpts{
-	balance:   big.Mul(big.NewInt(100000000), types.NewInt(build.FilecoinPrecision)),
-	sectors:   DefaultPresealsPerBootstrapMiner,
-	proofType: abi.RegisteredSealProof_StackedDrg2KiBV1_1, // default _concrete_ proof type for non-genesis miners (notice the _1) for new actors versions.
+	balance:    big.Mul(big.NewInt(100000000), types.NewInt(build.FilecoinPrecision)),
+	sectors:    DefaultPresealsPerBootstrapMiner,
+	sectorSize: abi.SectorSize(2 << 10), // 2KiB.
 }
 
 // OptBuilder is used to create an option after some other node is already
@@ -135,11 +136,13 @@ func ConstructorOpts(extra ...node.Option) NodeOpt {
 	}
 }
 
-// ProofType sets the proof type for this node. If you're using new actor
-// versions, this should be a _1 proof type.
-func ProofType(proofType abi.RegisteredSealProof) NodeOpt {
+// SectorSize sets the sector size for this miner. Start() will populate the
+// corresponding proof type depending on the network version (genesis network
+// version if the Ensemble is unstarted, or the current network version
+// if started).
+func SectorSize(sectorSize abi.SectorSize) NodeOpt {
 	return func(opts *nodeOpts) error {
-		opts.proofType = proofType
+		opts.sectorSize = sectorSize
 		return nil
 	}
 }

itests/kit/sectors.go

@@ -0,0 +1,16 @@
+package kit
+
+import (
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+)
+
+// EnableLargeSectors enables 512MiB sectors. This is useful in combination with
+// mock proofs, for testing larger transfers.
+func EnableLargeSectors() {
+	policy.SetSupportedProofTypes(
+		abi.RegisteredSealProof_StackedDrg2KiBV1,
+		abi.RegisteredSealProof_StackedDrg512MiBV1, // <==
+	)
+}