Merge remote-tracking branch 'origin/master' into chore/upgrade_go-multihash

.circleci/config.yml

@@ -820,6 +820,11 @@ workflows:
           suite: itest-deals_offline
           target: "./itests/deals_offline_test.go"
       
+      - test:
+          name: test-itest-deals_padding
+          suite: itest-deals_padding
+          target: "./itests/deals_padding_test.go"
+      
       - test:
           name: test-itest-deals_power
           suite: itest-deals_power

blockstore/splitstore/splitstore_test.go

@@ -24,6 +24,7 @@ import (
 func init() {
 	CompactionThreshold = 5
 	CompactionBoundary = 2
+	WarmupBoundary = 0
 	logging.SetLogLevel("splitstore", "DEBUG")
 }
 

blockstore/splitstore/splitstore_warmup.go

@@ -9,10 +9,17 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	cid "github.com/ipfs/go-cid"
 
+	"github.com/filecoin-project/go-state-types/abi"
 	bstore "github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+var (
+	// WarmupBoundary is the number of epochs to load state during warmup.
+	WarmupBoundary = build.Finality
+)
+
 // warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
 // this is necessary when we sync from a snapshot or when we enable the splitstore
 // on top of an existing blockstore (which becomes the coldstore).
@@ -43,12 +50,16 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error {
 // and headers all the way up to genesis.
 // objects are written in batches so as to minimize overhead.
 func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
+	var boundaryEpoch abi.ChainEpoch
 	epoch := curTs.Height()
+	if WarmupBoundary < epoch {
+		boundaryEpoch = epoch - WarmupBoundary
+	}
 	batchHot := make([]blocks.Block, 0, batchSize)
 	count := int64(0)
 	xcount := int64(0)
 	missing := int64(0)
-	err := s.walkChain(curTs, epoch, epoch+1, // we don't load messages/receipts in warmup
+	err := s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
 				return errStopWalk
@@ -69,7 +80,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 			if err != nil {
 				if err == bstore.ErrNotFound {
 					missing++
-					return nil
+					return errStopWalk
 				}
 				return err
 			}

go.mod

@@ -34,11 +34,12 @@ require (
 	github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v1.7.0
-	github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
+	github.com/filecoin-project/go-fil-commcid v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.6.0-rc1
+	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
+	github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-multistore v0.0.3
-	github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
+	github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498
 	github.com/filecoin-project/go-state-types v0.1.1-0.20210722133031-ad9bfe54c124
 	github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe
@@ -48,7 +49,7 @@ require (
 	github.com/filecoin-project/specs-actors/v2 v2.3.5
 	github.com/filecoin-project/specs-actors/v3 v3.1.1
 	github.com/filecoin-project/specs-actors/v4 v4.0.1
-	github.com/filecoin-project/specs-actors/v5 v5.0.3-0.20210722213508-58cb5de23d1f
+	github.com/filecoin-project/specs-actors/v5 v5.0.3
 	github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506
 	github.com/filecoin-project/test-vectors/schema v0.0.5
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1

go.sum

@@ -269,7 +269,6 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW
 github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
-github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U=
 github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 h1:U9Z+76pHCKBmtdxFV7JFZJj7OVm12I6dEKwtMVbq5p0=
 github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
@@ -280,11 +279,14 @@ github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRD
 github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ=
 github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
+github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8=
+github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
+github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
+github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
 github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
-github.com/filecoin-project/go-fil-markets v1.6.0-rc1 h1:kQtND2NXz/cfGkjq+f5MCtz2oZAQabQvQ/zu4fppIps=
+github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2 h1:L5KvfMqZMxST/T4Zu8khwX0K1farbB0w2957ckkbO8A=
-github.com/filecoin-project/go-fil-markets v1.6.0-rc1/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ=
+github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2/go.mod h1:ZuFDagROUV6GfvBU//KReTQDw+EZci4rH7jMYTD10vs=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -296,8 +298,9 @@ github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec h1:r
 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
 github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI=
 github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ=
-github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg=
 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak=
+github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 h1:0BogtftbcgyBx4lP2JWM00ZK7/pXmgnrDqKp9aLTgVs=
+github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
 github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 h1:G10ezOvpH1CLXQ19EA9VWNwyL0mg536ujSayjV0yg0k=
 github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts=
 github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I=
@@ -332,8 +335,8 @@ github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIP
 github.com/filecoin-project/specs-actors/v4 v4.0.1 h1:AiWrtvJZ63MHGe6rn7tPu4nSUY8bA1KDNszqJaD5+Fg=
 github.com/filecoin-project/specs-actors/v4 v4.0.1/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng=
 github.com/filecoin-project/specs-actors/v5 v5.0.0-20210512015452-4fe3889fff57/go.mod h1:283yBMMUSDB2abcjP/hhrwTkhb9h3sfM6KGrep/ZlBI=
-github.com/filecoin-project/specs-actors/v5 v5.0.3-0.20210722213508-58cb5de23d1f h1:hSgH0xcKkL+fCnEl/beqzn2loBN9+YUsxvVvefZ6cP4=
+github.com/filecoin-project/specs-actors/v5 v5.0.3 h1:hMQIGfkZ1kN+oVGaDXbpsu9YEUEyL/zWJSeoaZpruu4=
-github.com/filecoin-project/specs-actors/v5 v5.0.3-0.20210722213508-58cb5de23d1f/go.mod h1:E0yeEl6Scl6eWeeWmxwQsAufvOAC72H6ELyh2Y62H90=
+github.com/filecoin-project/specs-actors/v5 v5.0.3/go.mod h1:E0yeEl6Scl6eWeeWmxwQsAufvOAC72H6ELyh2Y62H90=
 github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 h1:Ur/l2+6qN+lQiqjozWWc5p9UDaAMDZKTlDS98oRnlIw=
 github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
 github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg=
@@ -706,7 +709,6 @@ github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHn
 github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM=
 github.com/ipfs/go-log/v2 v2.1.2/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM=
 github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
-github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
 github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU=
 github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g=
 github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA=

itests/batch_deal_test.go

@@ -90,7 +90,11 @@ func TestBatchDealInput(t *testing.T) {
 				res, _, _, err := kit.CreateImportFile(ctx, client, rseed, piece)
 				require.NoError(t, err)
 
-				deal := dh.StartDeal(ctx, res.Root, false, dealStartEpoch)
+				dp := dh.DefaultStartDealParams()
+				dp.Data.Root = res.Root
+				dp.DealStartEpoch = dealStartEpoch
+
+				deal := dh.StartDeal(ctx, dp)
 				dh.WaitDealSealed(ctx, deal, false, true, checkNoPadding)
 			}
 

itests/deals_concurrent_test.go

@@ -39,7 +39,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
 	// For these tests where the block time is artificially short, just use
 	// a deal start epoch that is guaranteed to be far enough in the future
 	// so that the deal starts sealing in time
-	startEpoch := abi.ChainEpoch(2 << 12)
+	startEpoch := abi.ChainEpoch(8 << 10)
 
 	runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) {
 		api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me
@@ -81,8 +81,6 @@ func TestDealCyclesConcurrent(t *testing.T) {
 
 	kit.QuietMiningLogs()
 
-	blockTime := 10 * time.Millisecond
-
 	// For these tests where the block time is artificially short, just use
 	// a deal start epoch that is guaranteed to be far enough in the future
 	// so that the deal starts sealing in time
@@ -90,7 +88,7 @@ func TestDealCyclesConcurrent(t *testing.T) {
 
 	runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) {
 		client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs())
-		ens.InterconnectAll().BeginMining(blockTime)
+		ens.InterconnectAll().BeginMining(250 * time.Millisecond)
 		dh := kit.NewDealHarness(t, client, miner, miner)
 
 		dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{
@@ -126,8 +124,6 @@ func TestSimultanenousTransferLimit(t *testing.T) {
 		policy.SetPreCommitChallengeDelay(oldDelay)
 	})
 
-	blockTime := 10 * time.Millisecond
-
 	// For these tests where the block time is artificially short, just use
 	// a deal start epoch that is guaranteed to be far enough in the future
 	// so that the deal starts sealing in time
@@ -142,7 +138,7 @@ func TestSimultanenousTransferLimit(t *testing.T) {
 			node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))),
 			node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle)),
 		))
-		ens.InterconnectAll().BeginMining(blockTime)
+		ens.InterconnectAll().BeginMining(250 * time.Millisecond)
 		dh := kit.NewDealHarness(t, client, miner, miner)
 
 		ctx, cancel := context.WithCancel(context.Background())
@@ -190,7 +186,21 @@ func TestSimultanenousTransferLimit(t *testing.T) {
 		cancel()
 		wg.Wait()
 
-		require.LessOrEqual(t, maxOngoing, graphsyncThrottle)
+		// The eventing systems across go-data-transfer and go-graphsync
+		// are racy, and that's why we can't enforce graphsyncThrottle exactly,
+		// without making this test racy.
+		//
+		// Essentially what could happen is that the graphsync layer starts the
+		// next transfer before the go-data-transfer FSM has the opportunity to
+		// move the previously completed transfer to the next stage, thus giving
+		// the appearance that more than graphsyncThrottle transfers are
+		// in progress.
+		//
+		// Concurrency (20) is x10 higher than graphsyncThrottle (2), so if all
+		// 20 transfers are not happening at once, we know the throttle is
+		// in effect. Thus we are a little bit lenient here to account for the
+		// above races and allow up to graphsyncThrottle*2.
+		require.LessOrEqual(t, maxOngoing, graphsyncThrottle*2)
 	}
 
 	runTest(t)

itests/deals_offline_test.go

@@ -6,32 +6,26 @@ import (
 	"testing"
 	"time"
 
+	commcid "github.com/filecoin-project/go-fil-commcid"
+	commp "github.com/filecoin-project/go-fil-commp-hashhash"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/itests/kit"
 	"github.com/stretchr/testify/require"
 )
 
 func TestOfflineDealFlow(t *testing.T) {
-	blocktime := 10 * time.Millisecond
 
-	// For these tests where the block time is artificially short, just use
+	runTest := func(t *testing.T, fastRet bool, upscale abi.PaddedPieceSize) {
-	// a deal start epoch that is guaranteed to be far enough in the future
-	// so that the deal starts sealing in time
-	startEpoch := abi.ChainEpoch(2 << 12)
-
-	runTest := func(t *testing.T, fastRet bool) {
 		ctx := context.Background()
-		client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs())
+		client, miner, ens := kit.EnsembleMinimal(t, kit.WithAllSubsystems()) // no mock proofs
-		ens.InterconnectAll().BeginMining(blocktime)
+		ens.InterconnectAll().BeginMining(250 * time.Millisecond)
 
 		dh := kit.NewDealHarness(t, client, miner, miner)
 
 		// Create a random file and import on the client.
-		res, inFile := client.CreateImportFile(ctx, 1, 0)
+		res, inFile := client.CreateImportFile(ctx, 1, 200)
 
 		// Get the piece size and commP
 		rootCid := res.Root
@@ -39,31 +33,32 @@ func TestOfflineDealFlow(t *testing.T) {
 		require.NoError(t, err)
 		t.Log("FILE CID:", rootCid)
 
-		// Create a storage deal with the miner
+		// test whether padding works as intended
-		maddr, err := miner.ActorAddress(ctx)
+		if upscale > 0 {
-		require.NoError(t, err)
+			newRawCp, err := commp.PadCommP(
+				pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:],
+				uint64(pieceInfo.PieceSize),
+				uint64(upscale),
+			)
+			require.NoError(t, err)
 
-		addr, err := client.WalletDefaultAddress(ctx)
+			pieceInfo.PieceSize = upscale
-		require.NoError(t, err)
+			pieceInfo.PieceCID, err = commcid.DataCommitmentV1ToCID(newRawCp)
+			require.NoError(t, err)
+		}
 
-		// Manual storage deal (offline deal)
+		dp := dh.DefaultStartDealParams()
-		dataRef := &storagemarket.DataRef{
+		dp.DealStartEpoch = abi.ChainEpoch(4 << 10)
+		dp.FastRetrieval = fastRet
+		// Replace with params for manual storage deal (offline deal)
+		dp.Data = &storagemarket.DataRef{
 			TransferType: storagemarket.TTManual,
 			Root:         rootCid,
 			PieceCid:     &pieceInfo.PieceCID,
 			PieceSize:    pieceInfo.PieceSize.Unpadded(),
 		}
 
-		proposalCid, err := client.ClientStartDeal(ctx, &api.StartDealParams{
+		proposalCid := dh.StartDeal(ctx, dp)
-			Data:              dataRef,
-			Wallet:            addr,
-			Miner:             maddr,
-			EpochPrice:        types.NewInt(1000000),
-			DealStartEpoch:    startEpoch,
-			MinBlocksDuration: uint64(build.MinDealDuration),
-			FastRetrieval:     fastRet,
-		})
-		require.NoError(t, err)
 
 		// Wait for the deal to reach StorageDealCheckForAcceptance on the client
 		cd, err := client.ClientGetDealInfo(ctx, *proposalCid)
@@ -96,6 +91,7 @@ func TestOfflineDealFlow(t *testing.T) {
 
 	}
 
-	t.Run("stdretrieval", func(t *testing.T) { runTest(t, false) })
+	t.Run("stdretrieval", func(t *testing.T) { runTest(t, false, 0) })
-	t.Run("fastretrieval", func(t *testing.T) { runTest(t, true) })
+	t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 0) })
+	t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 1024) })
 }

itests/deals_padding_test.go

@@ -0,0 +1,76 @@
+package itests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	commcid "github.com/filecoin-project/go-fil-commcid"
+	commp "github.com/filecoin-project/go-fil-commp-hashhash"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDealPadding(t *testing.T) {
+
+	kit.QuietMiningLogs()
+
+	var blockTime = 250 * time.Millisecond
+	startEpoch := abi.ChainEpoch(2 << 12)
+	policy.SetPreCommitChallengeDelay(2)
+
+	client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs.
+	ens.InterconnectAll().BeginMining(blockTime)
+	dh := kit.NewDealHarness(t, client, miner, miner)
+
+	ctx := context.Background()
+	client.WaitTillChain(ctx, kit.BlockMinedBy(miner.ActorAddr))
+
+	// Create a random file, would originally be a 256-byte sector
+	res, inFile := client.CreateImportFile(ctx, 1, 200)
+
+	// Get the piece size and commP
+	pieceInfo, err := client.ClientDealPieceCID(ctx, res.Root)
+	require.NoError(t, err)
+	t.Log("FILE CID:", res.Root)
+
+	runTest := func(t *testing.T, upscale abi.PaddedPieceSize) {
+		// test whether padding works as intended
+		newRawCp, err := commp.PadCommP(
+			pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:],
+			uint64(pieceInfo.PieceSize),
+			uint64(upscale),
+		)
+		require.NoError(t, err)
+
+		pcid, err := commcid.DataCommitmentV1ToCID(newRawCp)
+		require.NoError(t, err)
+
+		dp := dh.DefaultStartDealParams()
+		dp.Data.Root = res.Root
+		dp.Data.PieceCid = &pcid
+		dp.Data.PieceSize = upscale.Unpadded()
+		dp.DealStartEpoch = startEpoch
+		proposalCid := dh.StartDeal(ctx, dp)
+
+		// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+		time.Sleep(time.Second)
+
+		di, err := client.ClientGetDealInfo(ctx, *proposalCid)
+		require.NoError(t, err)
+		require.True(t, di.PieceCID.Equals(pcid))
+
+		dh.WaitDealSealed(ctx, proposalCid, false, false, nil)
+
+		// Retrieve the deal
+		outFile := dh.PerformRetrieval(ctx, proposalCid, res.Root, false)
+
+		kit.AssertFilesEqual(t, inFile, outFile)
+	}
+
+	t.Run("padQuarterSector", func(t *testing.T) { runTest(t, 512) })
+	t.Run("padHalfSector", func(t *testing.T) { runTest(t, 1024) })
+	t.Run("padFullSector", func(t *testing.T) { runTest(t, 2048) })
+}

itests/deals_power_test.go

@@ -50,7 +50,9 @@ func TestFirstDealEnablesMining(t *testing.T) {
 	}()
 
 	// now perform the deal.
-	deal := dh.StartDeal(ctx, ref.Root, false, 0)
+	dp := dh.DefaultStartDealParams()
+	dp.Data.Root = ref.Root
+	deal := dh.StartDeal(ctx, dp)
 
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)

itests/deals_publish_test.go

@@ -69,7 +69,10 @@ func TestPublishDealsBatching(t *testing.T) {
 		upds, err := client.ClientGetDealUpdates(ctx)
 		require.NoError(t, err)
 
-		dh.StartDeal(ctx, res.Root, false, startEpoch)
+		dp := dh.DefaultStartDealParams()
+		dp.Data.Root = res.Root
+		dp.DealStartEpoch = startEpoch
+		dh.StartDeal(ctx, dp)
 
 		// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 		time.Sleep(time.Second)

itests/deals_test.go

@@ -21,10 +21,8 @@ func TestDealsWithSealingAndRPC(t *testing.T) {
 		policy.SetPreCommitChallengeDelay(oldDelay)
 	})
 
-	var blockTime = 50 * time.Millisecond
-
 	client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs.
-	ens.InterconnectAll().BeginMining(blockTime)
+	ens.InterconnectAll().BeginMining(250 * time.Millisecond)
 	dh := kit.NewDealHarness(t, client, miner, miner)
 
 	t.Run("stdretrieval", func(t *testing.T) {

itests/kit/deals.go

@@ -88,7 +88,11 @@ func (dh *DealHarness) MakeOnlineDeal(ctx context.Context, params MakeFullDealPa
 		dh.t.Logf("deal-making continuing; current height is %d", ts.Height())
 	}
 
-	deal = dh.StartDeal(ctx, res.Root, params.FastRet, params.StartEpoch)
+	dp := dh.DefaultStartDealParams()
+	dp.Data.Root = res.Root
+	dp.DealStartEpoch = params.StartEpoch
+	dp.FastRetrieval = params.FastRet
+	deal = dh.StartDeal(ctx, dp)
 
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)
@@ -97,29 +101,28 @@ func (dh *DealHarness) MakeOnlineDeal(ctx context.Context, params MakeFullDealPa
 	return deal, res, path
 }
 
-// StartDeal starts a storage deal between the client and the miner.
+func (dh *DealHarness) DefaultStartDealParams() api.StartDealParams {
-func (dh *DealHarness) StartDeal(ctx context.Context, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid {
+	dp := api.StartDealParams{
-	maddr, err := dh.main.ActorAddress(ctx)
+		Data:              &storagemarket.DataRef{TransferType: storagemarket.TTGraphsync},
-	require.NoError(dh.t, err)
-
-	addr, err := dh.client.WalletDefaultAddress(ctx)
-	require.NoError(dh.t, err)
-
-	deal, err := dh.client.ClientStartDeal(ctx, &api.StartDealParams{
-		Data: &storagemarket.DataRef{
-			TransferType: storagemarket.TTGraphsync,
-			Root:         fcid,
-		},
-		Wallet:            addr,
-		Miner:             maddr,
 		EpochPrice:        types.NewInt(1000000),
-		DealStartEpoch:    startEpoch,
 		MinBlocksDuration: uint64(build.MinDealDuration),
-		FastRetrieval:     fastRet,
+	}
-	})
+
+	var err error
+	dp.Miner, err = dh.main.ActorAddress(context.Background())
 	require.NoError(dh.t, err)
 
-	return deal
+	dp.Wallet, err = dh.client.WalletDefaultAddress(context.Background())
+	require.NoError(dh.t, err)
+
+	return dp
+}
+
+// StartDeal starts a storage deal between the client and the miner.
+func (dh *DealHarness) StartDeal(ctx context.Context, dealParams api.StartDealParams) *cid.Cid {
+	dealProposalCid, err := dh.client.ClientStartDeal(ctx, &dealParams)
+	require.NoError(dh.t, err)
+	return dealProposalCid
 }
 
 // WaitDealSealed waits until the deal is sealed.