diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c77c2128..4d24e25b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -820,6 +820,11 @@ workflows: suite: itest-deals_offline target: "./itests/deals_offline_test.go" + - test: + name: test-itest-deals_padding + suite: itest-deals_padding + target: "./itests/deals_padding_test.go" + - test: name: test-itest-deals_power suite: itest-deals_power diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index b945eb90b..df9984d41 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -24,6 +24,7 @@ import ( func init() { CompactionThreshold = 5 CompactionBoundary = 2 + WarmupBoundary = 0 logging.SetLogLevel("splitstore", "DEBUG") } diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index 7c5769e22..2079a5474 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -9,10 +9,17 @@ import ( blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-state-types/abi" bstore "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" ) +var ( + // WarmupBoundary is the number of epochs to load state during warmup. + WarmupBoundary = build.Finality +) + // warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore; // this is necessary when we sync from a snapshot or when we enable the splitstore // on top of an existing blockstore (which becomes the coldstore). @@ -43,12 +50,16 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { // and headers all the way up to genesis. // objects are written in batches so as to minimize overhead. func (s *SplitStore) doWarmup(curTs *types.TipSet) error { + var boundaryEpoch abi.ChainEpoch epoch := curTs.Height() + if WarmupBoundary < epoch { + boundaryEpoch = epoch - WarmupBoundary + } batchHot := make([]blocks.Block, 0, batchSize) count := int64(0) xcount := int64(0) missing := int64(0) - err := s.walkChain(curTs, epoch, epoch+1, // we don't load messages/receipts in warmup + err := s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup func(c cid.Cid) error { if isUnitaryObject(c) { return errStopWalk @@ -69,7 +80,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error { if err != nil { if err == bstore.ErrNotFound { missing++ - return nil + return errStopWalk } return err } diff --git a/go.mod b/go.mod index 808a74ea1..d141bd16a 100644 --- a/go.mod +++ b/go.mod @@ -34,11 +34,12 @@ require ( github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.7.0 - github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a - github.com/filecoin-project/go-fil-markets v1.6.0-rc1 + github.com/filecoin-project/go-fil-commcid v0.1.0 + github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 + github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 - github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 + github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 github.com/filecoin-project/go-state-types v0.1.1-0.20210722133031-ad9bfe54c124 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe @@ -48,7 +49,7 @@ require ( github.com/filecoin-project/specs-actors/v2 v2.3.5 github.com/filecoin-project/specs-actors/v3 v3.1.1 github.com/filecoin-project/specs-actors/v4 v4.0.1 - github.com/filecoin-project/specs-actors/v5 v5.0.3-0.20210722213508-58cb5de23d1f + github.com/filecoin-project/specs-actors/v5 v5.0.3 github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 63773c2c7..8dd40864e 100644 --- a/go.sum +++ b/go.sum @@ -269,7 +269,6 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= -github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 h1:U9Z+76pHCKBmtdxFV7JFZJj7OVm12I6dEKwtMVbq5p0= github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= @@ -280,11 +279,14 @@ github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRD github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8= +github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.6.0-rc1 h1:kQtND2NXz/cfGkjq+f5MCtz2oZAQabQvQ/zu4fppIps= -github.com/filecoin-project/go-fil-markets v1.6.0-rc1/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= +github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2 h1:L5KvfMqZMxST/T4Zu8khwX0K1farbB0w2957ckkbO8A= +github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2/go.mod h1:ZuFDagROUV6GfvBU//KReTQDw+EZci4rH7jMYTD10vs= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -296,8 +298,9 @@ github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec h1:r github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= -github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 h1:0BogtftbcgyBx4lP2JWM00ZK7/pXmgnrDqKp9aLTgVs= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 h1:G10ezOvpH1CLXQ19EA9VWNwyL0mg536ujSayjV0yg0k= github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= @@ -332,8 +335,8 @@ github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIP github.com/filecoin-project/specs-actors/v4 v4.0.1 h1:AiWrtvJZ63MHGe6rn7tPu4nSUY8bA1KDNszqJaD5+Fg= github.com/filecoin-project/specs-actors/v4 v4.0.1/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= github.com/filecoin-project/specs-actors/v5 v5.0.0-20210512015452-4fe3889fff57/go.mod h1:283yBMMUSDB2abcjP/hhrwTkhb9h3sfM6KGrep/ZlBI= -github.com/filecoin-project/specs-actors/v5 v5.0.3-0.20210722213508-58cb5de23d1f h1:hSgH0xcKkL+fCnEl/beqzn2loBN9+YUsxvVvefZ6cP4= -github.com/filecoin-project/specs-actors/v5 v5.0.3-0.20210722213508-58cb5de23d1f/go.mod h1:E0yeEl6Scl6eWeeWmxwQsAufvOAC72H6ELyh2Y62H90= +github.com/filecoin-project/specs-actors/v5 v5.0.3 h1:hMQIGfkZ1kN+oVGaDXbpsu9YEUEyL/zWJSeoaZpruu4= +github.com/filecoin-project/specs-actors/v5 v5.0.3/go.mod h1:E0yeEl6Scl6eWeeWmxwQsAufvOAC72H6ELyh2Y62H90= github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 h1:Ur/l2+6qN+lQiqjozWWc5p9UDaAMDZKTlDS98oRnlIw= github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= @@ -706,7 +709,6 @@ github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHn github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.2/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= -github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= diff --git a/itests/batch_deal_test.go b/itests/batch_deal_test.go index 3881c917a..01622486a 100644 --- a/itests/batch_deal_test.go +++ b/itests/batch_deal_test.go @@ -90,7 +90,11 @@ func TestBatchDealInput(t *testing.T) { res, _, _, err := kit.CreateImportFile(ctx, client, rseed, piece) require.NoError(t, err) - deal := dh.StartDeal(ctx, res.Root, false, dealStartEpoch) + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.DealStartEpoch = dealStartEpoch + + deal := dh.StartDeal(ctx, dp) dh.WaitDealSealed(ctx, deal, false, true, checkNoPadding) } diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 241c9071d..69e1b4e7f 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -39,7 +39,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time - startEpoch := abi.ChainEpoch(2 << 12) + startEpoch := abi.ChainEpoch(8 << 10) runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) { api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me @@ -81,8 +81,6 @@ func TestDealCyclesConcurrent(t *testing.T) { kit.QuietMiningLogs() - blockTime := 10 * time.Millisecond - // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time @@ -90,7 +88,7 @@ func TestDealCyclesConcurrent(t *testing.T) { runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) - ens.InterconnectAll().BeginMining(blockTime) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ @@ -126,8 +124,6 @@ func TestSimultanenousTransferLimit(t *testing.T) { policy.SetPreCommitChallengeDelay(oldDelay) }) - blockTime := 10 * time.Millisecond - // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time @@ -142,7 +138,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))), node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle)), )) - ens.InterconnectAll().BeginMining(blockTime) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) ctx, cancel := context.WithCancel(context.Background()) @@ -190,7 +186,21 @@ func TestSimultanenousTransferLimit(t *testing.T) { cancel() wg.Wait() - require.LessOrEqual(t, maxOngoing, graphsyncThrottle) + // The eventing systems across go-data-transfer and go-graphsync + // are racy, and that's why we can't enforce graphsyncThrottle exactly, + // without making this test racy. + // + // Essentially what could happen is that the graphsync layer starts the + // next transfer before the go-data-transfer FSM has the opportunity to + // move the previously completed transfer to the next stage, thus giving + // the appearance that more than graphsyncThrottle transfers are + // in progress. + // + // Concurrency (20) is x10 higher than graphsyncThrottle (2), so if all + // 20 transfers are not happening at once, we know the throttle is + // in effect. Thus we are a little bit lenient here to account for the + // above races and allow up to graphsyncThrottle*2. + require.LessOrEqual(t, maxOngoing, graphsyncThrottle*2) } runTest(t) diff --git a/itests/deals_offline_test.go b/itests/deals_offline_test.go index ceae46fdf..003f12b11 100644 --- a/itests/deals_offline_test.go +++ b/itests/deals_offline_test.go @@ -6,32 +6,26 @@ import ( "testing" "time" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" ) func TestOfflineDealFlow(t *testing.T) { - blocktime := 10 * time.Millisecond - // For these tests where the block time is artificially short, just use - // a deal start epoch that is guaranteed to be far enough in the future - // so that the deal starts sealing in time - startEpoch := abi.ChainEpoch(2 << 12) - - runTest := func(t *testing.T, fastRet bool) { + runTest := func(t *testing.T, fastRet bool, upscale abi.PaddedPieceSize) { ctx := context.Background() - client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) - ens.InterconnectAll().BeginMining(blocktime) + client, miner, ens := kit.EnsembleMinimal(t, kit.WithAllSubsystems()) // no mock proofs + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) // Create a random file and import on the client. - res, inFile := client.CreateImportFile(ctx, 1, 0) + res, inFile := client.CreateImportFile(ctx, 1, 200) // Get the piece size and commP rootCid := res.Root @@ -39,31 +33,32 @@ func TestOfflineDealFlow(t *testing.T) { require.NoError(t, err) t.Log("FILE CID:", rootCid) - // Create a storage deal with the miner - maddr, err := miner.ActorAddress(ctx) - require.NoError(t, err) + // test whether padding works as intended + if upscale > 0 { + newRawCp, err := commp.PadCommP( + pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:], + uint64(pieceInfo.PieceSize), + uint64(upscale), + ) + require.NoError(t, err) - addr, err := client.WalletDefaultAddress(ctx) - require.NoError(t, err) + pieceInfo.PieceSize = upscale + pieceInfo.PieceCID, err = commcid.DataCommitmentV1ToCID(newRawCp) + require.NoError(t, err) + } - // Manual storage deal (offline deal) - dataRef := &storagemarket.DataRef{ + dp := dh.DefaultStartDealParams() + dp.DealStartEpoch = abi.ChainEpoch(4 << 10) + dp.FastRetrieval = fastRet + // Replace with params for manual storage deal (offline deal) + dp.Data = &storagemarket.DataRef{ TransferType: storagemarket.TTManual, Root: rootCid, PieceCid: &pieceInfo.PieceCID, PieceSize: pieceInfo.PieceSize.Unpadded(), } - proposalCid, err := client.ClientStartDeal(ctx, &api.StartDealParams{ - Data: dataRef, - Wallet: addr, - Miner: maddr, - EpochPrice: types.NewInt(1000000), - DealStartEpoch: startEpoch, - MinBlocksDuration: uint64(build.MinDealDuration), - FastRetrieval: fastRet, - }) - require.NoError(t, err) + proposalCid := dh.StartDeal(ctx, dp) // Wait for the deal to reach StorageDealCheckForAcceptance on the client cd, err := client.ClientGetDealInfo(ctx, *proposalCid) @@ -96,6 +91,7 @@ func TestOfflineDealFlow(t *testing.T) { } - t.Run("stdretrieval", func(t *testing.T) { runTest(t, false) }) - t.Run("fastretrieval", func(t *testing.T) { runTest(t, true) }) + t.Run("stdretrieval", func(t *testing.T) { runTest(t, false, 0) }) + t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 0) }) + t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 1024) }) } diff --git a/itests/deals_padding_test.go b/itests/deals_padding_test.go new file mode 100644 index 000000000..cd15d30d7 --- /dev/null +++ b/itests/deals_padding_test.go @@ -0,0 +1,76 @@ +package itests + +import ( + "context" + "testing" + "time" + + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/stretchr/testify/require" +) + +func TestDealPadding(t *testing.T) { + + kit.QuietMiningLogs() + + var blockTime = 250 * time.Millisecond + startEpoch := abi.ChainEpoch(2 << 12) + policy.SetPreCommitChallengeDelay(2) + + client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs. + ens.InterconnectAll().BeginMining(blockTime) + dh := kit.NewDealHarness(t, client, miner, miner) + + ctx := context.Background() + client.WaitTillChain(ctx, kit.BlockMinedBy(miner.ActorAddr)) + + // Create a random file, would originally be a 256-byte sector + res, inFile := client.CreateImportFile(ctx, 1, 200) + + // Get the piece size and commP + pieceInfo, err := client.ClientDealPieceCID(ctx, res.Root) + require.NoError(t, err) + t.Log("FILE CID:", res.Root) + + runTest := func(t *testing.T, upscale abi.PaddedPieceSize) { + // test whether padding works as intended + newRawCp, err := commp.PadCommP( + pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:], + uint64(pieceInfo.PieceSize), + uint64(upscale), + ) + require.NoError(t, err) + + pcid, err := commcid.DataCommitmentV1ToCID(newRawCp) + require.NoError(t, err) + + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.Data.PieceCid = &pcid + dp.Data.PieceSize = upscale.Unpadded() + dp.DealStartEpoch = startEpoch + proposalCid := dh.StartDeal(ctx, dp) + + // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(time.Second) + + di, err := client.ClientGetDealInfo(ctx, *proposalCid) + require.NoError(t, err) + require.True(t, di.PieceCID.Equals(pcid)) + + dh.WaitDealSealed(ctx, proposalCid, false, false, nil) + + // Retrieve the deal + outFile := dh.PerformRetrieval(ctx, proposalCid, res.Root, false) + + kit.AssertFilesEqual(t, inFile, outFile) + } + + t.Run("padQuarterSector", func(t *testing.T) { runTest(t, 512) }) + t.Run("padHalfSector", func(t *testing.T) { runTest(t, 1024) }) + t.Run("padFullSector", func(t *testing.T) { runTest(t, 2048) }) +} diff --git a/itests/deals_power_test.go b/itests/deals_power_test.go index 16ad8ae6a..0c29ad060 100644 --- a/itests/deals_power_test.go +++ b/itests/deals_power_test.go @@ -50,7 +50,9 @@ func TestFirstDealEnablesMining(t *testing.T) { }() // now perform the deal. - deal := dh.StartDeal(ctx, ref.Root, false, 0) + dp := dh.DefaultStartDealParams() + dp.Data.Root = ref.Root + deal := dh.StartDeal(ctx, dp) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) diff --git a/itests/deals_publish_test.go b/itests/deals_publish_test.go index 10592d8b4..6cefde6b9 100644 --- a/itests/deals_publish_test.go +++ b/itests/deals_publish_test.go @@ -69,7 +69,10 @@ func TestPublishDealsBatching(t *testing.T) { upds, err := client.ClientGetDealUpdates(ctx) require.NoError(t, err) - dh.StartDeal(ctx, res.Root, false, startEpoch) + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.DealStartEpoch = startEpoch + dh.StartDeal(ctx, dp) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) diff --git a/itests/deals_test.go b/itests/deals_test.go index f2e106f1f..a461586a1 100644 --- a/itests/deals_test.go +++ b/itests/deals_test.go @@ -21,10 +21,8 @@ func TestDealsWithSealingAndRPC(t *testing.T) { policy.SetPreCommitChallengeDelay(oldDelay) }) - var blockTime = 50 * time.Millisecond - client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs. - ens.InterconnectAll().BeginMining(blockTime) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) t.Run("stdretrieval", func(t *testing.T) { diff --git a/itests/kit/deals.go b/itests/kit/deals.go index 311db4b25..0832447f2 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -88,7 +88,11 @@ func (dh *DealHarness) MakeOnlineDeal(ctx context.Context, params MakeFullDealPa dh.t.Logf("deal-making continuing; current height is %d", ts.Height()) } - deal = dh.StartDeal(ctx, res.Root, params.FastRet, params.StartEpoch) + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.DealStartEpoch = params.StartEpoch + dp.FastRetrieval = params.FastRet + deal = dh.StartDeal(ctx, dp) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) @@ -97,29 +101,28 @@ func (dh *DealHarness) MakeOnlineDeal(ctx context.Context, params MakeFullDealPa return deal, res, path } -// StartDeal starts a storage deal between the client and the miner. -func (dh *DealHarness) StartDeal(ctx context.Context, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid { - maddr, err := dh.main.ActorAddress(ctx) - require.NoError(dh.t, err) - - addr, err := dh.client.WalletDefaultAddress(ctx) - require.NoError(dh.t, err) - - deal, err := dh.client.ClientStartDeal(ctx, &api.StartDealParams{ - Data: &storagemarket.DataRef{ - TransferType: storagemarket.TTGraphsync, - Root: fcid, - }, - Wallet: addr, - Miner: maddr, +func (dh *DealHarness) DefaultStartDealParams() api.StartDealParams { + dp := api.StartDealParams{ + Data: &storagemarket.DataRef{TransferType: storagemarket.TTGraphsync}, EpochPrice: types.NewInt(1000000), - DealStartEpoch: startEpoch, MinBlocksDuration: uint64(build.MinDealDuration), - FastRetrieval: fastRet, - }) + } + + var err error + dp.Miner, err = dh.main.ActorAddress(context.Background()) require.NoError(dh.t, err) - return deal + dp.Wallet, err = dh.client.WalletDefaultAddress(context.Background()) + require.NoError(dh.t, err) + + return dp +} + +// StartDeal starts a storage deal between the client and the miner. +func (dh *DealHarness) StartDeal(ctx context.Context, dealParams api.StartDealParams) *cid.Cid { + dealProposalCid, err := dh.client.ClientStartDeal(ctx, &dealParams) + require.NoError(dh.t, err) + return dealProposalCid } // WaitDealSealed waits until the deal is sealed.