diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c77c2128..4d24e25b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -820,6 +820,11 @@ workflows: suite: itest-deals_offline target: "./itests/deals_offline_test.go" + - test: + name: test-itest-deals_padding + suite: itest-deals_padding + target: "./itests/deals_padding_test.go" + - test: name: test-itest-deals_power suite: itest-deals_power diff --git a/go.mod b/go.mod index 1725754b4..c702c3a5c 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.7.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a + github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-fil-markets v1.6.0-rc1 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 diff --git a/go.sum b/go.sum index ee34abe5c..534aa8a6e 100644 --- a/go.sum +++ b/go.sum @@ -282,6 +282,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnman github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= github.com/filecoin-project/go-fil-markets v1.6.0-rc1 h1:kQtND2NXz/cfGkjq+f5MCtz2oZAQabQvQ/zu4fppIps= github.com/filecoin-project/go-fil-markets v1.6.0-rc1/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= @@ -817,6 +819,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -1217,8 +1221,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 241c9071d..3c2fbc918 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -39,7 +39,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time - startEpoch := abi.ChainEpoch(2 << 12) + startEpoch := abi.ChainEpoch(8 << 10) runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) { api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me @@ -81,8 +81,6 @@ func TestDealCyclesConcurrent(t *testing.T) { kit.QuietMiningLogs() - blockTime := 10 * time.Millisecond - // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time @@ -90,7 +88,7 @@ func TestDealCyclesConcurrent(t *testing.T) { runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) - ens.InterconnectAll().BeginMining(blockTime) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ @@ -126,8 +124,6 @@ func TestSimultanenousTransferLimit(t *testing.T) { policy.SetPreCommitChallengeDelay(oldDelay) }) - blockTime := 10 * time.Millisecond - // For these tests where the block time is artificially short, just use // a deal start epoch that is guaranteed to be far enough in the future // so that the deal starts sealing in time @@ -142,7 +138,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))), node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle)), )) - ens.InterconnectAll().BeginMining(blockTime) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) ctx, cancel := context.WithCancel(context.Background()) diff --git a/itests/deals_offline_test.go b/itests/deals_offline_test.go index 8e40fbcb7..003f12b11 100644 --- a/itests/deals_offline_test.go +++ b/itests/deals_offline_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" @@ -14,17 +16,11 @@ import ( ) func TestOfflineDealFlow(t *testing.T) { - blocktime := 10 * time.Millisecond - // For these tests where the block time is artificially short, just use - // a deal start epoch that is guaranteed to be far enough in the future - // so that the deal starts sealing in time - startEpoch := abi.ChainEpoch(2 << 12) - - runTest := func(t *testing.T, fastRet bool) { + runTest := func(t *testing.T, fastRet bool, upscale abi.PaddedPieceSize) { ctx := context.Background() - client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs()) - ens.InterconnectAll().BeginMining(blocktime) + client, miner, ens := kit.EnsembleMinimal(t, kit.WithAllSubsystems()) // no mock proofs + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) @@ -37,8 +33,22 @@ func TestOfflineDealFlow(t *testing.T) { require.NoError(t, err) t.Log("FILE CID:", rootCid) + // test whether padding works as intended + if upscale > 0 { + newRawCp, err := commp.PadCommP( + pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:], + uint64(pieceInfo.PieceSize), + uint64(upscale), + ) + require.NoError(t, err) + + pieceInfo.PieceSize = upscale + pieceInfo.PieceCID, err = commcid.DataCommitmentV1ToCID(newRawCp) + require.NoError(t, err) + } + dp := dh.DefaultStartDealParams() - dp.DealStartEpoch = startEpoch + dp.DealStartEpoch = abi.ChainEpoch(4 << 10) dp.FastRetrieval = fastRet // Replace with params for manual storage deal (offline deal) dp.Data = &storagemarket.DataRef{ @@ -81,6 +91,7 @@ func TestOfflineDealFlow(t *testing.T) { } - t.Run("stdretrieval", func(t *testing.T) { runTest(t, false) }) - t.Run("fastretrieval", func(t *testing.T) { runTest(t, true) }) + t.Run("stdretrieval", func(t *testing.T) { runTest(t, false, 0) }) + t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 0) }) + t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 1024) }) } diff --git a/itests/deals_padding_test.go b/itests/deals_padding_test.go new file mode 100644 index 000000000..cd15d30d7 --- /dev/null +++ b/itests/deals_padding_test.go @@ -0,0 +1,76 @@ +package itests + +import ( + "context" + "testing" + "time" + + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/stretchr/testify/require" +) + +func TestDealPadding(t *testing.T) { + + kit.QuietMiningLogs() + + var blockTime = 250 * time.Millisecond + startEpoch := abi.ChainEpoch(2 << 12) + policy.SetPreCommitChallengeDelay(2) + + client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs. + ens.InterconnectAll().BeginMining(blockTime) + dh := kit.NewDealHarness(t, client, miner, miner) + + ctx := context.Background() + client.WaitTillChain(ctx, kit.BlockMinedBy(miner.ActorAddr)) + + // Create a random file, would originally be a 256-byte sector + res, inFile := client.CreateImportFile(ctx, 1, 200) + + // Get the piece size and commP + pieceInfo, err := client.ClientDealPieceCID(ctx, res.Root) + require.NoError(t, err) + t.Log("FILE CID:", res.Root) + + runTest := func(t *testing.T, upscale abi.PaddedPieceSize) { + // test whether padding works as intended + newRawCp, err := commp.PadCommP( + pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:], + uint64(pieceInfo.PieceSize), + uint64(upscale), + ) + require.NoError(t, err) + + pcid, err := commcid.DataCommitmentV1ToCID(newRawCp) + require.NoError(t, err) + + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.Data.PieceCid = &pcid + dp.Data.PieceSize = upscale.Unpadded() + dp.DealStartEpoch = startEpoch + proposalCid := dh.StartDeal(ctx, dp) + + // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(time.Second) + + di, err := client.ClientGetDealInfo(ctx, *proposalCid) + require.NoError(t, err) + require.True(t, di.PieceCID.Equals(pcid)) + + dh.WaitDealSealed(ctx, proposalCid, false, false, nil) + + // Retrieve the deal + outFile := dh.PerformRetrieval(ctx, proposalCid, res.Root, false) + + kit.AssertFilesEqual(t, inFile, outFile) + } + + t.Run("padQuarterSector", func(t *testing.T) { runTest(t, 512) }) + t.Run("padHalfSector", func(t *testing.T) { runTest(t, 1024) }) + t.Run("padFullSector", func(t *testing.T) { runTest(t, 2048) }) +} diff --git a/itests/deals_test.go b/itests/deals_test.go index f2e106f1f..a461586a1 100644 --- a/itests/deals_test.go +++ b/itests/deals_test.go @@ -21,10 +21,8 @@ func TestDealsWithSealingAndRPC(t *testing.T) { policy.SetPreCommitChallengeDelay(oldDelay) }) - var blockTime = 50 * time.Millisecond - client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs. - ens.InterconnectAll().BeginMining(blockTime) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) dh := kit.NewDealHarness(t, client, miner, miner) t.Run("stdretrieval", func(t *testing.T) {