cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Tests for online and offline dealpadding

Browse Source 
Also bump the times on several flaky tests that can not complete in time
on a typical laptop ( and fail half the time on CircleCI )
This commit is contained in:
Peter Rabbitson 2021-07-24 04:06:02 +02:00
parent 2edf7fd25b
commit 8d873e3edc
7 changed files with 115 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.circleci/config.yml
View File

@ -820,6 +820,11 @@ workflows:
suite: itest-deals_offline
target: "./itests/deals_offline_test.go"
- test:
name: test-itest-deals_padding
suite: itest-deals_padding
target: "./itests/deals_padding_test.go"
- test:
name: test-itest-deals_power
suite: itest-deals_power

1
go.mod
View File

@ -35,6 +35,7 @@ require (
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v1.7.0
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
github.com/filecoin-project/go-fil-markets v1.6.0-rc1
github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
github.com/filecoin-project/go-multistore v0.0.3

7
go.sum
View File

@ -282,6 +282,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnman
github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
github.com/filecoin-project/go-fil-markets v1.6.0-rc1 h1:kQtND2NXz/cfGkjq+f5MCtz2oZAQabQvQ/zu4fppIps=
github.com/filecoin-project/go-fil-markets v1.6.0-rc1/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ=
@ -817,6 +819,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
@ -1217,8 +1221,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+
github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=

10
itests/deals_concurrent_test.go
View File

@ -39,7 +39,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
// For these tests where the block time is artificially short, just use
// a deal start epoch that is guaranteed to be far enough in the future
// so that the deal starts sealing in time
startEpoch := abi.ChainEpoch(2 << 12)
startEpoch := abi.ChainEpoch(8 << 10)
runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) {
api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me
@ -81,8 +81,6 @@ func TestDealCyclesConcurrent(t *testing.T) {
kit.QuietMiningLogs()
blockTime := 10 * time.Millisecond
// For these tests where the block time is artificially short, just use
// a deal start epoch that is guaranteed to be far enough in the future
// so that the deal starts sealing in time
@ -90,7 +88,7 @@ func TestDealCyclesConcurrent(t *testing.T) {
runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) {
client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs())
ens.InterconnectAll().BeginMining(blockTime)
ens.InterconnectAll().BeginMining(250 * time.Millisecond)
dh := kit.NewDealHarness(t, client, miner, miner)
dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{
@ -126,8 +124,6 @@ func TestSimultanenousTransferLimit(t *testing.T) {
policy.SetPreCommitChallengeDelay(oldDelay)
})
blockTime := 10 * time.Millisecond
// For these tests where the block time is artificially short, just use
// a deal start epoch that is guaranteed to be far enough in the future
// so that the deal starts sealing in time
@ -142,7 +138,7 @@ func TestSimultanenousTransferLimit(t *testing.T) {
node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))),
node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle)),
))
ens.InterconnectAll().BeginMining(blockTime)
ens.InterconnectAll().BeginMining(250 * time.Millisecond)
dh := kit.NewDealHarness(t, client, miner, miner)
ctx, cancel := context.WithCancel(context.Background())

35
itests/deals_offline_test.go
View File

@ -6,6 +6,8 @@ import (
"testing"
"time"
commcid "github.com/filecoin-project/go-fil-commcid"
commp "github.com/filecoin-project/go-fil-commp-hashhash"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
@ -14,17 +16,11 @@ import (
)
func TestOfflineDealFlow(t *testing.T) {
blocktime := 10 * time.Millisecond
// For these tests where the block time is artificially short, just use
// a deal start epoch that is guaranteed to be far enough in the future
// so that the deal starts sealing in time
startEpoch := abi.ChainEpoch(2 << 12)
runTest := func(t *testing.T, fastRet bool) {
runTest := func(t *testing.T, fastRet bool, upscale abi.PaddedPieceSize) {
ctx := context.Background()
client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs())
ens.InterconnectAll().BeginMining(blocktime)
client, miner, ens := kit.EnsembleMinimal(t, kit.WithAllSubsystems()) // no mock proofs
ens.InterconnectAll().BeginMining(250 * time.Millisecond)
dh := kit.NewDealHarness(t, client, miner, miner)
@ -37,8 +33,22 @@ func TestOfflineDealFlow(t *testing.T) {
require.NoError(t, err)
t.Log("FILE CID:", rootCid)
// test whether padding works as intended
if upscale > 0 {
newRawCp, err := commp.PadCommP(
pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:],
uint64(pieceInfo.PieceSize),
uint64(upscale),
)
require.NoError(t, err)
pieceInfo.PieceSize = upscale
pieceInfo.PieceCID, err = commcid.DataCommitmentV1ToCID(newRawCp)
require.NoError(t, err)
}
dp := dh.DefaultStartDealParams()
dp.DealStartEpoch = startEpoch
dp.DealStartEpoch = abi.ChainEpoch(4 << 10)
dp.FastRetrieval = fastRet
// Replace with params for manual storage deal (offline deal)
dp.Data = &storagemarket.DataRef{
@ -81,6 +91,7 @@ func TestOfflineDealFlow(t *testing.T) {
}
t.Run("stdretrieval", func(t *testing.T) { runTest(t, false) })
t.Run("fastretrieval", func(t *testing.T) { runTest(t, true) })
t.Run("stdretrieval", func(t *testing.T) { runTest(t, false, 0) })
t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 0) })
t.Run("fastretrieval", func(t *testing.T) { runTest(t, true, 1024) })
}

76
itests/deals_padding_test.go Normal file
View File

@ -0,0 +1,76 @@
package itests
import (
"context"
"testing"
"time"
commcid "github.com/filecoin-project/go-fil-commcid"
commp "github.com/filecoin-project/go-fil-commp-hashhash"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/stretchr/testify/require"
)
func TestDealPadding(t *testing.T) {
kit.QuietMiningLogs()
var blockTime = 250 * time.Millisecond
startEpoch := abi.ChainEpoch(2 << 12)
policy.SetPreCommitChallengeDelay(2)
client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs.
ens.InterconnectAll().BeginMining(blockTime)
dh := kit.NewDealHarness(t, client, miner, miner)
ctx := context.Background()
client.WaitTillChain(ctx, kit.BlockMinedBy(miner.ActorAddr))
// Create a random file, would originally be a 256-byte sector
res, inFile := client.CreateImportFile(ctx, 1, 200)
// Get the piece size and commP
pieceInfo, err := client.ClientDealPieceCID(ctx, res.Root)
require.NoError(t, err)
t.Log("FILE CID:", res.Root)
runTest := func(t *testing.T, upscale abi.PaddedPieceSize) {
// test whether padding works as intended
newRawCp, err := commp.PadCommP(
pieceInfo.PieceCID.Hash()[len(pieceInfo.PieceCID.Hash())-32:],
uint64(pieceInfo.PieceSize),
uint64(upscale),
)
require.NoError(t, err)
pcid, err := commcid.DataCommitmentV1ToCID(newRawCp)
require.NoError(t, err)
dp := dh.DefaultStartDealParams()
dp.Data.Root = res.Root
dp.Data.PieceCid = &pcid
dp.Data.PieceSize = upscale.Unpadded()
dp.DealStartEpoch = startEpoch
proposalCid := dh.StartDeal(ctx, dp)
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
di, err := client.ClientGetDealInfo(ctx, *proposalCid)
require.NoError(t, err)
require.True(t, di.PieceCID.Equals(pcid))
dh.WaitDealSealed(ctx, proposalCid, false, false, nil)
// Retrieve the deal
outFile := dh.PerformRetrieval(ctx, proposalCid, res.Root, false)
kit.AssertFilesEqual(t, inFile, outFile)
}
t.Run("padQuarterSector", func(t *testing.T) { runTest(t, 512) })
t.Run("padHalfSector", func(t *testing.T) { runTest(t, 1024) })
t.Run("padFullSector", func(t *testing.T) { runTest(t, 2048) })
}

4
itests/deals_test.go
View File

@ -21,10 +21,8 @@ func TestDealsWithSealingAndRPC(t *testing.T) {
policy.SetPreCommitChallengeDelay(oldDelay)
})
var blockTime = 50 * time.Millisecond
client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.WithAllSubsystems()) // no mock proofs.
ens.InterconnectAll().BeginMining(blockTime)
ens.InterconnectAll().BeginMining(250 * time.Millisecond)
dh := kit.NewDealHarness(t, client, miner, miner)
t.Run("stdretrieval", func(t *testing.T) {