lotus/itests/batch_deal_test.go

156 lines
4.2 KiB
Go
Raw Normal View History

2022-08-29 14:25:30 +00:00
// stm: #integration
package itests
import (
2021-05-20 15:12:42 +00:00
"context"
"fmt"
"sort"
"testing"
"time"
2022-06-14 15:00:51 +00:00
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
2021-06-18 18:45:29 +00:00
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
)
func TestBatchDealInput(t *testing.T) {
feat: Add additional test annotations (#8272) * Annotate api,proxy_util,blockstore_badger, policy tests * Annotate splitstore: bsbadger / markset * Annotate splitstore feature * Annotate union/timed blockstore tests * Annotate openrpc, diff_adt tests * Annotate error,drand,events tests * Annotate predicates_test * Fix annotations * Annotate tscache, gen tests * Annotate fundmanager test * Annotate repub and selection tests * Annotate statetree_test * Annotate forks_test * Annotate searchwait_test.go * Fix duplicated @@ symbols * Annotate chain stmgr/store tests * Annotate more (types) tests * More tests annotated * Annotate conformance chaos actor tests * Annotate more integration tests * Annotate journal system tests * Annotate more tests. * Annotate gas,head buffer behaviors * Fix markset annotations * doc: test annotations for the markets dagstore wrapper * Annotate miner_api test in dagstore * Annotate more test files * Remove bad annotations from fsrepo * Annotate wdpost system * Remove bad annotations * Renamce "conformance" to "chaos_actor" tests * doc: stm annotations for blockheader & election proof tests * Annotate remaining "A" tests * annotate: stm for error_test * memrepo_test.go * Annotate "b" file tests * message_test.go * doc: stm annotate for fsrepo_test * Annotate "c" file tests * Annotate "D" test files * message_test.go * doc: stm annotate for chain, node/config & client * docs: stm annotate node_test * Annotate u,v,wl tests * doc: stm annotations for various test files * Annotate "T" test files * doc: stm annotate for proxy_util_test & policy_test * doc: stm annotate for various tests * doc: final few stm annotations * Add mempool unit tests * Add two more memPool Add tests * Update submodules * Add check function tests * Add stm annotations, refactor test helper * Annotate api,proxy_util,blockstore_badger, policy tests * Annotate splitstore: bsbadger / markset solving merge conflicts * Annotate splitstore feature * Annotate union/timed blockstore tests * Annotate openrpc, diff_adt tests * Annotate error,drand,events tests * Annotate predicates_test * Fix annotations * Annotate tscache, gen tests * Annotate fundmanager test * Annotate statetree_test * Annotate forks_test * Annotate searchwait_test.go * Fix duplicated @@ symbols * Annotate chain stmgr/store tests * Annotate more (types) tests * More tests annotated * Annotate conformance chaos actor tests * Annotate more integration tests * Annotate journal system tests * Annotate more tests. * Annotate gas,head buffer behaviors solve merge conflict * Fix markset annotations * Annotate miner_api test in dagstore * Annotate more test files * doc: test annotations for the markets dagstore wrapper * Annotate wdpost system * Renamce "conformance" to "chaos_actor" tests * Annotate remaining "A" tests * doc: stm annotations for blockheader & election proof tests * annotate: stm for error_test * Annotate "b" file tests * memrepo_test.go * Annotate "c" file tests * message_test.go * Annotate "D" test files * doc: stm annotate for fsrepo_test * Annotate u,v,wl tests * message_test.go * doc: stm annotate for chain, node/config & client * docs: stm annotate node_test * Annotate "T" test files * doc: stm annotations for various test files * Add mempool unit tests solve merge conflict * doc: stm annotate for proxy_util_test & policy_test * doc: stm annotate for various tests * doc: final few stm annotations * Add two more memPool Add tests * Update submodules * Add check function tests solve conflict * Add stm annotations, refactor test helper solve merge conflict * Change CLI test kinds to "unit" * Fix double merged test * Fix ccupgrade_test merge * Fix lint issues * Add stm annotation to types_Test * Test vectors submodule * Add file annotation to burn_test Co-authored-by: Nikola Divic <divicnikola@gmail.com> Co-authored-by: TheMenko <themenkoprojects@gmail.com>
2022-03-16 17:37:34 +00:00
//stm: @MINER_SECTOR_STATUS_001, @MINER_SECTOR_LIST_001
2021-06-18 18:45:29 +00:00
kit.QuietMiningLogs()
var (
blockTime = 10 * time.Millisecond
// For these tests where the block time is artificially short, just use
// a deal start epoch that is guaranteed to be far enough in the future
// so that the deal starts sealing in time
dealStartEpoch = abi.ChainEpoch(2 << 12)
)
run := func(piece, deals, expectSectors int) func(t *testing.T) {
return func(t *testing.T) {
2023-03-27 13:34:59 +00:00
t.Logf("batchtest start")
2021-06-15 13:38:09 +00:00
ctx := context.Background()
publishPeriod := 10 * time.Second
maxDealsPerMsg := uint64(deals)
// Set max deals per publish deals message to maxDealsPerMsg
2021-06-18 18:45:29 +00:00
opts := kit.ConstructorOpts(node.Options(
2021-06-15 13:38:09 +00:00
node.Override(
new(*storageadapter.DealPublisher),
storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{
Period: publishPeriod,
MaxDealsPerMsg: maxDealsPerMsg,
})),
node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) {
return func() (sealiface.Config, error) {
2022-03-18 11:18:00 +00:00
cfg := config.DefaultStorageMiner()
sc := modules.ToSealingConfig(cfg.Dealmaking, cfg.Sealing)
sc.MaxWaitDealsSectors = 2
sc.MaxSealingSectors = 1
sc.MaxSealingSectorsForDeals = 3
sc.AlwaysKeepUnsealedCopy = true
sc.WaitDealsDelay = time.Hour
sc.BatchPreCommits = false
sc.AggregateCommits = false
return sc, nil
2021-06-15 13:38:09 +00:00
}, nil
}),
))
client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts, kit.ThroughRPC())
2021-06-15 13:38:09 +00:00
ens.InterconnectAll().BeginMining(blockTime)
dh := kit.NewDealHarness(t, client, miner, miner)
err := miner.MarketSetAsk(ctx, big.Zero(), big.Zero(), 200, 128, 32<<30)
require.NoError(t, err)
2023-03-27 13:34:59 +00:00
t.Logf("batchtest ask set")
checkNoPadding := func() {
sl, err := miner.SectorsListNonGenesis(ctx)
require.NoError(t, err)
sort.Slice(sl, func(i, j int) bool {
return sl[i] < sl[j]
})
for _, snum := range sl {
2021-06-15 13:38:09 +00:00
si, err := miner.SectorsStatus(ctx, snum, false)
require.NoError(t, err)
// fmt.Printf("S %d: %+v %s\n", snum, si.Deals, si.State)
for _, deal := range si.Deals {
if deal == 0 {
fmt.Printf("sector %d had a padding piece!\n", snum)
}
}
}
}
// Starts a deal and waits until it's published
runDealTillSeal := func(rseed int) {
2021-06-18 18:45:29 +00:00
res, _, _, err := kit.CreateImportFile(ctx, client, rseed, piece)
require.NoError(t, err)
dp := dh.DefaultStartDealParams()
dp.Data.Root = res.Root
dp.DealStartEpoch = dealStartEpoch
deal := dh.StartDeal(ctx, dp)
dh.WaitDealSealed(ctx, deal, false, true, checkNoPadding)
}
// Run maxDealsPerMsg deals in parallel
done := make(chan struct{}, maxDealsPerMsg)
for rseed := 0; rseed < int(maxDealsPerMsg); rseed++ {
rseed := rseed
go func() {
runDealTillSeal(rseed)
done <- struct{}{}
}()
}
2023-03-27 13:34:59 +00:00
t.Logf("batchtest deals started")
// Wait for maxDealsPerMsg of the deals to be published
for i := 0; i < int(maxDealsPerMsg); i++ {
<-done
}
2023-03-27 13:34:59 +00:00
t.Logf("batchtest deals published")
checkNoPadding()
2023-03-27 13:34:59 +00:00
t.Logf("batchtest no padding")
sl, err := miner.SectorsListNonGenesis(ctx)
require.NoError(t, err)
require.Equal(t, len(sl), expectSectors)
2023-03-27 13:34:59 +00:00
t.Logf("batchtest done")
}
}
t.Run("4-p1600B", run(1600, 4, 4))
t.Run("4-p513B", run(513, 4, 2))
if !testing.Short() {
t.Run("32-p257B", run(257, 32, 8))
// fixme: this appears to break data-transfer / markets in some really creative ways
//t.Run("32-p10B", run(10, 32, 2))
// t.Run("128-p10B", run(10, 128, 8))
}
}