cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

itests: Tests for storage path filters

Browse Source
This commit is contained in:
Łukasz Magiera 2022-07-12 12:42:08 +02:00
parent 0ca1cc2bcd
commit 59d069dc4d
9 changed files with 234 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.circleci/config.yml
View File

@ -893,6 +893,11 @@ workflows:
suite: itest-nonce
target: "./itests/nonce_test.go"
- test:
name: test-itest-path_type_filters
suite: itest-path_type_filters
target: "./itests/path_type_filters_test.go"
- test:
name: test-itest-paych_api
suite: itest-paych_api

5
itests/kit/blockminer.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
@ -184,7 +185,9 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
var target abi.ChainEpoch
reportSuccessFn := func(success bool, epoch abi.ChainEpoch, err error) {
require.NoError(bm.t, err)
if err != nil && !strings.Contains(err.Error(), "websocket connection closed") {
require.NoError(bm.t, err)
}
target = epoch
wait <- success
}

9
itests/kit/ensemble.go
View File

@ -546,7 +546,7 @@ func (n *Ensemble) Start() *Ensemble {
// using real proofs, therefore need real sectors.
if !n.bootstrapped && !n.options.mockProofs {
psd := m.PresealDir
noPaths := m.options.noMinerStorage
noPaths := m.options.noStorage
err := lr.SetStorage(func(sc *paths.StorageConfig) {
if noPaths {
@ -698,6 +698,13 @@ func (n *Ensemble) Start() *Ensemble {
lr, err := r.Lock(repo.Worker)
require.NoError(n.t, err)
if m.options.noStorage {
err := lr.SetStorage(func(sc *paths.StorageConfig) {
sc.StoragePaths = []paths.LocalPath{}
})
require.NoError(n.t, err)
}
ds, err := lr.Datastore(context.Background(), "/metadata")
require.NoError(n.t, err)

17
itests/kit/node_miner.go
View File

@ -167,9 +167,8 @@ func (tm *TestMiner) FlushSealingBatches(ctx context.Context) {
const metaFile = "sectorstore.json"
func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, weight uint64, seal, store bool) {
p, err := ioutil.TempDir("", "lotus-testsectors-")
require.NoError(t, err)
func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, conf func(*paths.LocalStorageMeta)) storiface.ID {
p := t.TempDir()
if err := os.MkdirAll(p, 0755); err != nil {
if !os.IsExist(err) {
@ -177,18 +176,20 @@ func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, weight uint64
}
}
_, err = os.Stat(filepath.Join(p, metaFile))
_, err := os.Stat(filepath.Join(p, metaFile))
if !os.IsNotExist(err) {
require.NoError(t, err)
}
cfg := &paths.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: weight,
CanSeal: seal,
CanStore: store,
Weight: 10,
CanSeal: false,
CanStore: false,
}
conf(cfg)
if !(cfg.CanStore || cfg.CanSeal) {
t.Fatal("must specify at least one of CanStore or cfg.CanSeal")
}
@ -201,4 +202,6 @@ func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, weight uint64
err = tm.StorageAddLocal(ctx, p)
require.NoError(t, err)
return cfg.ID
}

8
itests/kit/node_opts.go
View File

@ -43,7 +43,7 @@ type nodeOpts struct {
minerNoLocalSealing bool // use worker
minerAssigner string
disallowRemoteFinalize bool
noMinerStorage bool
noStorage bool
workerTasks []sealtasks.TaskType
workerStorageOpt func(paths.Store) paths.Store
@ -155,10 +155,10 @@ func PresealSectors(sectors int) NodeOpt {
}
}
// NoMinerStorage initializes miners with no writable storage paths (just read-only preseal paths)
func NoMinerStorage() NodeOpt {
// NoStorage initializes miners with no writable storage paths (just read-only preseal paths)
func NoStorage() NodeOpt {
return func(opts *nodeOpts) error {
opts.noMinerStorage = true
opts.noStorage = true
return nil
}
}

165
itests/path_type_filters_test.go Normal file
View File

@ -0,0 +1,165 @@
package itests
import (
"context"
"testing"
"time"
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
func TestPathTypeFilters(t *testing.T) {
runTest := func(t *testing.T, name string, asserts func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func())) {
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = logging.SetLogLevel("storageminer", "INFO")
var (
client kit.TestFullNode
miner kit.TestMiner
wiw, wdw kit.TestWorker
)
ens := kit.NewEnsemble(t, kit.LatestActorsAt(-1)).
FullNode(&client, kit.ThroughRPC()).
Miner(&miner, &client, kit.WithAllSubsystems(), kit.ThroughRPC(), kit.PresealSectors(2), kit.NoStorage()).
Worker(&miner, &wiw, kit.ThroughRPC(), kit.NoStorage(), kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWinningPoSt})).
Worker(&miner, &wdw, kit.ThroughRPC(), kit.NoStorage(), kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt})).
Start()
ens.InterconnectAll().BeginMiningMustPost(2 * time.Millisecond)
asserts(t, ctx, &miner, func() {
dh := kit.NewDealHarness(t, &client, &miner, &miner)
dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{N: 1})
})
})
}
runTest(t, "seal-to-stor-unseal-allowdeny", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
// allow all types in the sealing path
sealScratch := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanSeal = true
})
// unsealed storage
unsStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"unsealed"}
})
// other storage
sealStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.DenyTypes = []string{"unsealed"}
})
storlist, err := miner.StorageList(ctx)
require.NoError(t, err)
require.Len(t, storlist, 4) // 3 paths we've added + preseal
run()
storlist, err = miner.StorageList(ctx)
require.NoError(t, err)
require.Len(t, storlist[sealScratch], 0)
require.Len(t, storlist[unsStor], 1)
require.Len(t, storlist[sealStor], 1)
require.Equal(t, storiface.FTUnsealed, storlist[unsStor][0].SectorFileType)
require.Equal(t, storiface.FTSealed|storiface.FTCache, storlist[sealStor][0].SectorFileType)
})
runTest(t, "sealstor-unseal-allowdeny", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
// unsealed storage
unsStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.CanSeal = true
meta.AllowTypes = []string{"unsealed"}
})
// other storage
sealStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.CanSeal = true
meta.DenyTypes = []string{"unsealed"}
})
storlist, err := miner.StorageList(ctx)
require.NoError(t, err)
require.Len(t, storlist, 3) // 2 paths we've added + preseal
run()
storlist, err = miner.StorageList(ctx)
require.NoError(t, err)
require.Len(t, storlist[unsStor], 1)
require.Len(t, storlist[sealStor], 1)
require.Equal(t, storiface.FTUnsealed, storlist[unsStor][0].SectorFileType)
require.Equal(t, storiface.FTSealed|storiface.FTCache, storlist[sealStor][0].SectorFileType)
})
runTest(t, "seal-store-allseparate", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
// sealing stores
slU := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"unsealed"}
})
slS := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"sealed"}
})
slC := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"cache"}
})
// storage stores
stU := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"unsealed"}
})
stS := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"sealed"}
})
stC := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"cache"}
})
storlist, err := miner.StorageList(ctx)
require.NoError(t, err)
require.Len(t, storlist, 7) // 6 paths we've added + preseal
run()
storlist, err = miner.StorageList(ctx)
require.NoError(t, err)
require.Len(t, storlist[slU], 0)
require.Len(t, storlist[slS], 0)
require.Len(t, storlist[slC], 0)
require.Len(t, storlist[stU], 1)
require.Len(t, storlist[stS], 1)
require.Len(t, storlist[stC], 1)
require.Equal(t, storiface.FTUnsealed, storlist[stU][0].SectorFileType)
require.Equal(t, storiface.FTSealed, storlist[stS][0].SectorFileType)
require.Equal(t, storiface.FTCache, storlist[stC][0].SectorFileType)
})
}

11
itests/sector_finalize_early_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
)
func TestDealsWithFinalizeEarly(t *testing.T) {
@ -35,8 +36,14 @@ func TestDealsWithFinalizeEarly(t *testing.T) {
ctx := context.Background()
miner.AddStorage(ctx, t, 1000000000, true, false)
miner.AddStorage(ctx, t, 1000000000, false, true)
miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.Weight = 1000000000
meta.CanSeal = true
})
miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
meta.Weight = 1000000000
meta.CanStore = true
})
//stm: @STORAGE_LIST_001
sl, err := miner.StorageList(ctx)

2
itests/wdpost_no_miner_storage_test.go
View File

@ -31,7 +31,7 @@ func TestWindowPostNoMinerStorage(t *testing.T) {
)
ens := kit.NewEnsemble(t, kit.LatestActorsAt(-1)).
FullNode(&client, kit.ThroughRPC()).
Miner(&miner, &client, kit.WithAllSubsystems(), kit.ThroughRPC(), kit.PresealSectors(presealSectors), kit.NoMinerStorage()).
Miner(&miner, &client, kit.WithAllSubsystems(), kit.ThroughRPC(), kit.PresealSectors(presealSectors), kit.NoStorage()).
Worker(&miner, &wiw, kit.ThroughRPC(), kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWinningPoSt})).
Worker(&miner, &wdw, kit.ThroughRPC(), kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt})).
Worker(&miner, &sealw, kit.ThroughRPC(), kit.WithSealWorkerTasks).

28
storage/paths/local.go
View File

@ -44,6 +44,30 @@ type LocalStorageMeta struct {
// List of storage groups to which data from this path can be moved. If none
// are specified, allow to all
AllowTo []string
// AllowTypes lists sector file types which are allowed to be put into this
// path. If empty, all file types are allowed.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
AllowTypes []string
// DenyTypes lists sector file types which aren't allowed to be put into this
// path.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
}
// StorageConfig .lotusstorage/storage.json
@ -218,6 +242,8 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
CanStore: meta.CanStore,
Groups: meta.Groups,
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
}, fst)
if err != nil {
return xerrors.Errorf("declaring storage in index: %w", err)
@ -284,6 +310,8 @@ func (st *Local) Redeclare(ctx context.Context) error {
CanStore: meta.CanStore,
Groups: meta.Groups,
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
}, fst)
if err != nil {
return xerrors.Errorf("redeclaring storage in index: %w", err)