cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

make repo not depend on ffi

Browse Source
This commit is contained in:
Łukasz Magiera 2022-11-01 11:01:31 +00:00
parent 5f5cc794f0
commit ec89424c42
27 changed files with 166 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/lotus-miner/init.go
View File

@ -49,6 +49,7 @@ import (
"github.com/filecoin-project/lotus/journal"
"github.com/filecoin-project/lotus/journal/fsjournal"
storageminer "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
@ -218,7 +219,7 @@ var initCmd = &cli.Command{
return err
}
var localPaths []paths.LocalPath
var localPaths []storiface.LocalPath
if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 {
log.Infof("Setting up storage config with presealed sectors: %v", pssb)
@ -228,14 +229,14 @@ var initCmd = &cli.Command{
if err != nil {
return err
}
localPaths = append(localPaths, paths.LocalPath{
localPaths = append(localPaths, storiface.LocalPath{
Path: psp,
})
}
}
if !cctx.Bool("no-local-storage") {
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: true,
@ -249,12 +250,12 @@ var initCmd = &cli.Command{
return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err)
}
localPaths = append(localPaths, paths.LocalPath{
localPaths = append(localPaths, storiface.LocalPath{
Path: lr.Path(),
})
}
if err := lr.SetStorage(func(sc *paths.StorageConfig) {
if err := lr.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, localPaths...)
}); err != nil {
return xerrors.Errorf("set storage config: %w", err)
@ -471,7 +472,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
}
stor := paths.NewRemote(lstor, si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{})
smgr, err := sealer.New(ctx, lstor, stor, lr, si, sealer.Config{
smgr, err := sealer.New(ctx, lstor, stor, lr, si, config.SealerConfig{
ParallelFetchLimit: 10,
AllowAddPiece: true,
AllowPreCommit1: true,
@ -481,7 +482,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
AllowReplicaUpdate: true,
AllowProveReplicaUpdate2: true,
AllowRegenSectorKey: true,
}, wsts, smsts)
}, config.ProvingConfig{}, wsts, smsts)
if err != nil {
return err
}

14
cmd/lotus-miner/init_restore.go
View File

@ -27,7 +27,7 @@ import (
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
var restoreCmd = &cli.Command{
@ -52,7 +52,7 @@ var restoreCmd = &cli.Command{
ctx := lcli.ReqContext(cctx)
log.Info("Initializing lotus miner using a backup")
var storageCfg *paths.StorageConfig
var storageCfg *storiface.StorageConfig
if cctx.IsSet("storage-config") {
cf, err := homedir.Expand(cctx.String("storage-config"))
if err != nil {
@ -64,7 +64,7 @@ var restoreCmd = &cli.Command{
return xerrors.Errorf("reading storage config: %w", err)
}
storageCfg = &paths.StorageConfig{}
storageCfg = &storiface.StorageConfig{}
err = json.Unmarshal(cfb, storageCfg)
if err != nil {
return xerrors.Errorf("cannot unmarshal json for storage config: %w", err)
@ -95,7 +95,7 @@ var restoreCmd = &cli.Command{
},
}
func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfig *paths.StorageConfig, manageConfig func(*config.StorageMiner) error, after func(api lapi.FullNode, addr address.Address, peerid peer.ID, mi api.MinerInfo) error) error {
func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfig *storiface.StorageConfig, manageConfig func(*config.StorageMiner) error, after func(api lapi.FullNode, addr address.Address, peerid peer.ID, mi api.MinerInfo) error) error {
if cctx.NArg() != 1 {
return lcli.IncorrectNumArgs(cctx)
}
@ -214,7 +214,7 @@ func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfi
if strConfig != nil {
log.Info("Restoring storage path config")
err = lr.SetStorage(func(scfg *paths.StorageConfig) {
err = lr.SetStorage(func(scfg *storiface.StorageConfig) {
*scfg = *strConfig
})
if err != nil {
@ -223,8 +223,8 @@ func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfi
} else {
log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED")
// setting empty config to allow miner to be started
if err := lr.SetStorage(func(sc *paths.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{})
if err := lr.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, storiface.LocalPath{})
}); err != nil {
return xerrors.Errorf("set storage config: %w", err)
}

4
cmd/lotus-miner/init_service.go
View File

@ -17,7 +17,7 @@ import (
lcli "github.com/filecoin-project/lotus/cli"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
const (
@ -78,7 +78,7 @@ var serviceCmd = &cli.Command{
return xerrors.Errorf("please provide Lotus markets repo path via flag %s", FlagMarketsRepo)
}
if err := restore(ctx, cctx, repoPath, &paths.StorageConfig{}, func(cfg *config.StorageMiner) error {
if err := restore(ctx, cctx, repoPath, &storiface.StorageConfig{}, func(cfg *config.StorageMiner) error {
cfg.Subsystems.EnableMarkets = es.Contains(MarketsService)
cfg.Subsystems.EnableMining = false
cfg.Subsystems.EnableSealing = false

3
cmd/lotus-miner/storage.go
View File

@ -29,7 +29,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/tablewriter"
"github.com/filecoin-project/lotus/storage/paths"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
@ -148,7 +147,7 @@ over time
}
}
cfg := &paths.LocalStorageMeta{
cfg := &storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: cctx.Uint64("weight"),
CanSeal: cctx.Bool("seal"),

3
cmd/lotus-seed/seed/seed.go
View File

@ -27,7 +27,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet/key"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper/basicfs"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
@ -126,7 +125,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect
}
{
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 0, // read-only
CanSeal: false,

8
cmd/lotus-worker/main.go
View File

@ -447,10 +447,10 @@ var runCmd = &cli.Command{
return err
}
var localPaths []paths.LocalPath
var localPaths []storiface.LocalPath
if !cctx.Bool("no-local-storage") {
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: true,
@ -464,12 +464,12 @@ var runCmd = &cli.Command{
return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err)
}
localPaths = append(localPaths, paths.LocalPath{
localPaths = append(localPaths, storiface.LocalPath{
Path: lr.Path(),
})
}
if err := lr.SetStorage(func(sc *paths.StorageConfig) {
if err := lr.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, localPaths...)
}); err != nil {
return xerrors.Errorf("set storage config: %w", err)

8
cmd/lotus-worker/sealworker/rpc.go
View File

@ -92,8 +92,8 @@ func (w *Worker) StorageAddLocal(ctx context.Context, path string) error {
return xerrors.Errorf("opening local path: %w", err)
}
if err := w.Storage.SetStorage(func(sc *paths.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: path})
if err := w.Storage.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, storiface.LocalPath{Path: path})
}); err != nil {
return xerrors.Errorf("get storage config: %w", err)
}
@ -127,8 +127,8 @@ func (w *Worker) StorageDetachLocal(ctx context.Context, path string) error {
// drop from the persisted storage.json
var found bool
if err := w.Storage.SetStorage(func(sc *paths.StorageConfig) {
out := make([]paths.LocalPath, 0, len(sc.StoragePaths))
if err := w.Storage.SetStorage(func(sc *storiface.StorageConfig) {
out := make([]storiface.LocalPath, 0, len(sc.StoragePaths))
for _, storagePath := range sc.StoragePaths {
if storagePath.Path != path {
out = append(out, storagePath)

3
cmd/lotus-worker/storage.go
View File

@ -13,7 +13,6 @@ import (
"golang.org/x/xerrors"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -103,7 +102,7 @@ var storageAttachCmd = &cli.Command{
}
}
cfg := &paths.LocalStorageMeta{
cfg := &storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: cctx.Uint64("weight"),
CanSeal: cctx.Bool("seal"),

10
itests/kit/ensemble.go
View File

@ -586,11 +586,11 @@ func (n *Ensemble) Start() *Ensemble {
psd := m.PresealDir
noPaths := m.options.noStorage
err := lr.SetStorage(func(sc *paths.StorageConfig) {
err := lr.SetStorage(func(sc *storiface.StorageConfig) {
if noPaths {
sc.StoragePaths = []paths.LocalPath{}
sc.StoragePaths = []storiface.LocalPath{}
}
sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: psd})
sc.StoragePaths = append(sc.StoragePaths, storiface.LocalPath{Path: psd})
})
require.NoError(n.t, err)
@ -737,8 +737,8 @@ func (n *Ensemble) Start() *Ensemble {
require.NoError(n.t, err)
if m.options.noStorage {
err := lr.SetStorage(func(sc *paths.StorageConfig) {
sc.StoragePaths = []paths.LocalPath{}
err := lr.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = []storiface.LocalPath{}
})
require.NoError(n.t, err)
}

5
itests/kit/node_miner.go
View File

@ -26,7 +26,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/wallet/key"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/storage/paths"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -175,7 +174,7 @@ func (tm *TestMiner) FlushSealingBatches(ctx context.Context) {
const metaFile = "sectorstore.json"
func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, conf func(*paths.LocalStorageMeta)) storiface.ID {
func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, conf func(*storiface.LocalStorageMeta)) storiface.ID {
p := t.TempDir()
if err := os.MkdirAll(p, 0755); err != nil {
@ -189,7 +188,7 @@ func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, conf func(*pa
require.NoError(t, err)
}
cfg := &paths.LocalStorageMeta{
cfg := &storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: false,

5
itests/kit/node_worker.go
View File

@ -15,7 +15,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -38,7 +37,7 @@ type TestWorker struct {
options nodeOpts
}
func (tm *TestWorker) AddStorage(ctx context.Context, t *testing.T, conf func(*paths.LocalStorageMeta)) storiface.ID {
func (tm *TestWorker) AddStorage(ctx context.Context, t *testing.T, conf func(*storiface.LocalStorageMeta)) storiface.ID {
p := t.TempDir()
if err := os.MkdirAll(p, 0755); err != nil {
@ -52,7 +51,7 @@ func (tm *TestWorker) AddStorage(ctx context.Context, t *testing.T, conf func(*p
require.NoError(t, err)
}
cfg := &paths.LocalStorageMeta{
cfg := &storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: false,

7
itests/path_detach_redeclare_test.go
View File

@ -15,7 +15,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -74,7 +73,7 @@ func TestPathDetachRedeclare(t *testing.T) {
checkSectors(ctx, t, client, miner, 2, 2)
// attach a new path
newId := miner.AddStorage(ctx, t, func(cfg *paths.LocalStorageMeta) {
newId := miner.AddStorage(ctx, t, func(cfg *storiface.LocalStorageMeta) {
cfg.CanStore = true
})
@ -194,7 +193,7 @@ func TestPathDetachRedeclareWorker(t *testing.T) {
checkSectors(ctx, t, client, miner, 2, 2)
// attach a new path
newId := sealw.AddStorage(ctx, t, func(cfg *paths.LocalStorageMeta) {
newId := sealw.AddStorage(ctx, t, func(cfg *storiface.LocalStorageMeta) {
cfg.CanStore = true
})
@ -239,7 +238,7 @@ func TestPathDetachRedeclareWorker(t *testing.T) {
require.Len(t, local, 0)
// add a new one again, and move the sectors there
newId = sealw.AddStorage(ctx, t, func(cfg *paths.LocalStorageMeta) {
newId = sealw.AddStorage(ctx, t, func(cfg *storiface.LocalStorageMeta) {
cfg.CanStore = true
})

25
itests/path_type_filters_test.go
View File

@ -10,7 +10,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -45,7 +44,7 @@ func TestPathTypeFilters(t *testing.T) {
}
runTest(t, "invalid-type-alert", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
slU := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
slU := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"unsealed", "seeled"}
})
@ -79,18 +78,18 @@ func TestPathTypeFilters(t *testing.T) {
runTest(t, "seal-to-stor-unseal-allowdeny", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
// allow all types in the sealing path
sealScratch := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
sealScratch := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanSeal = true
})
// unsealed storage
unsStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
unsStor := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"unsealed"}
})
// other storage
sealStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
sealStor := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.DenyTypes = []string{"unsealed"}
})
@ -115,14 +114,14 @@ func TestPathTypeFilters(t *testing.T) {
runTest(t, "sealstor-unseal-allowdeny", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
// unsealed storage
unsStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
unsStor := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.CanSeal = true
meta.AllowTypes = []string{"unsealed"}
})
// other storage
sealStor := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
sealStor := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.CanSeal = true
meta.DenyTypes = []string{"unsealed"}
@ -147,29 +146,29 @@ func TestPathTypeFilters(t *testing.T) {
runTest(t, "seal-store-allseparate", func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func()) {
// sealing stores
slU := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
slU := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"unsealed"}
})
slS := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
slS := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"sealed"}
})
slC := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
slC := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanSeal = true
meta.AllowTypes = []string{"cache"}
})
// storage stores
stU := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
stU := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"unsealed"}
})
stS := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
stS := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"sealed"}
})
stC := miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
stC := miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.CanStore = true
meta.AllowTypes = []string{"cache"}
})

6
itests/sector_finalize_early_test.go
View File

@ -11,7 +11,7 @@ import (
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
func TestDealsWithFinalizeEarly(t *testing.T) {
@ -36,11 +36,11 @@ func TestDealsWithFinalizeEarly(t *testing.T) {
ctx := context.Background()
miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.Weight = 1000000000
meta.CanSeal = true
})
miner.AddStorage(ctx, t, func(meta *paths.LocalStorageMeta) {
miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) {
meta.Weight = 1000000000
meta.CanStore = true
})

2
node/config/doc_gen.go
View File

@ -894,7 +894,7 @@ If you see stuck Finalize tasks after enabling this setting, check
},
{
Name: "ResourceFiltering",
Type: "sealer.ResourceFilteringStrategy",
Type: "ResourceFilteringStrategy",
Comment: `ResourceFiltering instructs the system which resource filtering strategy
to use when evaluating tasks against this worker. An empty value defaults

10
node/config/storage.go
View File

@ -8,10 +8,10 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfig, error) {
func StorageFromFile(path string, def *storiface.StorageConfig) (*storiface.StorageConfig, error) {
file, err := os.Open(path)
switch {
case os.IsNotExist(err):
@ -27,8 +27,8 @@ func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfi
return StorageFromReader(file)
}
func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) {
var cfg paths.StorageConfig
func StorageFromReader(reader io.Reader) (*storiface.StorageConfig, error) {
var cfg storiface.StorageConfig
err := json.NewDecoder(reader).Decode(&cfg)
if err != nil {
return nil, err
@ -37,7 +37,7 @@ func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) {
return &cfg, nil
}
func WriteStorageFile(path string, config paths.StorageConfig) error {
func WriteStorageFile(path string, config storiface.StorageConfig) error {
b, err := json.MarshalIndent(config, "", " ")
if err != nil {
return xerrors.Errorf("marshaling storage config: %w", err)

12
node/repo/fsrepo.go
View File

@ -25,8 +25,8 @@ import (
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
const (
@ -572,26 +572,26 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
return nil
}
func (fsr *fsLockedRepo) GetStorage() (paths.StorageConfig, error) {
func (fsr *fsLockedRepo) GetStorage() (storiface.StorageConfig, error) {
fsr.storageLk.Lock()
defer fsr.storageLk.Unlock()
return fsr.getStorage(nil)
}
func (fsr *fsLockedRepo) getStorage(def *paths.StorageConfig) (paths.StorageConfig, error) {
func (fsr *fsLockedRepo) getStorage(def *storiface.StorageConfig) (storiface.StorageConfig, error) {
c, err := config.StorageFromFile(fsr.join(fsStorageConfig), def)
if err != nil {
return paths.StorageConfig{}, err
return storiface.StorageConfig{}, err
}
return *c, nil
}
func (fsr *fsLockedRepo) SetStorage(c func(*paths.StorageConfig)) error {
func (fsr *fsLockedRepo) SetStorage(c func(*storiface.StorageConfig)) error {
fsr.storageLk.Lock()
defer fsr.storageLk.Unlock()
sc, err := fsr.getStorage(&paths.StorageConfig{})
sc, err := fsr.getStorage(&storiface.StorageConfig{})
if err != nil {
return xerrors.Errorf("get storage: %w", err)
}

6
node/repo/interface.go
View File

@ -9,8 +9,8 @@ import (
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
// BlockstoreDomain represents the domain of a blockstore.
@ -73,8 +73,8 @@ type LockedRepo interface {
Config() (interface{}, error)
SetConfig(func(interface{})) error
GetStorage() (paths.StorageConfig, error)
SetStorage(func(*paths.StorageConfig)) error
GetStorage() (storiface.StorageConfig, error)
SetStorage(func(*storiface.StorageConfig)) error
Stat(path string) (fsutil.FsStat, error)
DiskUsage(path string) (int64, error)

17
node/repo/memrepo.go
View File

@ -18,7 +18,6 @@ import (
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -37,7 +36,7 @@ type MemRepo struct {
keystore map[string]types.KeyInfo
blockstore blockstore.Blockstore
sc *paths.StorageConfig
sc *storiface.StorageConfig
tempDir string
// holds the current config value
@ -59,13 +58,13 @@ func (lmem *lockedMemRepo) RepoType() RepoType {
return lmem.t
}
func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) {
func (lmem *lockedMemRepo) GetStorage() (storiface.StorageConfig, error) {
if err := lmem.checkToken(); err != nil {
return paths.StorageConfig{}, err
return storiface.StorageConfig{}, err
}
if lmem.mem.sc == nil {
lmem.mem.sc = &paths.StorageConfig{StoragePaths: []paths.LocalPath{
lmem.mem.sc = &storiface.StorageConfig{StoragePaths: []storiface.LocalPath{
{Path: lmem.Path()},
}}
}
@ -73,7 +72,7 @@ func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) {
return *lmem.mem.sc, nil
}
func (lmem *lockedMemRepo) SetStorage(c func(*paths.StorageConfig)) error {
func (lmem *lockedMemRepo) SetStorage(c func(*storiface.StorageConfig)) error {
if err := lmem.checkToken(); err != nil {
return err
}
@ -126,14 +125,14 @@ func (lmem *lockedMemRepo) Path() string {
}
func (lmem *lockedMemRepo) initSectorStore(t string) {
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), paths.StorageConfig{
StoragePaths: []paths.LocalPath{
if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), storiface.StorageConfig{
StoragePaths: []storiface.LocalPath{
{Path: t},
}}); err != nil {
panic(err)
}
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: true,

66
storage/paths/local.go
View File

@ -21,67 +21,9 @@ import (
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
// LocalStorageMeta [path]/sectorstore.json
type LocalStorageMeta struct {
ID storiface.ID
// A high weight means data is more likely to be stored in this path
Weight uint64 // 0 = readonly
// Intermediate data for the sealing process will be stored here
CanSeal bool
// Finalized sectors that will be proved over time will be stored here
CanStore bool
// MaxStorage specifies the maximum number of bytes to use for sector storage
// (0 = unlimited)
MaxStorage uint64
// List of storage groups this path belongs to
Groups []string
// List of storage groups to which data from this path can be moved. If none
// are specified, allow to all
AllowTo []string
// AllowTypes lists sector file types which are allowed to be put into this
// path. If empty, all file types are allowed.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
AllowTypes []string
// DenyTypes lists sector file types which aren't allowed to be put into this
// path.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
}
// StorageConfig .lotusstorage/storage.json
type StorageConfig struct {
StoragePaths []LocalPath
}
type LocalPath struct {
Path string
}
type LocalStorage interface {
GetStorage() (StorageConfig, error)
SetStorage(func(*StorageConfig)) error
GetStorage() (storiface.StorageConfig, error)
SetStorage(func(*storiface.StorageConfig)) error
Stat(path string) (fsutil.FsStat, error)
@ -213,7 +155,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
return xerrors.Errorf("reading storage metadata for %s: %w", p, err)
}
var meta LocalStorageMeta
var meta storiface.LocalStorageMeta
if err := json.Unmarshal(mb, &meta); err != nil {
return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p, err)
}
@ -309,7 +251,7 @@ func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMiss
return xerrors.Errorf("reading storage metadata for %s: %w", p.local, err)
}
var meta LocalStorageMeta
var meta storiface.LocalStorageMeta
if err := json.Unmarshal(mb, &meta); err != nil {
return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p.local, err)
}

8
storage/paths/local_test.go
View File

@ -19,18 +19,18 @@ const pathSize = 16 << 20
type TestingLocalStorage struct {
root string
c StorageConfig
c storiface.StorageConfig
}
func (t *TestingLocalStorage) DiskUsage(path string) (int64, error) {
return 1, nil
}
func (t *TestingLocalStorage) GetStorage() (StorageConfig, error) {
func (t *TestingLocalStorage) GetStorage() (storiface.StorageConfig, error) {
return t.c, nil
}
func (t *TestingLocalStorage) SetStorage(f func(*StorageConfig)) error {
func (t *TestingLocalStorage) SetStorage(f func(*storiface.StorageConfig)) error {
f(&t.c)
return nil
}
@ -51,7 +51,7 @@ func (t *TestingLocalStorage) init(subpath string) error {
metaFile := filepath.Join(path, MetaFile)
meta := &LocalStorageMeta{
meta := &storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 1,
CanSeal: true,

5
storage/paths/localstorage_cached.go
View File

@ -7,6 +7,7 @@ import (
lru "github.com/hashicorp/golang-lru"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
var StatTimeout = 5 * time.Second
@ -47,11 +48,11 @@ type diskUsageResult struct {
time time.Time
}
func (c *cachedLocalStorage) GetStorage() (StorageConfig, error) {
func (c *cachedLocalStorage) GetStorage() (storiface.StorageConfig, error) {
return c.base.GetStorage()
}
func (c *cachedLocalStorage) SetStorage(f func(*StorageConfig)) error {
func (c *cachedLocalStorage) SetStorage(f func(*storiface.StorageConfig)) error {
return c.base.SetStorage(f)
}

6
storage/paths/remote_test.go
View File

@ -38,7 +38,7 @@ func createTestStorage(t *testing.T, p string, seal bool, att ...*paths.Local) s
}
}
cfg := &paths.LocalStorageMeta{
cfg := &storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: seal,
@ -77,8 +77,8 @@ func TestMoveShared(t *testing.T) {
_ = lr.Close()
})
err = lr.SetStorage(func(config *paths.StorageConfig) {
*config = paths.StorageConfig{}
err = lr.SetStorage(func(config *storiface.StorageConfig) {
*config = storiface.StorageConfig{}
})
require.NoError(t, err)

10
storage/sealer/manager.go
View File

@ -3,7 +3,6 @@ package sealer
import (
"context"
"errors"
"github.com/filecoin-project/lotus/node/config"
"io"
"net/http"
"sort"
@ -20,6 +19,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
@ -191,8 +191,8 @@ func (m *Manager) AddLocalStorage(ctx context.Context, path string) error {
return xerrors.Errorf("opening local path: %w", err)
}
if err := m.ls.SetStorage(func(sc *paths.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: path})
if err := m.ls.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, storiface.LocalPath{Path: path})
}); err != nil {
return xerrors.Errorf("get storage config: %w", err)
}
@ -225,8 +225,8 @@ func (m *Manager) DetachLocalStorage(ctx context.Context, path string) error {
// drop from the persisted storage.json
var found bool
if err := m.ls.SetStorage(func(sc *paths.StorageConfig) {
out := make([]paths.LocalPath, 0, len(sc.StoragePaths))
if err := m.ls.SetStorage(func(sc *storiface.StorageConfig) {
out := make([]storiface.LocalPath, 0, len(sc.StoragePaths))
for _, storagePath := range sc.StoragePaths {
if storagePath.Path != path {
out = append(out, storagePath)

14
storage/sealer/manager_test.go
View File

@ -39,7 +39,7 @@ func init() {
logging.SetAllLoggers(logging.LevelDebug)
}
type testStorage paths.StorageConfig
type testStorage storiface.StorageConfig
func (t testStorage) DiskUsage(path string) (int64, error) {
return 1, nil // close enough
@ -50,7 +50,7 @@ func newTestStorage(t *testing.T) *testStorage {
require.NoError(t, err)
{
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 1,
CanSeal: true,
@ -63,7 +63,7 @@ func newTestStorage(t *testing.T) *testStorage {
}
return &testStorage{
StoragePaths: []paths.LocalPath{
StoragePaths: []storiface.LocalPath{
{Path: tp},
},
}
@ -82,12 +82,12 @@ func (t testStorage) cleanup() {
}
}
func (t testStorage) GetStorage() (paths.StorageConfig, error) {
return paths.StorageConfig(t), nil
func (t testStorage) GetStorage() (storiface.StorageConfig, error) {
return storiface.StorageConfig(t), nil
}
func (t *testStorage) SetStorage(f func(*paths.StorageConfig)) error {
f((*paths.StorageConfig)(t))
func (t *testStorage) SetStorage(f func(*storiface.StorageConfig)) error {
f((*storiface.StorageConfig)(t))
return nil
}

58
storage/sealer/storiface/storage.go
View File

@ -153,3 +153,61 @@ type SecDataHttpHeader struct {
Key string
Value string
}
// StorageConfig .lotusstorage/storage.json
type StorageConfig struct {
StoragePaths []LocalPath
}
type LocalPath struct {
Path string
}
// LocalStorageMeta [path]/sectorstore.json
type LocalStorageMeta struct {
ID ID
// A high weight means data is more likely to be stored in this path
Weight uint64 // 0 = readonly
// Intermediate data for the sealing process will be stored here
CanSeal bool
// Finalized sectors that will be proved over time will be stored here
CanStore bool
// MaxStorage specifies the maximum number of bytes to use for sector storage
// (0 = unlimited)
MaxStorage uint64
// List of storage groups this path belongs to
Groups []string
// List of storage groups to which data from this path can be moved. If none
// are specified, allow to all
AllowTo []string
// AllowTypes lists sector file types which are allowed to be put into this
// path. If empty, all file types are allowed.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
AllowTypes []string
// DenyTypes lists sector file types which aren't allowed to be put into this
// path.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
}

9
testplans/lotus-soup/testkit/role_miner.go
View File

@ -40,7 +40,6 @@ import (
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage/paths"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -198,9 +197,9 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) {
}
}
var localPaths []paths.LocalPath
var localPaths []storiface.LocalPath
b, err := json.MarshalIndent(&paths.LocalStorageMeta{
b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
ID: storiface.ID(uuid.New().String()),
Weight: 10,
CanSeal: true,
@ -214,11 +213,11 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) {
return nil, fmt.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err)
}
localPaths = append(localPaths, paths.LocalPath{
localPaths = append(localPaths, storiface.LocalPath{
Path: lr.Path(),
})
if err := lr.SetStorage(func(sc *paths.StorageConfig) {
if err := lr.SetStorage(func(sc *storiface.StorageConfig) {
sc.StoragePaths = append(sc.StoragePaths, localPaths...)
}); err != nil {
return nil, err