cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

make config not depend on ffi

Browse Source
This commit is contained in:
Łukasz Magiera 2022-10-31 17:15:09 +00:00
parent b540d10b48
commit 5f5cc794f0
9 changed files with 72 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
itests/kit/ensemble.go
View File

@ -632,7 +632,7 @@ func (n *Ensemble) Start() *Ensemble {
// disable resource filtering so that local worker gets assigned tasks
// regardless of system pressure.
node.Override(new(sectorstorage.Config), func() sectorstorage.Config {
node.Override(new(config.SealingConfig), func() config.SealingConfig {
scfg := config.DefaultStorageMiner()
if noLocal {
@ -645,8 +645,8 @@ func (n *Ensemble) Start() *Ensemble {
scfg.Storage.Assigner = assigner
scfg.Storage.DisallowRemoteFinalize = disallowRemoteFinalize
scfg.Storage.ResourceFiltering = sectorstorage.ResourceFilteringDisabled
return scfg.StorageManager()
scfg.Storage.ResourceFiltering = config.ResourceFilteringDisabled
return scfg.Sealing
}),
// upgrades

17
itests/wdpost_worker_config_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/storage/sealer"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/filecoin-project/lotus/storage/wdpost"
@ -35,10 +34,10 @@ func TestWindowPostNoBuiltinWindow(t *testing.T) {
kit.PresealSectors(sectors), // 2 sectors per partition, 2 partitions in all 48 deadlines
kit.LatestActorsAt(-1),
kit.ConstructorOpts(
node.Override(new(sealer.Config), func() sealer.Config {
c := config.DefaultStorageMiner().StorageManager()
c.DisableBuiltinWindowPoSt = true
return c
node.Override(new(*config.ProvingConfig), func() *config.ProvingConfig {
c := config.DefaultStorageMiner()
c.Proving.DisableBuiltinWindowPoSt = true
return &c.Proving
}),
node.Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(
config.DefaultStorageMiner().Fees,
@ -92,10 +91,10 @@ func TestWindowPostNoBuiltinWindowWithWorker(t *testing.T) {
kit.PresealSectors(sectors), // 2 sectors per partition, 2 partitions in all 48 deadlines
kit.LatestActorsAt(-1),
kit.ConstructorOpts(
node.Override(new(sealer.Config), func() sealer.Config {
c := config.DefaultStorageMiner().StorageManager()
c.DisableBuiltinWindowPoSt = true
return c
node.Override(new(*config.ProvingConfig), func() *config.ProvingConfig {
c := config.DefaultStorageMiner()
c.Proving.DisableBuiltinWindowPoSt = true
return &c.Proving
}),
node.Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(
config.DefaultStorageMiner().Fees,

3
node/builder_miner.go
View File

@ -224,7 +224,8 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)),
),
Override(new(sectorstorage.Config), cfg.StorageManager()),
Override(new(config.SealerConfig), cfg.Storage),
Override(new(config.ProvingConfig), cfg.Proving),
Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)),
)
}

17
node/config/def.go
View File

@ -15,7 +15,6 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/sealer"
)
const (
@ -162,7 +161,7 @@ func DefaultStorageMiner() *StorageMiner {
Assigner: "utilization",
// By default use the hardware resource filtering strategy.
ResourceFiltering: sealer.ResourceFilteringHardware,
ResourceFiltering: ResourceFilteringHardware,
},
Dealmaking: DealmakingConfig{
@ -274,3 +273,17 @@ func (dur Duration) MarshalText() ([]byte, error) {
d := time.Duration(dur)
return []byte(d.String()), nil
}
// ResourceFilteringStrategy is an enum indicating the kinds of resource
// filtering strategies that can be configured for workers.
type ResourceFilteringStrategy string
const (
// ResourceFilteringHardware specifies that available hardware resources
// should be evaluated when scheduling a task against the worker.
ResourceFilteringHardware = ResourceFilteringStrategy("hardware")
// ResourceFilteringDisabled disables resource filtering against this
// worker. The scheduler may assign any task to this worker.
ResourceFilteringDisabled = ResourceFilteringStrategy("disabled")
)

34
node/config/dep_test.go Normal file
View File

@ -0,0 +1,34 @@
package config
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
)
func goCmd() string {
var exeSuffix string
if runtime.GOOS == "windows" {
exeSuffix = ".exe"
}
path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix)
if _, err := os.Stat(path); err == nil {
return path
}
return "go"
}
func TestDoesntDependOnFFI(t *testing.T) {
deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/node/config").Output()
if err != nil {
t.Fatal(err)
}
for _, pkg := range strings.Fields(string(deps)) {
if pkg == "github.com/filecoin-project/filecoin-ffi" {
t.Fatal("config depends on filecoin-ffi")
}
}
}

26
node/config/storage.go
View File

@ -9,7 +9,6 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer"
)
func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfig, error) {
@ -50,28 +49,3 @@ func WriteStorageFile(path string, config paths.StorageConfig) error {
return nil
}
func (c *StorageMiner) StorageManager() sealer.Config {
return sealer.Config{
ParallelFetchLimit: c.Storage.ParallelFetchLimit,
AllowSectorDownload: c.Storage.AllowSectorDownload,
AllowAddPiece: c.Storage.AllowAddPiece,
AllowPreCommit1: c.Storage.AllowPreCommit1,
AllowPreCommit2: c.Storage.AllowPreCommit2,
AllowCommit: c.Storage.AllowCommit,
AllowUnseal: c.Storage.AllowUnseal,
AllowReplicaUpdate: c.Storage.AllowReplicaUpdate,
AllowProveReplicaUpdate2: c.Storage.AllowProveReplicaUpdate2,
AllowRegenSectorKey: c.Storage.AllowRegenSectorKey,
ResourceFiltering: c.Storage.ResourceFiltering,
DisallowRemoteFinalize: c.Storage.DisallowRemoteFinalize,
LocalWorkerName: c.Storage.LocalWorkerName,
Assigner: c.Storage.Assigner,
ParallelCheckLimit: c.Proving.ParallelCheckLimit,
DisableBuiltinWindowPoSt: c.Proving.DisableBuiltinWindowPoSt,
DisableBuiltinWinningPoSt: c.Proving.DisableBuiltinWinningPoSt,
}
}

3
node/config/types.go
View File

@ -4,7 +4,6 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/sealer"
)
// // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE
@ -452,7 +451,7 @@ type SealerConfig struct {
// ResourceFiltering instructs the system which resource filtering strategy
// to use when evaluating tasks against this worker. An empty value defaults
// to "hardware".
ResourceFiltering sealer.ResourceFilteringStrategy
ResourceFiltering ResourceFilteringStrategy
}
type BatchFeeConfig struct {

6
node/modules/storageminer.go
View File

@ -794,17 +794,17 @@ func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls paths.LocalStorag
return paths.NewLocal(ctx, ls, si, urls)
}
func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc sealer.Config) *paths.Remote {
func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc config.SealerConfig) *paths.Remote {
return paths.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit, &paths.DefaultPartialFileHandler{})
}
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc sealer.Config, ds dtypes.MetadataDS) (*sealer.Manager, error) {
func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc config.SealerConfig, pc config.ProvingConfig, ds dtypes.MetadataDS) (*sealer.Manager, error) {
ctx := helpers.LifecycleCtx(mctx, lc)
wsts := statestore.New(namespace.Wrap(ds, WorkerCallsPrefix))
smsts := statestore.New(namespace.Wrap(ds, ManagerWorkPrefix))
sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, wsts, smsts)
sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, pc, wsts, smsts)
if err != nil {
return nil, err
}

56
storage/sealer/manager.go
View File

@ -3,6 +3,7 @@ package sealer
import (
"context"
"errors"
"github.com/filecoin-project/lotus/node/config"
"io"
"net/http"
"sort"
@ -90,57 +91,12 @@ type result struct {
err error
}
// ResourceFilteringStrategy is an enum indicating the kinds of resource
// filtering strategies that can be configured for workers.
type ResourceFilteringStrategy string
const (
// ResourceFilteringHardware specifies that available hardware resources
// should be evaluated when scheduling a task against the worker.
ResourceFilteringHardware = ResourceFilteringStrategy("hardware")
// ResourceFilteringDisabled disables resource filtering against this
// worker. The scheduler may assign any task to this worker.
ResourceFilteringDisabled = ResourceFilteringStrategy("disabled")
)
type Config struct {
ParallelFetchLimit int
// Local worker config
AllowSectorDownload bool
AllowAddPiece bool
AllowPreCommit1 bool
AllowPreCommit2 bool
AllowCommit bool
AllowUnseal bool
AllowReplicaUpdate bool
AllowProveReplicaUpdate2 bool
AllowRegenSectorKey bool
LocalWorkerName string
// ResourceFiltering instructs the system which resource filtering strategy
// to use when evaluating tasks against this worker. An empty value defaults
// to "hardware".
ResourceFiltering ResourceFilteringStrategy
// PoSt config
ParallelCheckLimit int
DisableBuiltinWindowPoSt bool
DisableBuiltinWinningPoSt bool
DisallowRemoteFinalize bool
Assigner string
}
type StorageAuth http.Header
type WorkerStateStore *statestore.StateStore
type ManagerStateStore *statestore.StateStore
func New(ctx context.Context, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc Config, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) {
func New(ctx context.Context, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc config.SealerConfig, pc config.ProvingConfig, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) {
prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si})
if err != nil {
return nil, xerrors.Errorf("creating prover instance: %w", err)
@ -164,9 +120,9 @@ func New(ctx context.Context, lstor *paths.Local, stor paths.Store, ls paths.Loc
localProver: prover,
parallelCheckLimit: sc.ParallelCheckLimit,
disableBuiltinWindowPoSt: sc.DisableBuiltinWindowPoSt,
disableBuiltinWinningPoSt: sc.DisableBuiltinWinningPoSt,
parallelCheckLimit: pc.ParallelCheckLimit,
disableBuiltinWindowPoSt: pc.DisableBuiltinWindowPoSt,
disableBuiltinWinningPoSt: pc.DisableBuiltinWinningPoSt,
disallowRemoteFinalize: sc.DisallowRemoteFinalize,
work: mss,
@ -212,7 +168,7 @@ func New(ctx context.Context, lstor *paths.Local, stor paths.Store, ls paths.Loc
}
wcfg := WorkerConfig{
IgnoreResourceFiltering: sc.ResourceFiltering == ResourceFilteringDisabled,
IgnoreResourceFiltering: sc.ResourceFiltering == config.ResourceFilteringDisabled,
TaskTypes: localTasks,
Name: sc.LocalWorkerName,
}