cerc-io/lotus
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

toggle consideration of offline storage/retrieval deal proposals

Browse Source
This commit is contained in:
laser 2020-06-26 12:27:41 -07:00
parent e910a045d0
commit b9180a99d4
9 changed files with 238 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/api_storage.go
View File

@ -62,6 +62,10 @@ type StorageMiner interface {
DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error
DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)
DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error
DealsConsiderOfflineStorageDeals(context.Context) (bool, error)
DealsSetConsiderOfflineStorageDeals(context.Context, bool) error
DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)
DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
StorageAddLocal(ctx context.Context, path string) error
}

20
api/apistruct/struct.go
View File

@ -230,6 +230,10 @@ type StorageMinerStruct struct {
DealsSetConsiderOnlineStorageDeals func(context.Context, bool) error `perm:"admin"`
DealsConsiderOnlineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"`
DealsSetConsiderOnlineRetrievalDeals func(context.Context, bool) error `perm:"admin"`
DealsConsiderOfflineStorageDeals func(context.Context) (bool, error) `perm:"admin"`
DealsSetConsiderOfflineStorageDeals func(context.Context, bool) error `perm:"admin"`
DealsConsiderOfflineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"`
DealsSetConsiderOfflineRetrievalDeals func(context.Context, bool) error `perm:"admin"`
DealsPieceCidBlocklist func(context.Context) ([]cid.Cid, error) `perm:"admin"`
DealsSetPieceCidBlocklist func(context.Context, []cid.Cid) error `perm:"read"`
@ -904,6 +908,22 @@ func (c *StorageMinerStruct) DealsSetPieceCidBlocklist(ctx context.Context, cids
return c.Internal.DealsSetPieceCidBlocklist(ctx, cids)
}
func (c *StorageMinerStruct) DealsConsiderOfflineStorageDeals(ctx context.Context) (bool, error) {
return c.Internal.DealsConsiderOfflineStorageDeals(ctx)
}
func (c *StorageMinerStruct) DealsSetConsiderOfflineStorageDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetConsiderOfflineStorageDeals(ctx, b)
}
func (c *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(ctx context.Context) (bool, error) {
return c.Internal.DealsConsiderOfflineRetrievalDeals(ctx)
}
func (c *StorageMinerStruct) DealsSetConsiderOfflineRetrievalDeals(ctx context.Context, b bool) error {
return c.Internal.DealsSetConsiderOfflineRetrievalDeals(ctx, b)
}
func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error {
return c.Internal.StorageAddLocal(ctx, path)
}

33
cmd/lotus-storage-miner/market.go
View File

@ -70,13 +70,18 @@ var storageDealSelectionShowCmd = &cli.Command{
}
defer closer()
isAcceptingStorageDeals, err := smapi.DealsConsiderOnlineStorageDeals(lcli.DaemonContext(cctx))
onlineOk, err := smapi.DealsConsiderOnlineStorageDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
fmt.Printf("offline storage deals: %t\n", isAcceptingStorageDeals)
fmt.Printf("online storage deals: %t\n", isAcceptingStorageDeals)
offlineOk, err := smapi.DealsConsiderOfflineStorageDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
fmt.Printf("considering online storage deals: %t\n", onlineOk)
fmt.Printf("considering offline storage deals: %t\n", offlineOk)
return nil
},
@ -97,6 +102,11 @@ var storageDealSelectionResetCmd = &cli.Command{
return err
}
err = smapi.DealsSetConsiderOfflineStorageDeals(lcli.DaemonContext(cctx), true)
if err != nil {
return err
}
return nil
},
}
@ -104,6 +114,14 @@ var storageDealSelectionResetCmd = &cli.Command{
var storageDealSelectionRejectCmd = &cli.Command{
Name: "reject",
Usage: "Configure criteria which necessitate automatic rejection",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "online",
},
&cli.BoolFlag{
Name: "offline",
},
},
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
@ -111,10 +129,19 @@ var storageDealSelectionRejectCmd = &cli.Command{
}
defer closer()
if cctx.Bool("online") {
err = smapi.DealsSetConsiderOnlineStorageDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
if cctx.Bool("offline") {
err = smapi.DealsSetConsiderOfflineStorageDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
return nil
},

33
cmd/lotus-storage-miner/retrieval-deals.go
View File

@ -35,13 +35,18 @@ var retrievalDealSelectionShowCmd = &cli.Command{
}
defer closer()
isAcceptingRetrievalDeals, err := smapi.DealsConsiderOnlineRetrievalDeals(lcli.DaemonContext(cctx))
onlineOk, err := smapi.DealsConsiderOnlineRetrievalDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
fmt.Printf("offline retrieval deals: %t\n", isAcceptingRetrievalDeals)
fmt.Printf("online retrieval deals: %t\n", isAcceptingRetrievalDeals)
offlineOk, err := smapi.DealsConsiderOfflineRetrievalDeals(lcli.DaemonContext(cctx))
if err != nil {
return err
}
fmt.Printf("considering online retrieval deals: %t\n", onlineOk)
fmt.Printf("considering offline retrieval deals: %t\n", offlineOk)
return nil
},
@ -62,6 +67,11 @@ var retrievalDealSelectionResetCmd = &cli.Command{
return err
}
err = smapi.DealsSetConsiderOfflineRetrievalDeals(lcli.DaemonContext(cctx), true)
if err != nil {
return err
}
return nil
},
}
@ -69,6 +79,14 @@ var retrievalDealSelectionResetCmd = &cli.Command{
var retrievalDealSelectionRejectCmd = &cli.Command{
Name: "reject",
Usage: "Configure criteria which necessitate automatic rejection",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "online",
},
&cli.BoolFlag{
Name: "offline",
},
},
Action: func(cctx *cli.Context) error {
smapi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
@ -76,10 +94,19 @@ var retrievalDealSelectionRejectCmd = &cli.Command{
}
defer closer()
if cctx.Bool("online") {
err = smapi.DealsSetConsiderOnlineRetrievalDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
if cctx.Bool("offline") {
err = smapi.DealsSetConsiderOfflineRetrievalDeals(lcli.DaemonContext(cctx), false)
if err != nil {
return err
}
}
return nil
},

8
node/builder.go
View File

@ -313,12 +313,16 @@ func Online() Option {
Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver),
Override(new(*miner.Miner), modules.SetupBlockProducer),
Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc),
Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc),
Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc),
Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc),
Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc),
Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc),
Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc),
Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc),
Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc),
Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc),
Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc),
Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc),
),
)
}

4
node/config/def.go
View File

@ -35,7 +35,9 @@ type StorageMiner struct {
type DealmakingConfig struct {
ConsiderOnlineStorageDeals bool
ConsiderOfflineStorageDeals bool
ConsiderOnlineRetrievalDeals bool
ConsiderOfflineRetrievalDeals bool
PieceCidBlocklist []cid.Cid
}
@ -125,7 +127,9 @@ func DefaultStorageMiner() *StorageMiner {
Dealmaking: DealmakingConfig{
ConsiderOnlineStorageDeals: true,
ConsiderOfflineStorageDeals: true,
ConsiderOnlineRetrievalDeals: true,
ConsiderOfflineRetrievalDeals: true,
PieceCidBlocklist: []cid.Cid{},
},
}

20
node/impl/storminer.go
View File

@ -49,6 +49,10 @@ type StorageMinerAPI struct {
SetConsiderOnlineRetrievalDealsConfigFunc dtypes.SetConsiderOnlineRetrievalDealsConfigFunc
StorageDealPieceCidBlocklistConfigFunc dtypes.StorageDealPieceCidBlocklistConfigFunc
SetStorageDealPieceCidBlocklistConfigFunc dtypes.SetStorageDealPieceCidBlocklistConfigFunc
ConsiderOfflineStorageDealsConfigFunc dtypes.ConsiderOfflineStorageDealsConfigFunc
SetConsiderOfflineStorageDealsConfigFunc dtypes.SetConsiderOfflineStorageDealsConfigFunc
ConsiderOfflineRetrievalDealsConfigFunc dtypes.ConsiderOfflineRetrievalDealsConfigFunc
SetConsiderOfflineRetrievalDealsConfigFunc dtypes.SetConsiderOfflineRetrievalDealsConfigFunc
}
func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
@ -243,6 +247,22 @@ func (sm *StorageMinerAPI) DealsSetConsiderOnlineRetrievalDeals(ctx context.Cont
return sm.SetConsiderOnlineRetrievalDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsConsiderOfflineStorageDeals(ctx context.Context) (bool, error) {
return sm.ConsiderOfflineStorageDealsConfigFunc()
}
func (sm *StorageMinerAPI) DealsSetConsiderOfflineStorageDeals(ctx context.Context, b bool) error {
return sm.SetConsiderOfflineStorageDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsConsiderOfflineRetrievalDeals(ctx context.Context) (bool, error) {
return sm.ConsiderOfflineRetrievalDealsConfigFunc()
}
func (sm *StorageMinerAPI) DealsSetConsiderOfflineRetrievalDeals(ctx context.Context, b bool) error {
return sm.SetConsiderOfflineRetrievalDealsConfigFunc(b)
}
func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fname string) error {
fi, err := os.Open(fname)
if err != nil {

16
node/modules/dtypes/miner.go
View File

@ -34,3 +34,19 @@ type StorageDealPieceCidBlocklistConfigFunc func() ([]cid.Cid, error)
// SetStorageDealPieceCidBlocklistConfigFunc is a function which is used to set a
// list of CIDs for which the storage miner will reject deal proposals.
type SetStorageDealPieceCidBlocklistConfigFunc func([]cid.Cid) error
// ConsiderOfflineStorageDealsConfigFunc is a function which reads from miner
// config to determine if the user has disabled storage deals (or not).
type ConsiderOfflineStorageDealsConfigFunc func() (bool, error)
// SetConsiderOfflineStorageDealsConfigFunc is a function which is used to
// disable or enable storage deal acceptance.
type SetConsiderOfflineStorageDealsConfigFunc func(bool) error
// ConsiderOfflineRetrievalDealsConfigFunc is a function which reads from miner
// config to determine if the user has disabled retrieval acceptance (or not).
type ConsiderOfflineRetrievalDealsConfigFunc func() (bool, error)
// SetConsiderOfflineRetrievalDealsConfigFunc is a function which is used to
// disable or enable retrieval deal acceptance.
type SetConsiderOfflineRetrievalDealsConfigFunc func(bool) error

109
node/modules/storageminer.go
View File

@ -315,7 +315,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat
return storedAsk, nil
}
func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.ConsiderOnlineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc) (storagemarket.StorageProvider, error) {
func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc) (storagemarket.StorageProvider, error) {
net := smnet.NewFromLibp2pHost(h)
store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path()))
if err != nil {
@ -323,14 +323,24 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con
}
opt := storageimpl.CustomDealDecisionLogic(func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) {
b, err := isAcceptingFunc()
b, err := onlineOk()
if err != nil {
return false, "miner error", err
}
if !b {
log.Warnf("storage deal acceptance disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
return false, "miner is not accepting storage deals", nil
if deal.Ref != nil && deal.Ref.TransferType != storagemarket.TTManual && !b {
log.Warnf("online storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
return false, "miner is not considering online storage deals", nil
}
b, err = offlineOk()
if err != nil {
return false, "miner error", err
}
if deal.Ref != nil && deal.Ref.TransferType == storagemarket.TTManual && !b {
log.Warnf("offline storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
return false, "miner is not accepting offline storage deals", nil
}
blocklist, err := blocklistFunc()
@ -357,7 +367,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con
}
// RetrievalProvider creates a new retrieval provider attached to the provider blockstore
func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore, isAcceptingFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full)
maddr, err := minerAddrFromDS(ds)
@ -368,14 +378,23 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S
netwk := rmnet.NewFromLibp2pHost(h)
opt := retrievalimpl.DealDeciderOpt(func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) {
b, err := isAcceptingFunc()
b, err := onlineOk()
if err != nil {
return false, "miner error", err
}
if !b {
log.Warn("retrieval deal acceptance disabled; rejecting retrieval deal proposal from client")
return false, "miner is not accepting retrieval deals", nil
log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client")
return false, "miner is not accepting online retrieval deals", nil
}
b, err = offlineOk()
if err != nil {
return false, "miner error", err
}
if !b {
log.Info("offline retrieval has not been implemented yet")
}
return true, "", nil
@ -416,24 +435,6 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA
return sectorstorage.StorageAuth(headers), nil
}
func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineRetrievalDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOnlineRetrievalDeals
})
return
}, nil
}
func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineRetrievalDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOnlineRetrievalDeals = b
})
return
}, nil
}
func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
@ -452,6 +453,24 @@ func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsi
}, nil
}
func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineRetrievalDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOnlineRetrievalDeals
})
return
}, nil
}
func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineRetrievalDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOnlineRetrievalDeals = b
})
return
}, nil
}
func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlocklistConfigFunc, error) {
return func() (out []cid.Cid, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
@ -470,6 +489,42 @@ func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.Set
}, nil
}
func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineStorageDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOfflineStorageDeals
})
return
}, nil
}
func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineStorageDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOfflineStorageDeals = b
})
return
}, nil
}
func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineRetrievalDealsConfigFunc, error) {
return func() (out bool, err error) {
err = readCfg(r, func(cfg *config.StorageMiner) {
out = cfg.Dealmaking.ConsiderOfflineRetrievalDeals
})
return
}, nil
}
func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineRetrievalDealsConfigFunc, error) {
return func(b bool) (err error) {
err = mutateCfg(r, func(cfg *config.StorageMiner) {
cfg.Dealmaking.ConsiderOfflineRetrievalDeals = b
})
return
}, nil
}
func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error {
raw, err := r.Config()
if err != nil {