cleanup: miner: remove markets and deal-making from Lotus Miner (#12005)

* remove client CLI

* remove markets CLI from miner

* remove markets from all CLI

* remove client API

* update go mod

* remove EnableMarkets flag

* remove market subsystem

* remove dagstore

* remove index provider

* remove graphsync and data-transfer

* remove markets

* go mod tidy

* fix cbor gen deps

* remove deal making from config

* remove eol alert

* go mod tidy

* changes as per review

* make jen

* changes as per review

* merge master

* remove libp2p from config

* miner does not have libp2p conn in api test

node/modules/alerts.go

@@ -100,16 +100,6 @@ func CheckUDPBufferSize(wanted int) func(al *alerting.Alerting) {
 	}
 }
 
-func LegacyMarketsEOL(al *alerting.Alerting) {
-	// Add alert if lotus-miner legacy markets subsystem is still in use
-	alert := al.AddAlertType("system", "EOL")
-
-	// Alert with a message to migrate to Boost or similar markets subsystems
-	al.Raise(alert, map[string]string{
-		"message": "The lotus-miner legacy markets subsystem is deprecated and will be removed in a future release. Please migrate to [Boost](https://boost.filecoin.io) or similar markets subsystems.",
-	})
-}
-
 func CheckFvmConcurrency() func(al *alerting.Alerting) {
 	return func(al *alerting.Alerting) {
 		fvmConcurrency, ok := os.LookupEnv("LOTUS_FVM_CONCURRENCY")

node/modules/client.go

@@ -1,218 +0,0 @@
-package modules
-
-import (
-	"bytes"
-	"context"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	"github.com/libp2p/go-libp2p/core/host"
-	"go.uber.org/fx"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-data-transfer/v2/channelmonitor"
-	dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl"
-	dtnet "github.com/filecoin-project/go-data-transfer/v2/network"
-	dtgstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync"
-	"github.com/filecoin-project/go-fil-markets/discovery"
-	discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
-	rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
-	smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
-	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/market"
-	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/markets"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-	"github.com/filecoin-project/lotus/markets/retrievaladapter"
-	"github.com/filecoin-project/lotus/markets/storageadapter"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/impl/full"
-	payapi "github.com/filecoin-project/lotus/node/impl/paych"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/node/modules/helpers"
-	"github.com/filecoin-project/lotus/node/repo"
-	"github.com/filecoin-project/lotus/node/repo/imports"
-)
-
-func HandleMigrateClientFunds(lc fx.Lifecycle, mctx helpers.MetricsCtx, ds dtypes.MetadataDS, wallet full.WalletAPI, fundMgr *market.FundManager) {
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			addr, err := wallet.WalletDefaultAddress(ctx)
-			// nothing to be done if there is no default address
-			if err != nil {
-				return nil
-			}
-			b, err := ds.Get(helpers.LifecycleCtx(mctx, lc), datastore.NewKey("/marketfunds/client"))
-			if err != nil {
-				if xerrors.Is(err, datastore.ErrNotFound) {
-					return nil
-				}
-				log.Errorf("client funds migration - getting datastore value: %v", err)
-				return nil
-			}
-
-			var value abi.TokenAmount
-			if err = value.UnmarshalCBOR(bytes.NewReader(b)); err != nil {
-				log.Errorf("client funds migration - unmarshalling datastore value: %v", err)
-				return nil
-			}
-			_, err = fundMgr.Reserve(ctx, addr, addr, value)
-			if err != nil {
-				log.Errorf("client funds migration - reserving funds (wallet %s, addr %s, funds %d): %v",
-					addr, addr, value, err)
-				return nil
-			}
-
-			return ds.Delete(helpers.LifecycleCtx(mctx, lc), datastore.NewKey("/marketfunds/client"))
-		},
-	})
-}
-
-func ClientImportMgr(ds dtypes.MetadataDS, r repo.LockedRepo) (dtypes.ClientImportMgr, error) {
-	// store the imports under the repo's `imports` subdirectory.
-	dir := filepath.Join(r.Path(), "imports")
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return nil, xerrors.Errorf("failed to create directory %s: %w", dir, err)
-	}
-
-	ns := namespace.Wrap(ds, datastore.NewKey("/client"))
-	return imports.NewManager(ns, dir), nil
-}
-
-// TODO this should be removed.
-func ClientBlockstore() dtypes.ClientBlockstore {
-	// in most cases this is now unused in normal operations -- however, it's important to preserve for the IPFS use case
-	return blockstore.WrapIDStore(blockstore.FromDatastore(datastore.NewMapDatastore()))
-}
-
-// NewClientGraphsyncDataTransfer returns a data transfer manager that just
-// uses the clients's Client DAG service for transfers
-func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Graphsync, ds dtypes.MetadataDS, r repo.LockedRepo) (dtypes.ClientDataTransfer, error) {
-	// go-data-transfer protocol retries:
-	// 1s, 5s, 25s, 2m5s, 5m x 11 ~= 1 hour
-	dtRetryParams := dtnet.RetryParameters(time.Second, 5*time.Minute, 15, 5)
-	net := dtnet.NewFromLibp2pHost(h, dtRetryParams)
-
-	dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/client/transfers"))
-	transport := dtgstransport.NewTransport(h.ID(), gs)
-
-	// data-transfer push / pull channel restart configuration:
-	dtRestartConfig := dtimpl.ChannelRestartConfig(channelmonitor.Config{
-		// Disable Accept and Complete timeouts until this issue is resolved:
-		// https://github.com/filecoin-project/lotus/issues/6343#
-		// Wait for the other side to respond to an Open channel message
-		AcceptTimeout: 0,
-		// Wait for the other side to send a Complete message once all
-		// data has been sent / received
-		CompleteTimeout: 0,
-
-		// When an error occurs, wait a little while until all related errors
-		// have fired before sending a restart message
-		RestartDebounce: 10 * time.Second,
-		// After sending a restart, wait for at least 1 minute before sending another
-		RestartBackoff: time.Minute,
-		// After trying to restart 3 times, give up and fail the transfer
-		MaxConsecutiveRestarts: 3,
-	})
-	dt, err := dtimpl.NewDataTransfer(dtDs, net, transport, dtRestartConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	dt.OnReady(marketevents.ReadyLogger("client data transfer"))
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			dt.SubscribeToEvents(marketevents.DataTransferLogger)
-			return dt.Start(ctx)
-		},
-		OnStop: func(ctx context.Context) error {
-			return dt.Stop(ctx)
-		},
-	})
-	return dt, nil
-}
-
-// NewClientDatastore creates a datastore for the client to store its deals
-func NewClientDatastore(ds dtypes.MetadataDS) dtypes.ClientDatastore {
-	return namespace.Wrap(ds, datastore.NewKey("/deals/client"))
-}
-
-// StorageBlockstoreAccessor returns the default storage blockstore accessor
-// from the import manager.
-func StorageBlockstoreAccessor(importmgr dtypes.ClientImportMgr) storagemarket.BlockstoreAccessor {
-	return storageadapter.NewImportsBlockstoreAccessor(importmgr)
-}
-
-// RetrievalBlockstoreAccessor returns the default retrieval blockstore accessor
-// using the subdirectory `retrievals` under the repo.
-func RetrievalBlockstoreAccessor(r repo.LockedRepo) (retrievalmarket.BlockstoreAccessor, error) {
-	dir := filepath.Join(r.Path(), "retrievals")
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return nil, xerrors.Errorf("failed to create directory %s: %w", dir, err)
-	}
-	return retrievaladapter.NewCARBlockstoreAccessor(dir), nil
-}
-
-func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local,
-	deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, accessor storagemarket.BlockstoreAccessor, j journal.Journal) (storagemarket.StorageClient, error) {
-	// go-fil-markets protocol retries:
-	// 1s, 5s, 25s, 2m5s, 5m x 11 ~= 1 hour
-	marketsRetryParams := smnet.RetryParameters(time.Second, 5*time.Minute, 15, 5)
-	net := smnet.NewFromLibp2pHost(h, marketsRetryParams)
-
-	c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, accessor, storageimpl.DealPollingInterval(time.Second), storageimpl.MaxTraversalLinks(config.MaxTraversalLinks))
-	if err != nil {
-		return nil, err
-	}
-	c.OnReady(marketevents.ReadyLogger("storage client"))
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			c.SubscribeToEvents(marketevents.StorageClientLogger)
-
-			evtType := j.RegisterEventType("markets/storage/client", "state_change")
-			c.SubscribeToEvents(markets.StorageClientJournaler(j, evtType))
-
-			return c.Start(ctx)
-		},
-		OnStop: func(context.Context) error {
-			return c.Stop()
-		},
-	})
-	return c, nil
-}
-
-// RetrievalClient creates a new retrieval client attached to the client blockstore
-func RetrievalClient(forceOffChain bool) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
-	ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor *retrievaladapter.APIBlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
-	return func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
-		ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor *retrievaladapter.APIBlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
-		adapter := retrievaladapter.NewRetrievalClientNode(forceOffChain, payAPI, chainAPI, stateAPI)
-		network := rmnet.NewFromLibp2pHost(h)
-		ds = namespace.Wrap(ds, datastore.NewKey("/retrievals/client"))
-		client, err := retrievalimpl.NewClient(network, dt, adapter, resolver, ds, accessor)
-		if err != nil {
-			return nil, err
-		}
-		client.OnReady(marketevents.ReadyLogger("retrieval client"))
-		lc.Append(fx.Hook{
-			OnStart: func(ctx context.Context) error {
-				client.SubscribeToEvents(marketevents.RetrievalClientLogger)
-
-				evtType := j.RegisterEventType("markets/retrieval/client", "state_change")
-				client.SubscribeToEvents(markets.RetrievalClientJournaler(j, evtType))
-
-				return client.Start(ctx)
-			},
-		})
-		return client, nil
-	}
-}

node/modules/dtypes/miner.go

@@ -1,14 +1,11 @@
 package dtypes
 
 import (
-	"context"
 	"time"
 
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
@@ -89,11 +86,3 @@ type SetExpectedSealDurationFunc func(time.Duration) error
 // GetExpectedSealDurationFunc is a function which reads from miner
 // too determine how long sealing is expected to take
 type GetExpectedSealDurationFunc func() (time.Duration, error)
-
-type SetMaxDealStartDelayFunc func(time.Duration) error
-type GetMaxDealStartDelayFunc func() (time.Duration, error)
-
-type StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error)
-type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error)
-
-type RetrievalPricingFunc func(ctx context.Context, dealPricingParams retrievalmarket.PricingInput) (retrievalmarket.Ask, error)

node/modules/dtypes/storage.go

@@ -4,16 +4,8 @@ import (
 	bserv "github.com/ipfs/boxo/blockservice"
 	exchange "github.com/ipfs/boxo/exchange"
 	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-graphsync"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
-	dtnet "github.com/filecoin-project/go-data-transfer/v2/network"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation"
-	"github.com/filecoin-project/go-statestore"
 
 	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/node/repo/imports"
 )
 
 // MetadataDS stores metadata. By default it's namespaced under /metadata in
@@ -67,26 +59,3 @@ type (
 
 type ChainBitswap exchange.Interface
 type ChainBlockService bserv.BlockService
-
-type ClientImportMgr *imports.Manager
-type ClientBlockstore blockstore.BasicBlockstore
-type ClientDealStore *statestore.StateStore
-type ClientRequestValidator *requestvalidation.UnifiedRequestValidator
-type ClientDatastore datastore.Batching
-
-type Graphsync graphsync.GraphExchange
-
-// ClientDataTransfer is a data transfer manager for the client
-type ClientDataTransfer datatransfer.Manager
-
-type ProviderDealStore *statestore.StateStore
-type ProviderPieceStore piecestore.PieceStore
-
-type ProviderRequestValidator *requestvalidation.UnifiedRequestValidator
-
-// ProviderDataTransfer is a data transfer manager for the provider
-type ProviderDataTransfer datatransfer.Manager
-type ProviderTransferNetwork dtnet.DataTransferNetwork
-type ProviderTransport datatransfer.Transport
-type StagingBlockstore blockstore.BasicBlockstore
-type StagingGraphsync graphsync.GraphExchange

node/modules/graphsync.go

@@ -1,101 +0,0 @@
-package modules
-
-import (
-	"context"
-	"time"
-
-	"github.com/ipfs/go-graphsync"
-	graphsyncimpl "github.com/ipfs/go-graphsync/impl"
-	gsnet "github.com/ipfs/go-graphsync/network"
-	"github.com/ipfs/go-graphsync/storeutil"
-	"github.com/libp2p/go-libp2p/core/host"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"go.opencensus.io/stats"
-	"go.uber.org/fx"
-
-	"github.com/filecoin-project/lotus/metrics"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/node/modules/helpers"
-	"github.com/filecoin-project/lotus/node/repo"
-)
-
-// Graphsync creates a graphsync instance from the given loader and storer
-func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForRetrieval uint64) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, clientBs dtypes.ClientBlockstore, chainBs dtypes.ExposedBlockstore, h host.Host) (dtypes.Graphsync, error) {
-	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, clientBs dtypes.ClientBlockstore, chainBs dtypes.ExposedBlockstore, h host.Host) (dtypes.Graphsync, error) {
-		graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
-		lsys := storeutil.LinkSystemForBlockstore(clientBs)
-
-		gs := graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc),
-			graphsyncNetwork,
-			lsys,
-			graphsyncimpl.RejectAllRequestsByDefault(),
-			graphsyncimpl.MaxInProgressIncomingRequests(parallelTransfersForStorage),
-			graphsyncimpl.MaxInProgressOutgoingRequests(parallelTransfersForRetrieval),
-			graphsyncimpl.MaxLinksPerIncomingRequests(config.MaxTraversalLinks),
-			graphsyncimpl.MaxLinksPerOutgoingRequests(config.MaxTraversalLinks))
-		chainLinkSystem := storeutil.LinkSystemForBlockstore(chainBs)
-		err := gs.RegisterPersistenceOption("chainstore", chainLinkSystem)
-		if err != nil {
-			return nil, err
-		}
-		gs.RegisterIncomingRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) {
-			_, has := requestData.Extension("chainsync")
-			if has {
-				// TODO: we should confirm the selector is a reasonable one before we validate
-				// TODO: this code will get more complicated and should probably not live here eventually
-				hookActions.ValidateRequest()
-				hookActions.UsePersistenceOption("chainstore")
-			}
-		})
-		gs.RegisterOutgoingRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.OutgoingRequestHookActions) {
-			_, has := requestData.Extension("chainsync")
-			if has {
-				hookActions.UsePersistenceOption("chainstore")
-			}
-		})
-
-		graphsyncStats(mctx, lc, gs)
-
-		return gs, nil
-	}
-}
-
-func graphsyncStats(mctx helpers.MetricsCtx, lc fx.Lifecycle, gs dtypes.Graphsync) {
-	stopStats := make(chan struct{})
-	lc.Append(fx.Hook{
-		OnStart: func(context.Context) error {
-			go func() {
-				t := time.NewTicker(10 * time.Second)
-				for {
-					select {
-					case <-t.C:
-
-						st := gs.Stats()
-						stats.Record(mctx, metrics.GraphsyncReceivingPeersCount.M(int64(st.OutgoingRequests.TotalPeers)))
-						stats.Record(mctx, metrics.GraphsyncReceivingActiveCount.M(int64(st.OutgoingRequests.Active)))
-						stats.Record(mctx, metrics.GraphsyncReceivingCountCount.M(int64(st.OutgoingRequests.Pending)))
-						stats.Record(mctx, metrics.GraphsyncReceivingTotalMemoryAllocated.M(int64(st.IncomingResponses.TotalAllocatedAllPeers)))
-						stats.Record(mctx, metrics.GraphsyncReceivingTotalPendingAllocations.M(int64(st.IncomingResponses.TotalPendingAllocations)))
-						stats.Record(mctx, metrics.GraphsyncReceivingPeersPending.M(int64(st.IncomingResponses.NumPeersWithPendingAllocations)))
-						stats.Record(mctx, metrics.GraphsyncSendingPeersCount.M(int64(st.IncomingRequests.TotalPeers)))
-						stats.Record(mctx, metrics.GraphsyncSendingActiveCount.M(int64(st.IncomingRequests.Active)))
-						stats.Record(mctx, metrics.GraphsyncSendingCountCount.M(int64(st.IncomingRequests.Pending)))
-						stats.Record(mctx, metrics.GraphsyncSendingTotalMemoryAllocated.M(int64(st.OutgoingResponses.TotalAllocatedAllPeers)))
-						stats.Record(mctx, metrics.GraphsyncSendingTotalPendingAllocations.M(int64(st.OutgoingResponses.TotalPendingAllocations)))
-						stats.Record(mctx, metrics.GraphsyncSendingPeersPending.M(int64(st.OutgoingResponses.NumPeersWithPendingAllocations)))
-
-					case <-stopStats:
-						return
-					}
-				}
-			}()
-
-			return nil
-		},
-		OnStop: func(ctx context.Context) error {
-			close(stopStats)
-			return nil
-		},
-	})
-}

node/modules/services.go

@@ -6,8 +6,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/event"
 	"github.com/libp2p/go-libp2p/core/host"
@@ -17,9 +15,6 @@ import (
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-fil-markets/discovery"
-	discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
-
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -34,7 +29,6 @@ import (
 	"github.com/filecoin-project/lotus/journal"
 	"github.com/filecoin-project/lotus/journal/fsjournal"
 	"github.com/filecoin-project/lotus/lib/peermgr"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/hello"
 	"github.com/filecoin-project/lotus/node/impl/full"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -224,24 +218,6 @@ func RelayIndexerMessages(lc fx.Lifecycle, ps *pubsub.PubSub, nn dtypes.NetworkN
 	return nil
 }
 
-func NewLocalDiscovery(lc fx.Lifecycle, ds dtypes.MetadataDS) (*discoveryimpl.Local, error) {
-	local, err := discoveryimpl.NewLocal(namespace.Wrap(ds, datastore.NewKey("/deals/local")))
-	if err != nil {
-		return nil, err
-	}
-	local.OnReady(marketevents.ReadyLogger("discovery"))
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			return local.Start(ctx)
-		},
-	})
-	return local, nil
-}
-
-func RetrievalResolver(l *discoveryimpl.Local) discovery.PeerResolver {
-	return discoveryimpl.Multi(l)
-}
-
 type RandomBeaconParams struct {
 	fx.In
 

node/modules/storageminer.go

@@ -1,53 +1,28 @@
 package modules
 
 import (
-	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"net/http"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
-	graphsync "github.com/ipfs/go-graphsync/impl"
-	gsnet "github.com/ipfs/go-graphsync/network"
-	"github.com/ipfs/go-graphsync/storeutil"
-	provider "github.com/ipni/index-provider"
-	"github.com/libp2p/go-libp2p/core/host"
 	"go.uber.org/fx"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
-	dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl"
-	dtnet "github.com/filecoin-project/go-data-transfer/v2/network"
-	dtgstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync"
-	piecefilestore "github.com/filecoin-project/go-fil-markets/filestore"
-	piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
-	rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
-	"github.com/filecoin-project/go-fil-markets/shared"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
-	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
-	smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-paramfetch"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-statestore"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/api/v1api"
-	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/events"
@@ -55,11 +30,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/markets"
-	"github.com/filecoin-project/lotus/markets/dagstore"
-	"github.com/filecoin-project/lotus/markets/idxprov"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-	"github.com/filecoin-project/lotus/markets/pricing"
 	lotusminer "github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -332,163 +302,6 @@ func WindowPostScheduler(fc config.MinerFeeConfig, pc config.ProvingConfig) func
 	}
 }
 
-func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider, j journal.Journal) {
-	m.OnReady(marketevents.ReadyLogger("retrieval provider"))
-	lc.Append(fx.Hook{
-
-		OnStart: func(ctx context.Context) error {
-			m.SubscribeToEvents(marketevents.RetrievalProviderLogger)
-
-			evtType := j.RegisterEventType("markets/retrieval/provider", "state_change")
-			m.SubscribeToEvents(markets.RetrievalProviderJournaler(j, evtType))
-
-			return m.Start(ctx)
-		},
-		OnStop: func(context.Context) error {
-			return m.Stop()
-		},
-	})
-}
-
-func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider, j journal.Journal) {
-	ctx := helpers.LifecycleCtx(mctx, lc)
-	h.OnReady(marketevents.ReadyLogger("storage provider"))
-	lc.Append(fx.Hook{
-		OnStart: func(context.Context) error {
-			h.SubscribeToEvents(marketevents.StorageProviderLogger)
-
-			evtType := j.RegisterEventType("markets/storage/provider", "state_change")
-			h.SubscribeToEvents(markets.StorageProviderJournaler(j, evtType))
-
-			return h.Start(ctx)
-		},
-		OnStop: func(context.Context) error {
-			return h.Stop()
-		},
-	})
-}
-
-func HandleMigrateProviderFunds(lc fx.Lifecycle, ds dtypes.MetadataDS, node api.FullNode, minerAddress dtypes.MinerAddress) {
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			b, err := ds.Get(ctx, datastore.NewKey("/marketfunds/provider"))
-			if err != nil {
-				if xerrors.Is(err, datastore.ErrNotFound) {
-					return nil
-				}
-				return err
-			}
-
-			var value abi.TokenAmount
-			if err = value.UnmarshalCBOR(bytes.NewReader(b)); err != nil {
-				return err
-			}
-			ts, err := node.ChainHead(ctx)
-			if err != nil {
-				log.Errorf("provider funds migration - getting chain head: %v", err)
-				return nil
-			}
-
-			mi, err := node.StateMinerInfo(ctx, address.Address(minerAddress), ts.Key())
-			if err != nil {
-				log.Errorf("provider funds migration - getting miner info %s: %v", minerAddress, err)
-				return nil
-			}
-
-			_, err = node.MarketReserveFunds(ctx, mi.Worker, address.Address(minerAddress), value)
-			if err != nil {
-				log.Errorf("provider funds migration - reserving funds (wallet %s, addr %s, funds %d): %v",
-					mi.Worker, minerAddress, value, err)
-				return nil
-			}
-
-			return ds.Delete(ctx, datastore.NewKey("/marketfunds/provider"))
-		},
-	})
-}
-
-// NewProviderTransferNetwork sets up the libp2p2 protocol networking for data transfer
-func NewProviderTransferNetwork(h host.Host) dtypes.ProviderTransferNetwork {
-	return dtnet.NewFromLibp2pHost(h)
-}
-
-// NewProviderTransport sets up a data transfer transport over graphsync
-func NewProviderTransport(h host.Host, gs dtypes.StagingGraphsync) dtypes.ProviderTransport {
-	return dtgstransport.NewTransport(h.ID(), gs)
-}
-
-// NewProviderDataTransfer returns a data transfer manager
-func NewProviderDataTransfer(lc fx.Lifecycle, net dtypes.ProviderTransferNetwork, transport dtypes.ProviderTransport, ds dtypes.MetadataDS, r repo.LockedRepo) (dtypes.ProviderDataTransfer, error) {
-	dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/provider/transfers"))
-
-	dt, err := dtimpl.NewDataTransfer(dtDs, net, transport)
-	if err != nil {
-		return nil, err
-	}
-
-	dt.OnReady(marketevents.ReadyLogger("provider data transfer"))
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			dt.SubscribeToEvents(marketevents.DataTransferLogger)
-			return dt.Start(ctx)
-		},
-		OnStop: func(ctx context.Context) error {
-			return dt.Stop(ctx)
-		},
-	})
-	return dt, nil
-}
-
-// NewProviderPieceStore creates a statestore for storing metadata about pieces
-// shared by the storage and retrieval providers
-func NewProviderPieceStore(lc fx.Lifecycle, ds dtypes.MetadataDS) (dtypes.ProviderPieceStore, error) {
-	ps, err := piecestoreimpl.NewPieceStore(namespace.Wrap(ds, datastore.NewKey("/storagemarket")))
-	if err != nil {
-		return nil, err
-	}
-	ps.OnReady(marketevents.ReadyLogger("piecestore"))
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			return ps.Start(ctx)
-		},
-	})
-	return ps, nil
-}
-
-// StagingBlockstore creates a blockstore for staging blocks for a miner
-// in a storage deal, prior to sealing
-func StagingBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.StagingBlockstore, error) {
-	ctx := helpers.LifecycleCtx(mctx, lc)
-	stagingds, err := r.Datastore(ctx, "/staging")
-	if err != nil {
-		return nil, err
-	}
-
-	return blockstore.FromDatastore(stagingds), nil
-}
-
-// StagingGraphsync creates a graphsync instance which reads and writes blocks
-// to the StagingBlockstore
-func StagingGraphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePerPeer uint64, parallelTransfersForRetrieval uint64) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync {
-	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync {
-		graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
-		lsys := storeutil.LinkSystemForBlockstore(ibs)
-		gs := graphsync.New(helpers.LifecycleCtx(mctx, lc),
-			graphsyncNetwork,
-			lsys,
-			graphsync.RejectAllRequestsByDefault(),
-			graphsync.MaxInProgressIncomingRequests(parallelTransfersForRetrieval),
-			graphsync.MaxInProgressIncomingRequestsPerPeer(parallelTransfersForStoragePerPeer),
-			graphsync.MaxInProgressOutgoingRequests(parallelTransfersForStorage),
-			graphsync.MaxLinksPerIncomingRequests(config.MaxTraversalLinks),
-			graphsync.MaxLinksPerOutgoingRequests(config.MaxTraversalLinks))
-
-		graphsyncStats(mctx, lc, gs)
-
-		return gs
-	}
-}
-
 func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api v1api.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter, j journal.Journal) (*lotusminer.Miner, error) {
 	minerAddr, err := minerAddrFromDS(ds)
 	if err != nil {
@@ -512,273 +325,6 @@ func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api v1api.FullNod
 	return m, nil
 }
 
-func NewStorageAsk(ctx helpers.MetricsCtx, fapi v1api.FullNode, ds dtypes.MetadataDS, minerAddress dtypes.MinerAddress, spn storagemarket.StorageProviderNode) (*storedask.StoredAsk, error) {
-
-	mi, err := fapi.StateMinerInfo(ctx, address.Address(minerAddress), types.EmptyTSK)
-	if err != nil {
-		return nil, err
-	}
-
-	providerDs := namespace.Wrap(ds, datastore.NewKey("/deals/provider"))
-	// legacy this was mistake where this key was place -- so we move the legacy key if need be
-	err = shared.MoveKey(providerDs, "/latest-ask", "/storage-ask/latest")
-	if err != nil {
-		return nil, err
-	}
-	return storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress),
-		storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize)))
-}
-
-func BasicDealFilter(cfg config.DealmakingConfig, user dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
-	offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc,
-	verifiedOk dtypes.ConsiderVerifiedStorageDealsConfigFunc,
-	unverifiedOk dtypes.ConsiderUnverifiedStorageDealsConfigFunc,
-	blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc,
-	expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc,
-	startDelay dtypes.GetMaxDealStartDelayFunc,
-	spn storagemarket.StorageProviderNode,
-	r repo.LockedRepo,
-) dtypes.StorageDealFilter {
-	return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
-		offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc,
-		verifiedOk dtypes.ConsiderVerifiedStorageDealsConfigFunc,
-		unverifiedOk dtypes.ConsiderUnverifiedStorageDealsConfigFunc,
-		blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc,
-		expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc,
-		startDelay dtypes.GetMaxDealStartDelayFunc,
-		spn storagemarket.StorageProviderNode,
-		r repo.LockedRepo,
-	) dtypes.StorageDealFilter {
-
-		return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) {
-			b, err := onlineOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if deal.Ref != nil && deal.Ref.TransferType != storagemarket.TTManual && !b {
-				log.Warnf("online storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
-				return false, "miner is not considering online storage deals", nil
-			}
-
-			b, err = offlineOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if deal.Ref != nil && deal.Ref.TransferType == storagemarket.TTManual && !b {
-				log.Warnf("offline storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
-				return false, "miner is not accepting offline storage deals", nil
-			}
-
-			b, err = verifiedOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if deal.Proposal.VerifiedDeal && !b {
-				log.Warnf("verified storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
-				return false, "miner is not accepting verified storage deals", nil
-			}
-
-			b, err = unverifiedOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if !deal.Proposal.VerifiedDeal && !b {
-				log.Warnf("unverified storage deal consideration disabled; rejecting storage deal proposal from client: %s", deal.Client.String())
-				return false, "miner is not accepting unverified storage deals", nil
-			}
-
-			blocklist, err := blocklistFunc()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			for idx := range blocklist {
-				if deal.Proposal.PieceCID.Equals(blocklist[idx]) {
-					log.Warnf("piece CID in proposal %s is blocklisted; rejecting storage deal proposal from client: %s", deal.Proposal.PieceCID, deal.Client.String())
-					return false, fmt.Sprintf("miner has blocklisted piece CID %s", deal.Proposal.PieceCID), nil
-				}
-			}
-
-			sealDuration, err := expectedSealTimeFunc()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			sealEpochs := sealDuration / (time.Duration(build.BlockDelaySecs) * time.Second)
-			_, ht, err := spn.GetChainHead(ctx)
-			if err != nil {
-				return false, "failed to get chain head", err
-			}
-			earliest := abi.ChainEpoch(sealEpochs) + ht
-			if deal.Proposal.StartEpoch < earliest {
-				log.Warnw("proposed deal would start before sealing can be completed; rejecting storage deal proposal from client", "piece_cid", deal.Proposal.PieceCID, "client", deal.Client.String(), "seal_duration", sealDuration, "earliest", earliest, "curepoch", ht)
-				return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil
-			}
-
-			sd, err := startDelay()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			dir := filepath.Join(r.Path(), StagingAreaDirName)
-			diskUsageBytes, err := r.DiskUsage(dir)
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if cfg.MaxStagingDealsBytes != 0 && diskUsageBytes >= cfg.MaxStagingDealsBytes {
-				log.Errorw("proposed deal rejected because there are too many deals in the staging area at the moment", "MaxStagingDealsBytes", cfg.MaxStagingDealsBytes, "DiskUsageBytes", diskUsageBytes)
-				return false, "cannot accept deal as miner is overloaded at the moment - there are too many staging deals being processed", nil
-			}
-
-			// Reject if it's more than 7 days in the future
-			// TODO: read from cfg
-			maxStartEpoch := earliest + abi.ChainEpoch(uint64(sd.Seconds())/build.BlockDelaySecs)
-			if deal.Proposal.StartEpoch > maxStartEpoch {
-				return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil
-			}
-
-			if user != nil {
-				return user(ctx, deal)
-			}
-
-			return true, "", nil
-		}
-	}
-}
-
-func StorageProvider(minerAddress dtypes.MinerAddress,
-	storedAsk *storedask.StoredAsk,
-	h host.Host, ds dtypes.MetadataDS,
-	r repo.LockedRepo,
-	pieceStore dtypes.ProviderPieceStore,
-	indexer provider.Interface,
-	dataTransfer dtypes.ProviderDataTransfer,
-	spn storagemarket.StorageProviderNode,
-	df dtypes.StorageDealFilter,
-	dsw *dagstore.Wrapper,
-	meshCreator idxprov.MeshCreator,
-) (storagemarket.StorageProvider, error) {
-	net := smnet.NewFromLibp2pHost(h)
-
-	dir := filepath.Join(r.Path(), StagingAreaDirName)
-
-	// migrate temporary files that were created directly under the repo, by
-	// moving them to the new directory and symlinking them.
-	oldDir := r.Path()
-	if err := migrateDealStaging(oldDir, dir); err != nil {
-		return nil, xerrors.Errorf("failed to make deal staging directory %w", err)
-	}
-
-	store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(dir))
-	if err != nil {
-		return nil, err
-	}
-
-	opt := storageimpl.CustomDealDecisionLogic(storageimpl.DealDeciderFunc(df))
-
-	return storageimpl.NewProvider(
-		net,
-		namespace.Wrap(ds, datastore.NewKey("/deals/provider")),
-		store,
-		dsw,
-		indexer,
-		pieceStore,
-		dataTransfer,
-		spn,
-		address.Address(minerAddress),
-		storedAsk,
-		meshCreator,
-		opt,
-	)
-}
-
-func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
-	offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter {
-	return func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
-		offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter {
-		return func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) {
-			b, err := onlineOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if !b {
-				log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client")
-				return false, "miner is not accepting online retrieval deals", nil
-			}
-
-			b, err = offlineOk()
-			if err != nil {
-				return false, "miner error", err
-			}
-
-			if !b {
-				log.Info("offline retrieval has not been implemented yet")
-			}
-
-			if userFilter != nil {
-				return userFilter(ctx, state)
-			}
-
-			return true, "", nil
-		}
-	}
-}
-
-func RetrievalNetwork(h host.Host) rmnet.RetrievalMarketNetwork {
-	return rmnet.NewFromLibp2pHost(h)
-}
-
-// RetrievalPricingFunc configures the pricing function to use for retrieval deals.
-func RetrievalPricingFunc(cfg config.DealmakingConfig) func(_ dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
-	_ dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalPricingFunc {
-
-	return func(_ dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
-		_ dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalPricingFunc {
-		if cfg.RetrievalPricing.Strategy == config.RetrievalPricingExternalMode {
-			return pricing.ExternalRetrievalPricingFunc(cfg.RetrievalPricing.External.Path)
-		}
-
-		return retrievalimpl.DefaultPricingFunc(cfg.RetrievalPricing.Default.VerifiedDealsFreeTransfer)
-	}
-}
-
-// RetrievalProvider creates a new retrieval provider attached to the provider blockstore
-func RetrievalProvider(
-	maddr dtypes.MinerAddress,
-	adapter retrievalmarket.RetrievalProviderNode,
-	sa retrievalmarket.SectorAccessor,
-	netwk rmnet.RetrievalMarketNetwork,
-	ds dtypes.MetadataDS,
-	pieceStore dtypes.ProviderPieceStore,
-	dt dtypes.ProviderDataTransfer,
-	pricingFnc dtypes.RetrievalPricingFunc,
-	userFilter dtypes.RetrievalDealFilter,
-	dagStore *dagstore.Wrapper,
-) (retrievalmarket.RetrievalProvider, error) {
-	opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter))
-
-	retrievalmarket.DefaultPricePerByte = big.Zero() // todo: for whatever reason this is a global var in markets
-
-	return retrievalimpl.NewProvider(
-		address.Address(maddr),
-		adapter,
-		sa,
-		netwk,
-		pieceStore,
-		dagStore,
-		dt,
-		namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")),
-		retrievalimpl.RetrievalPricingFunc(pricingFnc),
-		opt,
-	)
-}
-
 var WorkerCallsPrefix = datastore.NewKey("/worker/calls")
 var ManagerWorkPrefix = datastore.NewKey("/stmgr/calls")
 
@@ -838,153 +384,6 @@ func StorageAuthWithURL(apiInfo string) interface{} {
 	}
 }
 
-func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) {
-	return func() (out bool, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.ConsiderOnlineStorageDeals
-		})
-		return
-	}, nil
-}
-
-func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineStorageDealsConfigFunc, error) {
-	return func(b bool) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ConsiderOnlineStorageDeals = b
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineRetrievalDealsConfigFunc, error) {
-	return func() (out bool, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.ConsiderOnlineRetrievalDeals
-		})
-		return
-	}, nil
-}
-
-func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineRetrievalDealsConfigFunc, error) {
-	return func(b bool) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ConsiderOnlineRetrievalDeals = b
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlocklistConfigFunc, error) {
-	return func() (out []cid.Cid, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.PieceCidBlocklist
-		})
-		return
-	}, nil
-}
-
-func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealPieceCidBlocklistConfigFunc, error) {
-	return func(blocklist []cid.Cid) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.PieceCidBlocklist = blocklist
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineStorageDealsConfigFunc, error) {
-	return func() (out bool, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.ConsiderOfflineStorageDeals
-		})
-		return
-	}, nil
-}
-
-func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineStorageDealsConfigFunc, error) {
-	return func(b bool) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ConsiderOfflineStorageDeals = b
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineRetrievalDealsConfigFunc, error) {
-	return func() (out bool, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.ConsiderOfflineRetrievalDeals
-		})
-		return
-	}, nil
-}
-
-func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineRetrievalDealsConfigFunc, error) {
-	return func(b bool) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ConsiderOfflineRetrievalDeals = b
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewConsiderVerifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderVerifiedStorageDealsConfigFunc, error) {
-	return func() (out bool, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.ConsiderVerifiedStorageDeals
-		})
-		return
-	}, nil
-}
-
-func NewSetConsideringVerifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderVerifiedStorageDealsConfigFunc, error) {
-	return func(b bool) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ConsiderVerifiedStorageDeals = b
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewConsiderUnverifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderUnverifiedStorageDealsConfigFunc, error) {
-	return func() (out bool, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = cfg.ConsiderUnverifiedStorageDeals
-		})
-		return
-	}, nil
-}
-
-func NewSetConsideringUnverifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderUnverifiedStorageDealsConfigFunc, error) {
-	return func(b bool) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ConsiderUnverifiedStorageDeals = b
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
 func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error) {
 	return func(cfg sealiface.Config) (err error) {
 		err = mutateSealingCfg(r, func(c config.SealingConfiger) {
@@ -1092,48 +491,6 @@ func NewGetSealConfigFunc(r repo.LockedRepo) (dtypes.GetSealingConfigFunc, error
 	}, nil
 }
 
-func NewSetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.SetExpectedSealDurationFunc, error) {
-	return func(delay time.Duration) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.ExpectedSealDuration = config.Duration(delay)
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewGetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.GetExpectedSealDurationFunc, error) {
-	return func() (out time.Duration, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = time.Duration(cfg.ExpectedSealDuration)
-		})
-		return
-	}, nil
-}
-
-func NewSetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.SetMaxDealStartDelayFunc, error) {
-	return func(delay time.Duration) (err error) {
-		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			cfg.MaxDealStartDelay = config.Duration(delay)
-			c.SetDealmakingConfig(cfg)
-		})
-		return
-	}, nil
-}
-
-func NewGetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.GetMaxDealStartDelayFunc, error) {
-	return func() (out time.Duration, err error) {
-		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg := c.GetDealmakingConfig()
-			out = time.Duration(cfg.MaxDealStartDelay)
-		})
-		return
-	}, nil
-}
-
 func readSealingCfg(r repo.LockedRepo, accessor func(config.DealmakingConfiger, config.SealingConfiger)) error {
 	raw, err := r.Config()
 	if err != nil {
@@ -1171,91 +528,6 @@ func mutateSealingCfg(r repo.LockedRepo, mutator func(config.SealingConfiger)) e
 	return multierr.Combine(typeErr, setConfigErr)
 }
 
-func readDealmakingCfg(r repo.LockedRepo, accessor func(config.DealmakingConfiger)) error {
-	raw, err := r.Config()
-	if err != nil {
-		return err
-	}
-
-	cfg, ok := raw.(config.DealmakingConfiger)
-	if !ok {
-		return xerrors.New("expected config with dealmaking config trait")
-	}
-
-	accessor(cfg)
-
-	return nil
-}
-
-func mutateDealmakingCfg(r repo.LockedRepo, mutator func(config.DealmakingConfiger)) error {
-	var typeErr error
-
-	setConfigErr := r.SetConfig(func(raw interface{}) {
-		cfg, ok := raw.(config.DealmakingConfiger)
-		if !ok {
-			typeErr = errors.New("expected config with dealmaking config trait")
-			return
-		}
-
-		mutator(cfg)
-	})
-
-	return multierr.Combine(typeErr, setConfigErr)
-}
-
-func migrateDealStaging(oldPath, newPath string) error {
-	dirInfo, err := os.Stat(newPath)
-	if err == nil {
-		if !dirInfo.IsDir() {
-			return xerrors.Errorf("%s is not a directory", newPath)
-		}
-		// The newPath exists already, below migration has already occurred.
-		return nil
-	}
-
-	// if the directory doesn't exist, create it
-	if os.IsNotExist(err) {
-		if err := os.MkdirAll(newPath, 0755); err != nil {
-			return xerrors.Errorf("failed to mk directory %s for deal staging: %w", newPath, err)
-		}
-	} else { // if we failed for other reasons, abort.
-		return err
-	}
-
-	// if this is the first time we created the directory, symlink all staged deals into it. "Migration"
-	// get a list of files in the miner repo
-	dirEntries, err := os.ReadDir(oldPath)
-	if err != nil {
-		return xerrors.Errorf("failed to list directory %s for deal staging: %w", oldPath, err)
-	}
-
-	for _, entry := range dirEntries {
-		// ignore directories, they are not the deals.
-		if entry.IsDir() {
-			continue
-		}
-		// the FileStore from fil-storage-market creates temporary staged deal files with the pattern "fstmp"
-		// https://github.com/filecoin-project/go-fil-markets/blob/00ff81e477d846ac0cb58a0c7d1c2e9afb5ee1db/filestore/filestore.go#L69
-		name := entry.Name()
-		if strings.Contains(name, "fstmp") {
-			// from the miner repo
-			oldPath := filepath.Join(oldPath, name)
-			// to its subdir "deal-staging"
-			newPath := filepath.Join(newPath, name)
-			// create a symbolic link in the new deal staging directory to preserve existing staged deals.
-			// all future staged deals will be created here.
-			if err := os.Rename(oldPath, newPath); err != nil {
-				return xerrors.Errorf("failed to move %s to %s: %w", oldPath, newPath, err)
-			}
-			if err := os.Symlink(newPath, oldPath); err != nil {
-				return xerrors.Errorf("failed to symlink %s to %s: %w", oldPath, newPath, err)
-			}
-			log.Infow("symlinked staged deal", "from", oldPath, "to", newPath)
-		}
-	}
-	return nil
-}
-
 func ExtractEnabledMinerSubsystems(cfg config.MinerSubsystemConfig) (res api.MinerSubsystems) {
 	if cfg.EnableMining {
 		res = append(res, api.SubsystemMining)
@@ -1266,8 +538,6 @@ func ExtractEnabledMinerSubsystems(cfg config.MinerSubsystemConfig) (res api.Min
 	if cfg.EnableSectorStorage {
 		res = append(res, api.SubsystemSectorStorage)
 	}
-	if cfg.EnableMarkets {
-		res = append(res, api.SubsystemMarkets)
-	}
+
 	return res
 }

node/modules/storageminer_dagstore.go

@@ -1,94 +0,0 @@
-package modules
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strconv"
-
-	"github.com/libp2p/go-libp2p/core/host"
-	"go.uber.org/fx"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/dagstore"
-
-	mdagstore "github.com/filecoin-project/lotus/markets/dagstore"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/node/repo"
-)
-
-const (
-	EnvDAGStoreCopyConcurrency = "LOTUS_DAGSTORE_COPY_CONCURRENCY"
-	DefaultDAGStoreDir         = "dagstore"
-)
-
-// NewMinerAPI creates a new MinerAPI adaptor for the dagstore mounts.
-func NewMinerAPI(cfg config.DAGStoreConfig) func(fx.Lifecycle, repo.LockedRepo, dtypes.ProviderPieceStore, mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) {
-	return func(lc fx.Lifecycle, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, sa mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) {
-		// caps the amount of concurrent calls to the storage, so that we don't
-		// spam it during heavy processes like bulk migration.
-		if v, ok := os.LookupEnv("LOTUS_DAGSTORE_MOUNT_CONCURRENCY"); ok {
-			concurrency, err := strconv.Atoi(v)
-			if err == nil {
-				cfg.MaxConcurrencyStorageCalls = concurrency
-			}
-		}
-
-		mountApi := mdagstore.NewMinerAPI(pieceStore, sa, cfg.MaxConcurrencyStorageCalls, cfg.MaxConcurrentUnseals)
-		ready := make(chan error, 1)
-		pieceStore.OnReady(func(err error) {
-			ready <- err
-		})
-		lc.Append(fx.Hook{
-			OnStart: func(ctx context.Context) error {
-				if err := <-ready; err != nil {
-					return fmt.Errorf("aborting dagstore start; piecestore failed to start: %s", err)
-				}
-				return mountApi.Start(ctx)
-			},
-			OnStop: func(context.Context) error {
-				return nil
-			},
-		})
-
-		return mountApi, nil
-	}
-}
-
-// DAGStore constructs a DAG store using the supplied minerAPI, and the
-// user configuration. It returns both the DAGStore and the Wrapper suitable for
-// passing to markets.
-func DAGStore(cfg config.DAGStoreConfig) func(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI, h host.Host) (*dagstore.DAGStore, *mdagstore.Wrapper, error) {
-	return func(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI, h host.Host) (*dagstore.DAGStore, *mdagstore.Wrapper, error) {
-		// fall back to default root directory if not explicitly set in the config.
-		if cfg.RootDir == "" {
-			cfg.RootDir = filepath.Join(r.Path(), DefaultDAGStoreDir)
-		}
-
-		v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency)
-		if ok {
-			concurrency, err := strconv.Atoi(v)
-			if err == nil {
-				cfg.MaxConcurrentReadyFetches = concurrency
-			}
-		}
-
-		dagst, w, err := mdagstore.NewDAGStore(cfg, minerAPI, h)
-		if err != nil {
-			return nil, nil, xerrors.Errorf("failed to create DAG store: %w", err)
-		}
-
-		lc.Append(fx.Hook{
-			OnStart: func(ctx context.Context) error {
-				return w.Start(ctx)
-			},
-			OnStop: func(context.Context) error {
-				return w.Close()
-			},
-		})
-
-		return dagst, w, nil
-	}
-}

node/modules/storageminer_idxprov.go

@@ -1,117 +0,0 @@
-package modules
-
-import (
-	"context"
-
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	provider "github.com/ipni/index-provider"
-	"github.com/ipni/index-provider/engine"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/libp2p/go-libp2p/core/host"
-	"go.uber.org/fx"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-type IdxProv struct {
-	fx.In
-
-	fx.Lifecycle
-	Datastore dtypes.MetadataDS
-}
-
-func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host, dt dtypes.ProviderDataTransfer, maddr dtypes.MinerAddress, ps *pubsub.PubSub, nn dtypes.NetworkName) (provider.Interface, error) {
-	return func(args IdxProv, marketHost host.Host, dt dtypes.ProviderDataTransfer, maddr dtypes.MinerAddress, ps *pubsub.PubSub, nn dtypes.NetworkName) (provider.Interface, error) {
-		topicName := cfg.TopicName
-		// If indexer topic name is left empty, infer it from the network name.
-		if topicName == "" {
-			// Use the same mechanism as the Dependency Injection (DI) to construct the topic name,
-			// so that we are certain it is consistent with the name allowed by the subscription
-			// filter.
-			//
-			// See: lp2p.GossipSub.
-			topicName = build.IndexerIngestTopic(nn)
-			log.Debugw("Inferred indexer topic from network name", "topic", topicName)
-		}
-
-		ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/index-provider"))
-		addrs := marketHost.Addrs()
-		addrsString := make([]string, 0, len(addrs))
-		for _, addr := range addrs {
-			addrsString = append(addrsString, addr.String())
-		}
-		var opts = []engine.Option{
-			engine.WithDatastore(ipds),
-			engine.WithHost(marketHost),
-			engine.WithRetrievalAddrs(addrsString...),
-			engine.WithEntriesCacheCapacity(cfg.EntriesCacheCapacity),
-			engine.WithChainedEntries(cfg.EntriesChunkSize),
-			engine.WithTopicName(topicName),
-			engine.WithPurgeCacheOnStart(cfg.PurgeCacheOnStart),
-		}
-
-		llog := log.With(
-			"idxProvEnabled", cfg.Enable,
-			"pid", marketHost.ID(),
-			"topic", topicName,
-			"retAddrs", marketHost.Addrs())
-		// If announcements to the network are enabled, then set options for datatransfer publisher.
-		if cfg.Enable {
-			// Join the indexer topic using the market's pubsub instance. Otherwise, the provider
-			// engine would create its own instance of pubsub down the line in dagsync, which has
-			// no validators by default.
-			t, err := ps.Join(topicName)
-			if err != nil {
-				llog.Errorw("Failed to join indexer topic", "err", err)
-				return nil, xerrors.Errorf("joining indexer topic %s: %w", topicName, err)
-			}
-
-			// Get the miner ID and set as extra gossip data.
-			// The extra data is required by the lotus-specific index-provider gossip message validators.
-			ma := address.Address(maddr)
-			opts = append(opts,
-				engine.WithPublisherKind(engine.DataTransferPublisher),
-				engine.WithDataTransfer(dt),
-				engine.WithExtraGossipData(ma.Bytes()),
-				engine.WithTopic(t),
-			)
-			llog = llog.With("extraGossipData", ma, "publisher", "data-transfer")
-		} else {
-			opts = append(opts, engine.WithPublisherKind(engine.NoPublisher))
-			llog = llog.With("publisher", "none")
-		}
-
-		// Instantiate the index provider engine.
-		e, err := engine.New(opts...)
-		if err != nil {
-			return nil, xerrors.Errorf("creating indexer provider engine: %w", err)
-		}
-		llog.Info("Instantiated index provider engine")
-
-		args.Lifecycle.Append(fx.Hook{
-			OnStart: func(ctx context.Context) error {
-				// Note that the OnStart context is cancelled after startup. Its use in e.Start is
-				// to start up gossipsub publishers and restore cache, all of  which are completed
-				// before e.Start returns. Therefore, it is fine to reuse the give context.
-				if err := e.Start(ctx); err != nil {
-					return xerrors.Errorf("starting indexer provider engine: %w", err)
-				}
-				log.Infof("Started index provider engine")
-				return nil
-			},
-			OnStop: func(_ context.Context) error {
-				if err := e.Shutdown(); err != nil {
-					return xerrors.Errorf("shutting down indexer provider engine: %w", err)
-				}
-				return nil
-			},
-		})
-		return e, nil
-	}
-}

node/modules/storageminer_idxprov_test.go

@@ -1,99 +0,0 @@
-package modules_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ipfs/go-datastore"
-	provider "github.com/ipni/index-provider"
-	"github.com/libp2p/go-libp2p"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/libp2p/go-libp2p/core/host"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/fx"
-
-	"github.com/filecoin-project/go-address"
-
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/modules"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-func Test_IndexProviderTopic(t *testing.T) {
-	tests := []struct {
-		name                 string
-		givenAllowedTopics   []string
-		givenConfiguredTopic string
-		givenNetworkName     dtypes.NetworkName
-		wantErr              string
-	}{
-		{
-			name:                 "Joins configured topic when allowed",
-			givenAllowedTopics:   []string{"fish"},
-			givenConfiguredTopic: "fish",
-		},
-		{
-			name:               "Joins topic inferred from network name when allowed",
-			givenAllowedTopics: []string{"/indexer/ingest/fish"},
-			givenNetworkName:   "fish",
-		},
-		{
-			name:                 "Fails to join configured topic when disallowed",
-			givenAllowedTopics:   []string{"/indexer/ingest/fish"},
-			givenConfiguredTopic: "lobster",
-			wantErr:              "joining indexer topic lobster: topic is not allowed by the subscription filter",
-		},
-		{
-			name:               "Fails to join topic inferred from network name when disallowed",
-			givenAllowedTopics: []string{"/indexer/ingest/fish"},
-			givenNetworkName:   "lobster",
-			wantErr:            "joining indexer topic /indexer/ingest/lobster: topic is not allowed by the subscription filter",
-		},
-	}
-
-	for _, test := range tests {
-		test := test
-		t.Run(test.name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel()
-
-			h, err := libp2p.New()
-			require.NoError(t, err)
-			defer func() {
-				require.NoError(t, h.Close())
-			}()
-
-			filter := pubsub.WithSubscriptionFilter(pubsub.NewAllowlistSubscriptionFilter(test.givenAllowedTopics...))
-			ps, err := pubsub.NewGossipSub(ctx, h, filter)
-			require.NoError(t, err)
-
-			app := fx.New(
-				fx.Provide(
-					func() host.Host { return h },
-					func() dtypes.NetworkName { return test.givenNetworkName },
-					func() dtypes.MinerAddress { return dtypes.MinerAddress(address.TestAddress) },
-					func() dtypes.ProviderDataTransfer { return nil },
-					func() *pubsub.PubSub { return ps },
-					func() dtypes.MetadataDS { return datastore.NewMapDatastore() },
-					modules.IndexProvider(config.IndexProviderConfig{
-						Enable:           true,
-						TopicName:        test.givenConfiguredTopic,
-						EntriesChunkSize: 16384,
-					}),
-				),
-				fx.Invoke(func(p provider.Interface) {}),
-			)
-			err = app.Start(ctx)
-
-			if test.wantErr == "" {
-				require.NoError(t, err)
-				err = app.Stop(ctx)
-				require.NoError(t, err)
-			} else {
-				require.True(t, strings.HasSuffix(err.Error(), test.wantErr))
-			}
-		})
-	}
-}