cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

pull dagstore migration into Lotus and simplify.

Browse Source 
Migration registers all shards with lazy init.

Shards are then initialized as they are retrieved for the
first time, or in bulk through a lotus-shed tool that will
be provided separately.
This commit is contained in:
Raúl Kripalani
2021-08-04 16:59:27 +01:00
parent e8e73e5374
commit 270bad8144
9 changed files with 315 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/config/def.go
-3
View File
@@ -189,9 +189,6 @@ func DefaultStorageMiner() *StorageMiner {
DealPublishControl: []string{},
},
// The default DAGStoreConfig doesn't define any paths for transients,
// indices and the datastore. Empty values will lead to these being
// placed under <repo>/dagStore.
DAGStore: DAGStoreConfig{
MaxConcurrentIndex: 5,
MaxConcurrentReadyFetches: 2,
node/config/types.go
+13 -17
View File
@@ -54,23 +54,19 @@ type StorageMiner struct {
}
type DAGStoreConfig struct {
// Path to the transients directory. The transients directory caches
// unsealed deals that have been fetched from the storage subsystem for
// serving retrievals. When empty or omitted, the default value applies.
// Default value: $LOTUS_MARKETS_PATH/dagStore/transients (split deployment)
// or $LOTUS_MINER_PATH/dagStore/transients (monolith deployment)
TransientsDir string
// Path to indices directory. When empty or omitted, the default value applies.
// Default value: $LOTUS_MARKETS_PATH/dagStore/index (split deployment)
// or $LOTUS_MINER_PATH/dagStore/index (monolith deployment)
IndexDir string
// Path to datastore directory. The datastore is a KV store tracking the
// state of shards known to the DAG store.
// Default value: $LOTUS_MARKETS_PATH/dagStore/datastore (split deployment)
// or $LOTUS_MINER_PATH/dagStore/datastore (monolith deployment)
DatastoreDir string
// Path to the dagstore root directory. This directory contains three
// subdirectories, which can be symlinked to alternative locations if
// need be:
//
// - ./transients: caches unsealed deals that have been fetched from the
// storage subsystem for serving retrievals.
// - ./indices: stores shard indices.
// - ./datastore: holds the KV store tracking the state of every shard
// known to the DAG store.
//
// Default value: $LOTUS_MARKETS_PATH/dagStore (split deployment) or
// $LOTUS_MINER_PATH/dagStore (monolith deployment)
RootDir string
// The maximum amount of indexing jobs that can run simultaneously.
// Default value: 5.
node/modules/storageminer.go
+23 -15
View File
@@ -19,16 +19,6 @@ import (
dtimpl "github.com/filecoin-project/go-data-transfer/impl"
dtnet "github.com/filecoin-project/go-data-transfer/network"
dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync"
piecefilestore "github.com/filecoin-project/go-fil-markets/filestore"
piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
"github.com/filecoin-project/go-fil-markets/shared"
"github.com/filecoin-project/go-fil-markets/storagemarket"
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-state-types/abi"
@@ -47,6 +37,17 @@ import (
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/routing"
piecefilestore "github.com/filecoin-project/go-fil-markets/filestore"
piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
"github.com/filecoin-project/go-fil-markets/shared"
"github.com/filecoin-project/go-fil-markets/storagemarket"
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@@ -590,13 +591,20 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
return nil, err
}
dagStorePath := filepath.Join(r.Path(), DefaultDAGStoreDir)
opt := storageimpl.CustomDealDecisionLogic(storageimpl.DealDeciderFunc(df))
shardMigrator := storageimpl.NewShardMigrator(address.Address(minerAddress), dagStorePath, dsw, pieceStore, spn)
return storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), store, dsw, pieceStore,
dataTransfer, spn, address.Address(minerAddress), storedAsk, shardMigrator, opt)
return storageimpl.NewProvider(
net,
namespace.Wrap(ds, datastore.NewKey("/deals/provider")),
store,
dsw,
pieceStore,
dataTransfer,
spn,
address.Address(minerAddress),
storedAsk,
opt,
)
}
func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc,
node/modules/storageminer_dagstore.go
+5 -11
View File
@@ -8,10 +8,11 @@ import (
"strconv"
"github.com/filecoin-project/dagstore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
mdagstore "github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -68,16 +69,9 @@ func DAGStore(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI) (
return nil, nil, err
}
// populate default directories if not explicitly set in the config.
defaultDir := filepath.Join(r.Path(), DefaultDAGStoreDir)
if cfg.TransientsDir == "" {
cfg.TransientsDir = filepath.Join(defaultDir, "transients")
}
if cfg.IndexDir == "" {
cfg.IndexDir = filepath.Join(defaultDir, "index")
}
if cfg.DatastoreDir == "" {
cfg.DatastoreDir = filepath.Join(defaultDir, "datastore")
// fall back to default root directory if not explicitly set in the config.
if cfg.RootDir == "" {
cfg.RootDir = filepath.Join(r.Path(), DefaultDAGStoreDir)
}
v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency)