cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

refactor dagstore + carv2 integration (+).

Browse Source 
- Integrate with config.toml.
- Export DAGStore to DI context in preparation for JSON-RPC
  and cli commands.
- Renames.
This commit is contained in:
Raúl Kripalani
2021-08-03 23:09:28 +01:00
parent 691da14996
commit c06c8541f9
17 changed files with 335 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/modules/client.go
+1 -1
View File
@@ -182,7 +182,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT
func RetrievalClient(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
carsPath := filepath.Join(r.Path(), dagStore, "retrieval-cars")
carsPath := filepath.Join(r.Path(), DefaultDAGStoreDir, "retrieval-cars")
if err := os.MkdirAll(carsPath, 0755); err != nil {
return nil, xerrors.Errorf("failed to create dir")
node/modules/storageminer.go
+13 -20
View File
@@ -7,9 +7,7 @@ import (
"fmt"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
@@ -17,22 +15,6 @@ import (
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
levelds "github.com/ipfs/go-ds-leveldb"
measure "github.com/ipfs/go-ds-measure"
graphsync "github.com/ipfs/go-graphsync/impl"
gsnet "github.com/ipfs/go-graphsync/network"
"github.com/ipfs/go-graphsync/storeutil"
"github.com/ipfs/go-merkledag"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/routing"
ldbopts "github.com/syndtr/goleveldb/leveldb/opt"
"github.com/filecoin-project/go-address"
dtimpl "github.com/filecoin-project/go-data-transfer/impl"
dtnet "github.com/filecoin-project/go-data-transfer/network"
@@ -52,6 +34,18 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/go-storedcounter"
"github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
graphsync "github.com/ipfs/go-graphsync/impl"
gsnet "github.com/ipfs/go-graphsync/network"
"github.com/ipfs/go-graphsync/storeutil"
"github.com/ipfs/go-merkledag"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/routing"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@@ -82,7 +76,6 @@ import (
)
var StorageCounterDSPrefix = "/storage/nextid"
var dagStore = "dagStore"
func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
maddrb, err := ds.Get(datastore.NewKey("miner-address"))
@@ -597,7 +590,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
return nil, err
}
dagStorePath := filepath.Join(r.Path(), dagStore)
dagStorePath := filepath.Join(r.Path(), DefaultDAGStoreDir)
opt := storageimpl.CustomDealDecisionLogic(storageimpl.DealDeciderFunc(df))
shardMigrator := storageimpl.NewShardMigrator(address.Address(minerAddress), dagStorePath, dsw, pieceStore, spn)
node/modules/storageminer_dagstore.go
+59 -57
View File
@@ -6,25 +6,39 @@ import (
"os"
"path/filepath"
"strconv"
"time"
"github.com/filecoin-project/dagstore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/lotus/markets/dagstore"
mdagstore "github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
"github.com/ipfs/go-datastore"
levelds "github.com/ipfs/go-ds-leveldb"
measure "github.com/ipfs/go-ds-measure"
ldbopts "github.com/syndtr/goleveldb/leveldb/opt"
"go.uber.org/fx"
"golang.org/x/xerrors"
)
func NewLotusAccessor(lc fx.Lifecycle,
pieceStore dtypes.ProviderPieceStore,
rpn retrievalmarket.RetrievalProviderNode,
) (dagstore.MinerAPI, error) {
mountApi := dagstore.NewMinerAPI(pieceStore, rpn)
const (
EnvDAGStoreCopyConcurrency = "LOTUS_DAGSTORE_COPY_CONCURRENCY"
DefaultDAGStoreDir = "dagStore"
)
// NewMinerAPI creates a new MinerAPI adaptor for the dagstore mounts.
func NewMinerAPI(lc fx.Lifecycle, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, rpn retrievalmarket.RetrievalProviderNode) (mdagstore.MinerAPI, error) {
cfg, err := extractDAGStoreConfig(r)
if err != nil {
return nil, err
}
// caps the amount of concurrent calls to the storage, so that we don't
// spam it during heavy processes like bulk migration.
if v, ok := os.LookupEnv("LOTUS_DAGSTORE_MOUNT_CONCURRENCY"); ok {
concurrency, err := strconv.Atoi(v)
if err == nil {
cfg.MaxConcurrencyStorageCalls = concurrency
}
}
mountApi := mdagstore.NewMinerAPI(pieceStore, rpn, cfg.MaxConcurrencyStorageCalls)
ready := make(chan error, 1)
pieceStore.OnReady(func(err error) {
ready <- err
@@ -44,72 +58,60 @@ func NewLotusAccessor(lc fx.Lifecycle,
return mountApi, nil
}
func DAGStoreWrapper(
lc fx.Lifecycle,
r repo.LockedRepo,
lotusAccessor dagstore.MinerAPI,
) (*dagstore.Wrapper, error) {
dir := filepath.Join(r.Path(), dagStore)
ds, err := newDAGStoreDatastore(dir)
// DAGStore constructs a DAG store using the supplied minerAPI, and the
// user configuration. It returns both the DAGStore and the Wrapper suitable for
// passing to markets.
func DAGStore(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI) (*dagstore.DAGStore, *mdagstore.Wrapper, error) {
cfg, err := extractDAGStoreConfig(r)
if err != nil {
return nil, err
return nil, nil, err
}
var maxCopies = 2
// TODO replace env with config.toml attribute.
v, ok := os.LookupEnv("LOTUS_DAGSTORE_COPY_CONCURRENCY")
// populate default directories if not explicitly set in the config.
defaultDir := filepath.Join(r.Path(), DefaultDAGStoreDir)
if cfg.TransientsDir == "" {
cfg.TransientsDir = filepath.Join(defaultDir, "transients")
}
if cfg.IndexDir == "" {
cfg.IndexDir = filepath.Join(defaultDir, "index")
}
if cfg.DatastoreDir == "" {
cfg.DatastoreDir = filepath.Join(defaultDir, "datastore")
}
v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency)
if ok {
concurrency, err := strconv.Atoi(v)
if err == nil {
maxCopies = concurrency
cfg.MaxConcurrentReadyFetches = concurrency
}
}
cfg := dagstore.MarketDAGStoreConfig{
TransientsDir: filepath.Join(dir, "transients"),
IndexDir: filepath.Join(dir, "index"),
Datastore: ds,
GCInterval: 1 * time.Minute,
MaxConcurrentIndex: 5,
MaxConcurrentReadyFetches: maxCopies,
}
dsw, err := dagstore.NewWrapper(cfg, lotusAccessor)
dagst, w, err := mdagstore.NewDAGStore(cfg, minerAPI)
if err != nil {
return nil, xerrors.Errorf("failed to create DAG store wrapper: %w", err)
return nil, nil, xerrors.Errorf("failed to create DAG store: %w", err)
}
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
return dsw.Start(ctx)
return w.Start(ctx)
},
OnStop: func(context.Context) error {
return dsw.Close()
return w.Close()
},
})
return dsw, nil
return dagst, w, nil
}
// newDAGStoreDatastore creates a datastore under the given base directory
// for dagstore metadata.
func newDAGStoreDatastore(baseDir string) (datastore.Batching, error) {
// Create a datastore directory under the base dir if it doesn't already exist
dsDir := filepath.Join(baseDir, "datastore")
if err := os.MkdirAll(dsDir, 0755); err != nil {
return nil, xerrors.Errorf("failed to create directory %s for DAG store datastore: %w", dsDir, err)
}
// Create a new LevelDB datastore
ds, err := levelds.NewDatastore(dsDir, &levelds.Options{
Compression: ldbopts.NoCompression,
NoSync: false,
Strict: ldbopts.StrictAll,
ReadOnly: false,
})
func extractDAGStoreConfig(r repo.LockedRepo) (config.DAGStoreConfig, error) {
cfg, err := r.Config()
if err != nil {
return nil, xerrors.Errorf("failed to open datastore for DAG store: %w", err)
return config.DAGStoreConfig{}, xerrors.Errorf("could not load config: %w", err)
}
// Keep statistics about the datastore
mds := measure.New("measure.", ds)
return mds, nil
mcfg, ok := cfg.(config.StorageMiner)
if !ok {
return config.DAGStoreConfig{}, xerrors.Errorf("config not expected type; expected config.StorageMiner, got: %T", cfg)
}
return mcfg.DAGStore, nil
}