diff --git a/markets/dagstore/wrapper.go b/markets/dagstore/wrapper.go index 5875aa275..620c97abe 100644 --- a/markets/dagstore/wrapper.go +++ b/markets/dagstore/wrapper.go @@ -90,7 +90,7 @@ func NewDAGStore(cfg config.DAGStoreConfig, mountApi MinerAPI) (*dagstore.DAGSto mountApi: mountApi, failureCh: failureCh, traceCh: traceCh, - gcInterval: cfg.GCInterval, + gcInterval: time.Duration(cfg.GCIntervalMillis) * time.Millisecond, } return dagst, w, nil diff --git a/markets/dagstore/wrapper_test.go b/markets/dagstore/wrapper_test.go index 1278253f0..c6e4e2f38 100644 --- a/markets/dagstore/wrapper_test.go +++ b/markets/dagstore/wrapper_test.go @@ -8,9 +8,10 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/node/config" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/mount" "github.com/filecoin-project/dagstore/shard" @@ -28,10 +29,10 @@ func TestWrapperAcquireRecovery(t *testing.T) { // Create a DAG store wrapper dagst, w, err := NewDAGStore(config.DAGStoreConfig{ - TransientsDir: t.TempDir(), - IndexDir: t.TempDir(), - DatastoreDir: t.TempDir(), - GCInterval: time.Millisecond, + TransientsDir: t.TempDir(), + IndexDir: t.TempDir(), + DatastoreDir: t.TempDir(), + GCIntervalMillis: 1, }, mockLotusMount{}) require.NoError(t, err) @@ -81,10 +82,10 @@ func TestWrapperBackground(t *testing.T) { // Create a DAG store wrapper dagst, w, err := NewDAGStore(config.DAGStoreConfig{ - TransientsDir: t.TempDir(), - IndexDir: t.TempDir(), - DatastoreDir: t.TempDir(), - GCInterval: time.Millisecond, + TransientsDir: t.TempDir(), + IndexDir: t.TempDir(), + DatastoreDir: t.TempDir(), + GCIntervalMillis: 1, }, mockLotusMount{}) require.NoError(t, err) diff --git a/node/config/def.go b/node/config/def.go index 06e48c731..9b76a2226 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -195,7 +195,7 @@ func DefaultStorageMiner() *StorageMiner { MaxConcurrentIndex: 5, MaxConcurrentReadyFetches: 2, MaxConcurrencyStorageCalls: 100, - GCInterval: time.Minute, + GCIntervalMillis: 60000, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/config/types.go b/node/config/types.go index ddcb27031..b3f038df4 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -1,11 +1,10 @@ package config import ( - "time" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/types" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/ipfs/go-cid" ) // // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE @@ -55,13 +54,41 @@ type StorageMiner struct { } type DAGStoreConfig struct { - TransientsDir string - IndexDir string - DatastoreDir string - MaxConcurrentIndex int - MaxConcurrentReadyFetches int + // Path to the transients directory. The transients directory caches + // unsealed deals that have been fetched from the storage subsystem for + // serving retrievals. When empty or omitted, the default value applies. + // Default value: $LOTUS_MARKETS_PATH/dagStore/transients (split deployment) + // or $LOTUS_MINER_PATH/dagStore/transients (monolith deployment) + TransientsDir string + + // Path to indices directory. When empty or omitted, the default value applies. + // Default value: $LOTUS_MARKETS_PATH/dagStore/index (split deployment) + // or $LOTUS_MINER_PATH/dagStore/index (monolith deployment) + IndexDir string + + // Path to datastore directory. The datastore is a KV store tracking the + // state of shards known to the DAG store. + // Default value: $LOTUS_MARKETS_PATH/dagStore/datastore (split deployment) + // or $LOTUS_MINER_PATH/dagStore/datastore (monolith deployment) + DatastoreDir string + + // The maximum amount of indexing jobs that can run simultaneously. + // Default value: 5. + MaxConcurrentIndex int + + // The maximum amount of unsealed deals that can be fetched simultaneously + // from the storage subsystem. + // Default value: 2. + MaxConcurrentReadyFetches int + + // The maximum number of simultaneous inflight API calls to the storage + // subsystem. + // Default value: 100. MaxConcurrencyStorageCalls int - GCInterval time.Duration + + // The number of milliseconds between calls to periodic dagstore GC. + // Default value: 60000 (60 seconds = 1 minute). + GCIntervalMillis int } type MinerSubsystemConfig struct { diff --git a/node/modules/storageminer_dagstore.go b/node/modules/storageminer_dagstore.go index 88b9db0e8..6f032ed34 100644 --- a/node/modules/storageminer_dagstore.go +++ b/node/modules/storageminer_dagstore.go @@ -9,12 +9,13 @@ import ( "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "go.uber.org/fx" + "golang.org/x/xerrors" + mdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" - "go.uber.org/fx" - "golang.org/x/xerrors" ) const ( @@ -109,7 +110,7 @@ func extractDAGStoreConfig(r repo.LockedRepo) (config.DAGStoreConfig, error) { if err != nil { return config.DAGStoreConfig{}, xerrors.Errorf("could not load config: %w", err) } - mcfg, ok := cfg.(config.StorageMiner) + mcfg, ok := cfg.(*config.StorageMiner) if !ok { return config.DAGStoreConfig{}, xerrors.Errorf("config not expected type; expected config.StorageMiner, got: %T", cfg) }