Merge remote-tracking branch 'origin/master' into feat/post-worker

.circleci/config.yml

@@ -628,6 +628,11 @@ jobs:
         default: "latest"
         description: A comma-separated string containing docker image tags to build and push (default = latest)
 
+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
     steps:
       - run:
           name: Confirm that environment variables are set
@@ -667,6 +672,7 @@ jobs:
 
             docker build \
               <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
               -f <<parameters.path>>/<<parameters.dockerfile>> \
               $docker_tag_args \
               <<parameters.path>>
@@ -1042,6 +1048,13 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
       - publish-packer-mainnet:
           requires:
             - build-all

.circleci/template.yml

@@ -628,6 +628,11 @@ jobs:
         default: "latest"
         description: A comma-separated string containing docker image tags to build and push (default = latest)
 
+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
     steps:
       - run:
           name: Confirm that environment variables are set
@@ -667,6 +672,7 @@ jobs:
 
             docker build \
               <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
               -f <<parameters.path>>/<<parameters.dockerfile>> \
               $docker_tag_args \
               <<parameters.path>>
@@ -862,6 +868,13 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
       - publish-packer-mainnet:
           requires:
             - build-all

Dockerfile.lotus

@@ -28,6 +28,14 @@ WORKDIR /opt/filecoin
 RUN make clean deps
 
 
+FROM builder-local AS builder-test
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+RUN make debug
+
+
 FROM builder-local AS builder
 MAINTAINER Lotus Development Team
 
@@ -198,6 +206,43 @@ RUN chown fc: /var/lib/lotus-worker
 RUN chown fc: /var/lib/lotus-wallet
 
 
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777
+
+###
+from base as lotus-test
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=builder-test /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
 VOLUME /var/tmp/filecoin-proof-parameters
 VOLUME /var/lib/lotus
 VOLUME /var/lib/lotus-miner

cli/auth.go

@@ -9,7 +9,6 @@ import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/api"
-	cliutil "github.com/filecoin-project/lotus/cli/util"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -128,7 +127,7 @@ var AuthApiInfoToken = &cli.Command{
 
 		// TODO: Log in audit log when it is implemented
 
-		currentEnv, _, _ := cliutil.EnvsForAPIInfos(t)
+		currentEnv, _, _ := t.APIInfoEnvVars()
 		fmt.Printf("%s=%s:%s\n", currentEnv, string(token), ainfo.Addr)
 		return nil
 	},

cli/util/api.go

@@ -28,63 +28,6 @@ const (
 	metadataTraceContext = "traceContext"
 )
 
-// flagsForAPI returns flags passed on the command line with the listen address
-// of the API server (only used by the tests), in the order of precedence they
-// should be applied for the requested kind of node.
-func flagsForAPI(t repo.RepoType) []string {
-	switch t {
-	case repo.FullNode:
-		return []string{"api-url"}
-	case repo.StorageMiner:
-		return []string{"miner-api-url"}
-	case repo.Worker:
-		return []string{"worker-api-url"}
-	case repo.Markets:
-		// support split markets-miner and monolith deployments.
-		return []string{"markets-api-url", "miner-api-url"}
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func flagsForRepo(t repo.RepoType) []string {
-	switch t {
-	case repo.FullNode:
-		return []string{"repo"}
-	case repo.StorageMiner:
-		return []string{"miner-repo"}
-	case repo.Worker:
-		return []string{"worker-repo"}
-	case repo.Markets:
-		// support split markets-miner and monolith deployments.
-		return []string{"markets-repo", "miner-repo"}
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-// EnvsForAPIInfos returns the environment variables to use in order of precedence
-// to determine the API endpoint of the specified node type.
-//
-// It returns the current variables and deprecated ones separately, so that
-// the user can log a warning when deprecated ones are found to be in use.
-func EnvsForAPIInfos(t repo.RepoType) (primary string, fallbacks []string, deprecated []string) {
-	switch t {
-	case repo.FullNode:
-		return "FULLNODE_API_INFO", nil, nil
-	case repo.StorageMiner:
-		// TODO remove deprecated deprecation period
-		return "MINER_API_INFO", nil, []string{"STORAGE_API_INFO"}
-	case repo.Worker:
-		return "WORKER_API_INFO", nil, nil
-	case repo.Markets:
-		// support split markets-miner and monolith deployments.
-		return "MARKETS_API_INFO", []string{"MINER_API_INFO"}, nil
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
 // GetAPIInfo returns the API endpoint to use for the specified kind of repo.
 //
 // The order of precedence is as follows:
@@ -96,8 +39,7 @@ func EnvsForAPIInfos(t repo.RepoType) (primary string, fallbacks []string, depre
 func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
 	// Check if there was a flag passed with the listen address of the API
 	// server (only used by the tests)
-	apiFlags := flagsForAPI(t)
+	for _, f := range t.APIFlags() {
-	for _, f := range apiFlags {
 		if !ctx.IsSet(f) {
 			continue
 		}
@@ -111,7 +53,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
 	// Note: it is not correct/intuitive to prefer environment variables over
 	// CLI flags (repo flags below).
 	//
-	primaryEnv, fallbacksEnvs, deprecatedEnvs := EnvsForAPIInfos(t)
+	primaryEnv, fallbacksEnvs, deprecatedEnvs := t.APIInfoEnvVars()
 	env, ok := os.LookupEnv(primaryEnv)
 	if ok {
 		return ParseApiInfo(env), nil
@@ -125,8 +67,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
 		}
 	}
 
-	repoFlags := flagsForRepo(t)
+	for _, f := range t.RepoFlags() {
-	for _, f := range repoFlags {
 		// cannot use ctx.IsSet because it ignores default values
 		path := ctx.String(f)
 		if path == "" {
@@ -175,13 +116,13 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
 		}
 	}
 
-	return APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t)
+	return APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t.Type())
 }
 
 func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) {
 	ainfo, err := GetAPIInfo(ctx, t)
 	if err != nil {
-		return "", nil, xerrors.Errorf("could not get API info for %s: %w", t, err)
+		return "", nil, xerrors.Errorf("could not get API info for %s: %w", t.Type(), err)
 	}
 
 	addr, err := ainfo.DialArgs(version)

cmd/lotus-shed/datastore.go

@@ -40,10 +40,10 @@ var datastoreListCmd = &cli.Command{
 	Name:        "list",
 	Description: "list datastore keys",
 	Flags: []cli.Flag{
-		&cli.IntFlag{
+		&cli.StringFlag{
 			Name:  "repo-type",
-			Usage: "node type (1 - full, 2 - storage, 3 - worker)",
+			Usage: "node type (FullNode, StorageMiner, Worker, Wallet)",
-			Value: 1,
+			Value: "FullNode",
 		},
 		&cli.BoolFlag{
 			Name:  "top-level",
@@ -71,7 +71,7 @@ var datastoreListCmd = &cli.Command{
 			return xerrors.Errorf("lotus repo doesn't exist")
 		}
 
-		lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type")))
+		lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type")))
 		if err != nil {
 			return err
 		}
@@ -109,10 +109,10 @@ var datastoreGetCmd = &cli.Command{
 	Name:        "get",
 	Description: "list datastore keys",
 	Flags: []cli.Flag{
-		&cli.IntFlag{
+		&cli.StringFlag{
 			Name:  "repo-type",
-			Usage: "node type (1 - full, 2 - storage, 3 - worker)",
+			Usage: "node type (FullNode, StorageMiner, Worker, Wallet)",
-			Value: 1,
+			Value: "FullNode",
 		},
 		&cli.StringFlag{
 			Name:  "enc",
@@ -137,7 +137,7 @@ var datastoreGetCmd = &cli.Command{
 			return xerrors.Errorf("lotus repo doesn't exist")
 		}
 
-		lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type")))
+		lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type")))
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/rpc.go

@@ -34,9 +34,11 @@ var rpcCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		rt := repo.FullNode
+		var rt repo.RepoType
 		if cctx.Bool("miner") {
 			rt = repo.StorageMiner
+		} else {
+			rt = repo.FullNode
 		}
 
 		addr, headers, err := lcli.GetRawAPI(cctx, rt, cctx.String("version"))

go.mod

@@ -112,7 +112,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-eventbus v0.2.1
-	github.com/libp2p/go-libp2p v0.18.0-rc6
+	github.com/libp2p/go-libp2p v0.18.0
 	github.com/libp2p/go-libp2p-connmgr v0.3.1
 	github.com/libp2p/go-libp2p-core v0.14.0
 	github.com/libp2p/go-libp2p-discovery v0.6.0

go.sum

@@ -1065,8 +1065,8 @@ github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76f
 github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw=
 github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8=
 github.com/libp2p/go-libp2p v0.18.0-rc3/go.mod h1:WYL+Xw1iuwi6rdfzw5VIEpD+HqzYucHZ6fcUuumbI3M=
-github.com/libp2p/go-libp2p v0.18.0-rc6 h1:IR6TVPYGo1wDY0tY61gyPQVxK1koOkXh49ejVfAnH7A=
+github.com/libp2p/go-libp2p v0.18.0 h1:moKKKG875KNGsCjZxTIFB75ihHiVjFeWg5I4aR1pDLk=
-github.com/libp2p/go-libp2p v0.18.0-rc6/go.mod h1:oOUOAlBrm1L0+jxT10h2TMUMTDz6pV3EvmkJ3beDYGQ=
+github.com/libp2p/go-libp2p v0.18.0/go.mod h1:+veaZ9z1SZQhmc5PW78jvnnxZ89Mgvmh4cggO11ETmw=
 github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo=
 github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E=
 github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I=

itests/batch_deal_test.go

@@ -51,7 +51,8 @@ func TestBatchDealInput(t *testing.T) {
 					})),
 				node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) {
 					return func() (sealiface.Config, error) {
-						sc := modules.ToSealingConfig(config.DefaultStorageMiner())
+						cfg := config.DefaultStorageMiner()
+						sc := modules.ToSealingConfig(cfg.Dealmaking, cfg.Sealing)
 						sc.MaxWaitDealsSectors = 2
 						sc.MaxSealingSectors = 1
 						sc.MaxSealingSectorsForDeals = 3

itests/kit/blockminer.go

@@ -192,7 +192,11 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 					InjectNulls: abi.ChainEpoch(nulls + i),
 					Done:        reportSuccessFn,
 				})
-				success = <-wait
+				select {
+				case success = <-wait:
+				case <-ctx.Done():
+					return
+				}
 				if !success {
 					// if we are mining a new null block and it brings us past deadline boundary we need to wait for miner to post
 					if ts.Height()+1+abi.ChainEpoch(nulls+i) >= dlinfo.Last() {

itests/sector_finalize_early_test.go

@@ -39,7 +39,7 @@ func TestDealsWithFinalizeEarly(t *testing.T) {
 			return func() (sealiface.Config, error) {
 				cf := config.DefaultStorageMiner()
 				cf.Sealing.FinalizeEarly = true
-				return modules.ToSealingConfig(cf), nil
+				return modules.ToSealingConfig(cf.Dealmaking, cf.Sealing), nil
 			}, nil
 		})))) // no mock proofs.
 	ens.InterconnectAll().BeginMining(blockTime)

itests/sector_miner_collateral_test.go

@@ -42,7 +42,8 @@ func TestMinerBalanceCollateral(t *testing.T) {
 		opts := kit.ConstructorOpts(
 			node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) {
 				return func() (sealiface.Config, error) {
-					sc := modules.ToSealingConfig(config.DefaultStorageMiner())
+					cfg := config.DefaultStorageMiner()
+					sc := modules.ToSealingConfig(cfg.Dealmaking, cfg.Sealing)
 
 					sc.MaxWaitDealsSectors = 4
 					sc.MaxSealingSectors = 4

lib/unixfs/filestore.go

@@ -1,4 +1,4 @@
-package client
+package unixfs
 
 import (
 	"context"
@@ -19,12 +19,15 @@ import (
 	"github.com/ipfs/go-merkledag"
 	"github.com/ipfs/go-unixfs/importer/balanced"
 	ihelper "github.com/ipfs/go-unixfs/importer/helpers"
+	mh "github.com/multiformats/go-multihash"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/build"
 )
 
-func unixFSCidBuilder() (cid.Builder, error) {
+var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
+
+func CidBuilder() (cid.Builder, error) {
 	prefix, err := merkledag.PrefixForCidVersion(1)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize UnixFS CID Builder: %w", err)
@@ -37,9 +40,9 @@ func unixFSCidBuilder() (cid.Builder, error) {
 	return b, nil
 }
 
-// createUnixFSFilestore takes a standard file whose path is src, forms a UnixFS DAG, and
+// CreateFilestore takes a standard file whose path is src, forms a UnixFS DAG, and
 // writes a CARv2 file with positional mapping (backed by the go-filestore library).
-func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) {
+func CreateFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) {
 	// This method uses a two-phase approach with a staging CAR blockstore and
 	// a final CAR blockstore.
 	//
@@ -80,7 +83,7 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath
 		return cid.Undef, xerrors.Errorf("failed to create temporary filestore: %w", err)
 	}
 
-	finalRoot1, err := buildUnixFS(ctx, file, fstore, true)
+	finalRoot1, err := Build(ctx, file, fstore, true)
 	if err != nil {
 		_ = fstore.Close()
 		return cid.Undef, xerrors.Errorf("failed to import file to store to compute root: %w", err)
@@ -102,7 +105,7 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath
 		return cid.Undef, xerrors.Errorf("failed to rewind file: %w", err)
 	}
 
-	finalRoot2, err := buildUnixFS(ctx, file, bs, true)
+	finalRoot2, err := Build(ctx, file, bs, true)
 	if err != nil {
 		_ = bs.Close()
 		return cid.Undef, xerrors.Errorf("failed to create UnixFS DAG with carv2 blockstore: %w", err)
@@ -119,10 +122,10 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath
 	return finalRoot1, nil
 }
 
-// buildUnixFS builds a UnixFS DAG out of the supplied reader,
+// Build builds a UnixFS DAG out of the supplied reader,
 // and imports the DAG into the supplied service.
-func buildUnixFS(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) {
+func Build(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) {
-	b, err := unixFSCidBuilder()
+	b, err := CidBuilder()
 	if err != nil {
 		return cid.Undef, err
 	}

lib/unixfs/filestore_test.go

@@ -1,5 +1,5 @@
 //stm: #unit
-package client
+package unixfs
 
 import (
 	"bytes"
@@ -21,8 +21,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-fil-markets/stores"
-
-	"github.com/filecoin-project/lotus/node/repo/imports"
 )
 
 // This test uses a full "dense" CARv2, and not a filestore (positional mapping).
@@ -42,7 +40,7 @@ func TestRoundtripUnixFS_Dense(t *testing.T) {
 		blockstore.UseWholeCIDs(true))
 	require.NoError(t, err)
 
-	root, err := buildUnixFS(ctx, bytes.NewBuffer(inputContents), bs, false)
+	root, err := Build(ctx, bytes.NewBuffer(inputContents), bs, false)
 	require.NoError(t, err)
 	require.NotEqual(t, cid.Undef, root)
 	require.NoError(t, bs.Finalize())
@@ -78,9 +76,6 @@ func TestRoundtripUnixFS_Dense(t *testing.T) {
 func TestRoundtripUnixFS_Filestore(t *testing.T) {
 	//stm: @CLIENT_DATA_IMPORT_001
 	ctx := context.Background()
-	a := &API{
-		Imports: &imports.Manager{},
-	}
 
 	inputPath, inputContents := genInputFile(t)
 	defer os.Remove(inputPath) //nolint:errcheck
@@ -88,7 +83,7 @@ func TestRoundtripUnixFS_Filestore(t *testing.T) {
 	dst := newTmpFile(t)
 	defer os.Remove(dst) //nolint:errcheck
 
-	root, err := a.createUnixFSFilestore(ctx, inputPath, dst)
+	root, err := CreateFilestore(ctx, inputPath, dst)
 	require.NoError(t, err)
 	require.NotEqual(t, cid.Undef, root)
 

node/builder.go

@@ -231,8 +231,12 @@ func IsType(t repo.RepoType) func(s *Settings) bool {
 }
 
 func isFullOrLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode }
-func isFullNode(s *Settings) bool       { return s.nodeType == repo.FullNode && !s.Lite }
+func isFullNode(s *Settings) bool {
-func isLiteNode(s *Settings) bool       { return s.nodeType == repo.FullNode && s.Lite }
+	return s.nodeType == repo.FullNode && !s.Lite
+}
+func isLiteNode(s *Settings) bool {
+	return s.nodeType == repo.FullNode && s.Lite
+}
 
 func Base() Option {
 	return Options(

node/builder_miner.go

@@ -154,8 +154,8 @@ func ConfigStorageMiner(c interface{}) Option {
 			Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)),
 
 			// DAG Store
-			Override(new(dagstore.MinerAPI), modules.NewMinerAPI),
+			Override(new(dagstore.MinerAPI), modules.NewMinerAPI(cfg.DAGStore)),
-			Override(DAGStoreKey, modules.DAGStore),
+			Override(DAGStoreKey, modules.DAGStore(cfg.DAGStore)),
 
 			// Markets (retrieval)
 			Override(new(dagstore.SectorAccessor), sectoraccessor.NewSectorAccessor),

node/config/dynamic_config.go

@@ -0,0 +1,27 @@
+package config
+
+type DealmakingConfiger interface {
+	GetDealmakingConfig() DealmakingConfig
+	SetDealmakingConfig(DealmakingConfig)
+}
+
+func (c *StorageMiner) GetDealmakingConfig() DealmakingConfig {
+	return c.Dealmaking
+}
+
+func (c *StorageMiner) SetDealmakingConfig(other DealmakingConfig) {
+	c.Dealmaking = other
+}
+
+type SealingConfiger interface {
+	GetSealingConfig() SealingConfig
+	SetSealingConfig(SealingConfig)
+}
+
+func (c *StorageMiner) GetSealingConfig() SealingConfig {
+	return c.Sealing
+}
+
+func (c *StorageMiner) SetSealingConfig(other SealingConfig) {
+	c.Sealing = other
+}

node/impl/client/client.go

@@ -42,7 +42,6 @@ import (
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/multiformats/go-multibase"
-	mh "github.com/multiformats/go-multihash"
 	"go.uber.org/fx"
 
 	"github.com/filecoin-project/go-address"
@@ -56,6 +55,7 @@ import (
 	"github.com/filecoin-project/go-fil-markets/storagemarket/network"
 	"github.com/filecoin-project/go-fil-markets/stores"
 
+	"github.com/filecoin-project/lotus/lib/unixfs"
 	"github.com/filecoin-project/lotus/markets/retrievaladapter"
 	"github.com/filecoin-project/lotus/markets/storageadapter"
 
@@ -79,7 +79,7 @@ import (
 
 var log = logging.Logger("client")
 
-var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
+var DefaultHashFunction = unixfs.DefaultHashFunction
 
 // 8 days ~=  SealDuration + PreCommit + MaxProveCommitDuration + 8 hour buffer
 const dealStartBufferHours uint64 = 8 * 24
@@ -548,7 +548,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
 		}()
 
 		// perform the unixfs chunking.
-		root, err = a.createUnixFSFilestore(ctx, ref.Path, carPath)
+		root, err = unixfs.CreateFilestore(ctx, ref.Path, carPath)
 		if err != nil {
 			return nil, xerrors.Errorf("failed to import file using unixfs: %w", err)
 		}
@@ -618,7 +618,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro
 	// once the DAG is formed and the root is calculated, we overwrite the
 	// inner carv1 header with the final root.
 
-	b, err := unixFSCidBuilder()
+	b, err := unixfs.CidBuilder()
 	if err != nil {
 		return cid.Undef, err
 	}
@@ -635,7 +635,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro
 		return cid.Undef, xerrors.Errorf("failed to create carv2 read/write blockstore: %w", err)
 	}
 
-	root, err := buildUnixFS(ctx, file, bs, false)
+	root, err := unixfs.Build(ctx, file, bs, false)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("failed to build unixfs dag: %w", err)
 	}
@@ -1364,7 +1364,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
 	defer os.Remove(tmp) //nolint:errcheck
 
 	// generate and import the UnixFS DAG into a filestore (positional reference) CAR.
-	root, err := a.createUnixFSFilestore(ctx, ref.Path, tmp)
+	root, err := unixfs.CreateFilestore(ctx, ref.Path, tmp)
 	if err != nil {
 		return xerrors.Errorf("failed to import file using unixfs: %w", err)
 	}

node/modules/storageminer.go

@@ -764,8 +764,9 @@ func StorageAuthWithURL(apiInfo string) func(ctx helpers.MetricsCtx, ca v0api.Co
 
 func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) {
 	return func() (out bool, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.ConsiderOnlineStorageDeals
+			cfg := c.GetDealmakingConfig()
+			out = cfg.ConsiderOnlineStorageDeals
 		})
 		return
 	}, nil
@@ -773,8 +774,10 @@ func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Consider
 
 func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineStorageDealsConfigFunc, error) {
 	return func(b bool) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ConsiderOnlineStorageDeals = b
+			cfg := c.GetDealmakingConfig()
+			cfg.ConsiderOnlineStorageDeals = b
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -782,8 +785,9 @@ func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsi
 
 func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineRetrievalDealsConfigFunc, error) {
 	return func() (out bool, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.ConsiderOnlineRetrievalDeals
+			cfg := c.GetDealmakingConfig()
+			out = cfg.ConsiderOnlineRetrievalDeals
 		})
 		return
 	}, nil
@@ -791,8 +795,10 @@ func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Consid
 
 func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineRetrievalDealsConfigFunc, error) {
 	return func(b bool) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ConsiderOnlineRetrievalDeals = b
+			cfg := c.GetDealmakingConfig()
+			cfg.ConsiderOnlineRetrievalDeals = b
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -800,8 +806,9 @@ func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Set
 
 func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlocklistConfigFunc, error) {
 	return func() (out []cid.Cid, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.PieceCidBlocklist
+			cfg := c.GetDealmakingConfig()
+			out = cfg.PieceCidBlocklist
 		})
 		return
 	}, nil
@@ -809,8 +816,10 @@ func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.Storag
 
 func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealPieceCidBlocklistConfigFunc, error) {
 	return func(blocklist []cid.Cid) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.PieceCidBlocklist = blocklist
+			cfg := c.GetDealmakingConfig()
+			cfg.PieceCidBlocklist = blocklist
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -818,8 +827,9 @@ func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.Set
 
 func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineStorageDealsConfigFunc, error) {
 	return func() (out bool, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.ConsiderOfflineStorageDeals
+			cfg := c.GetDealmakingConfig()
+			out = cfg.ConsiderOfflineStorageDeals
 		})
 		return
 	}, nil
@@ -827,8 +837,10 @@ func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Conside
 
 func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineStorageDealsConfigFunc, error) {
 	return func(b bool) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ConsiderOfflineStorageDeals = b
+			cfg := c.GetDealmakingConfig()
+			cfg.ConsiderOfflineStorageDeals = b
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -836,8 +848,9 @@ func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetCons
 
 func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineRetrievalDealsConfigFunc, error) {
 	return func() (out bool, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.ConsiderOfflineRetrievalDeals
+			cfg := c.GetDealmakingConfig()
+			out = cfg.ConsiderOfflineRetrievalDeals
 		})
 		return
 	}, nil
@@ -845,8 +858,10 @@ func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Consi
 
 func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineRetrievalDealsConfigFunc, error) {
 	return func(b bool) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ConsiderOfflineRetrievalDeals = b
+			cfg := c.GetDealmakingConfig()
+			cfg.ConsiderOfflineRetrievalDeals = b
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -854,8 +869,9 @@ func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Se
 
 func NewConsiderVerifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderVerifiedStorageDealsConfigFunc, error) {
 	return func() (out bool, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.ConsiderVerifiedStorageDeals
+			cfg := c.GetDealmakingConfig()
+			out = cfg.ConsiderVerifiedStorageDeals
 		})
 		return
 	}, nil
@@ -863,8 +879,10 @@ func NewConsiderVerifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Consid
 
 func NewSetConsideringVerifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderVerifiedStorageDealsConfigFunc, error) {
 	return func(b bool) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ConsiderVerifiedStorageDeals = b
+			cfg := c.GetDealmakingConfig()
+			cfg.ConsiderVerifiedStorageDeals = b
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -872,8 +890,9 @@ func NewSetConsideringVerifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetCon
 
 func NewConsiderUnverifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderUnverifiedStorageDealsConfigFunc, error) {
 	return func() (out bool, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = cfg.Dealmaking.ConsiderUnverifiedStorageDeals
+			cfg := c.GetDealmakingConfig()
+			out = cfg.ConsiderUnverifiedStorageDeals
 		})
 		return
 	}, nil
@@ -881,8 +900,10 @@ func NewConsiderUnverifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Cons
 
 func NewSetConsideringUnverifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderUnverifiedStorageDealsConfigFunc, error) {
 	return func(b bool) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ConsiderUnverifiedStorageDeals = b
+			cfg := c.GetDealmakingConfig()
+			cfg.ConsiderUnverifiedStorageDeals = b
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -890,8 +911,8 @@ func NewSetConsideringUnverifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetC
 
 func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error) {
 	return func(cfg sealiface.Config) (err error) {
-		err = mutateCfg(r, func(c *config.StorageMiner) {
+		err = mutateSealingCfg(r, func(c config.SealingConfiger) {
-			c.Sealing = config.SealingConfig{
+			newCfg := config.SealingConfig{
 				MaxWaitDealsSectors:             cfg.MaxWaitDealsSectors,
 				MaxSealingSectors:               cfg.MaxSealingSectors,
 				MaxSealingSectorsForDeals:       cfg.MaxSealingSectorsForDeals,
@@ -922,51 +943,54 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 				TerminateBatchMin:  cfg.TerminateBatchMin,
 				TerminateBatchWait: config.Duration(cfg.TerminateBatchWait),
 			}
+			c.SetSealingConfig(newCfg)
 		})
 		return
 	}, nil
 }
 
-func ToSealingConfig(cfg *config.StorageMiner) sealiface.Config {
+func ToSealingConfig(dealmakingCfg config.DealmakingConfig, sealingCfg config.SealingConfig) sealiface.Config {
 	return sealiface.Config{
-		MaxWaitDealsSectors:             cfg.Sealing.MaxWaitDealsSectors,
+		MaxWaitDealsSectors:             sealingCfg.MaxWaitDealsSectors,
-		MaxSealingSectors:               cfg.Sealing.MaxSealingSectors,
+		MaxSealingSectors:               sealingCfg.MaxSealingSectors,
-		MaxSealingSectorsForDeals:       cfg.Sealing.MaxSealingSectorsForDeals,
+		MaxSealingSectorsForDeals:       sealingCfg.MaxSealingSectorsForDeals,
-		StartEpochSealingBuffer:         abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer),
+		StartEpochSealingBuffer:         abi.ChainEpoch(dealmakingCfg.StartEpochSealingBuffer),
-		MakeNewSectorForDeals:           cfg.Dealmaking.MakeNewSectorForDeals,
+		MakeNewSectorForDeals:           dealmakingCfg.MakeNewSectorForDeals,
-		CommittedCapacitySectorLifetime: time.Duration(cfg.Sealing.CommittedCapacitySectorLifetime),
+		CommittedCapacitySectorLifetime: time.Duration(sealingCfg.CommittedCapacitySectorLifetime),
-		WaitDealsDelay:                  time.Duration(cfg.Sealing.WaitDealsDelay),
+		WaitDealsDelay:                  time.Duration(sealingCfg.WaitDealsDelay),
-		MakeCCSectorsAvailable:          cfg.Sealing.MakeCCSectorsAvailable,
+		MakeCCSectorsAvailable:          sealingCfg.MakeCCSectorsAvailable,
-		AlwaysKeepUnsealedCopy:          cfg.Sealing.AlwaysKeepUnsealedCopy,
+		AlwaysKeepUnsealedCopy:          sealingCfg.AlwaysKeepUnsealedCopy,
-		FinalizeEarly:                   cfg.Sealing.FinalizeEarly,
+		FinalizeEarly:                   sealingCfg.FinalizeEarly,
 
-		CollateralFromMinerBalance: cfg.Sealing.CollateralFromMinerBalance,
+		CollateralFromMinerBalance: sealingCfg.CollateralFromMinerBalance,
-		AvailableBalanceBuffer:     types.BigInt(cfg.Sealing.AvailableBalanceBuffer),
+		AvailableBalanceBuffer:     types.BigInt(sealingCfg.AvailableBalanceBuffer),
-		DisableCollateralFallback:  cfg.Sealing.DisableCollateralFallback,
+		DisableCollateralFallback:  sealingCfg.DisableCollateralFallback,
 
-		BatchPreCommits:     cfg.Sealing.BatchPreCommits,
+		BatchPreCommits:     sealingCfg.BatchPreCommits,
-		MaxPreCommitBatch:   cfg.Sealing.MaxPreCommitBatch,
+		MaxPreCommitBatch:   sealingCfg.MaxPreCommitBatch,
-		PreCommitBatchWait:  time.Duration(cfg.Sealing.PreCommitBatchWait),
+		PreCommitBatchWait:  time.Duration(sealingCfg.PreCommitBatchWait),
-		PreCommitBatchSlack: time.Duration(cfg.Sealing.PreCommitBatchSlack),
+		PreCommitBatchSlack: time.Duration(sealingCfg.PreCommitBatchSlack),
 
-		AggregateCommits:           cfg.Sealing.AggregateCommits,
+		AggregateCommits:           sealingCfg.AggregateCommits,
-		MinCommitBatch:             cfg.Sealing.MinCommitBatch,
+		MinCommitBatch:             sealingCfg.MinCommitBatch,
-		MaxCommitBatch:             cfg.Sealing.MaxCommitBatch,
+		MaxCommitBatch:             sealingCfg.MaxCommitBatch,
-		CommitBatchWait:            time.Duration(cfg.Sealing.CommitBatchWait),
+		CommitBatchWait:            time.Duration(sealingCfg.CommitBatchWait),
-		CommitBatchSlack:           time.Duration(cfg.Sealing.CommitBatchSlack),
+		CommitBatchSlack:           time.Duration(sealingCfg.CommitBatchSlack),
-		AggregateAboveBaseFee:      types.BigInt(cfg.Sealing.AggregateAboveBaseFee),
+		AggregateAboveBaseFee:      types.BigInt(sealingCfg.AggregateAboveBaseFee),
-		BatchPreCommitAboveBaseFee: types.BigInt(cfg.Sealing.BatchPreCommitAboveBaseFee),
+		BatchPreCommitAboveBaseFee: types.BigInt(sealingCfg.BatchPreCommitAboveBaseFee),
 
-		TerminateBatchMax:  cfg.Sealing.TerminateBatchMax,
+		TerminateBatchMax:  sealingCfg.TerminateBatchMax,
-		TerminateBatchMin:  cfg.Sealing.TerminateBatchMin,
+		TerminateBatchMin:  sealingCfg.TerminateBatchMin,
-		TerminateBatchWait: time.Duration(cfg.Sealing.TerminateBatchWait),
+		TerminateBatchWait: time.Duration(sealingCfg.TerminateBatchWait),
 	}
 }
 
 func NewGetSealConfigFunc(r repo.LockedRepo) (dtypes.GetSealingConfigFunc, error) {
 	return func() (out sealiface.Config, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readSealingCfg(r, func(dc config.DealmakingConfiger, sc config.SealingConfiger) {
-			out = ToSealingConfig(cfg)
+			scfg := sc.GetSealingConfig()
+			dcfg := dc.GetDealmakingConfig()
+			out = ToSealingConfig(dcfg, scfg)
 		})
 		return
 	}, nil
@@ -974,8 +998,10 @@ func NewGetSealConfigFunc(r repo.LockedRepo) (dtypes.GetSealingConfigFunc, error
 
 func NewSetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.SetExpectedSealDurationFunc, error) {
 	return func(delay time.Duration) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.ExpectedSealDuration = config.Duration(delay)
+			cfg := c.GetDealmakingConfig()
+			cfg.ExpectedSealDuration = config.Duration(delay)
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -983,8 +1009,9 @@ func NewSetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.SetExpectedSealDu
 
 func NewGetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.GetExpectedSealDurationFunc, error) {
 	return func() (out time.Duration, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = time.Duration(cfg.Dealmaking.ExpectedSealDuration)
+			cfg := c.GetDealmakingConfig()
+			out = time.Duration(cfg.ExpectedSealDuration)
 		})
 		return
 	}, nil
@@ -992,8 +1019,10 @@ func NewGetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.GetExpectedSealDu
 
 func NewSetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.SetMaxDealStartDelayFunc, error) {
 	return func(delay time.Duration) (err error) {
-		err = mutateCfg(r, func(cfg *config.StorageMiner) {
+		err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			cfg.Dealmaking.MaxDealStartDelay = config.Duration(delay)
+			cfg := c.GetDealmakingConfig()
+			cfg.MaxDealStartDelay = config.Duration(delay)
+			c.SetDealmakingConfig(cfg)
 		})
 		return
 	}, nil
@@ -1001,22 +1030,60 @@ func NewSetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.SetMaxDealStartDelay
 
 func NewGetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.GetMaxDealStartDelayFunc, error) {
 	return func() (out time.Duration, err error) {
-		err = readCfg(r, func(cfg *config.StorageMiner) {
+		err = readDealmakingCfg(r, func(c config.DealmakingConfiger) {
-			out = time.Duration(cfg.Dealmaking.MaxDealStartDelay)
+			cfg := c.GetDealmakingConfig()
+			out = time.Duration(cfg.MaxDealStartDelay)
 		})
 		return
 	}, nil
 }
 
-func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error {
+func readSealingCfg(r repo.LockedRepo, accessor func(config.DealmakingConfiger, config.SealingConfiger)) error {
 	raw, err := r.Config()
 	if err != nil {
 		return err
 	}
 
-	cfg, ok := raw.(*config.StorageMiner)
+	scfg, ok := raw.(config.SealingConfiger)
 	if !ok {
-		return xerrors.New("expected address of config.StorageMiner")
+		return xerrors.New("expected config with sealing config trait")
+	}
+
+	dcfg, ok := raw.(config.DealmakingConfiger)
+	if !ok {
+		return xerrors.New("expected config with dealmaking config trait")
+	}
+
+	accessor(dcfg, scfg)
+
+	return nil
+}
+
+func mutateSealingCfg(r repo.LockedRepo, mutator func(config.SealingConfiger)) error {
+	var typeErr error
+
+	setConfigErr := r.SetConfig(func(raw interface{}) {
+		cfg, ok := raw.(config.SealingConfiger)
+		if !ok {
+			typeErr = errors.New("expected config with sealing config trait")
+			return
+		}
+
+		mutator(cfg)
+	})
+
+	return multierr.Combine(typeErr, setConfigErr)
+}
+
+func readDealmakingCfg(r repo.LockedRepo, accessor func(config.DealmakingConfiger)) error {
+	raw, err := r.Config()
+	if err != nil {
+		return err
+	}
+
+	cfg, ok := raw.(config.DealmakingConfiger)
+	if !ok {
+		return xerrors.New("expected config with dealmaking config trait")
 	}
 
 	accessor(cfg)
@@ -1024,13 +1091,13 @@ func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error {
 	return nil
 }
 
-func mutateCfg(r repo.LockedRepo, mutator func(*config.StorageMiner)) error {
+func mutateDealmakingCfg(r repo.LockedRepo, mutator func(config.DealmakingConfiger)) error {
 	var typeErr error
 
 	setConfigErr := r.SetConfig(func(raw interface{}) {
-		cfg, ok := raw.(*config.StorageMiner)
+		cfg, ok := raw.(config.DealmakingConfiger)
 		if !ok {
-			typeErr = errors.New("expected miner config")
+			typeErr = errors.New("expected config with dealmaking config trait")
 			return
 		}
 

node/modules/storageminer_dagstore.go

@@ -25,88 +25,70 @@ const (
 )
 
 // NewMinerAPI creates a new MinerAPI adaptor for the dagstore mounts.
-func NewMinerAPI(lc fx.Lifecycle, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, sa mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) {
+func NewMinerAPI(cfg config.DAGStoreConfig) func(fx.Lifecycle, repo.LockedRepo, dtypes.ProviderPieceStore, mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) {
-	cfg, err := extractDAGStoreConfig(r)
+	return func(lc fx.Lifecycle, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, sa mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) {
-	if err != nil {
+		// caps the amount of concurrent calls to the storage, so that we don't
-		return nil, err
+		// spam it during heavy processes like bulk migration.
-	}
+		if v, ok := os.LookupEnv("LOTUS_DAGSTORE_MOUNT_CONCURRENCY"); ok {
-
+			concurrency, err := strconv.Atoi(v)
-	// caps the amount of concurrent calls to the storage, so that we don't
+			if err == nil {
-	// spam it during heavy processes like bulk migration.
+				cfg.MaxConcurrencyStorageCalls = concurrency
-	if v, ok := os.LookupEnv("LOTUS_DAGSTORE_MOUNT_CONCURRENCY"); ok {
-		concurrency, err := strconv.Atoi(v)
-		if err == nil {
-			cfg.MaxConcurrencyStorageCalls = concurrency
-		}
-	}
-
-	mountApi := mdagstore.NewMinerAPI(pieceStore, sa, cfg.MaxConcurrencyStorageCalls, cfg.MaxConcurrentUnseals)
-	ready := make(chan error, 1)
-	pieceStore.OnReady(func(err error) {
-		ready <- err
-	})
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			if err := <-ready; err != nil {
-				return fmt.Errorf("aborting dagstore start; piecestore failed to start: %s", err)
 			}
-			return mountApi.Start(ctx)
+		}
-		},
-		OnStop: func(context.Context) error {
-			return nil
-		},
-	})
 
-	return mountApi, nil
+		mountApi := mdagstore.NewMinerAPI(pieceStore, sa, cfg.MaxConcurrencyStorageCalls, cfg.MaxConcurrentUnseals)
+		ready := make(chan error, 1)
+		pieceStore.OnReady(func(err error) {
+			ready <- err
+		})
+		lc.Append(fx.Hook{
+			OnStart: func(ctx context.Context) error {
+				if err := <-ready; err != nil {
+					return fmt.Errorf("aborting dagstore start; piecestore failed to start: %s", err)
+				}
+				return mountApi.Start(ctx)
+			},
+			OnStop: func(context.Context) error {
+				return nil
+			},
+		})
+
+		return mountApi, nil
+	}
 }
 
 // DAGStore constructs a DAG store using the supplied minerAPI, and the
 // user configuration. It returns both the DAGStore and the Wrapper suitable for
 // passing to markets.
-func DAGStore(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI, h host.Host) (*dagstore.DAGStore, *mdagstore.Wrapper, error) {
+func DAGStore(cfg config.DAGStoreConfig) func(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI, h host.Host) (*dagstore.DAGStore, *mdagstore.Wrapper, error) {
-	cfg, err := extractDAGStoreConfig(r)
+	return func(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI, h host.Host) (*dagstore.DAGStore, *mdagstore.Wrapper, error) {
-	if err != nil {
+		// fall back to default root directory if not explicitly set in the config.
-		return nil, nil, err
+		if cfg.RootDir == "" {
-	}
+			cfg.RootDir = filepath.Join(r.Path(), DefaultDAGStoreDir)
-
-	// fall back to default root directory if not explicitly set in the config.
-	if cfg.RootDir == "" {
-		cfg.RootDir = filepath.Join(r.Path(), DefaultDAGStoreDir)
-	}
-
-	v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency)
-	if ok {
-		concurrency, err := strconv.Atoi(v)
-		if err == nil {
-			cfg.MaxConcurrentReadyFetches = concurrency
 		}
+
+		v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency)
+		if ok {
+			concurrency, err := strconv.Atoi(v)
+			if err == nil {
+				cfg.MaxConcurrentReadyFetches = concurrency
+			}
+		}
+
+		dagst, w, err := mdagstore.NewDAGStore(cfg, minerAPI, h)
+		if err != nil {
+			return nil, nil, xerrors.Errorf("failed to create DAG store: %w", err)
+		}
+
+		lc.Append(fx.Hook{
+			OnStart: func(ctx context.Context) error {
+				return w.Start(ctx)
+			},
+			OnStop: func(context.Context) error {
+				return w.Close()
+			},
+		})
+
+		return dagst, w, nil
 	}
-
-	dagst, w, err := mdagstore.NewDAGStore(cfg, minerAPI, h)
-	if err != nil {
-		return nil, nil, xerrors.Errorf("failed to create DAG store: %w", err)
-	}
-
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			return w.Start(ctx)
-		},
-		OnStop: func(context.Context) error {
-			return w.Close()
-		},
-	})
-
-	return dagst, w, nil
-}
-
-func extractDAGStoreConfig(r repo.LockedRepo) (config.DAGStoreConfig, error) {
-	cfg, err := r.Config()
-	if err != nil {
-		return config.DAGStoreConfig{}, xerrors.Errorf("could not load config: %w", err)
-	}
-	mcfg, ok := cfg.(*config.StorageMiner)
-	if !ok {
-		return config.DAGStoreConfig{}, xerrors.Errorf("config not expected type; expected config.StorageMiner, got: %T", cfg)
-	}
-	return mcfg.DAGStore, nil
 }

node/repo/fsrepo.go

@@ -41,47 +41,174 @@ const (
 	fsKeystore      = "keystore"
 )
 
-type RepoType int
+func NewRepoTypeFromString(t string) RepoType {
-
+	switch t {
-const (
+	case "FullNode":
-	_                 = iota // Default is invalid
+		return FullNode
-	FullNode RepoType = iota
+	case "StorageMiner":
-	StorageMiner
+		return StorageMiner
-	Worker
+	case "Worker":
-	Wallet
+		return Worker
-	Markets
+	case "Wallet":
-)
+		return Wallet
-
+	default:
-func (t RepoType) String() string {
+		panic("unknown RepoType")
-	s := [...]string{
-		"__invalid__",
-		"FullNode",
-		"StorageMiner",
-		"Worker",
-		"Wallet",
-		"Markets",
 	}
-	if t < 0 || int(t) > len(s) {
-		return "__invalid__"
-	}
-	return s[t]
 }
 
-func defConfForType(t RepoType) interface{} {
+type RepoType interface {
-	switch t {
+	Type() string
-	case FullNode:
+	Config() interface{}
-		return config.DefaultFullNode()
+
-	case StorageMiner, Markets:
+	// APIFlags returns flags passed on the command line with the listen address
-		// markets is a specialised miner service
+	// of the API server (only used by the tests), in the order of precedence they
-		// this taxonomy needs to be cleaned up
+	// should be applied for the requested kind of node.
-		return config.DefaultStorageMiner()
+	APIFlags() []string
-	case Worker:
+
-		return &struct{}{}
+	RepoFlags() []string
-	case Wallet:
+
-		return &struct{}{}
+	// APIInfoEnvVars returns the environment variables to use in order of precedence
-	default:
+	// to determine the API endpoint of the specified node type.
-		panic(fmt.Sprintf("unknown RepoType(%d)", int(t)))
+	//
-	}
+	// It returns the current variables and deprecated ones separately, so that
+	// the user can log a warning when deprecated ones are found to be in use.
+	APIInfoEnvVars() (string, []string, []string)
+}
+
+// SupportsStagingDeals is a trait for services that support staging deals
+type SupportsStagingDeals interface {
+	SupportsStagingDeals()
+}
+
+var FullNode fullNode
+
+type fullNode struct {
+}
+
+func (fullNode) Type() string {
+	return "FullNode"
+}
+
+func (fullNode) Config() interface{} {
+	return config.DefaultFullNode()
+}
+
+func (fullNode) APIFlags() []string {
+	return []string{"api-url"}
+}
+
+func (fullNode) RepoFlags() []string {
+	return []string{"repo"}
+}
+
+func (fullNode) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
+	return "FULLNODE_API_INFO", nil, nil
+}
+
+var StorageMiner storageMiner
+
+type storageMiner struct{}
+
+func (storageMiner) SupportsStagingDeals() {}
+
+func (storageMiner) Type() string {
+	return "StorageMiner"
+}
+
+func (storageMiner) Config() interface{} {
+	return config.DefaultStorageMiner()
+}
+
+func (storageMiner) APIFlags() []string {
+	return []string{"miner-api-url"}
+}
+
+func (storageMiner) RepoFlags() []string {
+	return []string{"miner-repo"}
+}
+
+func (storageMiner) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
+	// TODO remove deprecated deprecation period
+	return "MINER_API_INFO", nil, []string{"STORAGE_API_INFO"}
+}
+
+var Markets markets
+
+type markets struct{}
+
+func (markets) SupportsStagingDeals() {}
+
+func (markets) Type() string {
+	return "Markets"
+}
+
+func (markets) Config() interface{} {
+	return config.DefaultStorageMiner()
+}
+
+func (markets) APIFlags() []string {
+	// support split markets-miner and monolith deployments.
+	return []string{"markets-api-url", "miner-api-url"}
+}
+
+func (markets) RepoFlags() []string {
+	// support split markets-miner and monolith deployments.
+	return []string{"markets-repo", "miner-repo"}
+}
+
+func (markets) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
+	// support split markets-miner and monolith deployments.
+	return "MARKETS_API_INFO", []string{"MINER_API_INFO"}, nil
+}
+
+type worker struct {
+}
+
+var Worker worker
+
+func (worker) Type() string {
+	return "Worker"
+}
+
+func (worker) Config() interface{} {
+	return &struct{}{}
+}
+
+func (worker) APIFlags() []string {
+	return []string{"worker-api-url"}
+}
+
+func (worker) RepoFlags() []string {
+	return []string{"worker-repo"}
+}
+
+func (worker) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
+	return "WORKER_API_INFO", nil, nil
+}
+
+var Wallet wallet
+
+type wallet struct {
+}
+
+func (wallet) Type() string {
+	return "Wallet"
+}
+
+func (wallet) Config() interface{} {
+	return &struct{}{}
+}
+
+func (wallet) APIFlags() []string {
+	panic("not supported")
+}
+
+func (wallet) RepoFlags() []string {
+	panic("not supported")
+}
+
+func (wallet) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) {
+	panic("not supported")
 }
 
 var log = logging.Logger("repo")
@@ -165,7 +292,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error {
 		return err
 	}
 
-	comm, err := config.ConfigComment(defConfForType(t))
+	comm, err := config.ConfigComment(t.Config())
 	if err != nil {
 		return xerrors.Errorf("comment: %w", err)
 	}
@@ -406,7 +533,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) {
 }
 
 func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) {
-	return config.FromFile(fsr.configPath, defConfForType(fsr.repoType))
+	return config.FromFile(fsr.configPath, fsr.repoType.Config())
 }
 
 func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {

node/repo/memrepo.go

@@ -40,9 +40,6 @@ type MemRepo struct {
 	sc      *stores.StorageConfig
 	tempDir string
 
-	// given a repo type, produce the default config
-	configF func(t RepoType) interface{}
-
 	// holds the current config value
 	config struct {
 		sync.Mutex
@@ -108,7 +105,7 @@ func (lmem *lockedMemRepo) Path() string {
 		panic(err) // only used in tests, probably fine
 	}
 
-	if lmem.t == StorageMiner {
+	if _, ok := lmem.t.(SupportsStagingDeals); ok {
 		// this is required due to the method makeDealStaging from cmd/lotus-storage-miner/init.go
 		// deal-staging is the directory deal files are staged in before being sealed into sectors
 		// for offline deal flow.
@@ -152,7 +149,6 @@ var _ Repo = &MemRepo{}
 // MemRepoOptions contains options for memory repo
 type MemRepoOptions struct {
 	Ds       datastore.Datastore
-	ConfigF  func(RepoType) interface{}
 	KeyStore map[string]types.KeyInfo
 }
 
@@ -163,9 +159,6 @@ func NewMemory(opts *MemRepoOptions) *MemRepo {
 	if opts == nil {
 		opts = &MemRepoOptions{}
 	}
-	if opts.ConfigF == nil {
-		opts.ConfigF = defConfForType
-	}
 	if opts.Ds == nil {
 		opts.Ds = dssync.MutexWrap(datastore.NewMapDatastore())
 	}
@@ -177,7 +170,6 @@ func NewMemory(opts *MemRepoOptions) *MemRepo {
 		repoLock:   make(chan struct{}, 1),
 		blockstore: blockstore.WrapIDStore(blockstore.NewMemorySync()),
 		datastore:  opts.Ds,
-		configF:    opts.ConfigF,
 		keystore:   opts.KeyStore,
 	}
 }
@@ -296,7 +288,7 @@ func (lmem *lockedMemRepo) Config() (interface{}, error) {
 	defer lmem.mem.config.Unlock()
 
 	if lmem.mem.config.val == nil {
-		lmem.mem.config.val = lmem.mem.configF(lmem.t)
+		lmem.mem.config.val = lmem.t.Config()
 	}
 
 	return lmem.mem.config.val, nil
@@ -311,7 +303,7 @@ func (lmem *lockedMemRepo) SetConfig(c func(interface{})) error {
 	defer lmem.mem.config.Unlock()
 
 	if lmem.mem.config.val == nil {
-		lmem.mem.config.val = lmem.mem.configF(lmem.t)
+		lmem.mem.config.val = lmem.t.Config()
 	}
 
 	c(lmem.mem.config.val)