Merge pull request #11278 from filecoin-project/lotus_provider_config

LotusProvider database-oriented config system

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 orbs:
-  aws-cli: circleci/aws-cli@1.3.2
+  aws-cli: circleci/aws-cli@4.1.1
-  docker: circleci/docker@2.1.4
+  docker: circleci/docker@2.3.0
 
 executors:
   golang:

.circleci/template.yml

@@ -1,7 +1,7 @@
 version: 2.1
 orbs:
-  aws-cli: circleci/aws-cli@1.3.2
+  aws-cli: circleci/aws-cli@4.1.1
-  docker: circleci/docker@2.1.4
+  docker: circleci/docker@2.3.0
 
 executors:
   golang:

chain/gen/genesis/miners.go

@@ -247,7 +247,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 			}
 
 			params := &markettypes.PublishStorageDealsParams{}
-			for _, preseal := range m.Sectors {
+			for _, presealTmp := range m.Sectors {
+				preseal := presealTmp
 				preseal.Deal.VerifiedDeal = true
 				preseal.Deal.EndEpoch = minerInfos[i].presealExp
 				p := markettypes.ClientDealProposal{

chain/vectors/gen/main.go

@@ -146,7 +146,10 @@ func MakeUnsignedMessageVectors() []vectors.UnsignedMessageVector {
 		}
 
 		params := make([]byte, 32)
-		crand.Read(params)
+		_, err = crand.Read(params)
+		if err != nil {
+			panic(err)
+		}
 
 		msg := &types.Message{
 			To:         to,

cmd/lotus-bench/main.go

@@ -3,10 +3,10 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/json"
 	"fmt"
 	"math/big"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"sync"
@@ -546,7 +546,10 @@ var sealBenchCmd = &cli.Command{
 		}
 
 		var challenge [32]byte
-		rand.Read(challenge[:])
+		_, err = rand.Read(challenge[:])
+		if err != nil {
+			return err
+		}
 
 		beforePost := time.Now()
 
@@ -776,9 +779,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 		start := time.Now()
 		log.Infof("[%d] Writing piece into sector...", i)
 
-		r := rand.New(rand.NewSource(100 + int64(i)))
+		pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), rand.Reader)
-
-		pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r)
 		if err != nil {
 			return nil, nil, err
 		}

cmd/lotus-miner/init.go

@@ -16,7 +16,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
-	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/mitchellh/go-homedir"
@@ -122,6 +121,33 @@ var initCmd = &cli.Command{
 			Name:  "from",
 			Usage: "select which address to send actor creation message from",
 		},
+		&cli.StringFlag{
+			Name:    "db-host",
+			EnvVars: []string{"LOTUS_DB_HOST"},
+			Usage:   "Command separated list of hostnames for yugabyte cluster",
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-name",
+			EnvVars: []string{"LOTUS_DB_NAME"},
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-user",
+			EnvVars: []string{"LOTUS_DB_USER"},
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-password",
+			EnvVars: []string{"LOTUS_DB_PASSWORD"},
+			Value:   "yugabyte",
+		},
+		&cli.StringFlag{
+			Name:    "db-port",
+			EnvVars: []string{"LOTUS_DB_PORT"},
+			Hidden:  true,
+			Value:   "5433",
+		},
 	},
 	Subcommands: []*cli.Command{
 		restoreCmd,
@@ -466,8 +492,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix))
 
 			// TODO: run sector index init only for devnets. This is not needed for longer running networks
-			harmonyDB, err := harmonydb.New([]string{"127.0.0.1"}, "yugabyte", "yugabyte", "yugabyte", "5433", "",
+			harmonyDB, err := harmonydb.New([]string{cctx.String("db-host")}, cctx.String("db-name"), cctx.String("db-user"), cctx.String("db-password"), cctx.String("db-port"), "")
-				func(s string) { logging.Logger("harmonydb").Error(s) })
 			if err != nil {
 				return err
 			}

cmd/lotus-provider/config.go

@@ -1,24 +1,38 @@
 package main
 
 import (
+	"context"
+	"database/sql"
+	"errors"
 	"fmt"
+	"io"
+	"os"
+	"strings"
 
+	"github.com/BurntSushi/toml"
 	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
 )
 
 var configCmd = &cli.Command{
 	Name:  "config",
-	Usage: "Manage node config",
+	Usage: "Manage node config by layers. The layer 'base' will always be applied. ",
 	Subcommands: []*cli.Command{
 		configDefaultCmd,
 		configSetCmd,
 		configGetCmd,
+		configListCmd,
+		configViewCmd,
+		configRmCmd,
 	},
 }
 
 var configDefaultCmd = &cli.Command{
-	Name:  "default",
+	Name:    "default",
-	Usage: "Print default system config",
+	Aliases: []string{"defaults"},
+	Usage:   "Print default node config",
 	Flags: []cli.Flag{
 		&cli.BoolFlag{
 			Name:  "no-comment",
@@ -26,36 +40,203 @@ var configDefaultCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("[config]\nstatus = Coming Soon")
+		c := config.DefaultLotusProvider()
-		// [overlay.sealer1.tasks]\nsealer_task_enable = true
+
+		cb, err := config.ConfigUpdate(c, nil, config.Commented(!cctx.Bool("no-comment")), config.DefaultKeepUncommented(), config.NoEnv())
+		if err != nil {
+			return err
+		}
+
+		fmt.Print(string(cb))
+
 		return nil
 	},
 }
 
 var configSetCmd = &cli.Command{
-	Name:  "set",
+	Name:      "set",
-	Usage: "Set all config",
+	Aliases:   []string{"add"},
+	Usage:     "Set a config layer or the base by providing a filename or stdin.",
+	ArgsUsage: "a layer's file name",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "title",
+			Usage: "title of the config layer (req'd for stdin)",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("Coming soon")
+		args := cctx.Args()
+
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		name := cctx.String("title")
+		var stream io.Reader = os.Stdin
+		if args.Len() != 1 {
+			if cctx.String("title") == "" {
+				return errors.New("must have a title for stdin, or a file name")
+			}
+		} else {
+			stream, err = os.Open(args.First())
+			if err != nil {
+				return fmt.Errorf("cannot open file %s: %w", args.First(), err)
+			}
+			if name == "" {
+				name = strings.Split(args.First(), ".")[0]
+			}
+		}
+		bytes, err := io.ReadAll(stream)
+		if err != nil {
+			return fmt.Errorf("cannot read stream/file %w", err)
+		}
+
+		lp := config.DefaultLotusProvider() // ensure it's toml
+		_, err = toml.Decode(string(bytes), lp)
+		if err != nil {
+			return fmt.Errorf("cannot decode file: %w", err)
+		}
+		_ = lp
+
+		_, err = db.Exec(context.Background(),
+			`INSERT INTO harmony_config (title, config) VALUES ($1, $2) 
+			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes))
+		if err != nil {
+			return fmt.Errorf("unable to save config layer: %w", err)
+		}
+
+		fmt.Println("Layer " + name + " created/updated")
 		return nil
 	},
 }
 
 var configGetCmd = &cli.Command{
-	Name:  "get",
+	Name:      "get",
-	Usage: "Get all config",
+	Aliases:   []string{"cat", "show"},
-	Flags: []cli.Flag{
+	Usage:     "Get a config layer by name. You may want to pipe the output to a file, or use 'less'",
-		&cli.BoolFlag{
+	ArgsUsage: "layer name",
-			Name:  "no-comment",
-			Usage: "don't comment default values",
-		},
-		&cli.BoolFlag{
-			Name:  "no-doc",
-			Usage: "don't add value documentation",
-		},
-	},
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("Coming soon")
+		args := cctx.Args()
+		if args.Len() != 1 {
+			return fmt.Errorf("want 1 layer arg, got %d", args.Len())
+		}
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		var cfg string
+		err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, args.First()).Scan(&cfg)
+		if err != nil {
+			return err
+		}
+		fmt.Println(cfg)
+
 		return nil
 	},
 }
+
+var configListCmd = &cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
+	Usage:   "List config layers you can get.",
+	Flags:   []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+		var res []string
+		err = db.Select(context.Background(), &res, `SELECT title FROM harmony_config ORDER BY title`)
+		if err != nil {
+			return fmt.Errorf("unable to read from db: %w", err)
+		}
+		for _, r := range res {
+			fmt.Println(r)
+		}
+
+		return nil
+	},
+}
+
+var configRmCmd = &cli.Command{
+	Name:    "remove",
+	Aliases: []string{"rm", "del", "delete"},
+	Usage:   "Remove a named config layer.",
+	Flags:   []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		args := cctx.Args()
+		if args.Len() != 1 {
+			return errors.New("must have exactly 1 arg for the layer name")
+		}
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+		ct, err := db.Exec(context.Background(), `DELETE FROM harmony_config WHERE title=$1`, args.First())
+		if err != nil {
+			return fmt.Errorf("unable to read from db: %w", err)
+		}
+		if ct == 0 {
+			return fmt.Errorf("no layer named %s", args.First())
+		}
+
+		return nil
+	},
+}
+var configViewCmd = &cli.Command{
+	Name:      "interpret",
+	Aliases:   []string{"view", "stacked", "stack"},
+	Usage:     "Interpret stacked config layers by this version of lotus-provider.",
+	ArgsUsage: "a list of layers to be interpreted as the final config",
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "layers",
+			Usage:    "comma or space separated list of layers to be interpreted",
+			Value:    cli.NewStringSlice("base"),
+			Required: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+		lp, err := getConfig(cctx, db)
+		if err != nil {
+			return err
+		}
+
+		e := toml.NewEncoder(os.Stdout)
+		e.Indent = "  "
+		return e.Encode(lp)
+	},
+}
+
+func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
+	lp := config.DefaultLotusProvider()
+	have := []string{}
+	for _, layer := range cctx.StringSlice("layers") {
+		text := ""
+		err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
+		if err != nil {
+			if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
+				return nil, fmt.Errorf("missing layer '%s' ", layer)
+			}
+			return nil, fmt.Errorf("could not read layer '%s': %w", layer, err)
+		}
+		meta, err := toml.Decode(text, &lp)
+		if err != nil {
+			return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
+		}
+		for _, k := range meta.Keys() {
+			have = append(have, strings.Join(k, " "))
+		}
+	}
+	_ = have // FUTURE: verify that required fields are here.
+	// If config includes 3rd-party config, consider JSONSchema as a way that
+	// 3rd-parties can dynamically include config requirements and we can
+	// validate the config. Because of layering, we must validate @ startup.
+	return lp, nil
+}

cmd/lotus-provider/main.go

@@ -81,9 +81,40 @@ func main() {
 				Value:   "~/.lotusprovider", // should follow --repo default
 			},
 			&cli.StringFlag{
-				Name:    "repo",
+				Name:    "db-host",
-				EnvVars: []string{"LOTUS_PATH"},
+				EnvVars: []string{"LOTUS_DB_HOST"},
+				Usage:   "Command separated list of hostnames for yugabyte cluster",
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-name",
+				EnvVars: []string{"LOTUS_DB_NAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-user",
+				EnvVars: []string{"LOTUS_DB_USER"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-password",
+				EnvVars: []string{"LOTUS_DB_PASSWORD"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-port",
+				EnvVars: []string{"LOTUS_DB_PORT"},
 				Hidden:  true,
+				Value:   "5433",
+			},
+			&cli.StringFlag{
+				Name:    "layers",
+				EnvVars: []string{"LOTUS_LAYERS"},
+				Value:   "base",
+			},
+			&cli.StringFlag{
+				Name:    FlagRepoPath,
+				EnvVars: []string{"LOTUS_REPO_PATH"},
 				Value:   "~/.lotus",
 			},
 			cliutil.FlagVeryVerbose,
@@ -95,7 +126,7 @@ func main() {
 		After: func(c *cli.Context) error {
 			if r := recover(); r != nil {
 				// Generate report in LOTUS_PATH and re-raise panic
-				build.GeneratePanicReport(c.String("panic-reports"), c.String(FlagProviderRepo), c.App.Name)
+				build.GeneratePanicReport(c.String("panic-reports"), c.String(FlagRepoPath), c.App.Name)
 				panic(r)
 			}
 			return nil
@@ -107,5 +138,5 @@ func main() {
 }
 
 const (
-	FlagProviderRepo = "provider-repo"
+	FlagRepoPath = "repo-path"
 )

cmd/lotus-provider/run.go

@@ -54,37 +54,6 @@ var runCmd = &cli.Command{
 			Usage: "manage open file limit",
 			Value: true,
 		},
-		&cli.StringFlag{
-			Name:    "db-host",
-			EnvVars: []string{"LOTUS_DB_HOST"},
-			Usage:   "Command separated list of hostnames for yugabyte cluster",
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-name",
-			EnvVars: []string{"LOTUS_DB_NAME"},
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-user",
-			EnvVars: []string{"LOTUS_DB_USER"},
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-password",
-			EnvVars: []string{"LOTUS_DB_PASSWORD"},
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-port",
-			EnvVars: []string{"LOTUS_DB_PORT"},
-			Hidden:  true,
-			Value:   "5433",
-		},
-		&cli.StringFlag{
-			Name:  FlagProviderRepo,
-			Value: "~/lotusminer",
-		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if !cctx.Bool("enable-gpu-proving") {
@@ -118,7 +87,7 @@ var runCmd = &cli.Command{
 
 		// Open repo
 
-		repoPath := cctx.String(FlagProviderRepo)
+		repoPath := cctx.String(FlagRepoPath)
 		fmt.Println("repopath", repoPath)
 		r, err := repo.NewFS(repoPath)
 		if err != nil {
@@ -159,18 +128,10 @@ var runCmd = &cli.Command{
 			}
 		}
 
-		dbConfig := config.HarmonyDB{
+		db, err := makeDB(cctx)
-			Username: cctx.String("db-user"),
-			Password: cctx.String("db-password"),
-			Hosts:    strings.Split(cctx.String("db-host"), ","),
-			Database: cctx.String("db-name"),
-			Port:     cctx.String("db-port"),
-		}
-		db, err := harmonydb.NewFromConfig(dbConfig)
 		if err != nil {
 			return err
 		}
-
 		shutdownChan := make(chan struct{})
 
 		/* defaults break lockedRepo (below)
@@ -209,6 +170,11 @@ var runCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		lp, err := getConfig(cctx, db)
+		if err != nil {
+			return err
+		}
+		_ = lp // here is where the config feeds into task runners
 
 		taskEngine, err := harmonytask.New(db, []harmonytask.TaskInterface{}, address)
 		if err != nil {
@@ -256,3 +222,15 @@ var runCmd = &cli.Command{
 		return nil
 	},
 }
+
+func makeDB(cctx *cli.Context) (*harmonydb.DB, error) {
+	dbConfig := config.HarmonyDB{
+		Username: cctx.String("db-user"),
+		Password: cctx.String("db-password"),
+		Hosts:    strings.Split(cctx.String("db-host"), ","),
+		Database: cctx.String("db-name"),
+		Port:     cctx.String("db-port"),
+	}
+	return harmonydb.NewFromConfig(dbConfig)
+
+}

cmd/lotus-sim/simulation/stages/funding_stage.go

@@ -166,7 +166,8 @@ func (fs *FundingStage) PackMessages(ctx context.Context, bb *blockbuilder.Block
 		)
 	}()
 
-	for _, actor := range targets {
+	for _, actorTmp := range targets {
+		actor := actorTmp
 		switch {
 		case builtin.IsAccountActor(actor.Code):
 			if _, err := bb.PushMessage(&types.Message{

documentation/en/cli-lotus-miner.md

@@ -66,6 +66,10 @@ OPTIONS:
    --no-local-storage                                         don't use storageminer repo for sector storage (default: false)
    --gas-premium value                                        set gas premium for initialization messages in AttoFIL (default: "0")
    --from value                                               select which address to send actor creation message from
+   --db-host value                                            Command separated list of hostnames for yugabyte cluster (default: "yugabyte") [$LOTUS_DB_HOST]
+   --db-name value                                            (default: "yugabyte") [$LOTUS_DB_NAME]
+   --db-user value                                            (default: "yugabyte") [$LOTUS_DB_USER]
+   --db-password value                                        (default: "yugabyte") [$LOTUS_DB_PASSWORD]
    --help, -h                                                 show help
 ```
 

documentation/en/default-lotus-provider-config.toml

@@ -1,2 +1,190 @@
-[config]
+[Subsystems]
-status = Coming Soon
+  # type: bool
+  #EnableWindowPost = false
+
+  # type: bool
+  #EnableWinningPost = false
+
+
+[Fees]
+  # type: types.FIL
+  #DefaultMaxFee = "0.07 FIL"
+
+  # type: types.FIL
+  #MaxPreCommitGasFee = "0.025 FIL"
+
+  # type: types.FIL
+  #MaxCommitGasFee = "0.05 FIL"
+
+  # type: types.FIL
+  #MaxTerminateGasFee = "0.5 FIL"
+
+  # WindowPoSt is a high-value operation, so the default fee should be high.
+  #
+  # type: types.FIL
+  #MaxWindowPoStGasFee = "5 FIL"
+
+  # type: types.FIL
+  #MaxPublishDealsFee = "0.05 FIL"
+
+  [Fees.MaxPreCommitBatchGasFee]
+    # type: types.FIL
+    #Base = "0 FIL"
+
+    # type: types.FIL
+    #PerSector = "0.02 FIL"
+
+  [Fees.MaxCommitBatchGasFee]
+    # type: types.FIL
+    #Base = "0 FIL"
+
+    # type: types.FIL
+    #PerSector = "0.03 FIL"
+
+
+[Addresses]
+  # Addresses to send PreCommit messages from
+  #
+  # type: []string
+  #PreCommitControl = []
+
+  # Addresses to send Commit messages from
+  #
+  # type: []string
+  #CommitControl = []
+
+  # type: []string
+  #TerminateControl = []
+
+  # DisableOwnerFallback disables usage of the owner address for messages
+  # sent automatically
+  #
+  # type: bool
+  #DisableOwnerFallback = false
+
+  # DisableWorkerFallback disables usage of the worker address for messages
+  # sent automatically, if control addresses are configured.
+  # A control address that doesn't have enough funds will still be chosen
+  # over the worker address if this flag is set.
+  #
+  # type: bool
+  #DisableWorkerFallback = false
+
+
+[Proving]
+  # Maximum number of sector checks to run in parallel. (0 = unlimited)
+  # 
+  # WARNING: Setting this value too high may make the node crash by running out of stack
+  # WARNING: Setting this value too low may make sector challenge reading much slower, resulting in failed PoSt due
+  # to late submission.
+  # 
+  # After changing this option, confirm that the new value works in your setup by invoking
+  # 'lotus-miner proving compute window-post 0'
+  #
+  # type: int
+  #ParallelCheckLimit = 32
+
+  # Maximum amount of time a proving pre-check can take for a sector. If the check times out the sector will be skipped
+  # 
+  # WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+  # test challenge took longer than this timeout
+  # WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
+  # blocked (e.g. in case of disconnected NFS mount)
+  #
+  # type: Duration
+  #SingleCheckTimeout = "10m0s"
+
+  # Maximum amount of time a proving pre-check can take for an entire partition. If the check times out, sectors in
+  # the partition which didn't get checked on time will be skipped
+  # 
+  # WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+  # test challenge took longer than this timeout
+  # WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
+  # blocked or slow
+  #
+  # type: Duration
+  #PartitionCheckTimeout = "20m0s"
+
+  # Disable Window PoSt computation on the lotus-miner process even if no window PoSt workers are present.
+  # 
+  # WARNING: If no windowPoSt workers are connected, window PoSt WILL FAIL resulting in faulty sectors which will need
+  # to be recovered. Before enabling this option, make sure your PoSt workers work correctly.
+  # 
+  # After changing this option, confirm that the new value works in your setup by invoking
+  # 'lotus-miner proving compute window-post 0'
+  #
+  # type: bool
+  #DisableBuiltinWindowPoSt = false
+
+  # Disable Winning PoSt computation on the lotus-miner process even if no winning PoSt workers are present.
+  # 
+  # WARNING: If no WinningPoSt workers are connected, Winning PoSt WILL FAIL resulting in lost block rewards.
+  # Before enabling this option, make sure your PoSt workers work correctly.
+  #
+  # type: bool
+  #DisableBuiltinWinningPoSt = false
+
+  # Disable WindowPoSt provable sector readability checks.
+  # 
+  # In normal operation, when preparing to compute WindowPoSt, lotus-miner will perform a round of reading challenges
+  # from all sectors to confirm that those sectors can be proven. Challenges read in this process are discarded, as
+  # we're only interested in checking that sector data can be read.
+  # 
+  # When using builtin proof computation (no PoSt workers, and DisableBuiltinWindowPoSt is set to false), this process
+  # can save a lot of time and compute resources in the case that some sectors are not readable - this is caused by
+  # the builtin logic not skipping snark computation when some sectors need to be skipped.
+  # 
+  # When using PoSt workers, this process is mostly redundant, with PoSt workers challenges will be read once, and
+  # if challenges for some sectors aren't readable, those sectors will just get skipped.
+  # 
+  # Disabling sector pre-checks will slightly reduce IO load when proving sectors, possibly resulting in shorter
+  # time to produce window PoSt. In setups with good IO capabilities the effect of this option on proving time should
+  # be negligible.
+  # 
+  # NOTE: It likely is a bad idea to disable sector pre-checks in setups with no PoSt workers.
+  # 
+  # NOTE: Even when this option is enabled, recovering sectors will be checked before recovery declaration message is
+  # sent to the chain
+  # 
+  # After changing this option, confirm that the new value works in your setup by invoking
+  # 'lotus-miner proving compute window-post 0'
+  #
+  # type: bool
+  #DisableWDPoStPreChecks = false
+
+  # Maximum number of partitions to prove in a single SubmitWindowPoSt messace. 0 = network limit (10 in nv16)
+  # 
+  # A single partition may contain up to 2349 32GiB sectors, or 2300 64GiB sectors.
+  # 
+  # The maximum number of sectors which can be proven in a single PoSt message is 25000 in network version 16, which
+  # means that a single message can prove at most 10 partitions
+  # 
+  # Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+  # to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
+  # 
+  # Setting this value above the network limit has no effect
+  #
+  # type: int
+  #MaxPartitionsPerPoStMessage = 0
+
+  # In some cases when submitting DeclareFaultsRecovered messages,
+  # there may be too many recoveries to fit in a BlockGasLimit.
+  # In those cases it may be necessary to set this value to something low (eg 1);
+  # Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed,
+  # resulting in more total gas use (but each message will have lower gas limit)
+  #
+  # type: int
+  #MaxPartitionsPerRecoveryMessage = 0
+
+  # Enable single partition per PoSt Message for partitions containing recovery sectors
+  # 
+  # In cases when submitting PoSt messages which contain recovering sectors, the default network limit may still be
+  # too high to fit in the block gas limit. In those cases, it becomes useful to only house the single partition
+  # with recovering sectors in the post message
+  # 
+  # Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+  # to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
+  #
+  # type: bool
+  #SingleRecoveringPartitionPerPostMessage = false
+

lib/harmony/harmonydb/harmonydb.go

@@ -33,7 +33,6 @@ type DB struct {
 	cfg       *pgxpool.Config
 	schema    string
 	hostnames []string
-	log       func(string)
 }
 
 var logger = logging.Logger("harmonydb")
@@ -50,7 +49,6 @@ func NewFromConfig(cfg config.HarmonyDB) (*DB, error) {
 		cfg.Database,
 		cfg.Port,
 		"",
-		func(s string) { logger.Error(s) },
 	)
 }
 
@@ -63,7 +61,6 @@ func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error
 			cfg.Database,
 			cfg.Port,
 			id,
-			func(s string) { logger.Error(s) },
 		)
 	}
 }
@@ -71,7 +68,7 @@ func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error
 // New is to be called once per binary to establish the pool.
 // log() is for errors. It returns an upgraded database's connection.
 // This entry point serves both production and integration tests, so it's more DI.
-func New(hosts []string, username, password, database, port string, itestID ITestID, log func(string)) (*DB, error) {
+func New(hosts []string, username, password, database, port string, itestID ITestID) (*DB, error) {
 	itest := string(itestID)
 	connString := ""
 	if len(hosts) > 0 {
@@ -102,11 +99,11 @@ func New(hosts []string, username, password, database, port string, itestID ITes
 	}
 
 	cfg.ConnConfig.OnNotice = func(conn *pgconn.PgConn, n *pgconn.Notice) {
-		log("database notice: " + n.Message + ": " + n.Detail)
+		logger.Debug("database notice: " + n.Message + ": " + n.Detail)
 		DBMeasures.Errors.M(1)
 	}
 
-	db := DB{cfg: cfg, schema: schema, hostnames: hosts, log: log} // pgx populated in AddStatsAndConnect
+	db := DB{cfg: cfg, schema: schema, hostnames: hosts} // pgx populated in AddStatsAndConnect
 	if err := db.addStatsAndConnect(); err != nil {
 		return nil, err
 	}
@@ -172,10 +169,13 @@ func (db *DB) addStatsAndConnect() error {
 		return nil
 	}
 
+	// Timeout the first connection so we know if the DB is down.
+	ctx, ctxClose := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second))
+	defer ctxClose()
 	var err error
-	db.pgx, err = pgxpool.NewWithConfig(context.Background(), db.cfg)
+	db.pgx, err = pgxpool.NewWithConfig(ctx, db.cfg)
 	if err != nil {
-		db.log(fmt.Sprintf("Unable to connect to database: %v\n", err))
+		logger.Error(fmt.Sprintf("Unable to connect to database: %v\n", err))
 		return err
 	}
 	return nil
@@ -229,7 +229,7 @@ func (db *DB) upgrade() error {
 		applied TIMESTAMP DEFAULT current_timestamp
 	)`)
 	if err != nil {
-		db.log("Upgrade failed.")
+		logger.Error("Upgrade failed.")
 		return err
 	}
 
@@ -240,7 +240,7 @@ func (db *DB) upgrade() error {
 		var landedEntries []struct{ Entry string }
 		err = db.Select(context.Background(), &landedEntries, "SELECT entry FROM base")
 		if err != nil {
-			db.log("Cannot read entries: " + err.Error())
+			logger.Error("Cannot read entries: " + err.Error())
 			return err
 		}
 		for _, l := range landedEntries {
@@ -249,18 +249,23 @@ func (db *DB) upgrade() error {
 	}
 	dir, err := fs.ReadDir("sql")
 	if err != nil {
-		db.log("Cannot read fs entries: " + err.Error())
+		logger.Error("Cannot read fs entries: " + err.Error())
 		return err
 	}
 	sort.Slice(dir, func(i, j int) bool { return dir[i].Name() < dir[j].Name() })
+
+	if len(dir) == 0 {
+		logger.Error("No sql files found.")
+	}
 	for _, e := range dir {
 		name := e.Name()
 		if landed[name] || !strings.HasSuffix(name, ".sql") {
+			logger.Debug("DB Schema " + name + " already applied.")
 			continue
 		}
 		file, err := fs.ReadFile("sql/" + name)
 		if err != nil {
-			db.log("weird embed file read err")
+			logger.Error("weird embed file read err")
 			return err
 		}
 		for _, s := range strings.Split(string(file), ";") { // Implement the changes.
@@ -270,7 +275,7 @@ func (db *DB) upgrade() error {
 			_, err = db.pgx.Exec(context.Background(), s)
 			if err != nil {
 				msg := fmt.Sprintf("Could not upgrade! File %s, Query: %s, Returned: %s", name, s, err.Error())
-				db.log(msg)
+				logger.Error(msg)
 				return errors.New(msg) // makes devs lives easier by placing message at the end.
 			}
 		}
@@ -278,7 +283,7 @@ func (db *DB) upgrade() error {
 		// Mark Completed.
 		_, err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name)
 		if err != nil {
-			db.log("Cannot update base: " + err.Error())
+			logger.Error("Cannot update base: " + err.Error())
 			return fmt.Errorf("cannot insert into base: %w", err)
 		}
 	}

lib/harmony/harmonydb/sql/20230919.sql

@@ -0,0 +1,5 @@
+CREATE TABLE harmony_config (
+    id SERIAL PRIMARY KEY NOT NULL,
+    title VARCHAR(300) UNIQUE NOT NULL,
+    config TEXT NOT NULL
+);

node/config/cfgdocgen/gen.go

@@ -104,7 +104,7 @@ var Doc = map[string][]DocField{
 	for _, typeName := range outt {
 		typ := out[typeName]
 
-		fmt.Printf("\t\"%s\": []DocField{\n", typeName)
+		fmt.Printf("\t\"%s\": {\n", typeName)
 
 		for _, f := range typ {
 			fmt.Println("\t\t{")

node/config/def.go

@@ -342,3 +342,51 @@ func DefaultUserRaftConfig() *UserRaftConfig {
 
 	return &cfg
 }
+
+func DefaultLotusProvider() *LotusProviderConfig {
+	/*
+		reqs := map[string]*regexp.Regexp{}
+		for _, str := range LotusRequired {
+			reqs[str]=regexp.MustCompile("(?i)"+str)
+		}
+	*/
+	return &LotusProviderConfig{
+		Fees: LotusProviderFees{
+			DefaultMaxFee:      DefaultDefaultMaxFee,
+			MaxPreCommitGasFee: types.MustParseFIL("0.025"),
+			MaxCommitGasFee:    types.MustParseFIL("0.05"),
+
+			MaxPreCommitBatchGasFee: BatchFeeConfig{
+				Base:      types.MustParseFIL("0"),
+				PerSector: types.MustParseFIL("0.02"),
+			},
+			MaxCommitBatchGasFee: BatchFeeConfig{
+				Base:      types.MustParseFIL("0"),
+				PerSector: types.MustParseFIL("0.03"), // enough for 6 agg and 1nFIL base fee
+			},
+
+			MaxTerminateGasFee:  types.MustParseFIL("0.5"),
+			MaxWindowPoStGasFee: types.MustParseFIL("5"),
+			MaxPublishDealsFee:  types.MustParseFIL("0.05"),
+		},
+		Addresses: LotusProviderAddresses{
+			PreCommitControl: []string{},
+			CommitControl:    []string{},
+			TerminateControl: []string{},
+		},
+		Proving: ProvingConfig{
+			ParallelCheckLimit:    32,
+			PartitionCheckTimeout: Duration(20 * time.Minute),
+			SingleCheckTimeout:    Duration(10 * time.Minute),
+		},
+		/*
+			HarmonyDB: HarmonyDB{
+				Hosts:    []string{"127.0.0.1"},
+				Username: "yugabyte",
+				Password: "yugabyte",
+				Database: "yugabyte",
+				Port:     "5433",
+			},
+		*/
+	}
+}

node/config/def_test.go

@@ -71,6 +71,10 @@ func TestDefaultMinerRoundtrip(t *testing.T) {
 
 	fmt.Println(s)
 
+	// Differs between test envs
+	c.HarmonyDB = HarmonyDB{}
+	c2.(*StorageMiner).HarmonyDB = HarmonyDB{}
+
 	fmt.Println(c)
 	fmt.Println(c2)
 	require.True(t, reflect.DeepEqual(c, c2))

node/config/doc_gen.go

@@ -9,7 +9,7 @@ type DocField struct {
 }
 
 var Doc = map[string][]DocField{
-	"API": []DocField{
+	"API": {
 		{
 			Name: "ListenAddress",
 			Type: "string",
@@ -29,7 +29,7 @@ var Doc = map[string][]DocField{
 			Comment: ``,
 		},
 	},
-	"Backup": []DocField{
+	"Backup": {
 		{
 			Name: "DisableMetadataLog",
 			Type: "bool",
@@ -41,7 +41,7 @@ Note that in case of metadata corruption it might be much harder to recover
 your node if metadata log is disabled`,
 		},
 	},
-	"BatchFeeConfig": []DocField{
+	"BatchFeeConfig": {
 		{
 			Name: "Base",
 			Type: "types.FIL",
@@ -55,7 +55,7 @@ your node if metadata log is disabled`,
 			Comment: ``,
 		},
 	},
-	"Chainstore": []DocField{
+	"Chainstore": {
 		{
 			Name: "EnableSplitstore",
 			Type: "bool",
@@ -69,7 +69,7 @@ your node if metadata log is disabled`,
 			Comment: ``,
 		},
 	},
-	"Client": []DocField{
+	"Client": {
 		{
 			Name: "UseIpfs",
 			Type: "bool",
@@ -117,7 +117,7 @@ without existing payment channels with available funds will fail instead
 of automatically performing on-chain operations.`,
 		},
 	},
-	"Common": []DocField{
+	"Common": {
 		{
 			Name: "API",
 			Type: "API",
@@ -149,7 +149,7 @@ of automatically performing on-chain operations.`,
 			Comment: ``,
 		},
 	},
-	"DAGStoreConfig": []DocField{
+	"DAGStoreConfig": {
 		{
 			Name: "RootDir",
 			Type: "string",
@@ -206,7 +206,7 @@ representation, e.g. 1m, 5m, 1h.
 Default value: 1 minute.`,
 		},
 	},
-	"DealmakingConfig": []DocField{
+	"DealmakingConfig": {
 		{
 			Name: "ConsiderOnlineStorageDeals",
 			Type: "bool",
@@ -341,7 +341,7 @@ see https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#
 			Comment: ``,
 		},
 	},
-	"Events": []DocField{
+	"Events": {
 		{
 			Name: "DisableRealTimeFilterAPI",
 			Type: "bool",
@@ -394,7 +394,7 @@ the database must already exist and be writeable. If a relative path is provided
 relative to the CWD (current working directory).`,
 		},
 	},
-	"FaultReporterConfig": []DocField{
+	"FaultReporterConfig": {
 		{
 			Name: "EnableConsensusFaultReporter",
 			Type: "bool",
@@ -423,7 +423,7 @@ ReportConsensusFault messages. It will pay for gas fees, and receive any
 rewards. This address should have adequate funds to cover gas fees.`,
 		},
 	},
-	"FeeConfig": []DocField{
+	"FeeConfig": {
 		{
 			Name: "DefaultMaxFee",
 			Type: "types.FIL",
@@ -431,7 +431,7 @@ rewards. This address should have adequate funds to cover gas fees.`,
 			Comment: ``,
 		},
 	},
-	"FevmConfig": []DocField{
+	"FevmConfig": {
 		{
 			Name: "EnableEthRPC",
 			Type: "bool",
@@ -453,7 +453,7 @@ Set to 0 to keep all mappings`,
 			Comment: ``,
 		},
 	},
-	"FullNode": []DocField{
+	"FullNode": {
 		{
 			Name: "Client",
 			Type: "Client",
@@ -503,7 +503,7 @@ Set to 0 to keep all mappings`,
 			Comment: ``,
 		},
 	},
-	"HarmonyDB": []DocField{
+	"HarmonyDB": {
 		{
 			Name: "Hosts",
 			Type: "[]string",
@@ -536,7 +536,7 @@ in a cluster. Only 1 is required`,
 			Comment: `The port to find Yugabyte. Blank for default.`,
 		},
 	},
-	"IndexConfig": []DocField{
+	"IndexConfig": {
 		{
 			Name: "EnableMsgIndex",
 			Type: "bool",
@@ -545,7 +545,7 @@ in a cluster. Only 1 is required`,
 EnableMsgIndex enables indexing of messages on chain.`,
 		},
 	},
-	"IndexProviderConfig": []DocField{
+	"IndexProviderConfig": {
 		{
 			Name: "Enable",
 			Type: "bool",
@@ -590,7 +590,7 @@ starts. By default, the cache is rehydrated from previously cached entries store
 datastore if any is present.`,
 		},
 	},
-	"Libp2p": []DocField{
+	"Libp2p": {
 		{
 			Name: "ListenAddresses",
 			Type: "[]string",
@@ -657,7 +657,7 @@ count towards this limit.`,
 closed by the connection manager.`,
 		},
 	},
-	"Logging": []DocField{
+	"Logging": {
 		{
 			Name: "SubsystemLevels",
 			Type: "map[string]string",
@@ -665,7 +665,119 @@ closed by the connection manager.`,
 			Comment: `SubsystemLevels specify per-subsystem log levels`,
 		},
 	},
-	"MinerAddressConfig": []DocField{
+	"LotusProviderAddresses": {
+		{
+			Name: "PreCommitControl",
+			Type: "[]string",
+
+			Comment: `Addresses to send PreCommit messages from`,
+		},
+		{
+			Name: "CommitControl",
+			Type: "[]string",
+
+			Comment: `Addresses to send Commit messages from`,
+		},
+		{
+			Name: "TerminateControl",
+			Type: "[]string",
+
+			Comment: ``,
+		},
+		{
+			Name: "DisableOwnerFallback",
+			Type: "bool",
+
+			Comment: `DisableOwnerFallback disables usage of the owner address for messages
+sent automatically`,
+		},
+		{
+			Name: "DisableWorkerFallback",
+			Type: "bool",
+
+			Comment: `DisableWorkerFallback disables usage of the worker address for messages
+sent automatically, if control addresses are configured.
+A control address that doesn't have enough funds will still be chosen
+over the worker address if this flag is set.`,
+		},
+	},
+	"LotusProviderConfig": {
+		{
+			Name: "Subsystems",
+			Type: "ProviderSubsystemsConfig",
+
+			Comment: ``,
+		},
+		{
+			Name: "Fees",
+			Type: "LotusProviderFees",
+
+			Comment: ``,
+		},
+		{
+			Name: "Addresses",
+			Type: "LotusProviderAddresses",
+
+			Comment: ``,
+		},
+		{
+			Name: "Proving",
+			Type: "ProvingConfig",
+
+			Comment: ``,
+		},
+	},
+	"LotusProviderFees": {
+		{
+			Name: "DefaultMaxFee",
+			Type: "types.FIL",
+
+			Comment: ``,
+		},
+		{
+			Name: "MaxPreCommitGasFee",
+			Type: "types.FIL",
+
+			Comment: ``,
+		},
+		{
+			Name: "MaxCommitGasFee",
+			Type: "types.FIL",
+
+			Comment: ``,
+		},
+		{
+			Name: "MaxPreCommitBatchGasFee",
+			Type: "BatchFeeConfig",
+
+			Comment: `maxBatchFee = maxBase + maxPerSector * nSectors`,
+		},
+		{
+			Name: "MaxCommitBatchGasFee",
+			Type: "BatchFeeConfig",
+
+			Comment: ``,
+		},
+		{
+			Name: "MaxTerminateGasFee",
+			Type: "types.FIL",
+
+			Comment: ``,
+		},
+		{
+			Name: "MaxWindowPoStGasFee",
+			Type: "types.FIL",
+
+			Comment: `WindowPoSt is a high-value operation, so the default fee should be high.`,
+		},
+		{
+			Name: "MaxPublishDealsFee",
+			Type: "types.FIL",
+
+			Comment: ``,
+		},
+	},
+	"MinerAddressConfig": {
 		{
 			Name: "PreCommitControl",
 			Type: "[]string",
@@ -707,7 +819,7 @@ A control address that doesn't have enough funds will still be chosen
 over the worker address if this flag is set.`,
 		},
 	},
-	"MinerFeeConfig": []DocField{
+	"MinerFeeConfig": {
 		{
 			Name: "MaxPreCommitGasFee",
 			Type: "types.FIL",
@@ -757,7 +869,7 @@ over the worker address if this flag is set.`,
 			Comment: ``,
 		},
 	},
-	"MinerSubsystemConfig": []DocField{
+	"MinerSubsystemConfig": {
 		{
 			Name: "EnableMining",
 			Type: "bool",
@@ -803,7 +915,21 @@ This is useful to allow workers to bypass the lotus miner to access sector infor
 			Comment: ``,
 		},
 	},
-	"ProvingConfig": []DocField{
+	"ProviderSubsystemsConfig": {
+		{
+			Name: "EnableWindowPost",
+			Type: "bool",
+
+			Comment: ``,
+		},
+		{
+			Name: "EnableWinningPost",
+			Type: "bool",
+
+			Comment: ``,
+		},
+	},
+	"ProvingConfig": {
 		{
 			Name: "ParallelCheckLimit",
 			Type: "int",
@@ -930,7 +1056,7 @@ Note that setting this value lower may result in less efficient gas use - more m
 to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)`,
 		},
 	},
-	"Pubsub": []DocField{
+	"Pubsub": {
 		{
 			Name: "Bootstrapper",
 			Type: "bool",
@@ -990,7 +1116,7 @@ This property is used only if ElasticSearchTracer propery is set.`,
 			Comment: `Auth token that will be passed with logs to elasticsearch - used for weighted peers score.`,
 		},
 	},
-	"RetrievalPricing": []DocField{
+	"RetrievalPricing": {
 		{
 			Name: "Strategy",
 			Type: "string",
@@ -1010,7 +1136,7 @@ This property is used only if ElasticSearchTracer propery is set.`,
 			Comment: ``,
 		},
 	},
-	"RetrievalPricingDefault": []DocField{
+	"RetrievalPricingDefault": {
 		{
 			Name: "VerifiedDealsFreeTransfer",
 			Type: "bool",
@@ -1021,7 +1147,7 @@ This parameter is ONLY applicable if the retrieval pricing policy strategy has b
 default value is true`,
 		},
 	},
-	"RetrievalPricingExternal": []DocField{
+	"RetrievalPricingExternal": {
 		{
 			Name: "Path",
 			Type: "string",
@@ -1030,7 +1156,7 @@ default value is true`,
 This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".`,
 		},
 	},
-	"SealerConfig": []DocField{
+	"SealerConfig": {
 		{
 			Name: "ParallelFetchLimit",
 			Type: "int",
@@ -1131,7 +1257,7 @@ to use when evaluating tasks against this worker. An empty value defaults
 to "hardware".`,
 		},
 	},
-	"SealingConfig": []DocField{
+	"SealingConfig": {
 		{
 			Name: "MaxWaitDealsSectors",
 			Type: "uint64",
@@ -1337,7 +1463,7 @@ Submitting a smaller number of prove commits per epoch would reduce the possibil
 			Comment: ``,
 		},
 	},
-	"Splitstore": []DocField{
+	"Splitstore": {
 		{
 			Name: "ColdStoreType",
 			Type: "string",
@@ -1404,7 +1530,7 @@ is set.  Moving GC will not occur when total moving size exceeds
 HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
 		},
 	},
-	"StorageMiner": []DocField{
+	"StorageMiner": {
 		{
 			Name: "Subsystems",
 			Type: "MinerSubsystemConfig",
@@ -1466,7 +1592,7 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
 			Comment: ``,
 		},
 	},
-	"UserRaftConfig": []DocField{
+	"UserRaftConfig": {
 		{
 			Name: "ClusterModeEnabled",
 			Type: "bool",
@@ -1528,7 +1654,7 @@ copies that we keep as backups (renaming) after cleanup.`,
 			Comment: `Tracing enables propagation of contexts across binary boundaries.`,
 		},
 	},
-	"Wallet": []DocField{
+	"Wallet": {
 		{
 			Name: "RemoteBackend",
 			Type: "string",

node/config/load.go

@@ -124,6 +124,7 @@ func ValidateSplitstoreSet(cfgRaw string) error {
 type cfgUpdateOpts struct {
 	comment         bool
 	keepUncommented func(string) bool
+	noEnv           bool
 }
 
 // UpdateCfgOpt is a functional option for updating the config
@@ -149,6 +150,13 @@ func DefaultKeepUncommented() UpdateCfgOpt {
 	return KeepUncommented(MatchEnableSplitstoreField)
 }
 
+func NoEnv() UpdateCfgOpt {
+	return func(opts *cfgUpdateOpts) error {
+		opts.noEnv = true
+		return nil
+	}
+}
+
 // ConfigUpdate takes in a config and a default config and optionally comments out default values
 func ConfigUpdate(cfgCur, cfgDef interface{}, opts ...UpdateCfgOpt) ([]byte, error) {
 	var updateOpts cfgUpdateOpts
@@ -236,7 +244,9 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, opts ...UpdateCfgOpt) ([]byte, err
 						outLines = append(outLines, pad+"# type: "+doc.Type)
 					}
 
-					outLines = append(outLines, pad+"# env var: LOTUS_"+strings.ToUpper(strings.ReplaceAll(section, ".", "_"))+"_"+strings.ToUpper(lf[0]))
+					if !updateOpts.noEnv {
+						outLines = append(outLines, pad+"# env var: LOTUS_"+strings.ToUpper(strings.ReplaceAll(section, ".", "_"))+"_"+strings.ToUpper(lf[0]))
+					}
 				}
 			}
 

node/config/types.go

@@ -66,6 +66,19 @@ type StorageMiner struct {
 	HarmonyDB HarmonyDB
 }
 
+type LotusProviderConfig struct {
+	Subsystems ProviderSubsystemsConfig
+
+	Fees      LotusProviderFees
+	Addresses LotusProviderAddresses
+	Proving   ProvingConfig
+}
+
+type ProviderSubsystemsConfig struct {
+	EnableWindowPost  bool
+	EnableWinningPost bool
+}
+
 type DAGStoreConfig struct {
 	// Path to the dagstore root directory. This directory contains three
 	// subdirectories, which can be symlinked to alternative locations if
@@ -499,6 +512,20 @@ type MinerFeeConfig struct {
 	MaxMarketBalanceAddFee types.FIL
 }
 
+type LotusProviderFees struct {
+	DefaultMaxFee      types.FIL
+	MaxPreCommitGasFee types.FIL
+	MaxCommitGasFee    types.FIL
+
+	// maxBatchFee = maxBase + maxPerSector * nSectors
+	MaxPreCommitBatchGasFee BatchFeeConfig
+	MaxCommitBatchGasFee    BatchFeeConfig
+
+	MaxTerminateGasFee types.FIL
+	// WindowPoSt is a high-value operation, so the default fee should be high.
+	MaxWindowPoStGasFee types.FIL
+	MaxPublishDealsFee  types.FIL
+}
 type MinerAddressConfig struct {
 	// Addresses to send PreCommit messages from
 	PreCommitControl []string
@@ -517,6 +544,23 @@ type MinerAddressConfig struct {
 	DisableWorkerFallback bool
 }
 
+type LotusProviderAddresses struct {
+	// Addresses to send PreCommit messages from
+	PreCommitControl []string
+	// Addresses to send Commit messages from
+	CommitControl    []string
+	TerminateControl []string
+
+	// DisableOwnerFallback disables usage of the owner address for messages
+	// sent automatically
+	DisableOwnerFallback bool
+	// DisableWorkerFallback disables usage of the worker address for messages
+	// sent automatically, if control addresses are configured.
+	// A control address that doesn't have enough funds will still be chosen
+	// over the worker address if this flag is set.
+	DisableWorkerFallback bool
+}
+
 // API contains configs for API endpoint
 type API struct {
 	// Binding address for the Lotus API

storage/sealer/ffiwrapper/sealer_cgo.go

@@ -147,7 +147,8 @@ func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize,
 		})
 	}
 
-	if len(piecePromises) > 0 && len(piecePromises) == 1 { // weird for linter
+	/*  #nosec G601 -- length is verified */
+	if len(piecePromises) == 1 {
 		p := piecePromises[0]
 		return p()
 	}
@@ -348,7 +349,8 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storiface.SectorRef, exis
 	}
 	stagedFile = nil
 
-	if len(piecePromises) > 0 && len(piecePromises) == 1 { // weird for linter
+	/*  #nosec G601 -- length is verified */
+	if len(piecePromises) == 1 {
 		p := piecePromises[0]
 		return p()
 	}

storage/sealer/ffiwrapper/sealer_test.go

@@ -3,6 +3,7 @@ package ffiwrapper
 import (
 	"bytes"
 	"context"
+	crand "crypto/rand"
 	"fmt"
 	"io"
 	"io/fs"
@@ -971,7 +972,9 @@ func TestMulticoreSDR(t *testing.T) {
 
 func TestPoStChallengeAssumptions(t *testing.T) {
 	var r [32]byte
-	rand.Read(r[:])
+	if _, err := crand.Read(r[:]); err != nil {
+		panic(err)
+	}
 	r[31] &= 0x3f
 
 	// behaves like a pure function

storage/sealer/piece_provider_test.go

@@ -3,8 +3,8 @@ package sealer
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"io"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -195,7 +195,10 @@ type pieceProviderTestHarness struct {
 
 func generatePieceData(size uint64) []byte {
 	bz := make([]byte, size)
-	rand.Read(bz)
+	_, err := rand.Read(bz)
+	if err != nil {
+		panic(err)
+	}
 	return bz
 }