config

cmd/lotus-provider/config.go

@@ -1,24 +1,35 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"os"
+	"regexp"
+	"strings"
 
+	"github.com/BurntSushi/toml"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/kr/pretty"
 	"github.com/urfave/cli/v2"
 )
 
 var configCmd = &cli.Command{
 	Name:  "config",
-	Usage: "Manage node config",
+	Usage: "Manage node config by layers. The layer 'base' will always be applied. ",
 	Subcommands: []*cli.Command{
 		configDefaultCmd,
 		configSetCmd,
 		configGetCmd,
+		configListCmd,
+		configViewCmd,
 	},
 }
 
 var configDefaultCmd = &cli.Command{
 	Name:  "default",
-	Usage: "Print default system config",
+	Usage: "Print default node config",
 	Flags: []cli.Flag{
 		&cli.BoolFlag{
 			Name:  "no-comment",
@@ -26,36 +37,137 @@ var configDefaultCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("[config]\nstatus = Coming Soon")
+		c := config.DefaultLotusProvider()
-		// [overlay.sealer1.tasks]\nsealer_task_enable = true
+
+		cb, err := config.ConfigUpdate(c, nil, config.Commented(!cctx.Bool("no-comment")), config.DefaultKeepUncommented())
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(cb))
+
 		return nil
 	},
 }
 
 var configSetCmd = &cli.Command{
 	Name:      "set",
-	Usage: "Set all config",
+	Usage:     "Set a config layer or the base",
+	ArgsUsage: "a layer's name",
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("Coming soon")
+		args := cctx.Args()
+		if args.Len() != 1 {
+			return errors.New("Must have exactly 1 arg for the file name.")
+		}
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		fn := args.First()
+		bytes, err := os.ReadFile(fn)
+		if err != nil {
+			return fmt.Errorf("cannot read file %w", err)
+		}
+
+		lp := config.DefaultLotusProvider() // ensure it's toml
+		toml.Decode(string(bytes), lp)
+		_ = lp
+
+		name := strings.Split(fn, ".")[0]
+		_, err = db.Exec(context.Background(),
+			`INSERT INTO harmony_config (title, config) VALUES (?,?) 
+			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes))
+		if err != nil {
+			return fmt.Errorf("unable to save config layer: %w", err)
+		}
+
+		fmt.Println("Layer " + name + " created/updated")
 		return nil
 	},
 }
 
 var configGetCmd = &cli.Command{
 	Name:      "get",
-	Usage: "Get all config",
+	Usage:     "Get a config layer by name. You may want to pipe the output to a file, or use 'less'",
-	Flags: []cli.Flag{
+	ArgsUsage: "layer name",
-		&cli.BoolFlag{
-			Name:  "no-comment",
-			Usage: "don't comment default values",
-		},
-		&cli.BoolFlag{
-			Name:  "no-doc",
-			Usage: "don't add value documentation",
-		},
-	},
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("Coming soon")
+		args := cctx.Args()
+		if args.Len() != 1 {
+			return errors.New("Must have exactly 1 arg for the layer name.")
+		}
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		var cfg string
+		err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=?`, args.First()).Scan(&cfg)
+		if err != nil {
+			return err
+		}
+		fmt.Println(cfg)
+
 		return nil
 	},
 }
+
+var configListCmd = &cli.Command{
+	Name:  "list",
+	Usage: "List config layers you can get.",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+		var res []string
+		db.Select(context.Background(), &res, `SELECT title FROM harmony_confg ORDER BY title`)
+		for _, r := range res {
+			fmt.Println(r)
+		}
+
+		return nil
+	},
+}
+
+var configViewCmd = &cli.Command{
+	Name:      "view",
+	Usage:     "View stacked config layers as it will be interpreted.",
+	ArgsUsage: "a list of layers to be interpreted as the final config",
+	Action: func(cctx *cli.Context) error {
+		db, err := makeDB(cctx)
+		if err != nil {
+			return err
+		}
+		lp, err := getConfig(cctx, db)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(pretty.Sprint(lp))
+
+		return nil
+	},
+}
+
+func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
+	lp := config.DefaultLotusProvider()
+	have := []string{}
+	for _, layer := range regexp.MustCompile("[ |,]").Split(cctx.String("layers"), -1) {
+		text := ""
+		err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=?`, layer).Scan(&text)
+		if err != nil {
+			return nil, fmt.Errorf("could not read layer %s: %w", layer, err)
+		}
+		meta, err := toml.Decode(text, &lp)
+		if err != nil {
+			return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
+		}
+		for _, k := range meta.Keys() {
+			have = append(have, strings.Join(k, " "))
+		}
+	}
+	_ = have // TODO: verify that required fields are here.
+	return lp, nil
+}

cmd/lotus-provider/main.go

@@ -81,10 +81,36 @@ func main() {
 				Value:   "~/.lotusprovider", // should follow --repo default
 			},
 			&cli.StringFlag{
-				Name:    "repo",
+				Name:    "db-host",
-				EnvVars: []string{"LOTUS_PATH"},
+				EnvVars: []string{"LOTUS_DB_HOST"},
+				Usage:   "Command separated list of hostnames for yugabyte cluster",
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-name",
+				EnvVars: []string{"LOTUS_DB_NAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-user",
+				EnvVars: []string{"LOTUS_DB_USER"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-password",
+				EnvVars: []string{"LOTUS_DB_PASSWORD"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-port",
+				EnvVars: []string{"LOTUS_DB_PORT"},
 				Hidden:  true,
-				Value:   "~/.lotus",
+				Value:   "5433",
+			},
+			&cli.StringFlag{
+				Name:    "layers",
+				EnvVars: []string{"LOTUS_LAYERS"},
+				Value:   "base",
 			},
 			cliutil.FlagVeryVerbose,
 		},

cmd/lotus-provider/run.go

@@ -54,37 +54,6 @@ var runCmd = &cli.Command{
 			Usage: "manage open file limit",
 			Value: true,
 		},
-		&cli.StringFlag{
-			Name:    "db-host",
-			EnvVars: []string{"LOTUS_DB_HOST"},
-			Usage:   "Command separated list of hostnames for yugabyte cluster",
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-name",
-			EnvVars: []string{"LOTUS_DB_NAME"},
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-user",
-			EnvVars: []string{"LOTUS_DB_USER"},
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-password",
-			EnvVars: []string{"LOTUS_DB_PASSWORD"},
-			Value:   "yugabyte",
-		},
-		&cli.StringFlag{
-			Name:    "db-port",
-			EnvVars: []string{"LOTUS_DB_PORT"},
-			Hidden:  true,
-			Value:   "5433",
-		},
-		&cli.StringFlag{
-			Name:  FlagProviderRepo,
-			Value: "~/lotusminer",
-		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if !cctx.Bool("enable-gpu-proving") {
@@ -159,18 +128,10 @@ var runCmd = &cli.Command{
 			}
 		}
 
-		dbConfig := config.HarmonyDB{
+		db, err := makeDB(cctx)
-			Username: cctx.String("db-user"),
-			Password: cctx.String("db-password"),
-			Hosts:    strings.Split(cctx.String("db-host"), ","),
-			Database: cctx.String("db-name"),
-			Port:     cctx.String("db-port"),
-		}
-		db, err := harmonydb.NewFromConfig(dbConfig)
 		if err != nil {
 			return err
 		}
-
 		shutdownChan := make(chan struct{})
 
 		/* defaults break lockedRepo (below)
@@ -209,6 +170,11 @@ var runCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		lp, err := getConfig(cctx, db)
+		if err != nil {
+			return err
+		}
+		_ = lp // here is where the config feeds into task runners
 
 		taskEngine, err := harmonytask.New(db, []harmonytask.TaskInterface{}, address)
 		if err != nil {
@@ -256,3 +222,15 @@ var runCmd = &cli.Command{
 		return nil
 	},
 }
+
+func makeDB(cctx *cli.Context) (*harmonydb.DB, error) {
+	dbConfig := config.HarmonyDB{
+		Username: cctx.String("db-user"),
+		Password: cctx.String("db-password"),
+		Hosts:    strings.Split(cctx.String("db-host"), ","),
+		Database: cctx.String("db-name"),
+		Port:     cctx.String("db-port"),
+	}
+	return harmonydb.NewFromConfig(dbConfig)
+
+}

go.mod

@@ -273,6 +273,8 @@ require (
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
@@ -320,6 +322,7 @@ require (
 	github.com/quic-go/quic-go v0.33.0 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v2.18.12+incompatible // indirect

go.sum

@@ -984,6 +984,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@@ -1531,6 +1532,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=

lib/harmony/harmonydb/sql/20230919.sql

@@ -0,0 +1,5 @@
+CREATE TABLE harmony_config (
+    id SERIAL PRIMARY KEY NOT NULL,
+    title VARCHAR(300) UNIQUE NOT NULL,
+    config TEXT NOT NULL
+);

node/config/def.go

@@ -343,3 +343,46 @@ func DefaultUserRaftConfig() *UserRaftConfig {
 
 	return &cfg
 }
+
+func DefaultLotusProvider() *LotusProviderConfig {
+	/*
+		reqs := map[string]*regexp.Regexp{}
+		for _, str := range LotusRequired {
+			reqs[str]=regexp.MustCompile("(?i)"+str)
+		}
+	*/
+	return &LotusProviderConfig{
+		Fees: LotusProviderFees{
+			DefaultMaxFee:      DefaultDefaultMaxFee,
+			MaxPreCommitGasFee: types.MustParseFIL("0.025"),
+			MaxCommitGasFee:    types.MustParseFIL("0.05"),
+
+			MaxPreCommitBatchGasFee: BatchFeeConfig{
+				Base:      types.MustParseFIL("0"),
+				PerSector: types.MustParseFIL("0.02"),
+			},
+			MaxCommitBatchGasFee: BatchFeeConfig{
+				Base:      types.MustParseFIL("0"),
+				PerSector: types.MustParseFIL("0.03"), // enough for 6 agg and 1nFIL base fee
+			},
+
+			MaxTerminateGasFee:  types.MustParseFIL("0.5"),
+			MaxWindowPoStGasFee: types.MustParseFIL("5"),
+			MaxPublishDealsFee:  types.MustParseFIL("0.05"),
+		},
+		Addresses: LotusProviderAddresses{
+			PreCommitControl: []string{},
+			CommitControl:    []string{},
+			TerminateControl: []string{},
+		},
+		/*
+			HarmonyDB: HarmonyDB{
+				Hosts:    []string{"127.0.0.1"},
+				Username: "yugabyte",
+				Password: "yugabyte",
+				Database: "yugabyte",
+				Port:     "5433",
+			},
+		*/
+	}
+}

node/config/types.go

@@ -65,6 +65,14 @@ type StorageMiner struct {
 	HarmonyDB HarmonyDB
 }
 
+type LotusProviderConfig struct {
+	EnableWindowPost  bool
+	EnableWinningPost bool
+
+	Fees      LotusProviderFees
+	Addresses LotusProviderAddresses
+}
+
 type DAGStoreConfig struct {
 	// Path to the dagstore root directory. This directory contains three
 	// subdirectories, which can be symlinked to alternative locations if
@@ -499,6 +507,20 @@ type MinerFeeConfig struct {
 	MaxMarketBalanceAddFee types.FIL
 }
 
+type LotusProviderFees struct {
+	DefaultMaxFee      types.FIL
+	MaxPreCommitGasFee types.FIL
+	MaxCommitGasFee    types.FIL
+
+	// maxBatchFee = maxBase + maxPerSector * nSectors
+	MaxPreCommitBatchGasFee BatchFeeConfig
+	MaxCommitBatchGasFee    BatchFeeConfig
+
+	MaxTerminateGasFee types.FIL
+	// WindowPoSt is a high-value operation, so the default fee should be high.
+	MaxWindowPoStGasFee types.FIL
+	MaxPublishDealsFee  types.FIL
+}
 type MinerAddressConfig struct {
 	// Addresses to send PreCommit messages from
 	PreCommitControl []string
@@ -517,6 +539,23 @@ type MinerAddressConfig struct {
 	DisableWorkerFallback bool
 }
 
+type LotusProviderAddresses struct {
+	// Addresses to send PreCommit messages from
+	PreCommitControl []string
+	// Addresses to send Commit messages from
+	CommitControl    []string
+	TerminateControl []string
+
+	// DisableOwnerFallback disables usage of the owner address for messages
+	// sent automatically
+	DisableOwnerFallback bool
+	// DisableWorkerFallback disables usage of the worker address for messages
+	// sent automatically, if control addresses are configured.
+	// A control address that doesn't have enough funds will still be chosen
+	// over the worker address if this flag is set.
+	DisableWorkerFallback bool
+}
+
 // API contains configs for API endpoint
 type API struct {
 	// Binding address for the Lotus API