diff --git a/cmd/lotus-provider/config.go b/cmd/lotus-provider/config.go index b7a4c817a..c6e21599e 100644 --- a/cmd/lotus-provider/config.go +++ b/cmd/lotus-provider/config.go @@ -1,24 +1,35 @@ package main import ( + "context" + "errors" "fmt" + "os" + "regexp" + "strings" + "github.com/BurntSushi/toml" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/node/config" + "github.com/kr/pretty" "github.com/urfave/cli/v2" ) var configCmd = &cli.Command{ Name: "config", - Usage: "Manage node config", + Usage: "Manage node config by layers. The layer 'base' will always be applied. ", Subcommands: []*cli.Command{ configDefaultCmd, configSetCmd, configGetCmd, + configListCmd, + configViewCmd, }, } var configDefaultCmd = &cli.Command{ Name: "default", - Usage: "Print default system config", + Usage: "Print default node config", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "no-comment", @@ -26,36 +37,137 @@ var configDefaultCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - fmt.Println("[config]\nstatus = Coming Soon") - // [overlay.sealer1.tasks]\nsealer_task_enable = true + c := config.DefaultLotusProvider() + + cb, err := config.ConfigUpdate(c, nil, config.Commented(!cctx.Bool("no-comment")), config.DefaultKeepUncommented()) + if err != nil { + return err + } + + fmt.Println(string(cb)) + return nil }, } var configSetCmd = &cli.Command{ - Name: "set", - Usage: "Set all config", + Name: "set", + Usage: "Set a config layer or the base", + ArgsUsage: "a layer's name", Action: func(cctx *cli.Context) error { - fmt.Println("Coming soon") + args := cctx.Args() + if args.Len() != 1 { + return errors.New("Must have exactly 1 arg for the file name.") + } + db, err := makeDB(cctx) + if err != nil { + return err + } + + fn := args.First() + bytes, err := os.ReadFile(fn) + if err != nil { + return fmt.Errorf("cannot read file %w", err) + } + + lp := config.DefaultLotusProvider() // ensure it's toml + toml.Decode(string(bytes), lp) + _ = lp + + name := strings.Split(fn, ".")[0] + _, err = db.Exec(context.Background(), + `INSERT INTO harmony_config (title, config) VALUES (?,?) + ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes)) + if err != nil { + return fmt.Errorf("unable to save config layer: %w", err) + } + + fmt.Println("Layer " + name + " created/updated") return nil }, } var configGetCmd = &cli.Command{ - Name: "get", - Usage: "Get all config", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "no-comment", - Usage: "don't comment default values", - }, - &cli.BoolFlag{ - Name: "no-doc", - Usage: "don't add value documentation", - }, - }, + Name: "get", + Usage: "Get a config layer by name. You may want to pipe the output to a file, or use 'less'", + ArgsUsage: "layer name", Action: func(cctx *cli.Context) error { - fmt.Println("Coming soon") + args := cctx.Args() + if args.Len() != 1 { + return errors.New("Must have exactly 1 arg for the layer name.") + } + db, err := makeDB(cctx) + if err != nil { + return err + } + + var cfg string + err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=?`, args.First()).Scan(&cfg) + if err != nil { + return err + } + fmt.Println(cfg) + return nil }, } + +var configListCmd = &cli.Command{ + Name: "list", + Usage: "List config layers you can get.", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + db, err := makeDB(cctx) + if err != nil { + return err + } + var res []string + db.Select(context.Background(), &res, `SELECT title FROM harmony_confg ORDER BY title`) + for _, r := range res { + fmt.Println(r) + } + + return nil + }, +} + +var configViewCmd = &cli.Command{ + Name: "view", + Usage: "View stacked config layers as it will be interpreted.", + ArgsUsage: "a list of layers to be interpreted as the final config", + Action: func(cctx *cli.Context) error { + db, err := makeDB(cctx) + if err != nil { + return err + } + lp, err := getConfig(cctx, db) + if err != nil { + return err + } + + fmt.Println(pretty.Sprint(lp)) + + return nil + }, +} + +func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) { + lp := config.DefaultLotusProvider() + have := []string{} + for _, layer := range regexp.MustCompile("[ |,]").Split(cctx.String("layers"), -1) { + text := "" + err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=?`, layer).Scan(&text) + if err != nil { + return nil, fmt.Errorf("could not read layer %s: %w", layer, err) + } + meta, err := toml.Decode(text, &lp) + if err != nil { + return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err) + } + for _, k := range meta.Keys() { + have = append(have, strings.Join(k, " ")) + } + } + _ = have // TODO: verify that required fields are here. + return lp, nil +} diff --git a/cmd/lotus-provider/main.go b/cmd/lotus-provider/main.go index 380923b07..aa53251e5 100644 --- a/cmd/lotus-provider/main.go +++ b/cmd/lotus-provider/main.go @@ -81,10 +81,36 @@ func main() { Value: "~/.lotusprovider", // should follow --repo default }, &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, + Name: "db-host", + EnvVars: []string{"LOTUS_DB_HOST"}, + Usage: "Command separated list of hostnames for yugabyte cluster", + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-name", + EnvVars: []string{"LOTUS_DB_NAME"}, + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-user", + EnvVars: []string{"LOTUS_DB_USER"}, + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-password", + EnvVars: []string{"LOTUS_DB_PASSWORD"}, + Value: "yugabyte", + }, + &cli.StringFlag{ + Name: "db-port", + EnvVars: []string{"LOTUS_DB_PORT"}, Hidden: true, - Value: "~/.lotus", + Value: "5433", + }, + &cli.StringFlag{ + Name: "layers", + EnvVars: []string{"LOTUS_LAYERS"}, + Value: "base", }, cliutil.FlagVeryVerbose, }, diff --git a/cmd/lotus-provider/run.go b/cmd/lotus-provider/run.go index 45fbfcd95..95e5cf346 100644 --- a/cmd/lotus-provider/run.go +++ b/cmd/lotus-provider/run.go @@ -54,37 +54,6 @@ var runCmd = &cli.Command{ Usage: "manage open file limit", Value: true, }, - &cli.StringFlag{ - Name: "db-host", - EnvVars: []string{"LOTUS_DB_HOST"}, - Usage: "Command separated list of hostnames for yugabyte cluster", - Value: "yugabyte", - }, - &cli.StringFlag{ - Name: "db-name", - EnvVars: []string{"LOTUS_DB_NAME"}, - Value: "yugabyte", - }, - &cli.StringFlag{ - Name: "db-user", - EnvVars: []string{"LOTUS_DB_USER"}, - Value: "yugabyte", - }, - &cli.StringFlag{ - Name: "db-password", - EnvVars: []string{"LOTUS_DB_PASSWORD"}, - Value: "yugabyte", - }, - &cli.StringFlag{ - Name: "db-port", - EnvVars: []string{"LOTUS_DB_PORT"}, - Hidden: true, - Value: "5433", - }, - &cli.StringFlag{ - Name: FlagProviderRepo, - Value: "~/lotusminer", - }, }, Action: func(cctx *cli.Context) error { if !cctx.Bool("enable-gpu-proving") { @@ -159,18 +128,10 @@ var runCmd = &cli.Command{ } } - dbConfig := config.HarmonyDB{ - Username: cctx.String("db-user"), - Password: cctx.String("db-password"), - Hosts: strings.Split(cctx.String("db-host"), ","), - Database: cctx.String("db-name"), - Port: cctx.String("db-port"), - } - db, err := harmonydb.NewFromConfig(dbConfig) + db, err := makeDB(cctx) if err != nil { return err } - shutdownChan := make(chan struct{}) /* defaults break lockedRepo (below) @@ -209,6 +170,11 @@ var runCmd = &cli.Command{ if err != nil { return err } + lp, err := getConfig(cctx, db) + if err != nil { + return err + } + _ = lp // here is where the config feeds into task runners taskEngine, err := harmonytask.New(db, []harmonytask.TaskInterface{}, address) if err != nil { @@ -256,3 +222,15 @@ var runCmd = &cli.Command{ return nil }, } + +func makeDB(cctx *cli.Context) (*harmonydb.DB, error) { + dbConfig := config.HarmonyDB{ + Username: cctx.String("db-user"), + Password: cctx.String("db-password"), + Hosts: strings.Split(cctx.String("db-host"), ","), + Database: cctx.String("db-name"), + Port: cctx.String("db-port"), + } + return harmonydb.NewFromConfig(dbConfig) + +} diff --git a/go.mod b/go.mod index ac0950eed..ee89a61f3 100644 --- a/go.mod +++ b/go.mod @@ -273,6 +273,8 @@ require ( github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -320,6 +322,7 @@ require ( github.com/quic-go/quic-go v0.33.0 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rivo/uniseg v0.1.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect diff --git a/go.sum b/go.sum index 5fb7cda2d..db80d2f3c 100644 --- a/go.sum +++ b/go.sum @@ -984,6 +984,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -1531,6 +1532,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= diff --git a/lib/harmony/harmonydb/sql/20230919.sql b/lib/harmony/harmonydb/sql/20230919.sql new file mode 100644 index 000000000..84699a0d5 --- /dev/null +++ b/lib/harmony/harmonydb/sql/20230919.sql @@ -0,0 +1,5 @@ +CREATE TABLE harmony_config ( + id SERIAL PRIMARY KEY NOT NULL, + title VARCHAR(300) UNIQUE NOT NULL, + config TEXT NOT NULL +); \ No newline at end of file diff --git a/node/config/def.go b/node/config/def.go index aba7e340d..0b77a56f0 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -343,3 +343,46 @@ func DefaultUserRaftConfig() *UserRaftConfig { return &cfg } + +func DefaultLotusProvider() *LotusProviderConfig { + /* + reqs := map[string]*regexp.Regexp{} + for _, str := range LotusRequired { + reqs[str]=regexp.MustCompile("(?i)"+str) + } + */ + return &LotusProviderConfig{ + Fees: LotusProviderFees{ + DefaultMaxFee: DefaultDefaultMaxFee, + MaxPreCommitGasFee: types.MustParseFIL("0.025"), + MaxCommitGasFee: types.MustParseFIL("0.05"), + + MaxPreCommitBatchGasFee: BatchFeeConfig{ + Base: types.MustParseFIL("0"), + PerSector: types.MustParseFIL("0.02"), + }, + MaxCommitBatchGasFee: BatchFeeConfig{ + Base: types.MustParseFIL("0"), + PerSector: types.MustParseFIL("0.03"), // enough for 6 agg and 1nFIL base fee + }, + + MaxTerminateGasFee: types.MustParseFIL("0.5"), + MaxWindowPoStGasFee: types.MustParseFIL("5"), + MaxPublishDealsFee: types.MustParseFIL("0.05"), + }, + Addresses: LotusProviderAddresses{ + PreCommitControl: []string{}, + CommitControl: []string{}, + TerminateControl: []string{}, + }, + /* + HarmonyDB: HarmonyDB{ + Hosts: []string{"127.0.0.1"}, + Username: "yugabyte", + Password: "yugabyte", + Database: "yugabyte", + Port: "5433", + }, + */ + } +} diff --git a/node/config/types.go b/node/config/types.go index 21c92e47b..282f35037 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -65,6 +65,14 @@ type StorageMiner struct { HarmonyDB HarmonyDB } +type LotusProviderConfig struct { + EnableWindowPost bool + EnableWinningPost bool + + Fees LotusProviderFees + Addresses LotusProviderAddresses +} + type DAGStoreConfig struct { // Path to the dagstore root directory. This directory contains three // subdirectories, which can be symlinked to alternative locations if @@ -499,6 +507,20 @@ type MinerFeeConfig struct { MaxMarketBalanceAddFee types.FIL } +type LotusProviderFees struct { + DefaultMaxFee types.FIL + MaxPreCommitGasFee types.FIL + MaxCommitGasFee types.FIL + + // maxBatchFee = maxBase + maxPerSector * nSectors + MaxPreCommitBatchGasFee BatchFeeConfig + MaxCommitBatchGasFee BatchFeeConfig + + MaxTerminateGasFee types.FIL + // WindowPoSt is a high-value operation, so the default fee should be high. + MaxWindowPoStGasFee types.FIL + MaxPublishDealsFee types.FIL +} type MinerAddressConfig struct { // Addresses to send PreCommit messages from PreCommitControl []string @@ -517,6 +539,23 @@ type MinerAddressConfig struct { DisableWorkerFallback bool } +type LotusProviderAddresses struct { + // Addresses to send PreCommit messages from + PreCommitControl []string + // Addresses to send Commit messages from + CommitControl []string + TerminateControl []string + + // DisableOwnerFallback disables usage of the owner address for messages + // sent automatically + DisableOwnerFallback bool + // DisableWorkerFallback disables usage of the worker address for messages + // sent automatically, if control addresses are configured. + // A control address that doesn't have enough funds will still be chosen + // over the worker address if this flag is set. + DisableWorkerFallback bool +} + // API contains configs for API endpoint type API struct { // Binding address for the Lotus API