cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

config

Browse Source
This commit is contained in:
Andrew Jackson (Ajax) 2023-09-19 22:48:39 -05:00
parent 13b50ce6c8
commit a377033d22
8 changed files with 271 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
cmd/lotus-provider/config.go
View File

@ -1,24 +1,35 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"regexp"
"strings"
"github.com/BurntSushi/toml"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/node/config"
"github.com/kr/pretty"
"github.com/urfave/cli/v2"
)
var configCmd = &cli.Command{
Name: "config",
Usage: "Manage node config",
Usage: "Manage node config by layers. The layer 'base' will always be applied. ",
Subcommands: []*cli.Command{
configDefaultCmd,
configSetCmd,
configGetCmd,
configListCmd,
configViewCmd,
},
}
var configDefaultCmd = &cli.Command{
Name: "default",
Usage: "Print default system config",
Usage: "Print default node config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
@ -26,36 +37,137 @@ var configDefaultCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
fmt.Println("[config]\nstatus = Coming Soon")
// [overlay.sealer1.tasks]\nsealer_task_enable = true
c := config.DefaultLotusProvider()
cb, err := config.ConfigUpdate(c, nil, config.Commented(!cctx.Bool("no-comment")), config.DefaultKeepUncommented())
if err != nil {
return err
}
fmt.Println(string(cb))
return nil
},
}
var configSetCmd = &cli.Command{
Name: "set",
Usage: "Set all config",
Name: "set",
Usage: "Set a config layer or the base",
ArgsUsage: "a layer's name",
Action: func(cctx *cli.Context) error {
fmt.Println("Coming soon")
args := cctx.Args()
if args.Len() != 1 {
return errors.New("Must have exactly 1 arg for the file name.")
}
db, err := makeDB(cctx)
if err != nil {
return err
}
fn := args.First()
bytes, err := os.ReadFile(fn)
if err != nil {
return fmt.Errorf("cannot read file %w", err)
}
lp := config.DefaultLotusProvider() // ensure it's toml
toml.Decode(string(bytes), lp)
_ = lp
name := strings.Split(fn, ".")[0]
_, err = db.Exec(context.Background(),
`INSERT INTO harmony_config (title, config) VALUES (?,?)
ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes))
if err != nil {
return fmt.Errorf("unable to save config layer: %w", err)
}
fmt.Println("Layer " + name + " created/updated")
return nil
},
}
var configGetCmd = &cli.Command{
Name: "get",
Usage: "Get all config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
Usage: "don't comment default values",
},
&cli.BoolFlag{
Name: "no-doc",
Usage: "don't add value documentation",
},
},
Name: "get",
Usage: "Get a config layer by name. You may want to pipe the output to a file, or use 'less'",
ArgsUsage: "layer name",
Action: func(cctx *cli.Context) error {
fmt.Println("Coming soon")
args := cctx.Args()
if args.Len() != 1 {
return errors.New("Must have exactly 1 arg for the layer name.")
}
db, err := makeDB(cctx)
if err != nil {
return err
}
var cfg string
err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=?`, args.First()).Scan(&cfg)
if err != nil {
return err
}
fmt.Println(cfg)
return nil
},
}
var configListCmd = &cli.Command{
Name: "list",
Usage: "List config layers you can get.",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
db, err := makeDB(cctx)
if err != nil {
return err
}
var res []string
db.Select(context.Background(), &res, `SELECT title FROM harmony_confg ORDER BY title`)
for _, r := range res {
fmt.Println(r)
}
return nil
},
}
var configViewCmd = &cli.Command{
Name: "view",
Usage: "View stacked config layers as it will be interpreted.",
ArgsUsage: "a list of layers to be interpreted as the final config",
Action: func(cctx *cli.Context) error {
db, err := makeDB(cctx)
if err != nil {
return err
}
lp, err := getConfig(cctx, db)
if err != nil {
return err
}
fmt.Println(pretty.Sprint(lp))
return nil
},
}
func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
lp := config.DefaultLotusProvider()
have := []string{}
for _, layer := range regexp.MustCompile("[ |,]").Split(cctx.String("layers"), -1) {
text := ""
err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=?`, layer).Scan(&text)
if err != nil {
return nil, fmt.Errorf("could not read layer %s: %w", layer, err)
}
meta, err := toml.Decode(text, &lp)
if err != nil {
return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
}
for _, k := range meta.Keys() {
have = append(have, strings.Join(k, " "))
}
}
_ = have // TODO: verify that required fields are here.
return lp, nil
}

32
cmd/lotus-provider/main.go
View File

@ -81,10 +81,36 @@ func main() {
Value: "~/.lotusprovider", // should follow --repo default
},
&cli.StringFlag{
Name: "repo",
EnvVars: []string{"LOTUS_PATH"},
Name: "db-host",
EnvVars: []string{"LOTUS_DB_HOST"},
Usage: "Command separated list of hostnames for yugabyte cluster",
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-name",
EnvVars: []string{"LOTUS_DB_NAME"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-user",
EnvVars: []string{"LOTUS_DB_USER"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-password",
EnvVars: []string{"LOTUS_DB_PASSWORD"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-port",
EnvVars: []string{"LOTUS_DB_PORT"},
Hidden: true,
Value: "~/.lotus",
Value: "5433",
},
&cli.StringFlag{
Name: "layers",
EnvVars: []string{"LOTUS_LAYERS"},
Value: "base",
},
cliutil.FlagVeryVerbose,
},

58
cmd/lotus-provider/run.go
View File

@ -54,37 +54,6 @@ var runCmd = &cli.Command{
Usage: "manage open file limit",
Value: true,
},
&cli.StringFlag{
Name: "db-host",
EnvVars: []string{"LOTUS_DB_HOST"},
Usage: "Command separated list of hostnames for yugabyte cluster",
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-name",
EnvVars: []string{"LOTUS_DB_NAME"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-user",
EnvVars: []string{"LOTUS_DB_USER"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-password",
EnvVars: []string{"LOTUS_DB_PASSWORD"},
Value: "yugabyte",
},
&cli.StringFlag{
Name: "db-port",
EnvVars: []string{"LOTUS_DB_PORT"},
Hidden: true,
Value: "5433",
},
&cli.StringFlag{
Name: FlagProviderRepo,
Value: "~/lotusminer",
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Bool("enable-gpu-proving") {
@ -159,18 +128,10 @@ var runCmd = &cli.Command{
}
}
dbConfig := config.HarmonyDB{
Username: cctx.String("db-user"),
Password: cctx.String("db-password"),
Hosts: strings.Split(cctx.String("db-host"), ","),
Database: cctx.String("db-name"),
Port: cctx.String("db-port"),
}
db, err := harmonydb.NewFromConfig(dbConfig)
db, err := makeDB(cctx)
if err != nil {
return err
}
shutdownChan := make(chan struct{})
/* defaults break lockedRepo (below)
@ -209,6 +170,11 @@ var runCmd = &cli.Command{
if err != nil {
return err
}
lp, err := getConfig(cctx, db)
if err != nil {
return err
}
_ = lp // here is where the config feeds into task runners
taskEngine, err := harmonytask.New(db, []harmonytask.TaskInterface{}, address)
if err != nil {
@ -256,3 +222,15 @@ var runCmd = &cli.Command{
return nil
},
}
func makeDB(cctx *cli.Context) (*harmonydb.DB, error) {
dbConfig := config.HarmonyDB{
Username: cctx.String("db-user"),
Password: cctx.String("db-password"),
Hosts: strings.Split(cctx.String("db-host"), ","),
Database: cctx.String("db-name"),
Port: cctx.String("db-port"),
}
return harmonydb.NewFromConfig(dbConfig)
}

3
go.mod
View File

@ -273,6 +273,8 @@ require (
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
@ -320,6 +322,7 @@ require (
github.com/quic-go/quic-go v0.33.0 // indirect
github.com/quic-go/webtransport-go v0.5.3 // indirect
github.com/rivo/uniseg v0.1.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil v2.18.12+incompatible // indirect

2
go.sum
View File

@ -984,6 +984,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@ -1531,6 +1532,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=

5
lib/harmony/harmonydb/sql/20230919.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE harmony_config (
id SERIAL PRIMARY KEY NOT NULL,
title VARCHAR(300) UNIQUE NOT NULL,
config TEXT NOT NULL
);

43
node/config/def.go
View File

@ -343,3 +343,46 @@ func DefaultUserRaftConfig() *UserRaftConfig {
return &cfg
}
func DefaultLotusProvider() *LotusProviderConfig {
/*
reqs := map[string]*regexp.Regexp{}
for _, str := range LotusRequired {
reqs[str]=regexp.MustCompile("(?i)"+str)
}
*/
return &LotusProviderConfig{
Fees: LotusProviderFees{
DefaultMaxFee: DefaultDefaultMaxFee,
MaxPreCommitGasFee: types.MustParseFIL("0.025"),
MaxCommitGasFee: types.MustParseFIL("0.05"),
MaxPreCommitBatchGasFee: BatchFeeConfig{
Base: types.MustParseFIL("0"),
PerSector: types.MustParseFIL("0.02"),
},
MaxCommitBatchGasFee: BatchFeeConfig{
Base: types.MustParseFIL("0"),
PerSector: types.MustParseFIL("0.03"), // enough for 6 agg and 1nFIL base fee
},
MaxTerminateGasFee: types.MustParseFIL("0.5"),
MaxWindowPoStGasFee: types.MustParseFIL("5"),
MaxPublishDealsFee: types.MustParseFIL("0.05"),
},
Addresses: LotusProviderAddresses{
PreCommitControl: []string{},
CommitControl: []string{},
TerminateControl: []string{},
},
/*
HarmonyDB: HarmonyDB{
Hosts: []string{"127.0.0.1"},
Username: "yugabyte",
Password: "yugabyte",
Database: "yugabyte",
Port: "5433",
},
*/
}
}

39
node/config/types.go
View File

@ -65,6 +65,14 @@ type StorageMiner struct {
HarmonyDB HarmonyDB
}
type LotusProviderConfig struct {
EnableWindowPost bool
EnableWinningPost bool
Fees LotusProviderFees
Addresses LotusProviderAddresses
}
type DAGStoreConfig struct {
// Path to the dagstore root directory. This directory contains three
// subdirectories, which can be symlinked to alternative locations if
@ -499,6 +507,20 @@ type MinerFeeConfig struct {
MaxMarketBalanceAddFee types.FIL
}
type LotusProviderFees struct {
DefaultMaxFee types.FIL
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
// maxBatchFee = maxBase + maxPerSector * nSectors
MaxPreCommitBatchGasFee BatchFeeConfig
MaxCommitBatchGasFee BatchFeeConfig
MaxTerminateGasFee types.FIL
// WindowPoSt is a high-value operation, so the default fee should be high.
MaxWindowPoStGasFee types.FIL
MaxPublishDealsFee types.FIL
}
type MinerAddressConfig struct {
// Addresses to send PreCommit messages from
PreCommitControl []string
@ -517,6 +539,23 @@ type MinerAddressConfig struct {
DisableWorkerFallback bool
}
type LotusProviderAddresses struct {
// Addresses to send PreCommit messages from
PreCommitControl []string
// Addresses to send Commit messages from
CommitControl []string
TerminateControl []string
// DisableOwnerFallback disables usage of the owner address for messages
// sent automatically
DisableOwnerFallback bool
// DisableWorkerFallback disables usage of the worker address for messages
// sent automatically, if control addresses are configured.
// A control address that doesn't have enough funds will still be chosen
// over the worker address if this flag is set.
DisableWorkerFallback bool
}
// API contains configs for API endpoint
type API struct {
// Binding address for the Lotus API