From cf65804883c34097c35d94220a39f97893a4fceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 13:09:03 +0200 Subject: [PATCH 01/11] lotus daemon config utils --- cmd/lotus/config.go | 162 ++++++++++++++++++++++++++++++++++++++++++++ cmd/lotus/main.go | 1 + 2 files changed, 163 insertions(+) create mode 100644 cmd/lotus/config.go diff --git a/cmd/lotus/config.go b/cmd/lotus/config.go new file mode 100644 index 000000000..0f97da0ee --- /dev/null +++ b/cmd/lotus/config.go @@ -0,0 +1,162 @@ +package main + +import ( + "bytes" + "fmt" + "github.com/filecoin-project/lotus/node/repo" + "reflect" + "strings" + + "github.com/BurntSushi/toml" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/node/config" +) + +var configCmd = &cli.Command{ + Name: "config", + Usage: "Manage node config", + Subcommands: []*cli.Command{ + configDefaultCmd, + configUpdateCmd, + }, +} + +var configDefaultCmd = &cli.Command{ + Name: "default", + Usage: "Print default node config", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-comment", + Usage: "don't comment default values", + }, + }, + Action: func(cctx *cli.Context) error { + c := config.DefaultFullNode() + + if cctx.Bool("no-comment") { + buf := new(bytes.Buffer) + _, _ = buf.WriteString("# Default config:\n") + e := toml.NewEncoder(buf) + if err := e.Encode(c); err != nil { + return xerrors.Errorf("encoding default config: %w", err) + } + + fmt.Println(buf.String()) + return nil + } + + cb, err := config.ConfigComment(c) + if err != nil { + return err + } + + fmt.Println(string(cb)) + + return nil + }, +} + +var configUpdateCmd = &cli.Command{ + Name: "updated", + Usage: "Print updated node config", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-comment", + Usage: "don't comment default values", + }, + }, + Action: func(cctx *cli.Context) error { + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + + if !ok { + return xerrors.Errorf("repo not initialized") + } + + lr, err := r.LockRO(repo.FullNode) + if err != nil { + return xerrors.Errorf("locking repo: %w", err) + } + + cfgNode, err := lr.Config() + if err != nil { + _ = lr.Close() + return xerrors.Errorf("getting node config: %w", err) + } + + if err := lr.Close(); err != nil { + return err + } + + cfgDef := config.DefaultFullNode() + + var nodeStr, defStr string + { + buf := new(bytes.Buffer) + e := toml.NewEncoder(buf) + if err := e.Encode(cfgDef); err != nil { + return xerrors.Errorf("encoding default config: %w", err) + } + + defStr = buf.String() + } + + { + buf := new(bytes.Buffer) + e := toml.NewEncoder(buf) + if err := e.Encode(cfgNode); err != nil { + return xerrors.Errorf("encoding node config: %w", err) + } + + nodeStr = buf.String() + } + + if !cctx.Bool("no-comment") { + defLines := strings.Split(defStr, "\n") + defaults := map[string]struct{}{} + for i := range defLines { + l := strings.TrimSpace(defLines[i]) + if len(l) == 0 { + continue + } + if l[0] == '#' || l[0] == '[' { + continue + } + defaults[l] = struct{}{} + } + + nodeLines := strings.Split(nodeStr, "\n") + for i := range nodeLines { + if _, found := defaults[strings.TrimSpace(nodeLines[i])]; found { + nodeLines[i] = "#" + nodeLines[i] + } + } + + nodeStr = strings.Join(nodeLines, "\n") + } + + // sanity-check that the updated config parses the same way as the current one + { + cfgUpdated, err := config.FromReader(strings.NewReader(nodeStr), config.DefaultFullNode()) + if err != nil { + return xerrors.Errorf("parsing updated config: %w", err) + } + + if !reflect.DeepEqual(cfgNode, cfgUpdated) { + return xerrors.Errorf("updated config didn't match current config") + } + } + + fmt.Println(nodeStr) + return nil + }, +} diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 63d01f891..d803cce1e 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -29,6 +29,7 @@ func main() { local := []*cli.Command{ DaemonCmd, backupCmd, + configCmd, } if AdvanceBlockCmd != nil { local = append(local, AdvanceBlockCmd) From 97f5bb66c7ccb78ac26de09cda19efad53793c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 13:55:50 +0200 Subject: [PATCH 02/11] config: doc struct codegen --- Makefile | 5 +- cmd/lotus/config.go | 4 +- documentation/en/cli-lotus.md | 48 +++ node/config/cfgdocgen/gen.go | 131 +++++++ node/config/def.go | 254 -------------- node/config/doc_gen.go | 633 ++++++++++++++++++++++++++++++++++ node/config/types.go | 264 ++++++++++++++ 7 files changed, 1082 insertions(+), 257 deletions(-) create mode 100644 node/config/cfgdocgen/gen.go create mode 100644 node/config/doc_gen.go create mode 100644 node/config/types.go diff --git a/Makefile b/Makefile index a5ce8a99f..2e9fa7459 100644 --- a/Makefile +++ b/Makefile @@ -336,6 +336,9 @@ api-gen: goimports -w api .PHONY: api-gen +cfgdoc-gen: + go run ./node/config/cfgdocgen > ./node/config/doc_gen.go + appimage: lotus rm -rf appimage-builder-cache || true rm AppDir/io.filecoin.lotus.desktop || true @@ -373,7 +376,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin -gen: actors-gen type-gen method-gen docsgen api-gen circleci +gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci @echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli" .PHONY: gen diff --git a/cmd/lotus/config.go b/cmd/lotus/config.go index 0f97da0ee..2e035b315 100644 --- a/cmd/lotus/config.go +++ b/cmd/lotus/config.go @@ -28,7 +28,7 @@ var configDefaultCmd = &cli.Command{ Usage: "Print default node config", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "no-comment", + Name: "no-comment", Usage: "don't comment default values", }, }, @@ -63,7 +63,7 @@ var configUpdateCmd = &cli.Command{ Usage: "Print updated node config", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "no-comment", + Name: "no-comment", Usage: "don't comment default values", }, }, diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 6971ed6e7..cae648a0d 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -12,6 +12,7 @@ VERSION: COMMANDS: daemon Start a lotus daemon process backup Create node metadata backup + config Manage node config version Print version help, h Shows a list of commands or help for one command BASIC: @@ -108,6 +109,53 @@ OPTIONS: ``` +## lotus config +``` +NAME: + lotus config - Manage node config + +USAGE: + lotus config command [command options] [arguments...] + +COMMANDS: + default Print default node config + updated Print updated node config + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help (default: false) + --version, -v print the version (default: false) + +``` + +### lotus config default +``` +NAME: + lotus config default - Print default node config + +USAGE: + lotus config default [command options] [arguments...] + +OPTIONS: + --no-comment don't comment default values (default: false) + --help, -h show help (default: false) + +``` + +### lotus config updated +``` +NAME: + lotus config updated - Print updated node config + +USAGE: + lotus config updated [command options] [arguments...] + +OPTIONS: + --no-comment don't comment default values (default: false) + --help, -h show help (default: false) + +``` + ## lotus version ``` NAME: diff --git a/node/config/cfgdocgen/gen.go b/node/config/cfgdocgen/gen.go new file mode 100644 index 000000000..81159a486 --- /dev/null +++ b/node/config/cfgdocgen/gen.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "sort" + "strings" +) + +func run() error { + tfb, err := ioutil.ReadFile("./node/config/types.go") + if err != nil { + return err + } + + // could use the ast lib, but this is simpler + + type st int + const ( + stGlobal st = iota // looking for typedef + stType st = iota // in typedef + ) + + lines := strings.Split(string(tfb), "\n") + state := stGlobal + + type field struct { + Name string + Type string + Comment string + } + + var currentType string + var currentComment []string + + out := map[string][]field{} + + for l := range lines { + line := strings.TrimSpace(lines[l]) + + switch state { + case stGlobal: + if strings.HasPrefix(line, "type ") { + currentType = line + currentType = strings.TrimPrefix(currentType, "type") + currentType = strings.TrimSuffix(currentType, "{") + currentType = strings.TrimSpace(currentType) + currentType = strings.TrimSuffix(currentType, "struct") + currentType = strings.TrimSpace(currentType) + currentComment = nil + state = stType + continue + } + case stType: + if strings.HasPrefix(line, "// ") { + cline := strings.TrimSpace(strings.TrimPrefix(line, "//")) + currentComment = append(currentComment, cline) + continue + } + + comment := currentComment + currentComment = nil + + if strings.HasPrefix(line, "}") { + state = stGlobal + continue + } + + f := strings.Fields(line) + if len(f) < 2 { // empty or embedded struct + continue + } + + name := f[0] + typ := f[1] + + out[currentType] = append(out[currentType], field{ + Name: name, + Type: typ, + Comment: strings.Join(comment, "\n"), + }) + } + } + + var outt []string + for t := range out { + outt = append(outt, t) + } + sort.Strings(outt) + + fmt.Print(`// Code generated by github.com/filecoin-project/lotus/node/config/cfgdocgen. DO NOT EDIT. + +package config + +type DocField struct { + Name string + Type string + Comment string +} + +var Doc = map[string][]DocField{ +`) + + for _, typeName := range outt { + typ := out[typeName] + + fmt.Printf("\t\"%s\": []DocField{\n", typeName) + + for _, f := range typ { + fmt.Println("\t\t{") + fmt.Printf("\t\t\tName: \"%s\",\n", f.Name) + fmt.Printf("\t\t\tType: \"%s\",\n", f.Type) + fmt.Printf("\t\t\tComment: `%s`,\n", f.Comment) + fmt.Println("\t\t},") + } + + fmt.Printf("\t},\n") + } + + fmt.Println(`}`) + + return nil +} + +func main() { + if err := run(); err != nil { + fmt.Printf(err.Error()) + os.Exit(1) + } +} diff --git a/node/config/def.go b/node/config/def.go index 7042cf644..ef5dcd3ba 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -22,264 +22,10 @@ const ( RetrievalPricingExternalMode = "external" ) -// Common is common config between full node and miner -type Common struct { - API API - Backup Backup - Libp2p Libp2p - Pubsub Pubsub -} - -// FullNode is a full node config -type FullNode struct { - Common - Client Client - Metrics Metrics - Wallet Wallet - Fees FeeConfig - Chainstore Chainstore -} - -// // Common - -type Backup struct { - DisableMetadataLog bool -} - -// StorageMiner is a miner config -type StorageMiner struct { - Common - - Subsystems MinerSubsystemConfig - Dealmaking DealmakingConfig - Sealing SealingConfig - Storage sectorstorage.SealerConfig - Fees MinerFeeConfig - Addresses MinerAddressConfig -} - -type MinerSubsystemConfig struct { - EnableMining bool - EnableSealing bool - EnableSectorStorage bool - EnableMarkets bool - - SealerApiInfo string // if EnableSealing == false - SectorIndexApiInfo string // if EnableSectorStorage == false -} - -type DealmakingConfig struct { - ConsiderOnlineStorageDeals bool - ConsiderOfflineStorageDeals bool - ConsiderOnlineRetrievalDeals bool - ConsiderOfflineRetrievalDeals bool - ConsiderVerifiedStorageDeals bool - ConsiderUnverifiedStorageDeals bool - PieceCidBlocklist []cid.Cid - ExpectedSealDuration Duration - // Maximum amount of time proposed deal StartEpoch can be in future - MaxDealStartDelay Duration - // The amount of time to wait for more deals to arrive before - // publishing - PublishMsgPeriod Duration - // The maximum number of deals to include in a single PublishStorageDeals - // message - MaxDealsPerPublishMsg uint64 - // The maximum collateral that the provider will put up against a deal, - // as a multiplier of the minimum collateral bound - MaxProviderCollateralMultiplier uint64 - - // The maximum number of parallel online data transfers (storage+retrieval) - SimultaneousTransfers uint64 - - Filter string - RetrievalFilter string - - RetrievalPricing *RetrievalPricing -} - -type RetrievalPricing struct { - Strategy string // possible values: "default", "external" - - Default *RetrievalPricingDefault - External *RetrievalPricingExternal -} - -type RetrievalPricingExternal struct { - // Path of the external script that will be run to price a retrieval deal. - // This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external". - Path string -} - -type RetrievalPricingDefault struct { - // VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal - // of a payloadCid that belongs to a verified storage deal. - // This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default". - // default value is true - VerifiedDealsFreeTransfer bool -} - -type SealingConfig struct { - // 0 = no limit - MaxWaitDealsSectors uint64 - - // includes failed, 0 = no limit - MaxSealingSectors uint64 - - // includes failed, 0 = no limit - MaxSealingSectorsForDeals uint64 - - WaitDealsDelay Duration - - AlwaysKeepUnsealedCopy bool - - // Run sector finalization before submitting sector proof to the chain - FinalizeEarly bool - - // Whether to use available miner balance for sector collateral instead of sending it with each message - CollateralFromMinerBalance bool - // Minimum available balance to keep in the miner actor before sending it with messages - AvailableBalanceBuffer types.FIL - // Don't send collateral with messages even if there is no available balance in the miner actor - DisableCollateralFallback bool - - // enable / disable precommit batching (takes effect after nv13) - BatchPreCommits bool - // maximum precommit batch size - batches will be sent immediately above this size - MaxPreCommitBatch int - // how long to wait before submitting a batch after crossing the minimum batch size - PreCommitBatchWait Duration - // time buffer for forceful batch submission before sectors/deal in batch would start expiring - PreCommitBatchSlack Duration - - // enable / disable commit aggregation (takes effect after nv13) - AggregateCommits bool - // maximum batched commit size - batches will be sent immediately above this size - MinCommitBatch int - MaxCommitBatch int - // how long to wait before submitting a batch after crossing the minimum batch size - CommitBatchWait Duration - // time buffer for forceful batch submission before sectors/deals in batch would start expiring - CommitBatchSlack Duration - - // network BaseFee below which to stop doing commit aggregation, instead - // submitting proofs to the chain individually - AggregateAboveBaseFee types.FIL - - TerminateBatchMax uint64 - TerminateBatchMin uint64 - TerminateBatchWait Duration - - // Keep this many sectors in sealing pipeline, start CC if needed - // todo TargetSealingSectors uint64 - - // todo TargetSectors - stop auto-pleding new sectors after this many sectors are sealed, default CC upgrade for deals sectors if above -} - -type BatchFeeConfig struct { - Base types.FIL - PerSector types.FIL -} - func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount { return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector))) } -type MinerFeeConfig struct { - MaxPreCommitGasFee types.FIL - MaxCommitGasFee types.FIL - - // maxBatchFee = maxBase + maxPerSector * nSectors - MaxPreCommitBatchGasFee BatchFeeConfig - MaxCommitBatchGasFee BatchFeeConfig - - MaxTerminateGasFee types.FIL - MaxWindowPoStGasFee types.FIL - MaxPublishDealsFee types.FIL - MaxMarketBalanceAddFee types.FIL -} - -type MinerAddressConfig struct { - PreCommitControl []string - CommitControl []string - TerminateControl []string - DealPublishControl []string - - // DisableOwnerFallback disables usage of the owner address for messages - // sent automatically - DisableOwnerFallback bool - // DisableWorkerFallback disables usage of the worker address for messages - // sent automatically, if control addresses are configured. - // A control address that doesn't have enough funds will still be chosen - // over the worker address if this flag is set. - DisableWorkerFallback bool -} - -// API contains configs for API endpoint -type API struct { - ListenAddress string - RemoteListenAddress string - Timeout Duration -} - -// Libp2p contains configs for libp2p -type Libp2p struct { - ListenAddresses []string - AnnounceAddresses []string - NoAnnounceAddresses []string - BootstrapPeers []string - ProtectedPeers []string - - ConnMgrLow uint - ConnMgrHigh uint - ConnMgrGrace Duration -} - -type Pubsub struct { - Bootstrapper bool - DirectPeers []string - IPColocationWhitelist []string - RemoteTracer string -} - -type Chainstore struct { - EnableSplitstore bool - Splitstore Splitstore -} - -type Splitstore struct { - ColdStoreType string - HotStoreType string - MarkSetType string - - HotStoreMessageRetention uint64 -} - -// // Full Node - -type Metrics struct { - Nickname string - HeadNotifs bool -} - -type Client struct { - UseIpfs bool - IpfsOnlineMode bool - IpfsMAddr string - IpfsUseForRetrieval bool - SimultaneousTransfers uint64 -} - -type Wallet struct { - RemoteBackend string - EnableLedger bool - DisableLocal bool -} - -type FeeConfig struct { - DefaultMaxFee types.FIL -} - func defCommon() Common { return Common{ API: API{ diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go new file mode 100644 index 000000000..f8e00e895 --- /dev/null +++ b/node/config/doc_gen.go @@ -0,0 +1,633 @@ +// Code generated by github.com/filecoin-project/lotus/node/config/cfgdocgen. DO NOT EDIT. + +package config + +type DocField struct { + Name string + Type string + Comment string +} + +var Doc = map[string][]DocField{ + "API": []DocField{ + { + Name: "ListenAddress", + Type: "string", + Comment: ``, + }, + { + Name: "RemoteListenAddress", + Type: "string", + Comment: ``, + }, + { + Name: "Timeout", + Type: "Duration", + Comment: ``, + }, + }, + "Backup": []DocField{ + { + Name: "DisableMetadataLog", + Type: "bool", + Comment: ``, + }, + }, + "BatchFeeConfig": []DocField{ + { + Name: "Base", + Type: "types.FIL", + Comment: ``, + }, + { + Name: "PerSector", + Type: "types.FIL", + Comment: ``, + }, + }, + "Chainstore": []DocField{ + { + Name: "EnableSplitstore", + Type: "bool", + Comment: ``, + }, + { + Name: "Splitstore", + Type: "Splitstore", + Comment: ``, + }, + }, + "Client": []DocField{ + { + Name: "UseIpfs", + Type: "bool", + Comment: ``, + }, + { + Name: "IpfsOnlineMode", + Type: "bool", + Comment: ``, + }, + { + Name: "IpfsMAddr", + Type: "string", + Comment: ``, + }, + { + Name: "IpfsUseForRetrieval", + Type: "bool", + Comment: ``, + }, + { + Name: "SimultaneousTransfers", + Type: "uint64", + Comment: ``, + }, + }, + "Common": []DocField{ + { + Name: "API", + Type: "API", + Comment: ``, + }, + { + Name: "Backup", + Type: "Backup", + Comment: ``, + }, + { + Name: "Libp2p", + Type: "Libp2p", + Comment: ``, + }, + { + Name: "Pubsub", + Type: "Pubsub", + Comment: ``, + }, + }, + "DealmakingConfig": []DocField{ + { + Name: "ConsiderOnlineStorageDeals", + Type: "bool", + Comment: ``, + }, + { + Name: "ConsiderOfflineStorageDeals", + Type: "bool", + Comment: ``, + }, + { + Name: "ConsiderOnlineRetrievalDeals", + Type: "bool", + Comment: ``, + }, + { + Name: "ConsiderOfflineRetrievalDeals", + Type: "bool", + Comment: ``, + }, + { + Name: "ConsiderVerifiedStorageDeals", + Type: "bool", + Comment: ``, + }, + { + Name: "ConsiderUnverifiedStorageDeals", + Type: "bool", + Comment: ``, + }, + { + Name: "PieceCidBlocklist", + Type: "[]cid.Cid", + Comment: ``, + }, + { + Name: "ExpectedSealDuration", + Type: "Duration", + Comment: ``, + }, + { + Name: "MaxDealStartDelay", + Type: "Duration", + Comment: `Maximum amount of time proposed deal StartEpoch can be in future`, + }, + { + Name: "PublishMsgPeriod", + Type: "Duration", + Comment: `The amount of time to wait for more deals to arrive before +publishing`, + }, + { + Name: "MaxDealsPerPublishMsg", + Type: "uint64", + Comment: `The maximum number of deals to include in a single PublishStorageDeals +message`, + }, + { + Name: "MaxProviderCollateralMultiplier", + Type: "uint64", + Comment: `The maximum collateral that the provider will put up against a deal, +as a multiplier of the minimum collateral bound`, + }, + { + Name: "SimultaneousTransfers", + Type: "uint64", + Comment: `The maximum number of parallel online data transfers (storage+retrieval)`, + }, + { + Name: "Filter", + Type: "string", + Comment: ``, + }, + { + Name: "RetrievalFilter", + Type: "string", + Comment: ``, + }, + { + Name: "RetrievalPricing", + Type: "*RetrievalPricing", + Comment: ``, + }, + }, + "FeeConfig": []DocField{ + { + Name: "DefaultMaxFee", + Type: "types.FIL", + Comment: ``, + }, + }, + "FullNode": []DocField{ + { + Name: "Client", + Type: "Client", + Comment: ``, + }, + { + Name: "Metrics", + Type: "Metrics", + Comment: ``, + }, + { + Name: "Wallet", + Type: "Wallet", + Comment: ``, + }, + { + Name: "Fees", + Type: "FeeConfig", + Comment: ``, + }, + { + Name: "Chainstore", + Type: "Chainstore", + Comment: ``, + }, + }, + "Libp2p": []DocField{ + { + Name: "ListenAddresses", + Type: "[]string", + Comment: ``, + }, + { + Name: "AnnounceAddresses", + Type: "[]string", + Comment: ``, + }, + { + Name: "NoAnnounceAddresses", + Type: "[]string", + Comment: ``, + }, + { + Name: "BootstrapPeers", + Type: "[]string", + Comment: ``, + }, + { + Name: "ProtectedPeers", + Type: "[]string", + Comment: ``, + }, + { + Name: "ConnMgrLow", + Type: "uint", + Comment: ``, + }, + { + Name: "ConnMgrHigh", + Type: "uint", + Comment: ``, + }, + { + Name: "ConnMgrGrace", + Type: "Duration", + Comment: ``, + }, + }, + "Metrics": []DocField{ + { + Name: "Nickname", + Type: "string", + Comment: ``, + }, + { + Name: "HeadNotifs", + Type: "bool", + Comment: ``, + }, + }, + "MinerAddressConfig": []DocField{ + { + Name: "PreCommitControl", + Type: "[]string", + Comment: ``, + }, + { + Name: "CommitControl", + Type: "[]string", + Comment: ``, + }, + { + Name: "TerminateControl", + Type: "[]string", + Comment: ``, + }, + { + Name: "DealPublishControl", + Type: "[]string", + Comment: ``, + }, + { + Name: "DisableOwnerFallback", + Type: "bool", + Comment: `DisableOwnerFallback disables usage of the owner address for messages +sent automatically`, + }, + { + Name: "DisableWorkerFallback", + Type: "bool", + Comment: `DisableWorkerFallback disables usage of the worker address for messages +sent automatically, if control addresses are configured. +A control address that doesn't have enough funds will still be chosen +over the worker address if this flag is set.`, + }, + }, + "MinerFeeConfig": []DocField{ + { + Name: "MaxPreCommitGasFee", + Type: "types.FIL", + Comment: ``, + }, + { + Name: "MaxCommitGasFee", + Type: "types.FIL", + Comment: ``, + }, + { + Name: "MaxPreCommitBatchGasFee", + Type: "BatchFeeConfig", + Comment: `maxBatchFee = maxBase + maxPerSector * nSectors`, + }, + { + Name: "MaxCommitBatchGasFee", + Type: "BatchFeeConfig", + Comment: ``, + }, + { + Name: "MaxTerminateGasFee", + Type: "types.FIL", + Comment: ``, + }, + { + Name: "MaxWindowPoStGasFee", + Type: "types.FIL", + Comment: ``, + }, + { + Name: "MaxPublishDealsFee", + Type: "types.FIL", + Comment: ``, + }, + { + Name: "MaxMarketBalanceAddFee", + Type: "types.FIL", + Comment: ``, + }, + }, + "MinerSubsystemConfig": []DocField{ + { + Name: "EnableMining", + Type: "bool", + Comment: ``, + }, + { + Name: "EnableSealing", + Type: "bool", + Comment: ``, + }, + { + Name: "EnableSectorStorage", + Type: "bool", + Comment: ``, + }, + { + Name: "EnableMarkets", + Type: "bool", + Comment: ``, + }, + { + Name: "SealerApiInfo", + Type: "string", + Comment: ``, + }, + { + Name: "SectorIndexApiInfo", + Type: "string", + Comment: ``, + }, + }, + "Pubsub": []DocField{ + { + Name: "Bootstrapper", + Type: "bool", + Comment: ``, + }, + { + Name: "DirectPeers", + Type: "[]string", + Comment: ``, + }, + { + Name: "IPColocationWhitelist", + Type: "[]string", + Comment: ``, + }, + { + Name: "RemoteTracer", + Type: "string", + Comment: ``, + }, + }, + "RetrievalPricing": []DocField{ + { + Name: "Strategy", + Type: "string", + Comment: ``, + }, + { + Name: "Default", + Type: "*RetrievalPricingDefault", + Comment: ``, + }, + { + Name: "External", + Type: "*RetrievalPricingExternal", + Comment: ``, + }, + }, + "RetrievalPricingDefault": []DocField{ + { + Name: "VerifiedDealsFreeTransfer", + Type: "bool", + Comment: `VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal +of a payloadCid that belongs to a verified storage deal. +This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default". +default value is true`, + }, + }, + "RetrievalPricingExternal": []DocField{ + { + Name: "Path", + Type: "string", + Comment: `Path of the external script that will be run to price a retrieval deal. +This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".`, + }, + }, + "SealingConfig": []DocField{ + { + Name: "MaxWaitDealsSectors", + Type: "uint64", + Comment: `0 = no limit`, + }, + { + Name: "MaxSealingSectors", + Type: "uint64", + Comment: `includes failed, 0 = no limit`, + }, + { + Name: "MaxSealingSectorsForDeals", + Type: "uint64", + Comment: `includes failed, 0 = no limit`, + }, + { + Name: "WaitDealsDelay", + Type: "Duration", + Comment: ``, + }, + { + Name: "AlwaysKeepUnsealedCopy", + Type: "bool", + Comment: ``, + }, + { + Name: "FinalizeEarly", + Type: "bool", + Comment: `Run sector finalization before submitting sector proof to the chain`, + }, + { + Name: "CollateralFromMinerBalance", + Type: "bool", + Comment: `Whether to use available miner balance for sector collateral instead of sending it with each message`, + }, + { + Name: "AvailableBalanceBuffer", + Type: "types.FIL", + Comment: `Minimum available balance to keep in the miner actor before sending it with messages`, + }, + { + Name: "DisableCollateralFallback", + Type: "bool", + Comment: `Don't send collateral with messages even if there is no available balance in the miner actor`, + }, + { + Name: "BatchPreCommits", + Type: "bool", + Comment: `enable / disable precommit batching (takes effect after nv13)`, + }, + { + Name: "MaxPreCommitBatch", + Type: "int", + Comment: `maximum precommit batch size - batches will be sent immediately above this size`, + }, + { + Name: "PreCommitBatchWait", + Type: "Duration", + Comment: `how long to wait before submitting a batch after crossing the minimum batch size`, + }, + { + Name: "PreCommitBatchSlack", + Type: "Duration", + Comment: `time buffer for forceful batch submission before sectors/deal in batch would start expiring`, + }, + { + Name: "AggregateCommits", + Type: "bool", + Comment: `enable / disable commit aggregation (takes effect after nv13)`, + }, + { + Name: "MinCommitBatch", + Type: "int", + Comment: `maximum batched commit size - batches will be sent immediately above this size`, + }, + { + Name: "MaxCommitBatch", + Type: "int", + Comment: ``, + }, + { + Name: "CommitBatchWait", + Type: "Duration", + Comment: `how long to wait before submitting a batch after crossing the minimum batch size`, + }, + { + Name: "CommitBatchSlack", + Type: "Duration", + Comment: `time buffer for forceful batch submission before sectors/deals in batch would start expiring`, + }, + { + Name: "AggregateAboveBaseFee", + Type: "types.FIL", + Comment: `network BaseFee below which to stop doing commit aggregation, instead +submitting proofs to the chain individually`, + }, + { + Name: "TerminateBatchMax", + Type: "uint64", + Comment: ``, + }, + { + Name: "TerminateBatchMin", + Type: "uint64", + Comment: ``, + }, + { + Name: "TerminateBatchWait", + Type: "Duration", + Comment: ``, + }, + }, + "Splitstore": []DocField{ + { + Name: "ColdStoreType", + Type: "string", + Comment: ``, + }, + { + Name: "HotStoreType", + Type: "string", + Comment: ``, + }, + { + Name: "MarkSetType", + Type: "string", + Comment: ``, + }, + { + Name: "HotStoreMessageRetention", + Type: "uint64", + Comment: ``, + }, + }, + "StorageMiner": []DocField{ + { + Name: "Subsystems", + Type: "MinerSubsystemConfig", + Comment: ``, + }, + { + Name: "Dealmaking", + Type: "DealmakingConfig", + Comment: ``, + }, + { + Name: "Sealing", + Type: "SealingConfig", + Comment: ``, + }, + { + Name: "Storage", + Type: "sectorstorage.SealerConfig", + Comment: ``, + }, + { + Name: "Fees", + Type: "MinerFeeConfig", + Comment: ``, + }, + { + Name: "Addresses", + Type: "MinerAddressConfig", + Comment: ``, + }, + }, + "Wallet": []DocField{ + { + Name: "RemoteBackend", + Type: "string", + Comment: ``, + }, + { + Name: "EnableLedger", + Type: "bool", + Comment: ``, + }, + { + Name: "DisableLocal", + Type: "bool", + Comment: ``, + }, + }, +} diff --git a/node/config/types.go b/node/config/types.go new file mode 100644 index 000000000..7a2b26250 --- /dev/null +++ b/node/config/types.go @@ -0,0 +1,264 @@ +package config + +import ( + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" + sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" +) + +// // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE + +// Common is common config between full node and miner +type Common struct { + API API + Backup Backup + Libp2p Libp2p + Pubsub Pubsub +} + +// FullNode is a full node config +type FullNode struct { + Common + Client Client + Metrics Metrics + Wallet Wallet + Fees FeeConfig + Chainstore Chainstore +} + +// // Common + +type Backup struct { + DisableMetadataLog bool +} + +// StorageMiner is a miner config +type StorageMiner struct { + Common + + Subsystems MinerSubsystemConfig + Dealmaking DealmakingConfig + Sealing SealingConfig + Storage sectorstorage.SealerConfig + Fees MinerFeeConfig + Addresses MinerAddressConfig +} + +type MinerSubsystemConfig struct { + EnableMining bool + EnableSealing bool + EnableSectorStorage bool + EnableMarkets bool + + SealerApiInfo string // if EnableSealing == false + SectorIndexApiInfo string // if EnableSectorStorage == false +} + +type DealmakingConfig struct { + ConsiderOnlineStorageDeals bool + ConsiderOfflineStorageDeals bool + ConsiderOnlineRetrievalDeals bool + ConsiderOfflineRetrievalDeals bool + ConsiderVerifiedStorageDeals bool + ConsiderUnverifiedStorageDeals bool + PieceCidBlocklist []cid.Cid + ExpectedSealDuration Duration + // Maximum amount of time proposed deal StartEpoch can be in future + MaxDealStartDelay Duration + // The amount of time to wait for more deals to arrive before + // publishing + PublishMsgPeriod Duration + // The maximum number of deals to include in a single PublishStorageDeals + // message + MaxDealsPerPublishMsg uint64 + // The maximum collateral that the provider will put up against a deal, + // as a multiplier of the minimum collateral bound + MaxProviderCollateralMultiplier uint64 + + // The maximum number of parallel online data transfers (storage+retrieval) + SimultaneousTransfers uint64 + + Filter string + RetrievalFilter string + + RetrievalPricing *RetrievalPricing +} + +type RetrievalPricing struct { + Strategy string // possible values: "default", "external" + + Default *RetrievalPricingDefault + External *RetrievalPricingExternal +} + +type RetrievalPricingExternal struct { + // Path of the external script that will be run to price a retrieval deal. + // This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external". + Path string +} + +type RetrievalPricingDefault struct { + // VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal + // of a payloadCid that belongs to a verified storage deal. + // This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default". + // default value is true + VerifiedDealsFreeTransfer bool +} + +type SealingConfig struct { + // 0 = no limit + MaxWaitDealsSectors uint64 + + // includes failed, 0 = no limit + MaxSealingSectors uint64 + + // includes failed, 0 = no limit + MaxSealingSectorsForDeals uint64 + + WaitDealsDelay Duration + + AlwaysKeepUnsealedCopy bool + + // Run sector finalization before submitting sector proof to the chain + FinalizeEarly bool + + // Whether to use available miner balance for sector collateral instead of sending it with each message + CollateralFromMinerBalance bool + // Minimum available balance to keep in the miner actor before sending it with messages + AvailableBalanceBuffer types.FIL + // Don't send collateral with messages even if there is no available balance in the miner actor + DisableCollateralFallback bool + + // enable / disable precommit batching (takes effect after nv13) + BatchPreCommits bool + // maximum precommit batch size - batches will be sent immediately above this size + MaxPreCommitBatch int + // how long to wait before submitting a batch after crossing the minimum batch size + PreCommitBatchWait Duration + // time buffer for forceful batch submission before sectors/deal in batch would start expiring + PreCommitBatchSlack Duration + + // enable / disable commit aggregation (takes effect after nv13) + AggregateCommits bool + // maximum batched commit size - batches will be sent immediately above this size + MinCommitBatch int + MaxCommitBatch int + // how long to wait before submitting a batch after crossing the minimum batch size + CommitBatchWait Duration + // time buffer for forceful batch submission before sectors/deals in batch would start expiring + CommitBatchSlack Duration + + // network BaseFee below which to stop doing commit aggregation, instead + // submitting proofs to the chain individually + AggregateAboveBaseFee types.FIL + + TerminateBatchMax uint64 + TerminateBatchMin uint64 + TerminateBatchWait Duration + + // Keep this many sectors in sealing pipeline, start CC if needed + // todo TargetSealingSectors uint64 + + // todo TargetSectors - stop auto-pleding new sectors after this many sectors are sealed, default CC upgrade for deals sectors if above +} + +type BatchFeeConfig struct { + Base types.FIL + PerSector types.FIL +} + +type MinerFeeConfig struct { + MaxPreCommitGasFee types.FIL + MaxCommitGasFee types.FIL + + // maxBatchFee = maxBase + maxPerSector * nSectors + MaxPreCommitBatchGasFee BatchFeeConfig + MaxCommitBatchGasFee BatchFeeConfig + + MaxTerminateGasFee types.FIL + MaxWindowPoStGasFee types.FIL + MaxPublishDealsFee types.FIL + MaxMarketBalanceAddFee types.FIL +} + +type MinerAddressConfig struct { + PreCommitControl []string + CommitControl []string + TerminateControl []string + DealPublishControl []string + + // DisableOwnerFallback disables usage of the owner address for messages + // sent automatically + DisableOwnerFallback bool + // DisableWorkerFallback disables usage of the worker address for messages + // sent automatically, if control addresses are configured. + // A control address that doesn't have enough funds will still be chosen + // over the worker address if this flag is set. + DisableWorkerFallback bool +} + +// API contains configs for API endpoint +type API struct { + ListenAddress string + RemoteListenAddress string + Timeout Duration +} + +// Libp2p contains configs for libp2p +type Libp2p struct { + ListenAddresses []string + AnnounceAddresses []string + NoAnnounceAddresses []string + BootstrapPeers []string + ProtectedPeers []string + + ConnMgrLow uint + ConnMgrHigh uint + ConnMgrGrace Duration +} + +type Pubsub struct { + Bootstrapper bool + DirectPeers []string + IPColocationWhitelist []string + RemoteTracer string +} + +type Chainstore struct { + EnableSplitstore bool + Splitstore Splitstore +} + +type Splitstore struct { + ColdStoreType string + HotStoreType string + MarkSetType string + + HotStoreMessageRetention uint64 +} + +// // Full Node + +type Metrics struct { + Nickname string + HeadNotifs bool +} + +type Client struct { + UseIpfs bool + IpfsOnlineMode bool + IpfsMAddr string + IpfsUseForRetrieval bool + SimultaneousTransfers uint64 +} + +type Wallet struct { + RemoteBackend string + EnableLedger bool + DisableLocal bool +} + +type FeeConfig struct { + DefaultMaxFee types.FIL +} From 8b1f19e94cf8251ad91ce196f63f959e800217b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 14:55:19 +0200 Subject: [PATCH 03/11] Move doc-comment logic to the config pkg --- cmd/lotus/config.go | 79 ++----------------------- node/config/doc_gen.go | 2 +- node/config/load.go | 130 +++++++++++++++++++++++++++++++++++++---- node/config/types.go | 1 + 4 files changed, 127 insertions(+), 85 deletions(-) diff --git a/cmd/lotus/config.go b/cmd/lotus/config.go index 2e035b315..c965f2fc8 100644 --- a/cmd/lotus/config.go +++ b/cmd/lotus/config.go @@ -1,13 +1,8 @@ package main import ( - "bytes" "fmt" "github.com/filecoin-project/lotus/node/repo" - "reflect" - "strings" - - "github.com/BurntSushi/toml" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -35,19 +30,7 @@ var configDefaultCmd = &cli.Command{ Action: func(cctx *cli.Context) error { c := config.DefaultFullNode() - if cctx.Bool("no-comment") { - buf := new(bytes.Buffer) - _, _ = buf.WriteString("# Default config:\n") - e := toml.NewEncoder(buf) - if err := e.Encode(c); err != nil { - return xerrors.Errorf("encoding default config: %w", err) - } - - fmt.Println(buf.String()) - return nil - } - - cb, err := config.ConfigComment(c) + cb, err := config.ConfigUpdate(c, nil, !cctx.Bool("no-comment")) if err != nil { return err } @@ -99,64 +82,12 @@ var configUpdateCmd = &cli.Command{ cfgDef := config.DefaultFullNode() - var nodeStr, defStr string - { - buf := new(bytes.Buffer) - e := toml.NewEncoder(buf) - if err := e.Encode(cfgDef); err != nil { - return xerrors.Errorf("encoding default config: %w", err) - } - - defStr = buf.String() + updated, err := config.ConfigUpdate(cfgNode, cfgDef, !cctx.Bool("no-comment")) + if err != nil { + return err } - { - buf := new(bytes.Buffer) - e := toml.NewEncoder(buf) - if err := e.Encode(cfgNode); err != nil { - return xerrors.Errorf("encoding node config: %w", err) - } - - nodeStr = buf.String() - } - - if !cctx.Bool("no-comment") { - defLines := strings.Split(defStr, "\n") - defaults := map[string]struct{}{} - for i := range defLines { - l := strings.TrimSpace(defLines[i]) - if len(l) == 0 { - continue - } - if l[0] == '#' || l[0] == '[' { - continue - } - defaults[l] = struct{}{} - } - - nodeLines := strings.Split(nodeStr, "\n") - for i := range nodeLines { - if _, found := defaults[strings.TrimSpace(nodeLines[i])]; found { - nodeLines[i] = "#" + nodeLines[i] - } - } - - nodeStr = strings.Join(nodeLines, "\n") - } - - // sanity-check that the updated config parses the same way as the current one - { - cfgUpdated, err := config.FromReader(strings.NewReader(nodeStr), config.DefaultFullNode()) - if err != nil { - return xerrors.Errorf("parsing updated config: %w", err) - } - - if !reflect.DeepEqual(cfgNode, cfgUpdated) { - return xerrors.Errorf("updated config didn't match current config") - } - } - - fmt.Println(nodeStr) + fmt.Print(string(updated)) return nil }, } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index f8e00e895..1e9801b65 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -13,7 +13,7 @@ var Doc = map[string][]DocField{ { Name: "ListenAddress", Type: "string", - Comment: ``, + Comment: `Binding address for the Lotus API`, }, { Name: "RemoteListenAddress", diff --git a/node/config/load.go b/node/config/load.go index 61e6e8f97..e3415888a 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -5,6 +5,10 @@ import ( "fmt" "io" "os" + "reflect" + "regexp" + "strings" + "unicode" "github.com/BurntSushi/toml" "github.com/kelseyhightower/envconfig" @@ -42,15 +46,121 @@ func FromReader(reader io.Reader, def interface{}) (interface{}, error) { return cfg, nil } -func ConfigComment(t interface{}) ([]byte, error) { - buf := new(bytes.Buffer) - _, _ = buf.WriteString("# Default config:\n") - e := toml.NewEncoder(buf) - if err := e.Encode(t); err != nil { - return nil, xerrors.Errorf("encoding config: %w", err) +func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) { + var nodeStr, defStr string + if cfgDef != nil { + buf := new(bytes.Buffer) + e := toml.NewEncoder(buf) + if err := e.Encode(cfgDef); err != nil { + return nil, xerrors.Errorf("encoding default config: %w", err) + } + + defStr = buf.String() } - b := buf.Bytes() - b = bytes.ReplaceAll(b, []byte("\n"), []byte("\n#")) - b = bytes.ReplaceAll(b, []byte("#["), []byte("[")) - return b, nil + + { + buf := new(bytes.Buffer) + e := toml.NewEncoder(buf) + if err := e.Encode(cfgCur); err != nil { + return nil, xerrors.Errorf("encoding node config: %w", err) + } + + nodeStr = buf.String() + } + + if comment { + // create a map of default lines so we can comment those out later + defLines := strings.Split(defStr, "\n") + defaults := map[string]struct{}{} + for i := range defLines { + l := strings.TrimSpace(defLines[i]) + if len(l) == 0 { + continue + } + if l[0] == '#' || l[0] == '[' { + continue + } + defaults[l] = struct{}{} + } + + nodeLines := strings.Split(nodeStr, "\n") + var outLines []string + + sectionRx := regexp.MustCompile(`[\[.]([^.]+)]`) + var section string + + for i, line := range nodeLines { + // if this is a section, track it + trimmed := strings.TrimSpace(line) + if len(trimmed) > 0 { + if trimmed[0] == '[' { + m := sectionRx.FindSubmatch([]byte(trimmed)) + if len(m) != 2 { + return nil, xerrors.Errorf("section didn't match (line %d)", i) + } + section = string(m[1]) + + // never comment sections + outLines = append(outLines, line) + continue + } + } + + pad := strings.Repeat(" ", len(line) - len(strings.TrimLeftFunc(line, unicode.IsSpace))) + + // see if we have docs for this field + { + lf := strings.Fields(line) + if len(lf) > 1 { + var doc *DocField + for _, df := range Doc[section] { + if df.Name == lf[0] { + doc = &df + break + } + } + + if doc != nil { + // found docfield, emit doc comment + if len(doc.Comment) > 0 { + for _, docLine := range strings.Split(doc.Comment, "\n") { + outLines = append(outLines, pad + "# " + docLine) + } + outLines = append(outLines, pad + "#") + } + + outLines = append(outLines, pad + "# type: " + doc.Type) + } + } + } + + // if there is the same line in the default config, comment it out it output + if _, found := defaults[strings.TrimSpace(nodeLines[i])]; (cfgDef == nil || found) && len(line) > 0 { + line = pad + "#" + line[len(pad):] + outLines = append(outLines, line, "") + } else { + outLines = append(outLines, line) + } + } + + nodeStr = strings.Join(outLines, "\n") + } + + // sanity-check that the updated config parses the same way as the current one + if cfgDef != nil { + cfgUpdated, err := FromReader(strings.NewReader(nodeStr), cfgDef) + if err != nil { + return nil, xerrors.Errorf("parsing updated config: %w", err) + } + + if !reflect.DeepEqual(cfgCur, cfgUpdated) { + return nil, xerrors.Errorf("updated config didn't match current config") + } + } + + return []byte(nodeStr), nil +} + +func ConfigComment(t interface{}) ([]byte, error) { + return ConfigUpdate(t, nil, true) } diff --git a/node/config/types.go b/node/config/types.go index 7a2b26250..84a631962 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -200,6 +200,7 @@ type MinerAddressConfig struct { // API contains configs for API endpoint type API struct { + // Binding address for the Lotus API ListenAddress string RemoteListenAddress string Timeout Duration From 4757b014bc4e7b8c0646811abe149fde43cdca87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 14:57:42 +0200 Subject: [PATCH 04/11] Nicer lotus-miner config commands --- cmd/lotus-storage-miner/config.go | 79 +++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/config.go b/cmd/lotus-storage-miner/config.go index e5e4fc4c4..30c012b9e 100644 --- a/cmd/lotus-storage-miner/config.go +++ b/cmd/lotus-storage-miner/config.go @@ -2,21 +2,92 @@ package main import ( "fmt" - + "github.com/filecoin-project/lotus/node/repo" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/config" ) var configCmd = &cli.Command{ Name: "config", - Usage: "Output default configuration", + Usage: "Manage node config", + Subcommands: []*cli.Command{ + configDefaultCmd, + configUpdateCmd, + }, +} + +var configDefaultCmd = &cli.Command{ + Name: "default", + Usage: "Print default node config", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-comment", + Usage: "don't comment default values", + }, + }, Action: func(cctx *cli.Context) error { - comm, err := config.ConfigComment(config.DefaultStorageMiner()) + c := config.DefaultStorageMiner() + + cb, err := config.ConfigUpdate(c, nil, !cctx.Bool("no-comment")) if err != nil { return err } - fmt.Println(string(comm)) + + fmt.Println(string(cb)) + + return nil + }, +} + +var configUpdateCmd = &cli.Command{ + Name: "updated", + Usage: "Print updated node config", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-comment", + Usage: "don't comment default values", + }, + }, + Action: func(cctx *cli.Context) error { + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + + if !ok { + return xerrors.Errorf("repo not initialized") + } + + lr, err := r.LockRO(repo.StorageMiner) + if err != nil { + return xerrors.Errorf("locking repo: %w", err) + } + + cfgNode, err := lr.Config() + if err != nil { + _ = lr.Close() + return xerrors.Errorf("getting node config: %w", err) + } + + if err := lr.Close(); err != nil { + return err + } + + cfgDef := config.DefaultStorageMiner() + + updated, err := config.ConfigUpdate(cfgNode, cfgDef, !cctx.Bool("no-comment")) + if err != nil { + return err + } + + fmt.Print(string(updated)) return nil }, } From 3f4973cb41db41345119606b067c28fc4a38c156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 15:16:07 +0200 Subject: [PATCH 05/11] config: Fix doc finding logic --- node/config/cfgdocgen/gen.go | 4 +- node/config/doc_gen.go | 452 +++++++++++++++++------------------ node/config/doc_util.go | 44 ++++ node/config/load.go | 18 +- 4 files changed, 278 insertions(+), 240 deletions(-) create mode 100644 node/config/doc_util.go diff --git a/node/config/cfgdocgen/gen.go b/node/config/cfgdocgen/gen.go index 81159a486..b89a384f5 100644 --- a/node/config/cfgdocgen/gen.go +++ b/node/config/cfgdocgen/gen.go @@ -109,8 +109,8 @@ var Doc = map[string][]DocField{ for _, f := range typ { fmt.Println("\t\t{") - fmt.Printf("\t\t\tName: \"%s\",\n", f.Name) - fmt.Printf("\t\t\tType: \"%s\",\n", f.Type) + fmt.Printf("\t\t\tName: \"%s\",\n", f.Name) + fmt.Printf("\t\t\tType: \"%s\",\n", f.Type) fmt.Printf("\t\t\tComment: `%s`,\n", f.Comment) fmt.Println("\t\t},") } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 1e9801b65..4972a5197 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -11,304 +11,304 @@ type DocField struct { var Doc = map[string][]DocField{ "API": []DocField{ { - Name: "ListenAddress", - Type: "string", + Name: "ListenAddress", + Type: "string", Comment: `Binding address for the Lotus API`, }, { - Name: "RemoteListenAddress", - Type: "string", + Name: "RemoteListenAddress", + Type: "string", Comment: ``, }, { - Name: "Timeout", - Type: "Duration", + Name: "Timeout", + Type: "Duration", Comment: ``, }, }, "Backup": []DocField{ { - Name: "DisableMetadataLog", - Type: "bool", + Name: "DisableMetadataLog", + Type: "bool", Comment: ``, }, }, "BatchFeeConfig": []DocField{ { - Name: "Base", - Type: "types.FIL", + Name: "Base", + Type: "types.FIL", Comment: ``, }, { - Name: "PerSector", - Type: "types.FIL", + Name: "PerSector", + Type: "types.FIL", Comment: ``, }, }, "Chainstore": []DocField{ { - Name: "EnableSplitstore", - Type: "bool", + Name: "EnableSplitstore", + Type: "bool", Comment: ``, }, { - Name: "Splitstore", - Type: "Splitstore", + Name: "Splitstore", + Type: "Splitstore", Comment: ``, }, }, "Client": []DocField{ { - Name: "UseIpfs", - Type: "bool", + Name: "UseIpfs", + Type: "bool", Comment: ``, }, { - Name: "IpfsOnlineMode", - Type: "bool", + Name: "IpfsOnlineMode", + Type: "bool", Comment: ``, }, { - Name: "IpfsMAddr", - Type: "string", + Name: "IpfsMAddr", + Type: "string", Comment: ``, }, { - Name: "IpfsUseForRetrieval", - Type: "bool", + Name: "IpfsUseForRetrieval", + Type: "bool", Comment: ``, }, { - Name: "SimultaneousTransfers", - Type: "uint64", + Name: "SimultaneousTransfers", + Type: "uint64", Comment: ``, }, }, "Common": []DocField{ { - Name: "API", - Type: "API", + Name: "API", + Type: "API", Comment: ``, }, { - Name: "Backup", - Type: "Backup", + Name: "Backup", + Type: "Backup", Comment: ``, }, { - Name: "Libp2p", - Type: "Libp2p", + Name: "Libp2p", + Type: "Libp2p", Comment: ``, }, { - Name: "Pubsub", - Type: "Pubsub", + Name: "Pubsub", + Type: "Pubsub", Comment: ``, }, }, "DealmakingConfig": []DocField{ { - Name: "ConsiderOnlineStorageDeals", - Type: "bool", + Name: "ConsiderOnlineStorageDeals", + Type: "bool", Comment: ``, }, { - Name: "ConsiderOfflineStorageDeals", - Type: "bool", + Name: "ConsiderOfflineStorageDeals", + Type: "bool", Comment: ``, }, { - Name: "ConsiderOnlineRetrievalDeals", - Type: "bool", + Name: "ConsiderOnlineRetrievalDeals", + Type: "bool", Comment: ``, }, { - Name: "ConsiderOfflineRetrievalDeals", - Type: "bool", + Name: "ConsiderOfflineRetrievalDeals", + Type: "bool", Comment: ``, }, { - Name: "ConsiderVerifiedStorageDeals", - Type: "bool", + Name: "ConsiderVerifiedStorageDeals", + Type: "bool", Comment: ``, }, { - Name: "ConsiderUnverifiedStorageDeals", - Type: "bool", + Name: "ConsiderUnverifiedStorageDeals", + Type: "bool", Comment: ``, }, { - Name: "PieceCidBlocklist", - Type: "[]cid.Cid", + Name: "PieceCidBlocklist", + Type: "[]cid.Cid", Comment: ``, }, { - Name: "ExpectedSealDuration", - Type: "Duration", + Name: "ExpectedSealDuration", + Type: "Duration", Comment: ``, }, { - Name: "MaxDealStartDelay", - Type: "Duration", + Name: "MaxDealStartDelay", + Type: "Duration", Comment: `Maximum amount of time proposed deal StartEpoch can be in future`, }, { - Name: "PublishMsgPeriod", - Type: "Duration", + Name: "PublishMsgPeriod", + Type: "Duration", Comment: `The amount of time to wait for more deals to arrive before publishing`, }, { - Name: "MaxDealsPerPublishMsg", - Type: "uint64", + Name: "MaxDealsPerPublishMsg", + Type: "uint64", Comment: `The maximum number of deals to include in a single PublishStorageDeals message`, }, { - Name: "MaxProviderCollateralMultiplier", - Type: "uint64", + Name: "MaxProviderCollateralMultiplier", + Type: "uint64", Comment: `The maximum collateral that the provider will put up against a deal, as a multiplier of the minimum collateral bound`, }, { - Name: "SimultaneousTransfers", - Type: "uint64", + Name: "SimultaneousTransfers", + Type: "uint64", Comment: `The maximum number of parallel online data transfers (storage+retrieval)`, }, { - Name: "Filter", - Type: "string", + Name: "Filter", + Type: "string", Comment: ``, }, { - Name: "RetrievalFilter", - Type: "string", + Name: "RetrievalFilter", + Type: "string", Comment: ``, }, { - Name: "RetrievalPricing", - Type: "*RetrievalPricing", + Name: "RetrievalPricing", + Type: "*RetrievalPricing", Comment: ``, }, }, "FeeConfig": []DocField{ { - Name: "DefaultMaxFee", - Type: "types.FIL", + Name: "DefaultMaxFee", + Type: "types.FIL", Comment: ``, }, }, "FullNode": []DocField{ { - Name: "Client", - Type: "Client", + Name: "Client", + Type: "Client", Comment: ``, }, { - Name: "Metrics", - Type: "Metrics", + Name: "Metrics", + Type: "Metrics", Comment: ``, }, { - Name: "Wallet", - Type: "Wallet", + Name: "Wallet", + Type: "Wallet", Comment: ``, }, { - Name: "Fees", - Type: "FeeConfig", + Name: "Fees", + Type: "FeeConfig", Comment: ``, }, { - Name: "Chainstore", - Type: "Chainstore", + Name: "Chainstore", + Type: "Chainstore", Comment: ``, }, }, "Libp2p": []DocField{ { - Name: "ListenAddresses", - Type: "[]string", + Name: "ListenAddresses", + Type: "[]string", Comment: ``, }, { - Name: "AnnounceAddresses", - Type: "[]string", + Name: "AnnounceAddresses", + Type: "[]string", Comment: ``, }, { - Name: "NoAnnounceAddresses", - Type: "[]string", + Name: "NoAnnounceAddresses", + Type: "[]string", Comment: ``, }, { - Name: "BootstrapPeers", - Type: "[]string", + Name: "BootstrapPeers", + Type: "[]string", Comment: ``, }, { - Name: "ProtectedPeers", - Type: "[]string", + Name: "ProtectedPeers", + Type: "[]string", Comment: ``, }, { - Name: "ConnMgrLow", - Type: "uint", + Name: "ConnMgrLow", + Type: "uint", Comment: ``, }, { - Name: "ConnMgrHigh", - Type: "uint", + Name: "ConnMgrHigh", + Type: "uint", Comment: ``, }, { - Name: "ConnMgrGrace", - Type: "Duration", + Name: "ConnMgrGrace", + Type: "Duration", Comment: ``, }, }, "Metrics": []DocField{ { - Name: "Nickname", - Type: "string", + Name: "Nickname", + Type: "string", Comment: ``, }, { - Name: "HeadNotifs", - Type: "bool", + Name: "HeadNotifs", + Type: "bool", Comment: ``, }, }, "MinerAddressConfig": []DocField{ { - Name: "PreCommitControl", - Type: "[]string", + Name: "PreCommitControl", + Type: "[]string", Comment: ``, }, { - Name: "CommitControl", - Type: "[]string", + Name: "CommitControl", + Type: "[]string", Comment: ``, }, { - Name: "TerminateControl", - Type: "[]string", + Name: "TerminateControl", + Type: "[]string", Comment: ``, }, { - Name: "DealPublishControl", - Type: "[]string", + Name: "DealPublishControl", + Type: "[]string", Comment: ``, }, { - Name: "DisableOwnerFallback", - Type: "bool", + Name: "DisableOwnerFallback", + Type: "bool", Comment: `DisableOwnerFallback disables usage of the owner address for messages sent automatically`, }, { - Name: "DisableWorkerFallback", - Type: "bool", + Name: "DisableWorkerFallback", + Type: "bool", Comment: `DisableWorkerFallback disables usage of the worker address for messages sent automatically, if control addresses are configured. A control address that doesn't have enough funds will still be chosen @@ -317,121 +317,121 @@ over the worker address if this flag is set.`, }, "MinerFeeConfig": []DocField{ { - Name: "MaxPreCommitGasFee", - Type: "types.FIL", + Name: "MaxPreCommitGasFee", + Type: "types.FIL", Comment: ``, }, { - Name: "MaxCommitGasFee", - Type: "types.FIL", + Name: "MaxCommitGasFee", + Type: "types.FIL", Comment: ``, }, { - Name: "MaxPreCommitBatchGasFee", - Type: "BatchFeeConfig", + Name: "MaxPreCommitBatchGasFee", + Type: "BatchFeeConfig", Comment: `maxBatchFee = maxBase + maxPerSector * nSectors`, }, { - Name: "MaxCommitBatchGasFee", - Type: "BatchFeeConfig", + Name: "MaxCommitBatchGasFee", + Type: "BatchFeeConfig", Comment: ``, }, { - Name: "MaxTerminateGasFee", - Type: "types.FIL", + Name: "MaxTerminateGasFee", + Type: "types.FIL", Comment: ``, }, { - Name: "MaxWindowPoStGasFee", - Type: "types.FIL", + Name: "MaxWindowPoStGasFee", + Type: "types.FIL", Comment: ``, }, { - Name: "MaxPublishDealsFee", - Type: "types.FIL", + Name: "MaxPublishDealsFee", + Type: "types.FIL", Comment: ``, }, { - Name: "MaxMarketBalanceAddFee", - Type: "types.FIL", + Name: "MaxMarketBalanceAddFee", + Type: "types.FIL", Comment: ``, }, }, "MinerSubsystemConfig": []DocField{ { - Name: "EnableMining", - Type: "bool", + Name: "EnableMining", + Type: "bool", Comment: ``, }, { - Name: "EnableSealing", - Type: "bool", + Name: "EnableSealing", + Type: "bool", Comment: ``, }, { - Name: "EnableSectorStorage", - Type: "bool", + Name: "EnableSectorStorage", + Type: "bool", Comment: ``, }, { - Name: "EnableMarkets", - Type: "bool", + Name: "EnableMarkets", + Type: "bool", Comment: ``, }, { - Name: "SealerApiInfo", - Type: "string", + Name: "SealerApiInfo", + Type: "string", Comment: ``, }, { - Name: "SectorIndexApiInfo", - Type: "string", + Name: "SectorIndexApiInfo", + Type: "string", Comment: ``, }, }, "Pubsub": []DocField{ { - Name: "Bootstrapper", - Type: "bool", + Name: "Bootstrapper", + Type: "bool", Comment: ``, }, { - Name: "DirectPeers", - Type: "[]string", + Name: "DirectPeers", + Type: "[]string", Comment: ``, }, { - Name: "IPColocationWhitelist", - Type: "[]string", + Name: "IPColocationWhitelist", + Type: "[]string", Comment: ``, }, { - Name: "RemoteTracer", - Type: "string", + Name: "RemoteTracer", + Type: "string", Comment: ``, }, }, "RetrievalPricing": []DocField{ { - Name: "Strategy", - Type: "string", + Name: "Strategy", + Type: "string", Comment: ``, }, { - Name: "Default", - Type: "*RetrievalPricingDefault", + Name: "Default", + Type: "*RetrievalPricingDefault", Comment: ``, }, { - Name: "External", - Type: "*RetrievalPricingExternal", + Name: "External", + Type: "*RetrievalPricingExternal", Comment: ``, }, }, "RetrievalPricingDefault": []DocField{ { - Name: "VerifiedDealsFreeTransfer", - Type: "bool", + Name: "VerifiedDealsFreeTransfer", + Type: "bool", Comment: `VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal of a payloadCid that belongs to a verified storage deal. This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default". @@ -440,193 +440,193 @@ default value is true`, }, "RetrievalPricingExternal": []DocField{ { - Name: "Path", - Type: "string", + Name: "Path", + Type: "string", Comment: `Path of the external script that will be run to price a retrieval deal. This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".`, }, }, "SealingConfig": []DocField{ { - Name: "MaxWaitDealsSectors", - Type: "uint64", + Name: "MaxWaitDealsSectors", + Type: "uint64", Comment: `0 = no limit`, }, { - Name: "MaxSealingSectors", - Type: "uint64", + Name: "MaxSealingSectors", + Type: "uint64", Comment: `includes failed, 0 = no limit`, }, { - Name: "MaxSealingSectorsForDeals", - Type: "uint64", + Name: "MaxSealingSectorsForDeals", + Type: "uint64", Comment: `includes failed, 0 = no limit`, }, { - Name: "WaitDealsDelay", - Type: "Duration", + Name: "WaitDealsDelay", + Type: "Duration", Comment: ``, }, { - Name: "AlwaysKeepUnsealedCopy", - Type: "bool", + Name: "AlwaysKeepUnsealedCopy", + Type: "bool", Comment: ``, }, { - Name: "FinalizeEarly", - Type: "bool", + Name: "FinalizeEarly", + Type: "bool", Comment: `Run sector finalization before submitting sector proof to the chain`, }, { - Name: "CollateralFromMinerBalance", - Type: "bool", + Name: "CollateralFromMinerBalance", + Type: "bool", Comment: `Whether to use available miner balance for sector collateral instead of sending it with each message`, }, { - Name: "AvailableBalanceBuffer", - Type: "types.FIL", + Name: "AvailableBalanceBuffer", + Type: "types.FIL", Comment: `Minimum available balance to keep in the miner actor before sending it with messages`, }, { - Name: "DisableCollateralFallback", - Type: "bool", + Name: "DisableCollateralFallback", + Type: "bool", Comment: `Don't send collateral with messages even if there is no available balance in the miner actor`, }, { - Name: "BatchPreCommits", - Type: "bool", + Name: "BatchPreCommits", + Type: "bool", Comment: `enable / disable precommit batching (takes effect after nv13)`, }, { - Name: "MaxPreCommitBatch", - Type: "int", + Name: "MaxPreCommitBatch", + Type: "int", Comment: `maximum precommit batch size - batches will be sent immediately above this size`, }, { - Name: "PreCommitBatchWait", - Type: "Duration", + Name: "PreCommitBatchWait", + Type: "Duration", Comment: `how long to wait before submitting a batch after crossing the minimum batch size`, }, { - Name: "PreCommitBatchSlack", - Type: "Duration", + Name: "PreCommitBatchSlack", + Type: "Duration", Comment: `time buffer for forceful batch submission before sectors/deal in batch would start expiring`, }, { - Name: "AggregateCommits", - Type: "bool", + Name: "AggregateCommits", + Type: "bool", Comment: `enable / disable commit aggregation (takes effect after nv13)`, }, { - Name: "MinCommitBatch", - Type: "int", + Name: "MinCommitBatch", + Type: "int", Comment: `maximum batched commit size - batches will be sent immediately above this size`, }, { - Name: "MaxCommitBatch", - Type: "int", + Name: "MaxCommitBatch", + Type: "int", Comment: ``, }, { - Name: "CommitBatchWait", - Type: "Duration", + Name: "CommitBatchWait", + Type: "Duration", Comment: `how long to wait before submitting a batch after crossing the minimum batch size`, }, { - Name: "CommitBatchSlack", - Type: "Duration", + Name: "CommitBatchSlack", + Type: "Duration", Comment: `time buffer for forceful batch submission before sectors/deals in batch would start expiring`, }, { - Name: "AggregateAboveBaseFee", - Type: "types.FIL", + Name: "AggregateAboveBaseFee", + Type: "types.FIL", Comment: `network BaseFee below which to stop doing commit aggregation, instead submitting proofs to the chain individually`, }, { - Name: "TerminateBatchMax", - Type: "uint64", + Name: "TerminateBatchMax", + Type: "uint64", Comment: ``, }, { - Name: "TerminateBatchMin", - Type: "uint64", + Name: "TerminateBatchMin", + Type: "uint64", Comment: ``, }, { - Name: "TerminateBatchWait", - Type: "Duration", + Name: "TerminateBatchWait", + Type: "Duration", Comment: ``, }, }, "Splitstore": []DocField{ { - Name: "ColdStoreType", - Type: "string", + Name: "ColdStoreType", + Type: "string", Comment: ``, }, { - Name: "HotStoreType", - Type: "string", + Name: "HotStoreType", + Type: "string", Comment: ``, }, { - Name: "MarkSetType", - Type: "string", + Name: "MarkSetType", + Type: "string", Comment: ``, }, { - Name: "HotStoreMessageRetention", - Type: "uint64", + Name: "HotStoreMessageRetention", + Type: "uint64", Comment: ``, }, }, "StorageMiner": []DocField{ { - Name: "Subsystems", - Type: "MinerSubsystemConfig", + Name: "Subsystems", + Type: "MinerSubsystemConfig", Comment: ``, }, { - Name: "Dealmaking", - Type: "DealmakingConfig", + Name: "Dealmaking", + Type: "DealmakingConfig", Comment: ``, }, { - Name: "Sealing", - Type: "SealingConfig", + Name: "Sealing", + Type: "SealingConfig", Comment: ``, }, { - Name: "Storage", - Type: "sectorstorage.SealerConfig", + Name: "Storage", + Type: "sectorstorage.SealerConfig", Comment: ``, }, { - Name: "Fees", - Type: "MinerFeeConfig", + Name: "Fees", + Type: "MinerFeeConfig", Comment: ``, }, { - Name: "Addresses", - Type: "MinerAddressConfig", + Name: "Addresses", + Type: "MinerAddressConfig", Comment: ``, }, }, "Wallet": []DocField{ { - Name: "RemoteBackend", - Type: "string", + Name: "RemoteBackend", + Type: "string", Comment: ``, }, { - Name: "EnableLedger", - Type: "bool", + Name: "EnableLedger", + Type: "bool", Comment: ``, }, { - Name: "DisableLocal", - Type: "bool", + Name: "DisableLocal", + Type: "bool", Comment: ``, }, }, diff --git a/node/config/doc_util.go b/node/config/doc_util.go new file mode 100644 index 000000000..ee70a9cfd --- /dev/null +++ b/node/config/doc_util.go @@ -0,0 +1,44 @@ +package config + +import ( + "fmt" + "strings" +) + +func findDoc(root interface{}, section, name string) *DocField { + rt := fmt.Sprintf("%T", root)[len("*config."):] + + doc := findDocSect(rt, section, name) + if doc != nil { + return doc + } + + return findDocSect("Common", section, name) +} + +func findDocSect(root string, section, name string) *DocField { + path := strings.Split(section, ".") + + docSection := Doc[root] + for _, e := range path { + if docSection == nil { + return nil + } + + for _, field := range docSection { + if field.Name == e { + docSection = Doc[field.Type] + break + } + + } + } + + for _, df := range docSection { + if df.Name == name { + return &df + } + } + + return nil +} diff --git a/node/config/load.go b/node/config/load.go index e3415888a..569d65393 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -86,7 +86,7 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) { nodeLines := strings.Split(nodeStr, "\n") var outLines []string - sectionRx := regexp.MustCompile(`[\[.]([^.]+)]`) + sectionRx := regexp.MustCompile(`\[(.+)]`) var section string for i, line := range nodeLines { @@ -106,30 +106,24 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) { } } - pad := strings.Repeat(" ", len(line) - len(strings.TrimLeftFunc(line, unicode.IsSpace))) + pad := strings.Repeat(" ", len(line)-len(strings.TrimLeftFunc(line, unicode.IsSpace))) // see if we have docs for this field { lf := strings.Fields(line) if len(lf) > 1 { - var doc *DocField - for _, df := range Doc[section] { - if df.Name == lf[0] { - doc = &df - break - } - } + doc := findDoc(cfgCur, section, lf[0]) if doc != nil { // found docfield, emit doc comment if len(doc.Comment) > 0 { for _, docLine := range strings.Split(doc.Comment, "\n") { - outLines = append(outLines, pad + "# " + docLine) + outLines = append(outLines, pad+"# "+docLine) } - outLines = append(outLines, pad + "#") + outLines = append(outLines, pad+"#") } - outLines = append(outLines, pad + "# type: " + doc.Type) + outLines = append(outLines, pad+"# type: "+doc.Type) } } } From ed7f70e68e685cb1055c2c2ba5918c5966860a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 15:18:20 +0200 Subject: [PATCH 06/11] config doc gen: generate formatted code --- node/config/cfgdocgen/gen.go | 4 +- node/config/doc_gen.go | 565 +++++++++++++++++++++-------------- 2 files changed, 341 insertions(+), 228 deletions(-) diff --git a/node/config/cfgdocgen/gen.go b/node/config/cfgdocgen/gen.go index b89a384f5..50c0c291c 100644 --- a/node/config/cfgdocgen/gen.go +++ b/node/config/cfgdocgen/gen.go @@ -109,8 +109,8 @@ var Doc = map[string][]DocField{ for _, f := range typ { fmt.Println("\t\t{") - fmt.Printf("\t\t\tName: \"%s\",\n", f.Name) - fmt.Printf("\t\t\tType: \"%s\",\n", f.Type) + fmt.Printf("\t\t\tName: \"%s\",\n", f.Name) + fmt.Printf("\t\t\tType: \"%s\",\n\n", f.Type) fmt.Printf("\t\t\tComment: `%s`,\n", f.Comment) fmt.Println("\t\t},") } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 4972a5197..104984a64 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -11,304 +11,359 @@ type DocField struct { var Doc = map[string][]DocField{ "API": []DocField{ { - Name: "ListenAddress", - Type: "string", + Name: "ListenAddress", + Type: "string", + Comment: `Binding address for the Lotus API`, }, { - Name: "RemoteListenAddress", - Type: "string", + Name: "RemoteListenAddress", + Type: "string", + Comment: ``, }, { - Name: "Timeout", - Type: "Duration", + Name: "Timeout", + Type: "Duration", + Comment: ``, }, }, "Backup": []DocField{ { - Name: "DisableMetadataLog", - Type: "bool", + Name: "DisableMetadataLog", + Type: "bool", + Comment: ``, }, }, "BatchFeeConfig": []DocField{ { - Name: "Base", - Type: "types.FIL", + Name: "Base", + Type: "types.FIL", + Comment: ``, }, { - Name: "PerSector", - Type: "types.FIL", + Name: "PerSector", + Type: "types.FIL", + Comment: ``, }, }, "Chainstore": []DocField{ { - Name: "EnableSplitstore", - Type: "bool", + Name: "EnableSplitstore", + Type: "bool", + Comment: ``, }, { - Name: "Splitstore", - Type: "Splitstore", + Name: "Splitstore", + Type: "Splitstore", + Comment: ``, }, }, "Client": []DocField{ { - Name: "UseIpfs", - Type: "bool", + Name: "UseIpfs", + Type: "bool", + Comment: ``, }, { - Name: "IpfsOnlineMode", - Type: "bool", + Name: "IpfsOnlineMode", + Type: "bool", + Comment: ``, }, { - Name: "IpfsMAddr", - Type: "string", + Name: "IpfsMAddr", + Type: "string", + Comment: ``, }, { - Name: "IpfsUseForRetrieval", - Type: "bool", + Name: "IpfsUseForRetrieval", + Type: "bool", + Comment: ``, }, { - Name: "SimultaneousTransfers", - Type: "uint64", + Name: "SimultaneousTransfers", + Type: "uint64", + Comment: ``, }, }, "Common": []DocField{ { - Name: "API", - Type: "API", + Name: "API", + Type: "API", + Comment: ``, }, { - Name: "Backup", - Type: "Backup", + Name: "Backup", + Type: "Backup", + Comment: ``, }, { - Name: "Libp2p", - Type: "Libp2p", + Name: "Libp2p", + Type: "Libp2p", + Comment: ``, }, { - Name: "Pubsub", - Type: "Pubsub", + Name: "Pubsub", + Type: "Pubsub", + Comment: ``, }, }, "DealmakingConfig": []DocField{ { - Name: "ConsiderOnlineStorageDeals", - Type: "bool", + Name: "ConsiderOnlineStorageDeals", + Type: "bool", + Comment: ``, }, { - Name: "ConsiderOfflineStorageDeals", - Type: "bool", + Name: "ConsiderOfflineStorageDeals", + Type: "bool", + Comment: ``, }, { - Name: "ConsiderOnlineRetrievalDeals", - Type: "bool", + Name: "ConsiderOnlineRetrievalDeals", + Type: "bool", + Comment: ``, }, { - Name: "ConsiderOfflineRetrievalDeals", - Type: "bool", + Name: "ConsiderOfflineRetrievalDeals", + Type: "bool", + Comment: ``, }, { - Name: "ConsiderVerifiedStorageDeals", - Type: "bool", + Name: "ConsiderVerifiedStorageDeals", + Type: "bool", + Comment: ``, }, { - Name: "ConsiderUnverifiedStorageDeals", - Type: "bool", + Name: "ConsiderUnverifiedStorageDeals", + Type: "bool", + Comment: ``, }, { - Name: "PieceCidBlocklist", - Type: "[]cid.Cid", + Name: "PieceCidBlocklist", + Type: "[]cid.Cid", + Comment: ``, }, { - Name: "ExpectedSealDuration", - Type: "Duration", + Name: "ExpectedSealDuration", + Type: "Duration", + Comment: ``, }, { - Name: "MaxDealStartDelay", - Type: "Duration", + Name: "MaxDealStartDelay", + Type: "Duration", + Comment: `Maximum amount of time proposed deal StartEpoch can be in future`, }, { - Name: "PublishMsgPeriod", - Type: "Duration", + Name: "PublishMsgPeriod", + Type: "Duration", + Comment: `The amount of time to wait for more deals to arrive before publishing`, }, { - Name: "MaxDealsPerPublishMsg", - Type: "uint64", + Name: "MaxDealsPerPublishMsg", + Type: "uint64", + Comment: `The maximum number of deals to include in a single PublishStorageDeals message`, }, { - Name: "MaxProviderCollateralMultiplier", - Type: "uint64", + Name: "MaxProviderCollateralMultiplier", + Type: "uint64", + Comment: `The maximum collateral that the provider will put up against a deal, as a multiplier of the minimum collateral bound`, }, { - Name: "SimultaneousTransfers", - Type: "uint64", + Name: "SimultaneousTransfers", + Type: "uint64", + Comment: `The maximum number of parallel online data transfers (storage+retrieval)`, }, { - Name: "Filter", - Type: "string", + Name: "Filter", + Type: "string", + Comment: ``, }, { - Name: "RetrievalFilter", - Type: "string", + Name: "RetrievalFilter", + Type: "string", + Comment: ``, }, { - Name: "RetrievalPricing", - Type: "*RetrievalPricing", + Name: "RetrievalPricing", + Type: "*RetrievalPricing", + Comment: ``, }, }, "FeeConfig": []DocField{ { - Name: "DefaultMaxFee", - Type: "types.FIL", + Name: "DefaultMaxFee", + Type: "types.FIL", + Comment: ``, }, }, "FullNode": []DocField{ { - Name: "Client", - Type: "Client", + Name: "Client", + Type: "Client", + Comment: ``, }, { - Name: "Metrics", - Type: "Metrics", + Name: "Metrics", + Type: "Metrics", + Comment: ``, }, { - Name: "Wallet", - Type: "Wallet", + Name: "Wallet", + Type: "Wallet", + Comment: ``, }, { - Name: "Fees", - Type: "FeeConfig", + Name: "Fees", + Type: "FeeConfig", + Comment: ``, }, { - Name: "Chainstore", - Type: "Chainstore", + Name: "Chainstore", + Type: "Chainstore", + Comment: ``, }, }, "Libp2p": []DocField{ { - Name: "ListenAddresses", - Type: "[]string", + Name: "ListenAddresses", + Type: "[]string", + Comment: ``, }, { - Name: "AnnounceAddresses", - Type: "[]string", + Name: "AnnounceAddresses", + Type: "[]string", + Comment: ``, }, { - Name: "NoAnnounceAddresses", - Type: "[]string", + Name: "NoAnnounceAddresses", + Type: "[]string", + Comment: ``, }, { - Name: "BootstrapPeers", - Type: "[]string", + Name: "BootstrapPeers", + Type: "[]string", + Comment: ``, }, { - Name: "ProtectedPeers", - Type: "[]string", + Name: "ProtectedPeers", + Type: "[]string", + Comment: ``, }, { - Name: "ConnMgrLow", - Type: "uint", + Name: "ConnMgrLow", + Type: "uint", + Comment: ``, }, { - Name: "ConnMgrHigh", - Type: "uint", + Name: "ConnMgrHigh", + Type: "uint", + Comment: ``, }, { - Name: "ConnMgrGrace", - Type: "Duration", + Name: "ConnMgrGrace", + Type: "Duration", + Comment: ``, }, }, "Metrics": []DocField{ { - Name: "Nickname", - Type: "string", + Name: "Nickname", + Type: "string", + Comment: ``, }, { - Name: "HeadNotifs", - Type: "bool", + Name: "HeadNotifs", + Type: "bool", + Comment: ``, }, }, "MinerAddressConfig": []DocField{ { - Name: "PreCommitControl", - Type: "[]string", + Name: "PreCommitControl", + Type: "[]string", + Comment: ``, }, { - Name: "CommitControl", - Type: "[]string", + Name: "CommitControl", + Type: "[]string", + Comment: ``, }, { - Name: "TerminateControl", - Type: "[]string", + Name: "TerminateControl", + Type: "[]string", + Comment: ``, }, { - Name: "DealPublishControl", - Type: "[]string", + Name: "DealPublishControl", + Type: "[]string", + Comment: ``, }, { - Name: "DisableOwnerFallback", - Type: "bool", + Name: "DisableOwnerFallback", + Type: "bool", + Comment: `DisableOwnerFallback disables usage of the owner address for messages sent automatically`, }, { - Name: "DisableWorkerFallback", - Type: "bool", + Name: "DisableWorkerFallback", + Type: "bool", + Comment: `DisableWorkerFallback disables usage of the worker address for messages sent automatically, if control addresses are configured. A control address that doesn't have enough funds will still be chosen @@ -317,121 +372,143 @@ over the worker address if this flag is set.`, }, "MinerFeeConfig": []DocField{ { - Name: "MaxPreCommitGasFee", - Type: "types.FIL", + Name: "MaxPreCommitGasFee", + Type: "types.FIL", + Comment: ``, }, { - Name: "MaxCommitGasFee", - Type: "types.FIL", + Name: "MaxCommitGasFee", + Type: "types.FIL", + Comment: ``, }, { - Name: "MaxPreCommitBatchGasFee", - Type: "BatchFeeConfig", + Name: "MaxPreCommitBatchGasFee", + Type: "BatchFeeConfig", + Comment: `maxBatchFee = maxBase + maxPerSector * nSectors`, }, { - Name: "MaxCommitBatchGasFee", - Type: "BatchFeeConfig", + Name: "MaxCommitBatchGasFee", + Type: "BatchFeeConfig", + Comment: ``, }, { - Name: "MaxTerminateGasFee", - Type: "types.FIL", + Name: "MaxTerminateGasFee", + Type: "types.FIL", + Comment: ``, }, { - Name: "MaxWindowPoStGasFee", - Type: "types.FIL", + Name: "MaxWindowPoStGasFee", + Type: "types.FIL", + Comment: ``, }, { - Name: "MaxPublishDealsFee", - Type: "types.FIL", + Name: "MaxPublishDealsFee", + Type: "types.FIL", + Comment: ``, }, { - Name: "MaxMarketBalanceAddFee", - Type: "types.FIL", + Name: "MaxMarketBalanceAddFee", + Type: "types.FIL", + Comment: ``, }, }, "MinerSubsystemConfig": []DocField{ { - Name: "EnableMining", - Type: "bool", + Name: "EnableMining", + Type: "bool", + Comment: ``, }, { - Name: "EnableSealing", - Type: "bool", + Name: "EnableSealing", + Type: "bool", + Comment: ``, }, { - Name: "EnableSectorStorage", - Type: "bool", + Name: "EnableSectorStorage", + Type: "bool", + Comment: ``, }, { - Name: "EnableMarkets", - Type: "bool", + Name: "EnableMarkets", + Type: "bool", + Comment: ``, }, { - Name: "SealerApiInfo", - Type: "string", + Name: "SealerApiInfo", + Type: "string", + Comment: ``, }, { - Name: "SectorIndexApiInfo", - Type: "string", + Name: "SectorIndexApiInfo", + Type: "string", + Comment: ``, }, }, "Pubsub": []DocField{ { - Name: "Bootstrapper", - Type: "bool", + Name: "Bootstrapper", + Type: "bool", + Comment: ``, }, { - Name: "DirectPeers", - Type: "[]string", + Name: "DirectPeers", + Type: "[]string", + Comment: ``, }, { - Name: "IPColocationWhitelist", - Type: "[]string", + Name: "IPColocationWhitelist", + Type: "[]string", + Comment: ``, }, { - Name: "RemoteTracer", - Type: "string", + Name: "RemoteTracer", + Type: "string", + Comment: ``, }, }, "RetrievalPricing": []DocField{ { - Name: "Strategy", - Type: "string", + Name: "Strategy", + Type: "string", + Comment: ``, }, { - Name: "Default", - Type: "*RetrievalPricingDefault", + Name: "Default", + Type: "*RetrievalPricingDefault", + Comment: ``, }, { - Name: "External", - Type: "*RetrievalPricingExternal", + Name: "External", + Type: "*RetrievalPricingExternal", + Comment: ``, }, }, "RetrievalPricingDefault": []DocField{ { - Name: "VerifiedDealsFreeTransfer", - Type: "bool", + Name: "VerifiedDealsFreeTransfer", + Type: "bool", + Comment: `VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal of a payloadCid that belongs to a verified storage deal. This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default". @@ -440,193 +517,229 @@ default value is true`, }, "RetrievalPricingExternal": []DocField{ { - Name: "Path", - Type: "string", + Name: "Path", + Type: "string", + Comment: `Path of the external script that will be run to price a retrieval deal. This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".`, }, }, "SealingConfig": []DocField{ { - Name: "MaxWaitDealsSectors", - Type: "uint64", + Name: "MaxWaitDealsSectors", + Type: "uint64", + Comment: `0 = no limit`, }, { - Name: "MaxSealingSectors", - Type: "uint64", + Name: "MaxSealingSectors", + Type: "uint64", + Comment: `includes failed, 0 = no limit`, }, { - Name: "MaxSealingSectorsForDeals", - Type: "uint64", + Name: "MaxSealingSectorsForDeals", + Type: "uint64", + Comment: `includes failed, 0 = no limit`, }, { - Name: "WaitDealsDelay", - Type: "Duration", + Name: "WaitDealsDelay", + Type: "Duration", + Comment: ``, }, { - Name: "AlwaysKeepUnsealedCopy", - Type: "bool", + Name: "AlwaysKeepUnsealedCopy", + Type: "bool", + Comment: ``, }, { - Name: "FinalizeEarly", - Type: "bool", + Name: "FinalizeEarly", + Type: "bool", + Comment: `Run sector finalization before submitting sector proof to the chain`, }, { - Name: "CollateralFromMinerBalance", - Type: "bool", + Name: "CollateralFromMinerBalance", + Type: "bool", + Comment: `Whether to use available miner balance for sector collateral instead of sending it with each message`, }, { - Name: "AvailableBalanceBuffer", - Type: "types.FIL", + Name: "AvailableBalanceBuffer", + Type: "types.FIL", + Comment: `Minimum available balance to keep in the miner actor before sending it with messages`, }, { - Name: "DisableCollateralFallback", - Type: "bool", + Name: "DisableCollateralFallback", + Type: "bool", + Comment: `Don't send collateral with messages even if there is no available balance in the miner actor`, }, { - Name: "BatchPreCommits", - Type: "bool", + Name: "BatchPreCommits", + Type: "bool", + Comment: `enable / disable precommit batching (takes effect after nv13)`, }, { - Name: "MaxPreCommitBatch", - Type: "int", + Name: "MaxPreCommitBatch", + Type: "int", + Comment: `maximum precommit batch size - batches will be sent immediately above this size`, }, { - Name: "PreCommitBatchWait", - Type: "Duration", + Name: "PreCommitBatchWait", + Type: "Duration", + Comment: `how long to wait before submitting a batch after crossing the minimum batch size`, }, { - Name: "PreCommitBatchSlack", - Type: "Duration", + Name: "PreCommitBatchSlack", + Type: "Duration", + Comment: `time buffer for forceful batch submission before sectors/deal in batch would start expiring`, }, { - Name: "AggregateCommits", - Type: "bool", + Name: "AggregateCommits", + Type: "bool", + Comment: `enable / disable commit aggregation (takes effect after nv13)`, }, { - Name: "MinCommitBatch", - Type: "int", + Name: "MinCommitBatch", + Type: "int", + Comment: `maximum batched commit size - batches will be sent immediately above this size`, }, { - Name: "MaxCommitBatch", - Type: "int", + Name: "MaxCommitBatch", + Type: "int", + Comment: ``, }, { - Name: "CommitBatchWait", - Type: "Duration", + Name: "CommitBatchWait", + Type: "Duration", + Comment: `how long to wait before submitting a batch after crossing the minimum batch size`, }, { - Name: "CommitBatchSlack", - Type: "Duration", + Name: "CommitBatchSlack", + Type: "Duration", + Comment: `time buffer for forceful batch submission before sectors/deals in batch would start expiring`, }, { - Name: "AggregateAboveBaseFee", - Type: "types.FIL", + Name: "AggregateAboveBaseFee", + Type: "types.FIL", + Comment: `network BaseFee below which to stop doing commit aggregation, instead submitting proofs to the chain individually`, }, { - Name: "TerminateBatchMax", - Type: "uint64", + Name: "TerminateBatchMax", + Type: "uint64", + Comment: ``, }, { - Name: "TerminateBatchMin", - Type: "uint64", + Name: "TerminateBatchMin", + Type: "uint64", + Comment: ``, }, { - Name: "TerminateBatchWait", - Type: "Duration", + Name: "TerminateBatchWait", + Type: "Duration", + Comment: ``, }, }, "Splitstore": []DocField{ { - Name: "ColdStoreType", - Type: "string", + Name: "ColdStoreType", + Type: "string", + Comment: ``, }, { - Name: "HotStoreType", - Type: "string", + Name: "HotStoreType", + Type: "string", + Comment: ``, }, { - Name: "MarkSetType", - Type: "string", + Name: "MarkSetType", + Type: "string", + Comment: ``, }, { - Name: "HotStoreMessageRetention", - Type: "uint64", + Name: "HotStoreMessageRetention", + Type: "uint64", + Comment: ``, }, }, "StorageMiner": []DocField{ { - Name: "Subsystems", - Type: "MinerSubsystemConfig", + Name: "Subsystems", + Type: "MinerSubsystemConfig", + Comment: ``, }, { - Name: "Dealmaking", - Type: "DealmakingConfig", + Name: "Dealmaking", + Type: "DealmakingConfig", + Comment: ``, }, { - Name: "Sealing", - Type: "SealingConfig", + Name: "Sealing", + Type: "SealingConfig", + Comment: ``, }, { - Name: "Storage", - Type: "sectorstorage.SealerConfig", + Name: "Storage", + Type: "sectorstorage.SealerConfig", + Comment: ``, }, { - Name: "Fees", - Type: "MinerFeeConfig", + Name: "Fees", + Type: "MinerFeeConfig", + Comment: ``, }, { - Name: "Addresses", - Type: "MinerAddressConfig", + Name: "Addresses", + Type: "MinerAddressConfig", + Comment: ``, }, }, "Wallet": []DocField{ { - Name: "RemoteBackend", - Type: "string", + Name: "RemoteBackend", + Type: "string", + Comment: ``, }, { - Name: "EnableLedger", - Type: "bool", + Name: "EnableLedger", + Type: "bool", + Comment: ``, }, { - Name: "DisableLocal", - Type: "bool", + Name: "DisableLocal", + Type: "bool", + Comment: ``, }, }, From a26712ebc459bf5ea4ba9c2328ee305655f9bac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 15:40:30 +0200 Subject: [PATCH 07/11] config: Bring some docs from filecoin-docs --- node/config/doc_gen.go | 75 ++++++++++++++++++++++------------- node/config/types.go | 88 ++++++++++++++++++++++++++++++++---------- 2 files changed, 115 insertions(+), 48 deletions(-) diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 104984a64..ea68dc344 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -34,7 +34,8 @@ var Doc = map[string][]DocField{ Name: "DisableMetadataLog", Type: "bool", - Comment: ``, + Comment: `Note that in case of metadata corruption it might be much harder to recover +your node if metadata log is disabled`, }, }, "BatchFeeConfig": []DocField{ @@ -94,7 +95,8 @@ var Doc = map[string][]DocField{ Name: "SimultaneousTransfers", Type: "uint64", - Comment: ``, + Comment: `The maximum number of simultaneous data transfers between the client +and storage providers`, }, }, "Common": []DocField{ @@ -128,49 +130,51 @@ var Doc = map[string][]DocField{ Name: "ConsiderOnlineStorageDeals", Type: "bool", - Comment: ``, + Comment: `When enabled, the miner can accept online deals`, }, { Name: "ConsiderOfflineStorageDeals", Type: "bool", - Comment: ``, + Comment: `When enabled, the miner can accept offline deals`, }, { Name: "ConsiderOnlineRetrievalDeals", Type: "bool", - Comment: ``, + Comment: `When enabled, the miner can accept retrieval deals`, }, { Name: "ConsiderOfflineRetrievalDeals", Type: "bool", - Comment: ``, + Comment: `When enabled, the miner can accept offline retrieval deals`, }, { Name: "ConsiderVerifiedStorageDeals", Type: "bool", - Comment: ``, + Comment: `When enabled, the miner can accept verified deals`, }, { Name: "ConsiderUnverifiedStorageDeals", Type: "bool", - Comment: ``, + Comment: `When enabled, the miner can accept unverified deals`, }, { Name: "PieceCidBlocklist", Type: "[]cid.Cid", - Comment: ``, + Comment: `A list of Data CIDs to reject when making deals`, }, { Name: "ExpectedSealDuration", Type: "Duration", - Comment: ``, + Comment: `Maximum expected amount of time getting the deal into a sealed sector will take +This includes the time the deal will need to get transferred and published +before being assigned to a sector`, }, { Name: "MaxDealStartDelay", @@ -182,8 +186,8 @@ var Doc = map[string][]DocField{ Name: "PublishMsgPeriod", Type: "Duration", - Comment: `The amount of time to wait for more deals to arrive before -publishing`, + Comment: `When a deal is ready to publish, the amount of time to wait for more +deals to be ready to publish before publishing them all as a batch`, }, { Name: "MaxDealsPerPublishMsg", @@ -209,13 +213,15 @@ as a multiplier of the minimum collateral bound`, Name: "Filter", Type: "string", - Comment: ``, + Comment: `A command used for fine-grained evaluation of storage deals +see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`, }, { Name: "RetrievalFilter", Type: "string", - Comment: ``, + Comment: `A command used for fine-grained evaluation of retrieval deals +see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`, }, { Name: "RetrievalPricing", @@ -269,19 +275,23 @@ as a multiplier of the minimum collateral bound`, Name: "ListenAddresses", Type: "[]string", - Comment: ``, + Comment: `Binding address for the libp2p host - 0 means random port. +Format: multiaddress; see https://multiformats.io/multiaddr/`, }, { Name: "AnnounceAddresses", Type: "[]string", - Comment: ``, + Comment: `Addresses to explicitally announce to other peers. If not specified, +all interface addresses are announced +Format: multiaddress`, }, { Name: "NoAnnounceAddresses", Type: "[]string", - Comment: ``, + Comment: `Addresses to not announce +Format: multiaddress`, }, { Name: "BootstrapPeers", @@ -333,13 +343,13 @@ as a multiplier of the minimum collateral bound`, Name: "PreCommitControl", Type: "[]string", - Comment: ``, + Comment: `Addresses to send PreCommit messages from`, }, { Name: "CommitControl", Type: "[]string", - Comment: ``, + Comment: `Addresses to send Commit messages from`, }, { Name: "TerminateControl", @@ -405,7 +415,7 @@ over the worker address if this flag is set.`, Name: "MaxWindowPoStGasFee", Type: "types.FIL", - Comment: ``, + Comment: `WindowPoSt is a high-value operation, so the default fee should be high.`, }, { Name: "MaxPublishDealsFee", @@ -463,13 +473,18 @@ over the worker address if this flag is set.`, Name: "Bootstrapper", Type: "bool", - Comment: ``, + Comment: `Run the node in bootstrap-node mode`, }, { Name: "DirectPeers", Type: "[]string", - Comment: ``, + Comment: `DirectPeers specifies peers with direct peering agreements. These peers are +connected outside of the mesh, with all (valid) message unconditionally +forwarded to them. The router will maintain open connections to these peers. +Note that the peering agreement should be reciprocal with direct peers +symmetrically configured at both ends. +Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...`, }, { Name: "IPColocationWhitelist", @@ -529,31 +544,37 @@ This parameter is ONLY applicable if the retrieval pricing policy strategy has b Name: "MaxWaitDealsSectors", Type: "uint64", - Comment: `0 = no limit`, + Comment: `Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time. +If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created. +If more than MaxWaitDealsSectors deals are accepted in parallel, only MaxWaitDealsSectors deals will be processed in parallel +Note that setting this number too high in relation to deal ingestion rate may result in poor sector packing efficiency +0 = no limit`, }, { Name: "MaxSealingSectors", Type: "uint64", - Comment: `includes failed, 0 = no limit`, + Comment: `Upper bound on how many sectors can be sealing at the same time when creating new CC sectors (0 = unlimited)`, }, { Name: "MaxSealingSectorsForDeals", Type: "uint64", - Comment: `includes failed, 0 = no limit`, + Comment: `Upper bound on how many sectors can be sealing at the same time when creating new sectors with deals (0 = unlimited)`, }, { Name: "WaitDealsDelay", Type: "Duration", - Comment: ``, + Comment: `Period of time that a newly created sector will wait for more deals to be packed in to before it starts to seal. +Sectors which are fully filled will start sealing immediately`, }, { Name: "AlwaysKeepUnsealedCopy", Type: "bool", - Comment: ``, + Comment: `Whether to keep unsealed copies of deal data regardless of whether the client requested that. This lets the miner +avoid the relatively high cost of unsealing the data later, at the cost of more storage space`, }, { Name: "FinalizeEarly", diff --git a/node/config/types.go b/node/config/types.go index 84a631962..5cb76b9ea 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -30,6 +30,11 @@ type FullNode struct { // // Common type Backup struct { + // When set to true disables metadata log (.lotus/kvlog). This can save disk + // space by reducing metadata redundancy. + // + // Note that in case of metadata corruption it might be much harder to recover + // your node if metadata log is disabled DisableMetadataLog bool } @@ -56,18 +61,28 @@ type MinerSubsystemConfig struct { } type DealmakingConfig struct { - ConsiderOnlineStorageDeals bool - ConsiderOfflineStorageDeals bool - ConsiderOnlineRetrievalDeals bool - ConsiderOfflineRetrievalDeals bool - ConsiderVerifiedStorageDeals bool + // When enabled, the miner can accept online deals + ConsiderOnlineStorageDeals bool + // When enabled, the miner can accept offline deals + ConsiderOfflineStorageDeals bool + // When enabled, the miner can accept retrieval deals + ConsiderOnlineRetrievalDeals bool + // When enabled, the miner can accept offline retrieval deals + ConsiderOfflineRetrievalDeals bool + // When enabled, the miner can accept verified deals + ConsiderVerifiedStorageDeals bool + // When enabled, the miner can accept unverified deals ConsiderUnverifiedStorageDeals bool - PieceCidBlocklist []cid.Cid - ExpectedSealDuration Duration + // A list of Data CIDs to reject when making deals + PieceCidBlocklist []cid.Cid + // Maximum expected amount of time getting the deal into a sealed sector will take + // This includes the time the deal will need to get transferred and published + // before being assigned to a sector + ExpectedSealDuration Duration // Maximum amount of time proposed deal StartEpoch can be in future MaxDealStartDelay Duration - // The amount of time to wait for more deals to arrive before - // publishing + // When a deal is ready to publish, the amount of time to wait for more + // deals to be ready to publish before publishing them all as a batch PublishMsgPeriod Duration // The maximum number of deals to include in a single PublishStorageDeals // message @@ -79,7 +94,11 @@ type DealmakingConfig struct { // The maximum number of parallel online data transfers (storage+retrieval) SimultaneousTransfers uint64 - Filter string + // A command used for fine-grained evaluation of storage deals + // see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details + Filter string + // A command used for fine-grained evaluation of retrieval deals + // see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details RetrievalFilter string RetrievalPricing *RetrievalPricing @@ -107,17 +126,25 @@ type RetrievalPricingDefault struct { } type SealingConfig struct { + // Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time. + // If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created. + // If more than MaxWaitDealsSectors deals are accepted in parallel, only MaxWaitDealsSectors deals will be processed in parallel + // Note that setting this number too high in relation to deal ingestion rate may result in poor sector packing efficiency // 0 = no limit MaxWaitDealsSectors uint64 - // includes failed, 0 = no limit + // Upper bound on how many sectors can be sealing at the same time when creating new CC sectors (0 = unlimited) MaxSealingSectors uint64 - // includes failed, 0 = no limit + // Upper bound on how many sectors can be sealing at the same time when creating new sectors with deals (0 = unlimited) MaxSealingSectorsForDeals uint64 + // Period of time that a newly created sector will wait for more deals to be packed in to before it starts to seal. + // Sectors which are fully filled will start sealing immediately WaitDealsDelay Duration + // Whether to keep unsealed copies of deal data regardless of whether the client requested that. This lets the miner + // avoid the relatively high cost of unsealing the data later, at the cost of more storage space AlwaysKeepUnsealedCopy bool // Run sector finalization before submitting sector proof to the chain @@ -176,14 +203,17 @@ type MinerFeeConfig struct { MaxPreCommitBatchGasFee BatchFeeConfig MaxCommitBatchGasFee BatchFeeConfig - MaxTerminateGasFee types.FIL + MaxTerminateGasFee types.FIL + // WindowPoSt is a high-value operation, so the default fee should be high. MaxWindowPoStGasFee types.FIL MaxPublishDealsFee types.FIL MaxMarketBalanceAddFee types.FIL } type MinerAddressConfig struct { - PreCommitControl []string + // Addresses to send PreCommit messages from + PreCommitControl []string + // Addresses to send Commit messages from CommitControl []string TerminateControl []string DealPublishControl []string @@ -208,8 +238,15 @@ type API struct { // Libp2p contains configs for libp2p type Libp2p struct { - ListenAddresses []string - AnnounceAddresses []string + // Binding address for the libp2p host - 0 means random port. + // Format: multiaddress; see https://multiformats.io/multiaddr/ + ListenAddresses []string + // Addresses to explicitally announce to other peers. If not specified, + // all interface addresses are announced + // Format: multiaddress + AnnounceAddresses []string + // Addresses to not announce + // Format: multiaddress NoAnnounceAddresses []string BootstrapPeers []string ProtectedPeers []string @@ -220,7 +257,14 @@ type Libp2p struct { } type Pubsub struct { - Bootstrapper bool + // Run the node in bootstrap-node mode + Bootstrapper bool + // DirectPeers specifies peers with direct peering agreements. These peers are + // connected outside of the mesh, with all (valid) message unconditionally + // forwarded to them. The router will maintain open connections to these peers. + // Note that the peering agreement should be reciprocal with direct peers + // symmetrically configured at both ends. + // Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K... DirectPeers []string IPColocationWhitelist []string RemoteTracer string @@ -247,10 +291,12 @@ type Metrics struct { } type Client struct { - UseIpfs bool - IpfsOnlineMode bool - IpfsMAddr string - IpfsUseForRetrieval bool + UseIpfs bool + IpfsOnlineMode bool + IpfsMAddr string + IpfsUseForRetrieval bool + // The maximum number of simultaneous data transfers between the client + // and storage providers SimultaneousTransfers uint64 } From 4b8a5068bdf365f1b55aa6be8f0cf50d3f069a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 15:44:52 +0200 Subject: [PATCH 08/11] cli docsgen --- documentation/en/cli-lotus-miner.md | 42 ++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 37384c349..2ba693bff 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -13,7 +13,7 @@ COMMANDS: init Initialize a lotus miner repo run Start a lotus miner process stop Stop a running lotus miner - config Output default configuration + config Manage node config backup Create node metadata backup version Print version help, h Shows a list of commands or help for one command @@ -145,13 +145,47 @@ OPTIONS: ## lotus-miner config ``` NAME: - lotus-miner config - Output default configuration + lotus-miner config - Manage node config USAGE: - lotus-miner config [command options] [arguments...] + lotus-miner config command [command options] [arguments...] + +COMMANDS: + default Print default node config + updated Print updated node config + help, h Shows a list of commands or help for one command OPTIONS: - --help, -h show help (default: false) + --help, -h show help (default: false) + --version, -v print the version (default: false) + +``` + +### lotus-miner config default +``` +NAME: + lotus-miner config default - Print default node config + +USAGE: + lotus-miner config default [command options] [arguments...] + +OPTIONS: + --no-comment don't comment default values (default: false) + --help, -h show help (default: false) + +``` + +### lotus-miner config updated +``` +NAME: + lotus-miner config updated - Print updated node config + +USAGE: + lotus-miner config updated [command options] [arguments...] + +OPTIONS: + --no-comment don't comment default values (default: false) + --help, -h show help (default: false) ``` From ac2a1bfeabc117164a8e1ca7e49868fe806239d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 15:53:22 +0200 Subject: [PATCH 09/11] fix lint --- cmd/lotus-storage-miner/config.go | 5 +++-- cmd/lotus/config.go | 3 ++- node/config/cfgdocgen/gen.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/config.go b/cmd/lotus-storage-miner/config.go index 30c012b9e..652426583 100644 --- a/cmd/lotus-storage-miner/config.go +++ b/cmd/lotus-storage-miner/config.go @@ -2,11 +2,12 @@ package main import ( "fmt" - "github.com/filecoin-project/lotus/node/repo" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/repo" ) var configCmd = &cli.Command{ @@ -51,7 +52,7 @@ var configUpdateCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - r, err := repo.NewFS(cctx.String("repo")) + r, err := repo.NewFS(cctx.String(FlagMinerRepo)) if err != nil { return err } diff --git a/cmd/lotus/config.go b/cmd/lotus/config.go index c965f2fc8..fcb7e2b08 100644 --- a/cmd/lotus/config.go +++ b/cmd/lotus/config.go @@ -2,11 +2,12 @@ package main import ( "fmt" - "github.com/filecoin-project/lotus/node/repo" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/repo" ) var configCmd = &cli.Command{ diff --git a/node/config/cfgdocgen/gen.go b/node/config/cfgdocgen/gen.go index 50c0c291c..8d0efb65e 100644 --- a/node/config/cfgdocgen/gen.go +++ b/node/config/cfgdocgen/gen.go @@ -125,7 +125,7 @@ var Doc = map[string][]DocField{ func main() { if err := run(); err != nil { - fmt.Printf(err.Error()) + fmt.Println(err.Error()) os.Exit(1) } } From 80af6de3165c65d6908d28e6d1df60c70387d3b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 16:02:45 +0200 Subject: [PATCH 10/11] config: Add note on codegen to types.go --- node/config/types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/config/types.go b/node/config/types.go index 5cb76b9ea..63a493f51 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -8,6 +8,8 @@ import ( ) // // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE +// // +// // After making edits here, run 'make cfgdoc-gen' (or 'make gen') // Common is common config between full node and miner type Common struct { From 91c6aca7e70c9b1df5a2958b0c7448ac43b509f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Jul 2021 16:59:55 +0200 Subject: [PATCH 11/11] config: Add newlines consistently --- node/config/load.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/node/config/load.go b/node/config/load.go index 569d65393..082106044 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -131,9 +131,10 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) { // if there is the same line in the default config, comment it out it output if _, found := defaults[strings.TrimSpace(nodeLines[i])]; (cfgDef == nil || found) && len(line) > 0 { line = pad + "#" + line[len(pad):] - outLines = append(outLines, line, "") - } else { - outLines = append(outLines, line) + } + outLines = append(outLines, line) + if len(line) > 0 { + outLines = append(outLines, "") } }