cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into feat/splitstore-badger-markset

Browse Source
This commit is contained in:
Łukasz Magiera 2021-07-23 19:37:45 +02:00 committed by GitHub
commit 5b4d5c2eb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1630 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -336,6 +336,9 @@ api-gen:
goimports -w api goimports -w api
.PHONY: api-gen .PHONY: api-gen
cfgdoc-gen:
go run ./node/config/cfgdocgen > ./node/config/doc_gen.go
appimage: lotus appimage: lotus
rm -rf appimage-builder-cache || true rm -rf appimage-builder-cache || true
rm AppDir/io.filecoin.lotus.desktop || true rm AppDir/io.filecoin.lotus.desktop || true
@ -373,7 +376,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
gen: actors-gen type-gen method-gen docsgen api-gen circleci gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli" @echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
.PHONY: gen .PHONY: gen

78
cmd/lotus-storage-miner/config.go
View File

@ -4,19 +4,91 @@ import (
"fmt" "fmt"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/repo"
) )
var configCmd = &cli.Command{ var configCmd = &cli.Command{
Name: "config", Name: "config",
Usage: "Output default configuration", Usage: "Manage node config",
Subcommands: []*cli.Command{
configDefaultCmd,
configUpdateCmd,
},
}
var configDefaultCmd = &cli.Command{
Name: "default",
Usage: "Print default node config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
Usage: "don't comment default values",
},
},
Action: func(cctx *cli.Context) error { Action: func(cctx *cli.Context) error {
comm, err := config.ConfigComment(config.DefaultStorageMiner()) c := config.DefaultStorageMiner()
cb, err := config.ConfigUpdate(c, nil, !cctx.Bool("no-comment"))
if err != nil { if err != nil {
return err return err
} }
fmt.Println(string(comm))
fmt.Println(string(cb))
return nil
},
}
var configUpdateCmd = &cli.Command{
Name: "updated",
Usage: "Print updated node config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
Usage: "don't comment default values",
},
},
Action: func(cctx *cli.Context) error {
r, err := repo.NewFS(cctx.String(FlagMinerRepo))
if err != nil {
return err
}
ok, err := r.Exists()
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("repo not initialized")
}
lr, err := r.LockRO(repo.StorageMiner)
if err != nil {
return xerrors.Errorf("locking repo: %w", err)
}
cfgNode, err := lr.Config()
if err != nil {
_ = lr.Close()
return xerrors.Errorf("getting node config: %w", err)
}
if err := lr.Close(); err != nil {
return err
}
cfgDef := config.DefaultStorageMiner()
updated, err := config.ConfigUpdate(cfgNode, cfgDef, !cctx.Bool("no-comment"))
if err != nil {
return err
}
fmt.Print(string(updated))
return nil return nil
}, },
} }

94
cmd/lotus/config.go Normal file
View File

@ -0,0 +1,94 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/repo"
)
var configCmd = &cli.Command{
Name: "config",
Usage: "Manage node config",
Subcommands: []*cli.Command{
configDefaultCmd,
configUpdateCmd,
},
}
var configDefaultCmd = &cli.Command{
Name: "default",
Usage: "Print default node config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
Usage: "don't comment default values",
},
},
Action: func(cctx *cli.Context) error {
c := config.DefaultFullNode()
cb, err := config.ConfigUpdate(c, nil, !cctx.Bool("no-comment"))
if err != nil {
return err
}
fmt.Println(string(cb))
return nil
},
}
var configUpdateCmd = &cli.Command{
Name: "updated",
Usage: "Print updated node config",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-comment",
Usage: "don't comment default values",
},
},
Action: func(cctx *cli.Context) error {
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return err
}
ok, err := r.Exists()
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("repo not initialized")
}
lr, err := r.LockRO(repo.FullNode)
if err != nil {
return xerrors.Errorf("locking repo: %w", err)
}
cfgNode, err := lr.Config()
if err != nil {
_ = lr.Close()
return xerrors.Errorf("getting node config: %w", err)
}
if err := lr.Close(); err != nil {
return err
}
cfgDef := config.DefaultFullNode()
updated, err := config.ConfigUpdate(cfgNode, cfgDef, !cctx.Bool("no-comment"))
if err != nil {
return err
}
fmt.Print(string(updated))
return nil
},
}

1
cmd/lotus/main.go
View File

@ -29,6 +29,7 @@ func main() {
local := []*cli.Command{ local := []*cli.Command{
DaemonCmd, DaemonCmd,
backupCmd, backupCmd,
configCmd,
} }
if AdvanceBlockCmd != nil { if AdvanceBlockCmd != nil {
local = append(local, AdvanceBlockCmd) local = append(local, AdvanceBlockCmd)

42
documentation/en/cli-lotus-miner.md
View File

@ -13,7 +13,7 @@ COMMANDS:
init Initialize a lotus miner repo init Initialize a lotus miner repo
run Start a lotus miner process run Start a lotus miner process
stop Stop a running lotus miner stop Stop a running lotus miner
config Output default configuration config Manage node config
backup Create node metadata backup backup Create node metadata backup
version Print version version Print version
help, h Shows a list of commands or help for one command help, h Shows a list of commands or help for one command
@ -145,13 +145,47 @@ OPTIONS:
## lotus-miner config ## lotus-miner config
``` ```
NAME: NAME:
lotus-miner config - Output default configuration lotus-miner config - Manage node config
USAGE: USAGE:
lotus-miner config [command options] [arguments...] lotus-miner config command [command options] [arguments...]
COMMANDS:
default Print default node config
updated Print updated node config
help, h Shows a list of commands or help for one command
OPTIONS: OPTIONS:
--help, -h show help (default: false) --help, -h show help (default: false)
--version, -v print the version (default: false)
```
### lotus-miner config default
```
NAME:
lotus-miner config default - Print default node config
USAGE:
lotus-miner config default [command options] [arguments...]
OPTIONS:
--no-comment don't comment default values (default: false)
--help, -h show help (default: false)
```
### lotus-miner config updated
```
NAME:
lotus-miner config updated - Print updated node config
USAGE:
lotus-miner config updated [command options] [arguments...]
OPTIONS:
--no-comment don't comment default values (default: false)
--help, -h show help (default: false)
``` ```

48
documentation/en/cli-lotus.md
View File

@ -12,6 +12,7 @@ VERSION:
COMMANDS: COMMANDS:
daemon Start a lotus daemon process daemon Start a lotus daemon process
backup Create node metadata backup backup Create node metadata backup
config Manage node config
version Print version version Print version
help, h Shows a list of commands or help for one command help, h Shows a list of commands or help for one command
BASIC: BASIC:
@ -108,6 +109,53 @@ OPTIONS:
``` ```
## lotus config
```
NAME:
lotus config - Manage node config
USAGE:
lotus config command [command options] [arguments...]
COMMANDS:
default Print default node config
updated Print updated node config
help, h Shows a list of commands or help for one command
OPTIONS:
--help, -h show help (default: false)
--version, -v print the version (default: false)
```
### lotus config default
```
NAME:
lotus config default - Print default node config
USAGE:
lotus config default [command options] [arguments...]
OPTIONS:
--no-comment don't comment default values (default: false)
--help, -h show help (default: false)
```
### lotus config updated
```
NAME:
lotus config updated - Print updated node config
USAGE:
lotus config updated [command options] [arguments...]
OPTIONS:
--no-comment don't comment default values (default: false)
--help, -h show help (default: false)
```
## lotus version ## lotus version
``` ```
NAME: NAME:

131
node/config/cfgdocgen/gen.go Normal file
View File

@ -0,0 +1,131 @@
package main
import (
"fmt"
"io/ioutil"
"os"
"sort"
"strings"
)
func run() error {
tfb, err := ioutil.ReadFile("./node/config/types.go")
if err != nil {
return err
}
// could use the ast lib, but this is simpler
type st int
const (
stGlobal st = iota // looking for typedef
stType st = iota // in typedef
)
lines := strings.Split(string(tfb), "\n")
state := stGlobal
type field struct {
Name string
Type string
Comment string
}
var currentType string
var currentComment []string
out := map[string][]field{}
for l := range lines {
line := strings.TrimSpace(lines[l])
switch state {
case stGlobal:
if strings.HasPrefix(line, "type ") {
currentType = line
currentType = strings.TrimPrefix(currentType, "type")
currentType = strings.TrimSuffix(currentType, "{")
currentType = strings.TrimSpace(currentType)
currentType = strings.TrimSuffix(currentType, "struct")
currentType = strings.TrimSpace(currentType)
currentComment = nil
state = stType
continue
}
case stType:
if strings.HasPrefix(line, "// ") {
cline := strings.TrimSpace(strings.TrimPrefix(line, "//"))
currentComment = append(currentComment, cline)
continue
}
comment := currentComment
currentComment = nil
if strings.HasPrefix(line, "}") {
state = stGlobal
continue
}
f := strings.Fields(line)
if len(f) < 2 { // empty or embedded struct
continue
}
name := f[0]
typ := f[1]
out[currentType] = append(out[currentType], field{
Name: name,
Type: typ,
Comment: strings.Join(comment, "\n"),
})
}
}
var outt []string
for t := range out {
outt = append(outt, t)
}
sort.Strings(outt)
fmt.Print(`// Code generated by github.com/filecoin-project/lotus/node/config/cfgdocgen. DO NOT EDIT.
package config
type DocField struct {
Name string
Type string
Comment string
}
var Doc = map[string][]DocField{
`)
for _, typeName := range outt {
typ := out[typeName]
fmt.Printf("\t\"%s\": []DocField{\n", typeName)
for _, f := range typ {
fmt.Println("\t\t{")
fmt.Printf("\t\t\tName: \"%s\",\n", f.Name)
fmt.Printf("\t\t\tType: \"%s\",\n\n", f.Type)
fmt.Printf("\t\t\tComment: `%s`,\n", f.Comment)
fmt.Println("\t\t},")
}
fmt.Printf("\t},\n")
}
fmt.Println(`}`)
return nil
}
func main() {
if err := run(); err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
}

254
node/config/def.go
View File

@ -22,264 +22,10 @@ const (
RetrievalPricingExternalMode = "external" RetrievalPricingExternalMode = "external"
) )
// Common is common config between full node and miner
type Common struct {
API API
Backup Backup
Libp2p Libp2p
Pubsub Pubsub
}
// FullNode is a full node config
type FullNode struct {
Common
Client Client
Metrics Metrics
Wallet Wallet
Fees FeeConfig
Chainstore Chainstore
}
// // Common
type Backup struct {
DisableMetadataLog bool
}
// StorageMiner is a miner config
type StorageMiner struct {
Common
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
}
type MinerSubsystemConfig struct {
EnableMining bool
EnableSealing bool
EnableSectorStorage bool
EnableMarkets bool
SealerApiInfo string // if EnableSealing == false
SectorIndexApiInfo string // if EnableSectorStorage == false
}
type DealmakingConfig struct {
ConsiderOnlineStorageDeals bool
ConsiderOfflineStorageDeals bool
ConsiderOnlineRetrievalDeals bool
ConsiderOfflineRetrievalDeals bool
ConsiderVerifiedStorageDeals bool
ConsiderUnverifiedStorageDeals bool
PieceCidBlocklist []cid.Cid
ExpectedSealDuration Duration
// Maximum amount of time proposed deal StartEpoch can be in future
MaxDealStartDelay Duration
// The amount of time to wait for more deals to arrive before
// publishing
PublishMsgPeriod Duration
// The maximum number of deals to include in a single PublishStorageDeals
// message
MaxDealsPerPublishMsg uint64
// The maximum collateral that the provider will put up against a deal,
// as a multiplier of the minimum collateral bound
MaxProviderCollateralMultiplier uint64
// The maximum number of parallel online data transfers (storage+retrieval)
SimultaneousTransfers uint64
Filter string
RetrievalFilter string
RetrievalPricing *RetrievalPricing
}
type RetrievalPricing struct {
Strategy string // possible values: "default", "external"
Default *RetrievalPricingDefault
External *RetrievalPricingExternal
}
type RetrievalPricingExternal struct {
// Path of the external script that will be run to price a retrieval deal.
// This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".
Path string
}
type RetrievalPricingDefault struct {
// VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal
// of a payloadCid that belongs to a verified storage deal.
// This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default".
// default value is true
VerifiedDealsFreeTransfer bool
}
type SealingConfig struct {
// 0 = no limit
MaxWaitDealsSectors uint64
// includes failed, 0 = no limit
MaxSealingSectors uint64
// includes failed, 0 = no limit
MaxSealingSectorsForDeals uint64
WaitDealsDelay Duration
AlwaysKeepUnsealedCopy bool
// Run sector finalization before submitting sector proof to the chain
FinalizeEarly bool
// Whether to use available miner balance for sector collateral instead of sending it with each message
CollateralFromMinerBalance bool
// Minimum available balance to keep in the miner actor before sending it with messages
AvailableBalanceBuffer types.FIL
// Don't send collateral with messages even if there is no available balance in the miner actor
DisableCollateralFallback bool
// enable / disable precommit batching (takes effect after nv13)
BatchPreCommits bool
// maximum precommit batch size - batches will be sent immediately above this size
MaxPreCommitBatch int
// how long to wait before submitting a batch after crossing the minimum batch size
PreCommitBatchWait Duration
// time buffer for forceful batch submission before sectors/deal in batch would start expiring
PreCommitBatchSlack Duration
// enable / disable commit aggregation (takes effect after nv13)
AggregateCommits bool
// maximum batched commit size - batches will be sent immediately above this size
MinCommitBatch int
MaxCommitBatch int
// how long to wait before submitting a batch after crossing the minimum batch size
CommitBatchWait Duration
// time buffer for forceful batch submission before sectors/deals in batch would start expiring
CommitBatchSlack Duration
// network BaseFee below which to stop doing commit aggregation, instead
// submitting proofs to the chain individually
AggregateAboveBaseFee types.FIL
TerminateBatchMax uint64
TerminateBatchMin uint64
TerminateBatchWait Duration
// Keep this many sectors in sealing pipeline, start CC if needed
// todo TargetSealingSectors uint64
// todo TargetSectors - stop auto-pleding new sectors after this many sectors are sealed, default CC upgrade for deals sectors if above
}
type BatchFeeConfig struct {
Base types.FIL
PerSector types.FIL
}
func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount { func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount {
return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector))) return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector)))
} }
type MinerFeeConfig struct {
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
// maxBatchFee = maxBase + maxPerSector * nSectors
MaxPreCommitBatchGasFee BatchFeeConfig
MaxCommitBatchGasFee BatchFeeConfig
MaxTerminateGasFee types.FIL
MaxWindowPoStGasFee types.FIL
MaxPublishDealsFee types.FIL
MaxMarketBalanceAddFee types.FIL
}
type MinerAddressConfig struct {
PreCommitControl []string
CommitControl []string
TerminateControl []string
DealPublishControl []string
// DisableOwnerFallback disables usage of the owner address for messages
// sent automatically
DisableOwnerFallback bool
// DisableWorkerFallback disables usage of the worker address for messages
// sent automatically, if control addresses are configured.
// A control address that doesn't have enough funds will still be chosen
// over the worker address if this flag is set.
DisableWorkerFallback bool
}
// API contains configs for API endpoint
type API struct {
ListenAddress string
RemoteListenAddress string
Timeout Duration
}
// Libp2p contains configs for libp2p
type Libp2p struct {
ListenAddresses []string
AnnounceAddresses []string
NoAnnounceAddresses []string
BootstrapPeers []string
ProtectedPeers []string
ConnMgrLow uint
ConnMgrHigh uint
ConnMgrGrace Duration
}
type Pubsub struct {
Bootstrapper bool
DirectPeers []string
IPColocationWhitelist []string
RemoteTracer string
}
type Chainstore struct {
EnableSplitstore bool
Splitstore Splitstore
}
type Splitstore struct {
ColdStoreType string
HotStoreType string
MarkSetType string
HotStoreMessageRetention uint64
}
// // Full Node
type Metrics struct {
Nickname string
HeadNotifs bool
}
type Client struct {
UseIpfs bool
IpfsOnlineMode bool
IpfsMAddr string
IpfsUseForRetrieval bool
SimultaneousTransfers uint64
}
type Wallet struct {
RemoteBackend string
EnableLedger bool
DisableLocal bool
}
type FeeConfig struct {
DefaultMaxFee types.FIL
}
func defCommon() Common { func defCommon() Common {
return Common{ return Common{
API: API{ API: API{

767
node/config/doc_gen.go Normal file
View File

@ -0,0 +1,767 @@
// Code generated by github.com/filecoin-project/lotus/node/config/cfgdocgen. DO NOT EDIT.
package config
type DocField struct {
Name string
Type string
Comment string
}
var Doc = map[string][]DocField{
"API": []DocField{
{
Name: "ListenAddress",
Type: "string",
Comment: `Binding address for the Lotus API`,
},
{
Name: "RemoteListenAddress",
Type: "string",
Comment: ``,
},
{
Name: "Timeout",
Type: "Duration",
Comment: ``,
},
},
"Backup": []DocField{
{
Name: "DisableMetadataLog",
Type: "bool",
Comment: `Note that in case of metadata corruption it might be much harder to recover
your node if metadata log is disabled`,
},
},
"BatchFeeConfig": []DocField{
{
Name: "Base",
Type: "types.FIL",
Comment: ``,
},
{
Name: "PerSector",
Type: "types.FIL",
Comment: ``,
},
},
"Chainstore": []DocField{
{
Name: "EnableSplitstore",
Type: "bool",
Comment: ``,
},
{
Name: "Splitstore",
Type: "Splitstore",
Comment: ``,
},
},
"Client": []DocField{
{
Name: "UseIpfs",
Type: "bool",
Comment: ``,
},
{
Name: "IpfsOnlineMode",
Type: "bool",
Comment: ``,
},
{
Name: "IpfsMAddr",
Type: "string",
Comment: ``,
},
{
Name: "IpfsUseForRetrieval",
Type: "bool",
Comment: ``,
},
{
Name: "SimultaneousTransfers",
Type: "uint64",
Comment: `The maximum number of simultaneous data transfers between the client
and storage providers`,
},
},
"Common": []DocField{
{
Name: "API",
Type: "API",
Comment: ``,
},
{
Name: "Backup",
Type: "Backup",
Comment: ``,
},
{
Name: "Libp2p",
Type: "Libp2p",
Comment: ``,
},
{
Name: "Pubsub",
Type: "Pubsub",
Comment: ``,
},
},
"DealmakingConfig": []DocField{
{
Name: "ConsiderOnlineStorageDeals",
Type: "bool",
Comment: `When enabled, the miner can accept online deals`,
},
{
Name: "ConsiderOfflineStorageDeals",
Type: "bool",
Comment: `When enabled, the miner can accept offline deals`,
},
{
Name: "ConsiderOnlineRetrievalDeals",
Type: "bool",
Comment: `When enabled, the miner can accept retrieval deals`,
},
{
Name: "ConsiderOfflineRetrievalDeals",
Type: "bool",
Comment: `When enabled, the miner can accept offline retrieval deals`,
},
{
Name: "ConsiderVerifiedStorageDeals",
Type: "bool",
Comment: `When enabled, the miner can accept verified deals`,
},
{
Name: "ConsiderUnverifiedStorageDeals",
Type: "bool",
Comment: `When enabled, the miner can accept unverified deals`,
},
{
Name: "PieceCidBlocklist",
Type: "[]cid.Cid",
Comment: `A list of Data CIDs to reject when making deals`,
},
{
Name: "ExpectedSealDuration",
Type: "Duration",
Comment: `Maximum expected amount of time getting the deal into a sealed sector will take
This includes the time the deal will need to get transferred and published
before being assigned to a sector`,
},
{
Name: "MaxDealStartDelay",
Type: "Duration",
Comment: `Maximum amount of time proposed deal StartEpoch can be in future`,
},
{
Name: "PublishMsgPeriod",
Type: "Duration",
Comment: `When a deal is ready to publish, the amount of time to wait for more
deals to be ready to publish before publishing them all as a batch`,
},
{
Name: "MaxDealsPerPublishMsg",
Type: "uint64",
Comment: `The maximum number of deals to include in a single PublishStorageDeals
message`,
},
{
Name: "MaxProviderCollateralMultiplier",
Type: "uint64",
Comment: `The maximum collateral that the provider will put up against a deal,
as a multiplier of the minimum collateral bound`,
},
{
Name: "SimultaneousTransfers",
Type: "uint64",
Comment: `The maximum number of parallel online data transfers (storage+retrieval)`,
},
{
Name: "Filter",
Type: "string",
Comment: `A command used for fine-grained evaluation of storage deals
see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
},
{
Name: "RetrievalFilter",
Type: "string",
Comment: `A command used for fine-grained evaluation of retrieval deals
see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
},
{
Name: "RetrievalPricing",
Type: "*RetrievalPricing",
Comment: ``,
},
},
"FeeConfig": []DocField{
{
Name: "DefaultMaxFee",
Type: "types.FIL",
Comment: ``,
},
},
"FullNode": []DocField{
{
Name: "Client",
Type: "Client",
Comment: ``,
},
{
Name: "Metrics",
Type: "Metrics",
Comment: ``,
},
{
Name: "Wallet",
Type: "Wallet",
Comment: ``,
},
{
Name: "Fees",
Type: "FeeConfig",
Comment: ``,
},
{
Name: "Chainstore",
Type: "Chainstore",
Comment: ``,
},
},
"Libp2p": []DocField{
{
Name: "ListenAddresses",
Type: "[]string",
Comment: `Binding address for the libp2p host - 0 means random port.
Format: multiaddress; see https://multiformats.io/multiaddr/`,
},
{
Name: "AnnounceAddresses",
Type: "[]string",
Comment: `Addresses to explicitally announce to other peers. If not specified,
all interface addresses are announced
Format: multiaddress`,
},
{
Name: "NoAnnounceAddresses",
Type: "[]string",
Comment: `Addresses to not announce
Format: multiaddress`,
},
{
Name: "BootstrapPeers",
Type: "[]string",
Comment: ``,
},
{
Name: "ProtectedPeers",
Type: "[]string",
Comment: ``,
},
{
Name: "ConnMgrLow",
Type: "uint",
Comment: ``,
},
{
Name: "ConnMgrHigh",
Type: "uint",
Comment: ``,
},
{
Name: "ConnMgrGrace",
Type: "Duration",
Comment: ``,
},
},
"Metrics": []DocField{
{
Name: "Nickname",
Type: "string",
Comment: ``,
},
{
Name: "HeadNotifs",
Type: "bool",
Comment: ``,
},
},
"MinerAddressConfig": []DocField{
{
Name: "PreCommitControl",
Type: "[]string",
Comment: `Addresses to send PreCommit messages from`,
},
{
Name: "CommitControl",
Type: "[]string",
Comment: `Addresses to send Commit messages from`,
},
{
Name: "TerminateControl",
Type: "[]string",
Comment: ``,
},
{
Name: "DealPublishControl",
Type: "[]string",
Comment: ``,
},
{
Name: "DisableOwnerFallback",
Type: "bool",
Comment: `DisableOwnerFallback disables usage of the owner address for messages
sent automatically`,
},
{
Name: "DisableWorkerFallback",
Type: "bool",
Comment: `DisableWorkerFallback disables usage of the worker address for messages
sent automatically, if control addresses are configured.
A control address that doesn't have enough funds will still be chosen
over the worker address if this flag is set.`,
},
},
"MinerFeeConfig": []DocField{
{
Name: "MaxPreCommitGasFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxCommitGasFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxPreCommitBatchGasFee",
Type: "BatchFeeConfig",
Comment: `maxBatchFee = maxBase + maxPerSector * nSectors`,
},
{
Name: "MaxCommitBatchGasFee",
Type: "BatchFeeConfig",
Comment: ``,
},
{
Name: "MaxTerminateGasFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxWindowPoStGasFee",
Type: "types.FIL",
Comment: `WindowPoSt is a high-value operation, so the default fee should be high.`,
},
{
Name: "MaxPublishDealsFee",
Type: "types.FIL",
Comment: ``,
},
{
Name: "MaxMarketBalanceAddFee",
Type: "types.FIL",
Comment: ``,
},
},
"MinerSubsystemConfig": []DocField{
{
Name: "EnableMining",
Type: "bool",
Comment: ``,
},
{
Name: "EnableSealing",
Type: "bool",
Comment: ``,
},
{
Name: "EnableSectorStorage",
Type: "bool",
Comment: ``,
},
{
Name: "EnableMarkets",
Type: "bool",
Comment: ``,
},
{
Name: "SealerApiInfo",
Type: "string",
Comment: ``,
},
{
Name: "SectorIndexApiInfo",
Type: "string",
Comment: ``,
},
},
"Pubsub": []DocField{
{
Name: "Bootstrapper",
Type: "bool",
Comment: `Run the node in bootstrap-node mode`,
},
{
Name: "DirectPeers",
Type: "[]string",
Comment: `DirectPeers specifies peers with direct peering agreements. These peers are
connected outside of the mesh, with all (valid) message unconditionally
forwarded to them. The router will maintain open connections to these peers.
Note that the peering agreement should be reciprocal with direct peers
symmetrically configured at both ends.
Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...`,
},
{
Name: "IPColocationWhitelist",
Type: "[]string",
Comment: ``,
},
{
Name: "RemoteTracer",
Type: "string",
Comment: ``,
},
},
"RetrievalPricing": []DocField{
{
Name: "Strategy",
Type: "string",
Comment: ``,
},
{
Name: "Default",
Type: "*RetrievalPricingDefault",
Comment: ``,
},
{
Name: "External",
Type: "*RetrievalPricingExternal",
Comment: ``,
},
},
"RetrievalPricingDefault": []DocField{
{
Name: "VerifiedDealsFreeTransfer",
Type: "bool",
Comment: `VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal
of a payloadCid that belongs to a verified storage deal.
This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default".
default value is true`,
},
},
"RetrievalPricingExternal": []DocField{
{
Name: "Path",
Type: "string",
Comment: `Path of the external script that will be run to price a retrieval deal.
This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".`,
},
},
"SealingConfig": []DocField{
{
Name: "MaxWaitDealsSectors",
Type: "uint64",
Comment: `Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time.
If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created.
If more than MaxWaitDealsSectors deals are accepted in parallel, only MaxWaitDealsSectors deals will be processed in parallel
Note that setting this number too high in relation to deal ingestion rate may result in poor sector packing efficiency
0 = no limit`,
},
{
Name: "MaxSealingSectors",
Type: "uint64",
Comment: `Upper bound on how many sectors can be sealing at the same time when creating new CC sectors (0 = unlimited)`,
},
{
Name: "MaxSealingSectorsForDeals",
Type: "uint64",
Comment: `Upper bound on how many sectors can be sealing at the same time when creating new sectors with deals (0 = unlimited)`,
},
{
Name: "WaitDealsDelay",
Type: "Duration",
Comment: `Period of time that a newly created sector will wait for more deals to be packed in to before it starts to seal.
Sectors which are fully filled will start sealing immediately`,
},
{
Name: "AlwaysKeepUnsealedCopy",
Type: "bool",
Comment: `Whether to keep unsealed copies of deal data regardless of whether the client requested that. This lets the miner
avoid the relatively high cost of unsealing the data later, at the cost of more storage space`,
},
{
Name: "FinalizeEarly",
Type: "bool",
Comment: `Run sector finalization before submitting sector proof to the chain`,
},
{
Name: "CollateralFromMinerBalance",
Type: "bool",
Comment: `Whether to use available miner balance for sector collateral instead of sending it with each message`,
},
{
Name: "AvailableBalanceBuffer",
Type: "types.FIL",
Comment: `Minimum available balance to keep in the miner actor before sending it with messages`,
},
{
Name: "DisableCollateralFallback",
Type: "bool",
Comment: `Don't send collateral with messages even if there is no available balance in the miner actor`,
},
{
Name: "BatchPreCommits",
Type: "bool",
Comment: `enable / disable precommit batching (takes effect after nv13)`,
},
{
Name: "MaxPreCommitBatch",
Type: "int",
Comment: `maximum precommit batch size - batches will be sent immediately above this size`,
},
{
Name: "PreCommitBatchWait",
Type: "Duration",
Comment: `how long to wait before submitting a batch after crossing the minimum batch size`,
},
{
Name: "PreCommitBatchSlack",
Type: "Duration",
Comment: `time buffer for forceful batch submission before sectors/deal in batch would start expiring`,
},
{
Name: "AggregateCommits",
Type: "bool",
Comment: `enable / disable commit aggregation (takes effect after nv13)`,
},
{
Name: "MinCommitBatch",
Type: "int",
Comment: `maximum batched commit size - batches will be sent immediately above this size`,
},
{
Name: "MaxCommitBatch",
Type: "int",
Comment: ``,
},
{
Name: "CommitBatchWait",
Type: "Duration",
Comment: `how long to wait before submitting a batch after crossing the minimum batch size`,
},
{
Name: "CommitBatchSlack",
Type: "Duration",
Comment: `time buffer for forceful batch submission before sectors/deals in batch would start expiring`,
},
{
Name: "AggregateAboveBaseFee",
Type: "types.FIL",
Comment: `network BaseFee below which to stop doing commit aggregation, instead
submitting proofs to the chain individually`,
},
{
Name: "TerminateBatchMax",
Type: "uint64",
Comment: ``,
},
{
Name: "TerminateBatchMin",
Type: "uint64",
Comment: ``,
},
{
Name: "TerminateBatchWait",
Type: "Duration",
Comment: ``,
},
},
"Splitstore": []DocField{
{
Name: "ColdStoreType",
Type: "string",
Comment: ``,
},
{
Name: "HotStoreType",
Type: "string",
Comment: ``,
},
{
Name: "MarkSetType",
Type: "string",
Comment: ``,
},
{
Name: "HotStoreMessageRetention",
Type: "uint64",
Comment: ``,
},
},
"StorageMiner": []DocField{
{
Name: "Subsystems",
Type: "MinerSubsystemConfig",
Comment: ``,
},
{
Name: "Dealmaking",
Type: "DealmakingConfig",
Comment: ``,
},
{
Name: "Sealing",
Type: "SealingConfig",
Comment: ``,
},
{
Name: "Storage",
Type: "sectorstorage.SealerConfig",
Comment: ``,
},
{
Name: "Fees",
Type: "MinerFeeConfig",
Comment: ``,
},
{
Name: "Addresses",
Type: "MinerAddressConfig",
Comment: ``,
},
},
"Wallet": []DocField{
{
Name: "RemoteBackend",
Type: "string",
Comment: ``,
},
{
Name: "EnableLedger",
Type: "bool",
Comment: ``,
},
{
Name: "DisableLocal",
Type: "bool",
Comment: ``,
},
},
}

44
node/config/doc_util.go Normal file
View File

@ -0,0 +1,44 @@
package config
import (
"fmt"
"strings"
)
func findDoc(root interface{}, section, name string) *DocField {
rt := fmt.Sprintf("%T", root)[len("*config."):]
doc := findDocSect(rt, section, name)
if doc != nil {
return doc
}
return findDocSect("Common", section, name)
}
func findDocSect(root string, section, name string) *DocField {
path := strings.Split(section, ".")
docSection := Doc[root]
for _, e := range path {
if docSection == nil {
return nil
}
for _, field := range docSection {
if field.Name == e {
docSection = Doc[field.Type]
break
}
}
}
for _, df := range docSection {
if df.Name == name {
return &df
}
}
return nil
}

125
node/config/load.go
View File

@ -5,6 +5,10 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"reflect"
"regexp"
"strings"
"unicode"
"github.com/BurntSushi/toml" "github.com/BurntSushi/toml"
"github.com/kelseyhightower/envconfig" "github.com/kelseyhightower/envconfig"
@ -42,15 +46,116 @@ func FromReader(reader io.Reader, def interface{}) (interface{}, error) {
return cfg, nil return cfg, nil
} }
func ConfigComment(t interface{}) ([]byte, error) { func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) {
buf := new(bytes.Buffer) var nodeStr, defStr string
_, _ = buf.WriteString("# Default config:\n") if cfgDef != nil {
e := toml.NewEncoder(buf) buf := new(bytes.Buffer)
if err := e.Encode(t); err != nil { e := toml.NewEncoder(buf)
return nil, xerrors.Errorf("encoding config: %w", err) if err := e.Encode(cfgDef); err != nil {
return nil, xerrors.Errorf("encoding default config: %w", err)
}
defStr = buf.String()
} }
b := buf.Bytes()
b = bytes.ReplaceAll(b, []byte("\n"), []byte("\n#")) {
b = bytes.ReplaceAll(b, []byte("#["), []byte("[")) buf := new(bytes.Buffer)
return b, nil e := toml.NewEncoder(buf)
if err := e.Encode(cfgCur); err != nil {
return nil, xerrors.Errorf("encoding node config: %w", err)
}
nodeStr = buf.String()
}
if comment {
// create a map of default lines so we can comment those out later
defLines := strings.Split(defStr, "\n")
defaults := map[string]struct{}{}
for i := range defLines {
l := strings.TrimSpace(defLines[i])
if len(l) == 0 {
continue
}
if l[0] == '#' || l[0] == '[' {
continue
}
defaults[l] = struct{}{}
}
nodeLines := strings.Split(nodeStr, "\n")
var outLines []string
sectionRx := regexp.MustCompile(`\[(.+)]`)
var section string
for i, line := range nodeLines {
// if this is a section, track it
trimmed := strings.TrimSpace(line)
if len(trimmed) > 0 {
if trimmed[0] == '[' {
m := sectionRx.FindSubmatch([]byte(trimmed))
if len(m) != 2 {
return nil, xerrors.Errorf("section didn't match (line %d)", i)
}
section = string(m[1])
// never comment sections
outLines = append(outLines, line)
continue
}
}
pad := strings.Repeat(" ", len(line)-len(strings.TrimLeftFunc(line, unicode.IsSpace)))
// see if we have docs for this field
{
lf := strings.Fields(line)
if len(lf) > 1 {
doc := findDoc(cfgCur, section, lf[0])
if doc != nil {
// found docfield, emit doc comment
if len(doc.Comment) > 0 {
for _, docLine := range strings.Split(doc.Comment, "\n") {
outLines = append(outLines, pad+"# "+docLine)
}
outLines = append(outLines, pad+"#")
}
outLines = append(outLines, pad+"# type: "+doc.Type)
}
}
}
// if there is the same line in the default config, comment it out it output
if _, found := defaults[strings.TrimSpace(nodeLines[i])]; (cfgDef == nil || found) && len(line) > 0 {
line = pad + "#" + line[len(pad):]
}
outLines = append(outLines, line)
if len(line) > 0 {
outLines = append(outLines, "")
}
}
nodeStr = strings.Join(outLines, "\n")
}
// sanity-check that the updated config parses the same way as the current one
if cfgDef != nil {
cfgUpdated, err := FromReader(strings.NewReader(nodeStr), cfgDef)
if err != nil {
return nil, xerrors.Errorf("parsing updated config: %w", err)
}
if !reflect.DeepEqual(cfgCur, cfgUpdated) {
return nil, xerrors.Errorf("updated config didn't match current config")
}
}
return []byte(nodeStr), nil
}
func ConfigComment(t interface{}) ([]byte, error) {
return ConfigUpdate(t, nil, true)
} }

313
node/config/types.go Normal file
View File

@ -0,0 +1,313 @@
package config
import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/types"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
)
// // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE
// //
// // After making edits here, run 'make cfgdoc-gen' (or 'make gen')
// Common is common config between full node and miner
type Common struct {
API API
Backup Backup
Libp2p Libp2p
Pubsub Pubsub
}
// FullNode is a full node config
type FullNode struct {
Common
Client Client
Metrics Metrics
Wallet Wallet
Fees FeeConfig
Chainstore Chainstore
}
// // Common
type Backup struct {
// When set to true disables metadata log (.lotus/kvlog). This can save disk
// space by reducing metadata redundancy.
//
// Note that in case of metadata corruption it might be much harder to recover
// your node if metadata log is disabled
DisableMetadataLog bool
}
// StorageMiner is a miner config
type StorageMiner struct {
Common
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
}
type MinerSubsystemConfig struct {
EnableMining bool
EnableSealing bool
EnableSectorStorage bool
EnableMarkets bool
SealerApiInfo string // if EnableSealing == false
SectorIndexApiInfo string // if EnableSectorStorage == false
}
type DealmakingConfig struct {
// When enabled, the miner can accept online deals
ConsiderOnlineStorageDeals bool
// When enabled, the miner can accept offline deals
ConsiderOfflineStorageDeals bool
// When enabled, the miner can accept retrieval deals
ConsiderOnlineRetrievalDeals bool
// When enabled, the miner can accept offline retrieval deals
ConsiderOfflineRetrievalDeals bool
// When enabled, the miner can accept verified deals
ConsiderVerifiedStorageDeals bool
// When enabled, the miner can accept unverified deals
ConsiderUnverifiedStorageDeals bool
// A list of Data CIDs to reject when making deals
PieceCidBlocklist []cid.Cid
// Maximum expected amount of time getting the deal into a sealed sector will take
// This includes the time the deal will need to get transferred and published
// before being assigned to a sector
ExpectedSealDuration Duration
// Maximum amount of time proposed deal StartEpoch can be in future
MaxDealStartDelay Duration
// When a deal is ready to publish, the amount of time to wait for more
// deals to be ready to publish before publishing them all as a batch
PublishMsgPeriod Duration
// The maximum number of deals to include in a single PublishStorageDeals
// message
MaxDealsPerPublishMsg uint64
// The maximum collateral that the provider will put up against a deal,
// as a multiplier of the minimum collateral bound
MaxProviderCollateralMultiplier uint64
// The maximum number of parallel online data transfers (storage+retrieval)
SimultaneousTransfers uint64
// A command used for fine-grained evaluation of storage deals
// see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
Filter string
// A command used for fine-grained evaluation of retrieval deals
// see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
RetrievalFilter string
RetrievalPricing *RetrievalPricing
}
type RetrievalPricing struct {
Strategy string // possible values: "default", "external"
Default *RetrievalPricingDefault
External *RetrievalPricingExternal
}
type RetrievalPricingExternal struct {
// Path of the external script that will be run to price a retrieval deal.
// This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external".
Path string
}
type RetrievalPricingDefault struct {
// VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal
// of a payloadCid that belongs to a verified storage deal.
// This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default".
// default value is true
VerifiedDealsFreeTransfer bool
}
type SealingConfig struct {
// Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time.
// If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created.
// If more than MaxWaitDealsSectors deals are accepted in parallel, only MaxWaitDealsSectors deals will be processed in parallel
// Note that setting this number too high in relation to deal ingestion rate may result in poor sector packing efficiency
// 0 = no limit
MaxWaitDealsSectors uint64
// Upper bound on how many sectors can be sealing at the same time when creating new CC sectors (0 = unlimited)
MaxSealingSectors uint64
// Upper bound on how many sectors can be sealing at the same time when creating new sectors with deals (0 = unlimited)
MaxSealingSectorsForDeals uint64
// Period of time that a newly created sector will wait for more deals to be packed in to before it starts to seal.
// Sectors which are fully filled will start sealing immediately
WaitDealsDelay Duration
// Whether to keep unsealed copies of deal data regardless of whether the client requested that. This lets the miner
// avoid the relatively high cost of unsealing the data later, at the cost of more storage space
AlwaysKeepUnsealedCopy bool
// Run sector finalization before submitting sector proof to the chain
FinalizeEarly bool
// Whether to use available miner balance for sector collateral instead of sending it with each message
CollateralFromMinerBalance bool
// Minimum available balance to keep in the miner actor before sending it with messages
AvailableBalanceBuffer types.FIL
// Don't send collateral with messages even if there is no available balance in the miner actor
DisableCollateralFallback bool
// enable / disable precommit batching (takes effect after nv13)
BatchPreCommits bool
// maximum precommit batch size - batches will be sent immediately above this size
MaxPreCommitBatch int
// how long to wait before submitting a batch after crossing the minimum batch size
PreCommitBatchWait Duration
// time buffer for forceful batch submission before sectors/deal in batch would start expiring
PreCommitBatchSlack Duration
// enable / disable commit aggregation (takes effect after nv13)
AggregateCommits bool
// maximum batched commit size - batches will be sent immediately above this size
MinCommitBatch int
MaxCommitBatch int
// how long to wait before submitting a batch after crossing the minimum batch size
CommitBatchWait Duration
// time buffer for forceful batch submission before sectors/deals in batch would start expiring
CommitBatchSlack Duration
// network BaseFee below which to stop doing commit aggregation, instead
// submitting proofs to the chain individually
AggregateAboveBaseFee types.FIL
TerminateBatchMax uint64
TerminateBatchMin uint64
TerminateBatchWait Duration
// Keep this many sectors in sealing pipeline, start CC if needed
// todo TargetSealingSectors uint64
// todo TargetSectors - stop auto-pleding new sectors after this many sectors are sealed, default CC upgrade for deals sectors if above
}
type BatchFeeConfig struct {
Base types.FIL
PerSector types.FIL
}
type MinerFeeConfig struct {
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
// maxBatchFee = maxBase + maxPerSector * nSectors
MaxPreCommitBatchGasFee BatchFeeConfig
MaxCommitBatchGasFee BatchFeeConfig
MaxTerminateGasFee types.FIL
// WindowPoSt is a high-value operation, so the default fee should be high.
MaxWindowPoStGasFee types.FIL
MaxPublishDealsFee types.FIL
MaxMarketBalanceAddFee types.FIL
}
type MinerAddressConfig struct {
// Addresses to send PreCommit messages from
PreCommitControl []string
// Addresses to send Commit messages from
CommitControl []string
TerminateControl []string
DealPublishControl []string
// DisableOwnerFallback disables usage of the owner address for messages
// sent automatically
DisableOwnerFallback bool
// DisableWorkerFallback disables usage of the worker address for messages
// sent automatically, if control addresses are configured.
// A control address that doesn't have enough funds will still be chosen
// over the worker address if this flag is set.
DisableWorkerFallback bool
}
// API contains configs for API endpoint
type API struct {
// Binding address for the Lotus API
ListenAddress string
RemoteListenAddress string
Timeout Duration
}
// Libp2p contains configs for libp2p
type Libp2p struct {
// Binding address for the libp2p host - 0 means random port.
// Format: multiaddress; see https://multiformats.io/multiaddr/
ListenAddresses []string
// Addresses to explicitally announce to other peers. If not specified,
// all interface addresses are announced
// Format: multiaddress
AnnounceAddresses []string
// Addresses to not announce
// Format: multiaddress
NoAnnounceAddresses []string
BootstrapPeers []string
ProtectedPeers []string
ConnMgrLow uint
ConnMgrHigh uint
ConnMgrGrace Duration
}
type Pubsub struct {
// Run the node in bootstrap-node mode
Bootstrapper bool
// DirectPeers specifies peers with direct peering agreements. These peers are
// connected outside of the mesh, with all (valid) message unconditionally
// forwarded to them. The router will maintain open connections to these peers.
// Note that the peering agreement should be reciprocal with direct peers
// symmetrically configured at both ends.
// Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
DirectPeers []string
IPColocationWhitelist []string
RemoteTracer string
}
type Chainstore struct {
EnableSplitstore bool
Splitstore Splitstore
}
type Splitstore struct {
ColdStoreType string
HotStoreType string
MarkSetType string
HotStoreMessageRetention uint64
}
// // Full Node
type Metrics struct {
Nickname string
HeadNotifs bool
}
type Client struct {
UseIpfs bool
IpfsOnlineMode bool
IpfsMAddr string
IpfsUseForRetrieval bool
// The maximum number of simultaneous data transfers between the client
// and storage providers
SimultaneousTransfers uint64
}
type Wallet struct {
RemoteBackend string
EnableLedger bool
DisableLocal bool
}
type FeeConfig struct {
DefaultMaxFee types.FIL
}