Merge branch 'master' into libp2p-pubsub-tracer

node/config/def.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"encoding"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/ipfs/go-cid"
@@ -24,6 +26,16 @@ const (
 	RetrievalPricingExternalMode = "external"
 )
 
+// MaxTraversalLinks configures the maximum number of links to traverse in a DAG while calculating
+// CommP and traversing a DAG with graphsync; invokes a budget on DAG depth and density.
+var MaxTraversalLinks uint64 = 32 * (1 << 20)
+
+func init() {
+	if envMaxTraversal, err := strconv.ParseUint(os.Getenv("LOTUS_MAX_TRAVERSAL_LINKS"), 10, 64); err == nil {
+		MaxTraversalLinks = envMaxTraversal
+	}
+}
+
 func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount {
 	return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector)))
 }
@@ -65,7 +77,8 @@ func DefaultFullNode() *FullNode {
 			DefaultMaxFee: DefaultDefaultMaxFee,
 		},
 		Client: Client{
-			SimultaneousTransfers: DefaultSimultaneousTransfers,
+			SimultaneousTransfersForStorage:   DefaultSimultaneousTransfers,
+			SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers,
 		},
 		Chainstore: Chainstore{
 			EnableSplitstore: false,
@@ -101,7 +114,7 @@ func DefaultStorageMiner() *StorageMiner {
 			PreCommitBatchWait:  Duration(24 * time.Hour),           // this should be less than 31.5 hours, which is the expiration of a precommit ticket
 			PreCommitBatchSlack: Duration(3 * time.Hour),            // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration
 
-			CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * policy.GetMaxSectorExpirationExtension()),
+			CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * uint64(policy.GetMaxSectorExpirationExtension()) * uint64(time.Second)),
 
 			AggregateCommits: true,
 			MinCommitBatch:   miner5.MinAggregatedSectors, // per FIP13, we must have at least four proofs to aggregate, where 4 is the cross over point where aggregation wins out on single provecommit gas costs
@@ -109,7 +122,8 @@ func DefaultStorageMiner() *StorageMiner {
 			CommitBatchWait:  Duration(24 * time.Hour),    // this can be up to 30 days
 			CommitBatchSlack: Duration(1 * time.Hour),     // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration
 
-			AggregateAboveBaseFee: types.FIL(types.BigMul(types.PicoFil, types.NewInt(150))), // 0.15 nFIL
+			BatchPreCommitAboveBaseFee: types.FIL(types.BigMul(types.PicoFil, types.NewInt(320))), // 0.32 nFIL
+			AggregateAboveBaseFee:      types.FIL(types.BigMul(types.PicoFil, types.NewInt(320))), // 0.32 nFIL
 
 			TerminateBatchMin:  1,
 			TerminateBatchMax:  100,
@@ -146,7 +160,10 @@ func DefaultStorageMiner() *StorageMiner {
 			MaxDealsPerPublishMsg:           8,
 			MaxProviderCollateralMultiplier: 2,
 
-			SimultaneousTransfers: DefaultSimultaneousTransfers,
+			SimultaneousTransfersForStorage:   DefaultSimultaneousTransfers,
+			SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers,
+
+			StartEpochSealingBuffer: 480, // 480 epochs buffer == 4 hours from adding deal to sector to sector being sealed
 
 			RetrievalPricing: &RetrievalPricing{
 				Strategy: RetrievalPricingDefaultMode,

node/config/doc_gen.go

@@ -92,11 +92,18 @@ your node if metadata log is disabled`,
 			Comment: ``,
 		},
 		{
-			Name: "SimultaneousTransfers",
+			Name: "SimultaneousTransfersForStorage",
 			Type: "uint64",
 
 			Comment: `The maximum number of simultaneous data transfers between the client
-and storage providers`,
+and storage providers for storage deals`,
+		},
+		{
+			Name: "SimultaneousTransfersForRetrieval",
+			Type: "uint64",
+
+			Comment: `The maximum number of simultaneous data transfers between the client
+and storage providers for retrieval deals`,
 		},
 	},
 	"Common": []DocField{
@@ -253,10 +260,29 @@ message`,
 as a multiplier of the minimum collateral bound`,
 		},
 		{
-			Name: "SimultaneousTransfers",
+			Name: "MaxStagingDealsBytes",
+			Type: "int64",
+
+			Comment: `The maximum allowed disk usage size in bytes of staging deals not yet
+passed to the sealing node by the markets service. 0 is unlimited.`,
+		},
+		{
+			Name: "SimultaneousTransfersForStorage",
 			Type: "uint64",
 
-			Comment: `The maximum number of parallel online data transfers (storage+retrieval)`,
+			Comment: `The maximum number of parallel online data transfers for storage deals`,
+		},
+		{
+			Name: "SimultaneousTransfersForRetrieval",
+			Type: "uint64",
+
+			Comment: `The maximum number of parallel online data transfers for retrieval deals`,
+		},
+		{
+			Name: "StartEpochSealingBuffer",
+			Type: "uint64",
+
+			Comment: `Minimum start epoch buffer to give time for sealing of sector with deal.`,
 		},
 		{
 			Name: "Filter",
@@ -348,23 +374,36 @@ Format: multiaddress`,
 
 			Comment: ``,
 		},
+		{
+			Name: "DisableNatPortMap",
+			Type: "bool",
+
+			Comment: `When not disabled (default), lotus asks NAT devices (e.g., routers), to
+open up an external port and forward it to the port lotus is running on.
+When this works (i.e., when your router supports NAT port forwarding),
+it makes the local lotus node accessible from the public internet`,
+		},
 		{
 			Name: "ConnMgrLow",
 			Type: "uint",
 
-			Comment: ``,
+			Comment: `ConnMgrLow is the number of connections that the basic connection manager
+will trim down to.`,
 		},
 		{
 			Name: "ConnMgrHigh",
 			Type: "uint",
 
-			Comment: ``,
+			Comment: `ConnMgrHigh is the number of connections that, when exceeded, will trigger
+a connection GC operation. Note: protected/recently formed connections don't
+count towards this limit.`,
 		},
 		{
 			Name: "ConnMgrGrace",
 			Type: "Duration",
 
-			Comment: ``,
+			Comment: `ConnMgrGrace is a time duration that new connections are immune from being
+closed by the connection manager.`,
 		},
 	},
 	"MinerAddressConfig": []DocField{
@@ -720,6 +759,13 @@ avoid the relatively high cost of unsealing the data later, at the cost of more
 
 			Comment: `time buffer for forceful batch submission before sectors/deals in batch would start expiring`,
 		},
+		{
+			Name: "BatchPreCommitAboveBaseFee",
+			Type: "types.FIL",
+
+			Comment: `network BaseFee below which to stop doing precommit batching, instead
+sending precommit messages to the chain individually`,
+		},
 		{
 			Name: "AggregateAboveBaseFee",
 			Type: "types.FIL",

node/config/load.go

@@ -125,6 +125,8 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) {
 
 						outLines = append(outLines, pad+"# type: "+doc.Type)
 					}
+
+					outLines = append(outLines, pad+"# env var: LOTUS_"+strings.ToUpper(strings.ReplaceAll(section, ".", "_"))+"_"+strings.ToUpper(lf[0]))
 				}
 			}
 

node/config/types.go

@@ -126,9 +126,15 @@ type DealmakingConfig struct {
 	// The maximum collateral that the provider will put up against a deal,
 	// as a multiplier of the minimum collateral bound
 	MaxProviderCollateralMultiplier uint64
-
-	// The maximum number of parallel online data transfers (storage+retrieval)
-	SimultaneousTransfers uint64
+	// The maximum allowed disk usage size in bytes of staging deals not yet
+	// passed to the sealing node by the markets service. 0 is unlimited.
+	MaxStagingDealsBytes int64
+	// The maximum number of parallel online data transfers for storage deals
+	SimultaneousTransfersForStorage uint64
+	// The maximum number of parallel online data transfers for retrieval deals
+	SimultaneousTransfersForRetrieval uint64
+	// Minimum start epoch buffer to give time for sealing of sector with deal.
+	StartEpochSealingBuffer uint64
 
 	// A command used for fine-grained evaluation of storage deals
 	// see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
@@ -217,6 +223,10 @@ type SealingConfig struct {
 	// time buffer for forceful batch submission before sectors/deals in batch would start expiring
 	CommitBatchSlack Duration
 
+	// network BaseFee below which to stop doing precommit batching, instead
+	// sending precommit messages to the chain individually
+	BatchPreCommitAboveBaseFee types.FIL
+
 	// network BaseFee below which to stop doing commit aggregation, instead
 	// submitting proofs to the chain individually
 	AggregateAboveBaseFee types.FIL
@@ -292,8 +302,21 @@ type Libp2p struct {
 	BootstrapPeers      []string
 	ProtectedPeers      []string
 
-	ConnMgrLow   uint
-	ConnMgrHigh  uint
+	// When not disabled (default), lotus asks NAT devices (e.g., routers), to
+	// open up an external port and forward it to the port lotus is running on.
+	// When this works (i.e., when your router supports NAT port forwarding),
+	// it makes the local lotus node accessible from the public internet
+	DisableNatPortMap bool
+
+	// ConnMgrLow is the number of connections that the basic connection manager
+	// will trim down to.
+	ConnMgrLow uint
+	// ConnMgrHigh is the number of connections that, when exceeded, will trigger
+	// a connection GC operation. Note: protected/recently formed connections don't
+	// count towards this limit.
+	ConnMgrHigh uint
+	// ConnMgrGrace is a time duration that new connections are immune from being
+	// closed by the connection manager.
 	ConnMgrGrace Duration
 }
 
@@ -356,8 +379,11 @@ type Client struct {
 	IpfsMAddr           string
 	IpfsUseForRetrieval bool
 	// The maximum number of simultaneous data transfers between the client
-	// and storage providers
-	SimultaneousTransfers uint64
+	// and storage providers for storage deals
+	SimultaneousTransfersForStorage uint64
+	// The maximum number of simultaneous data transfers between the client
+	// and storage providers for retrieval deals
+	SimultaneousTransfersForRetrieval uint64
 }
 
 type Wallet struct {