Merge branch 'master' into libp2p-pubsub-tracer

node/builder.go

@@ -3,7 +3,6 @@ package node
 import (
 	"context"
 	"errors"
-	"os"
 	"time"
 
 	metricsi "github.com/ipfs/go-metrics-interface"
@@ -28,10 +27,12 @@ import (
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/beacon"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/journal/alerting"
 	"github.com/filecoin-project/lotus/lib/peermgr"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
@@ -82,6 +83,9 @@ const (
 	// System processes.
 	InitMemoryWatchdog
 
+	// health checks
+	CheckFDLimit
+
 	// libp2p
 	PstoreAddSelfKeysKey
 	StartListeningKey
@@ -146,6 +150,9 @@ func defaults() []Option {
 		// global system journal.
 		Override(new(journal.DisabledEvents), journal.EnvDisabledEvents),
 		Override(new(journal.Journal), modules.OpenFilesystemJournal),
+		Override(new(*alerting.Alerting), alerting.NewAlertingSystem),
+
+		Override(CheckFDLimit, modules.CheckFdLimit(build.DefaultFDLimit)),
 
 		Override(new(system.MemoryConstraints), modules.MemoryConstraints),
 		Override(InitMemoryWatchdog, modules.MemoryWatchdog),
@@ -187,7 +194,6 @@ var LibP2P = Options(
 	Override(new(routing.Routing), lp2p.Routing),
 
 	// Services
-	Override(NatPortMapKey, lp2p.NatPortMap),
 	Override(BandwidthReporterKey, lp2p.BandwidthCounter),
 	Override(AutoNATSvcKey, lp2p.AutoNATService),
 
@@ -269,6 +275,8 @@ func ConfigCommon(cfg *config.Common, enableLibp2pNode bool) Option {
 			Override(AddrsFactoryKey, lp2p.AddrsFactory(
 				cfg.Libp2p.AnnounceAddresses,
 				cfg.Libp2p.NoAnnounceAddresses)),
+
+			If(!cfg.Libp2p.DisableNatPortMap, Override(NatPortMapKey, lp2p.NatPortMap)),
 		),
 		Override(new(dtypes.MetadataDS), modules.Datastore(cfg.Backup.DisableMetadataLog)),
 	)
@@ -284,59 +292,9 @@ func Repo(r repo.Repo) Option {
 		if err != nil {
 			return err
 		}
-
-		var cfg *config.Chainstore
-		switch settings.nodeType {
-		case repo.FullNode:
-			cfgp, ok := c.(*config.FullNode)
-			if !ok {
-				return xerrors.Errorf("invalid config from repo, got: %T", c)
-			}
-			cfg = &cfgp.Chainstore
-		default:
-			cfg = &config.Chainstore{}
-		}
-
 		return Options(
 			Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
 
-			Override(new(dtypes.UniversalBlockstore), modules.UniversalBlockstore),
-
-			If(cfg.EnableSplitstore,
-				If(cfg.Splitstore.ColdStoreType == "universal",
-					Override(new(dtypes.ColdBlockstore), From(new(dtypes.UniversalBlockstore)))),
-				If(cfg.Splitstore.ColdStoreType == "discard",
-					Override(new(dtypes.ColdBlockstore), modules.DiscardColdBlockstore)),
-				If(cfg.Splitstore.HotStoreType == "badger",
-					Override(new(dtypes.HotBlockstore), modules.BadgerHotBlockstore)),
-				Override(new(dtypes.SplitBlockstore), modules.SplitBlockstore(cfg)),
-				Override(new(dtypes.BasicChainBlockstore), modules.ChainSplitBlockstore),
-				Override(new(dtypes.BasicStateBlockstore), modules.StateSplitBlockstore),
-				Override(new(dtypes.BaseBlockstore), From(new(dtypes.SplitBlockstore))),
-				Override(new(dtypes.ExposedBlockstore), modules.ExposedSplitBlockstore),
-				Override(new(dtypes.GCReferenceProtector), modules.SplitBlockstoreGCReferenceProtector),
-			),
-			If(!cfg.EnableSplitstore,
-				Override(new(dtypes.BasicChainBlockstore), modules.ChainFlatBlockstore),
-				Override(new(dtypes.BasicStateBlockstore), modules.StateFlatBlockstore),
-				Override(new(dtypes.BaseBlockstore), From(new(dtypes.UniversalBlockstore))),
-				Override(new(dtypes.ExposedBlockstore), From(new(dtypes.UniversalBlockstore))),
-				Override(new(dtypes.GCReferenceProtector), modules.NoopGCReferenceProtector),
-			),
-
-			Override(new(dtypes.ChainBlockstore), From(new(dtypes.BasicChainBlockstore))),
-			Override(new(dtypes.StateBlockstore), From(new(dtypes.BasicStateBlockstore))),
-
-			If(os.Getenv("LOTUS_ENABLE_CHAINSTORE_FALLBACK") == "1",
-				Override(new(dtypes.ChainBlockstore), modules.FallbackChainBlockstore),
-				Override(new(dtypes.StateBlockstore), modules.FallbackStateBlockstore),
-				Override(SetupFallbackBlockstoresKey, modules.InitFallbackBlockstores),
-			),
-
-			Override(new(dtypes.ClientImportMgr), modules.ClientImportMgr),
-
-			Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
-
 			Override(new(ci.PrivKey), lp2p.PrivKey),
 			Override(new(ci.PubKey), ci.PrivKey.GetPublic),
 			Override(new(peer.ID), peer.IDFromPublicKey),
@@ -416,6 +374,13 @@ func WithRepoType(repoType repo.RepoType) func(s *Settings) error {
 	}
 }
 
+func WithEnableLibp2pNode(enable bool) func(s *Settings) error {
+	return func(s *Settings) error {
+		s.enableLibp2pNode = enable
+		return nil
+	}
+}
+
 func WithInvokesKey(i invoke, resApi interface{}) func(s *Settings) error {
 	return func(s *Settings) error {
 		s.invokes[i] = fx.Populate(resApi)

node/builder_chain.go

@@ -1,6 +1,8 @@
 package node
 
 import (
+	"os"
+
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
@@ -12,6 +14,8 @@ import (
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/beacon"
+	"github.com/filecoin-project/lotus/chain/consensus"
+	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/exchange"
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	"github.com/filecoin-project/lotus/chain/market"
@@ -46,7 +50,7 @@ var ChainNode = Options(
 
 	// Consensus settings
 	Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig),
-	Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()),
+	Override(new(stmgr.UpgradeSchedule), filcns.DefaultUpgradeSchedule()),
 	Override(new(dtypes.NetworkName), modules.NetworkName),
 	Override(new(modules.Genesis), modules.ErrorGenesis),
 	Override(new(dtypes.AfterGenesisSet), modules.SetGenesis),
@@ -65,6 +69,10 @@ var ChainNode = Options(
 	Override(new(vm.SyscallBuilder), vm.Syscalls),
 
 	// Consensus: Chain storage/access
+	Override(new(chain.Genesis), chain.LoadGenesis),
+	Override(new(store.WeightFunc), filcns.Weight),
+	Override(new(stmgr.Executor), filcns.NewTipSetExecutor()),
+	Override(new(consensus.Consensus), filcns.NewFilecoinExpectedConsensus),
 	Override(new(*store.ChainStore), modules.ChainStore),
 	Override(new(*stmgr.StateManager), modules.StateManager),
 	Override(new(dtypes.ChainBitswap), modules.ChainBitswap),
@@ -92,7 +100,7 @@ var ChainNode = Options(
 	Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)),
 
 	// Shared graphsync (markets, serving chain)
-	Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfers)),
+	Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfersForStorage, config.DefaultFullNode().Client.SimultaneousTransfersForRetrieval)),
 
 	// Service: Wallet
 	Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
@@ -167,6 +175,43 @@ func ConfigFullNode(c interface{}) Option {
 	return Options(
 		ConfigCommon(&cfg.Common, enableLibp2pNode),
 
+		Override(new(dtypes.UniversalBlockstore), modules.UniversalBlockstore),
+
+		If(cfg.Chainstore.EnableSplitstore,
+			If(cfg.Chainstore.Splitstore.ColdStoreType == "universal",
+				Override(new(dtypes.ColdBlockstore), From(new(dtypes.UniversalBlockstore)))),
+			If(cfg.Chainstore.Splitstore.ColdStoreType == "discard",
+				Override(new(dtypes.ColdBlockstore), modules.DiscardColdBlockstore)),
+			If(cfg.Chainstore.Splitstore.HotStoreType == "badger",
+				Override(new(dtypes.HotBlockstore), modules.BadgerHotBlockstore)),
+			Override(new(dtypes.SplitBlockstore), modules.SplitBlockstore(&cfg.Chainstore)),
+			Override(new(dtypes.BasicChainBlockstore), modules.ChainSplitBlockstore),
+			Override(new(dtypes.BasicStateBlockstore), modules.StateSplitBlockstore),
+			Override(new(dtypes.BaseBlockstore), From(new(dtypes.SplitBlockstore))),
+			Override(new(dtypes.ExposedBlockstore), modules.ExposedSplitBlockstore),
+			Override(new(dtypes.GCReferenceProtector), modules.SplitBlockstoreGCReferenceProtector),
+		),
+		If(!cfg.Chainstore.EnableSplitstore,
+			Override(new(dtypes.BasicChainBlockstore), modules.ChainFlatBlockstore),
+			Override(new(dtypes.BasicStateBlockstore), modules.StateFlatBlockstore),
+			Override(new(dtypes.BaseBlockstore), From(new(dtypes.UniversalBlockstore))),
+			Override(new(dtypes.ExposedBlockstore), From(new(dtypes.UniversalBlockstore))),
+			Override(new(dtypes.GCReferenceProtector), modules.NoopGCReferenceProtector),
+		),
+
+		Override(new(dtypes.ChainBlockstore), From(new(dtypes.BasicChainBlockstore))),
+		Override(new(dtypes.StateBlockstore), From(new(dtypes.BasicStateBlockstore))),
+
+		If(os.Getenv("LOTUS_ENABLE_CHAINSTORE_FALLBACK") == "1",
+			Override(new(dtypes.ChainBlockstore), modules.FallbackChainBlockstore),
+			Override(new(dtypes.StateBlockstore), modules.FallbackStateBlockstore),
+			Override(SetupFallbackBlockstoresKey, modules.InitFallbackBlockstores),
+		),
+
+		Override(new(dtypes.ClientImportMgr), modules.ClientImportMgr),
+
+		Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
+
 		If(cfg.Client.UseIpfs,
 			Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, cfg.Client.IpfsOnlineMode)),
 			Override(new(storagemarket.BlockstoreAccessor), modules.IpfsStorageBlockstoreAccessor),
@@ -174,7 +219,7 @@ func ConfigFullNode(c interface{}) Option {
 				Override(new(retrievalmarket.BlockstoreAccessor), modules.IpfsRetrievalBlockstoreAccessor),
 			),
 		),
-		Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)),
+		Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfersForStorage, cfg.Client.SimultaneousTransfersForRetrieval)),
 
 		If(cfg.Wallet.RemoteBackend != "",
 			Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)),

node/builder_miner.go

@@ -15,6 +15,7 @@ import (
 	storage2 "github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
@@ -74,6 +75,8 @@ func ConfigStorageMiner(c interface{}) Option {
 	return Options(
 		ConfigCommon(&cfg.Common, enableLibp2pNode),
 
+		Override(CheckFDLimit, modules.CheckFdLimit(build.MinerFDLimit)), // recommend at least 100k FD limit to miners
+
 		Override(new(api.MinerSubsystems), modules.ExtractEnabledMinerSubsystems(cfg.Subsystems)),
 		Override(new(stores.LocalStorage), From(new(repo.LockedRepo))),
 		Override(new(*stores.Local), modules.LocalStorage),
@@ -133,7 +136,7 @@ func ConfigStorageMiner(c interface{}) Option {
 		If(cfg.Subsystems.EnableMarkets,
 			// Markets
 			Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
-			Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(cfg.Dealmaking.SimultaneousTransfers)),
+			Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(cfg.Dealmaking.SimultaneousTransfersForStorage, cfg.Dealmaking.SimultaneousTransfersForRetrieval)),
 			Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore),
 			Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
 
@@ -162,7 +165,7 @@ func ConfigStorageMiner(c interface{}) Option {
 			// Markets (storage)
 			Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer),
 			Override(new(*storedask.StoredAsk), modules.NewStorageAsk),
-			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)),
+			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
 			Override(new(storagemarket.StorageProvider), modules.StorageProvider),
 			Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})),
 			Override(HandleMigrateProviderFundsKey, modules.HandleMigrateProviderFunds),
@@ -189,15 +192,16 @@ func ConfigStorageMiner(c interface{}) Option {
 			Override(new(dtypes.GetMaxDealStartDelayFunc), modules.NewGetMaxDealStartDelayFunc),
 
 			If(cfg.Dealmaking.Filter != "",
-				Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))),
+				Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))),
 			),
 
 			If(cfg.Dealmaking.RetrievalFilter != "",
 				Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))),
 			),
 			Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(&cfg.Fees, storageadapter.PublishMsgConfig{
-				Period:         time.Duration(cfg.Dealmaking.PublishMsgPeriod),
-				MaxDealsPerMsg: cfg.Dealmaking.MaxDealsPerPublishMsg,
+				Period:                  time.Duration(cfg.Dealmaking.PublishMsgPeriod),
+				MaxDealsPerMsg:          cfg.Dealmaking.MaxDealsPerPublishMsg,
+				StartEpochSealingBuffer: cfg.Dealmaking.StartEpochSealingBuffer,
 			})),
 			Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)),
 		),

node/config/def.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"encoding"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/ipfs/go-cid"
@@ -24,6 +26,16 @@ const (
 	RetrievalPricingExternalMode = "external"
 )
 
+// MaxTraversalLinks configures the maximum number of links to traverse in a DAG while calculating
+// CommP and traversing a DAG with graphsync; invokes a budget on DAG depth and density.
+var MaxTraversalLinks uint64 = 32 * (1 << 20)
+
+func init() {
+	if envMaxTraversal, err := strconv.ParseUint(os.Getenv("LOTUS_MAX_TRAVERSAL_LINKS"), 10, 64); err == nil {
+		MaxTraversalLinks = envMaxTraversal
+	}
+}
+
 func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount {
 	return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector)))
 }
@@ -65,7 +77,8 @@ func DefaultFullNode() *FullNode {
 			DefaultMaxFee: DefaultDefaultMaxFee,
 		},
 		Client: Client{
-			SimultaneousTransfers: DefaultSimultaneousTransfers,
+			SimultaneousTransfersForStorage:   DefaultSimultaneousTransfers,
+			SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers,
 		},
 		Chainstore: Chainstore{
 			EnableSplitstore: false,
@@ -101,7 +114,7 @@ func DefaultStorageMiner() *StorageMiner {
 			PreCommitBatchWait:  Duration(24 * time.Hour),           // this should be less than 31.5 hours, which is the expiration of a precommit ticket
 			PreCommitBatchSlack: Duration(3 * time.Hour),            // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration
 
-			CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * policy.GetMaxSectorExpirationExtension()),
+			CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * uint64(policy.GetMaxSectorExpirationExtension()) * uint64(time.Second)),
 
 			AggregateCommits: true,
 			MinCommitBatch:   miner5.MinAggregatedSectors, // per FIP13, we must have at least four proofs to aggregate, where 4 is the cross over point where aggregation wins out on single provecommit gas costs
@@ -109,7 +122,8 @@ func DefaultStorageMiner() *StorageMiner {
 			CommitBatchWait:  Duration(24 * time.Hour),    // this can be up to 30 days
 			CommitBatchSlack: Duration(1 * time.Hour),     // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration
 
-			AggregateAboveBaseFee: types.FIL(types.BigMul(types.PicoFil, types.NewInt(150))), // 0.15 nFIL
+			BatchPreCommitAboveBaseFee: types.FIL(types.BigMul(types.PicoFil, types.NewInt(320))), // 0.32 nFIL
+			AggregateAboveBaseFee:      types.FIL(types.BigMul(types.PicoFil, types.NewInt(320))), // 0.32 nFIL
 
 			TerminateBatchMin:  1,
 			TerminateBatchMax:  100,
@@ -146,7 +160,10 @@ func DefaultStorageMiner() *StorageMiner {
 			MaxDealsPerPublishMsg:           8,
 			MaxProviderCollateralMultiplier: 2,
 
-			SimultaneousTransfers: DefaultSimultaneousTransfers,
+			SimultaneousTransfersForStorage:   DefaultSimultaneousTransfers,
+			SimultaneousTransfersForRetrieval: DefaultSimultaneousTransfers,
+
+			StartEpochSealingBuffer: 480, // 480 epochs buffer == 4 hours from adding deal to sector to sector being sealed
 
 			RetrievalPricing: &RetrievalPricing{
 				Strategy: RetrievalPricingDefaultMode,

node/config/doc_gen.go

@@ -92,11 +92,18 @@ your node if metadata log is disabled`,
 			Comment: ``,
 		},
 		{
-			Name: "SimultaneousTransfers",
+			Name: "SimultaneousTransfersForStorage",
 			Type: "uint64",
 
 			Comment: `The maximum number of simultaneous data transfers between the client
-and storage providers`,
+and storage providers for storage deals`,
+		},
+		{
+			Name: "SimultaneousTransfersForRetrieval",
+			Type: "uint64",
+
+			Comment: `The maximum number of simultaneous data transfers between the client
+and storage providers for retrieval deals`,
 		},
 	},
 	"Common": []DocField{
@@ -253,10 +260,29 @@ message`,
 as a multiplier of the minimum collateral bound`,
 		},
 		{
-			Name: "SimultaneousTransfers",
+			Name: "MaxStagingDealsBytes",
+			Type: "int64",
+
+			Comment: `The maximum allowed disk usage size in bytes of staging deals not yet
+passed to the sealing node by the markets service. 0 is unlimited.`,
+		},
+		{
+			Name: "SimultaneousTransfersForStorage",
 			Type: "uint64",
 
-			Comment: `The maximum number of parallel online data transfers (storage+retrieval)`,
+			Comment: `The maximum number of parallel online data transfers for storage deals`,
+		},
+		{
+			Name: "SimultaneousTransfersForRetrieval",
+			Type: "uint64",
+
+			Comment: `The maximum number of parallel online data transfers for retrieval deals`,
+		},
+		{
+			Name: "StartEpochSealingBuffer",
+			Type: "uint64",
+
+			Comment: `Minimum start epoch buffer to give time for sealing of sector with deal.`,
 		},
 		{
 			Name: "Filter",
@@ -348,23 +374,36 @@ Format: multiaddress`,
 
 			Comment: ``,
 		},
+		{
+			Name: "DisableNatPortMap",
+			Type: "bool",
+
+			Comment: `When not disabled (default), lotus asks NAT devices (e.g., routers), to
+open up an external port and forward it to the port lotus is running on.
+When this works (i.e., when your router supports NAT port forwarding),
+it makes the local lotus node accessible from the public internet`,
+		},
 		{
 			Name: "ConnMgrLow",
 			Type: "uint",
 
-			Comment: ``,
+			Comment: `ConnMgrLow is the number of connections that the basic connection manager
+will trim down to.`,
 		},
 		{
 			Name: "ConnMgrHigh",
 			Type: "uint",
 
-			Comment: ``,
+			Comment: `ConnMgrHigh is the number of connections that, when exceeded, will trigger
+a connection GC operation. Note: protected/recently formed connections don't
+count towards this limit.`,
 		},
 		{
 			Name: "ConnMgrGrace",
 			Type: "Duration",
 
-			Comment: ``,
+			Comment: `ConnMgrGrace is a time duration that new connections are immune from being
+closed by the connection manager.`,
 		},
 	},
 	"MinerAddressConfig": []DocField{
@@ -720,6 +759,13 @@ avoid the relatively high cost of unsealing the data later, at the cost of more
 
 			Comment: `time buffer for forceful batch submission before sectors/deals in batch would start expiring`,
 		},
+		{
+			Name: "BatchPreCommitAboveBaseFee",
+			Type: "types.FIL",
+
+			Comment: `network BaseFee below which to stop doing precommit batching, instead
+sending precommit messages to the chain individually`,
+		},
 		{
 			Name: "AggregateAboveBaseFee",
 			Type: "types.FIL",

node/config/load.go

@@ -125,6 +125,8 @@ func ConfigUpdate(cfgCur, cfgDef interface{}, comment bool) ([]byte, error) {
 
 						outLines = append(outLines, pad+"# type: "+doc.Type)
 					}
+
+					outLines = append(outLines, pad+"# env var: LOTUS_"+strings.ToUpper(strings.ReplaceAll(section, ".", "_"))+"_"+strings.ToUpper(lf[0]))
 				}
 			}
 

node/config/types.go

@@ -126,9 +126,15 @@ type DealmakingConfig struct {
 	// The maximum collateral that the provider will put up against a deal,
 	// as a multiplier of the minimum collateral bound
 	MaxProviderCollateralMultiplier uint64
-
-	// The maximum number of parallel online data transfers (storage+retrieval)
-	SimultaneousTransfers uint64
+	// The maximum allowed disk usage size in bytes of staging deals not yet
+	// passed to the sealing node by the markets service. 0 is unlimited.
+	MaxStagingDealsBytes int64
+	// The maximum number of parallel online data transfers for storage deals
+	SimultaneousTransfersForStorage uint64
+	// The maximum number of parallel online data transfers for retrieval deals
+	SimultaneousTransfersForRetrieval uint64
+	// Minimum start epoch buffer to give time for sealing of sector with deal.
+	StartEpochSealingBuffer uint64
 
 	// A command used for fine-grained evaluation of storage deals
 	// see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
@@ -217,6 +223,10 @@ type SealingConfig struct {
 	// time buffer for forceful batch submission before sectors/deals in batch would start expiring
 	CommitBatchSlack Duration
 
+	// network BaseFee below which to stop doing precommit batching, instead
+	// sending precommit messages to the chain individually
+	BatchPreCommitAboveBaseFee types.FIL
+
 	// network BaseFee below which to stop doing commit aggregation, instead
 	// submitting proofs to the chain individually
 	AggregateAboveBaseFee types.FIL
@@ -292,8 +302,21 @@ type Libp2p struct {
 	BootstrapPeers      []string
 	ProtectedPeers      []string
 
-	ConnMgrLow   uint
-	ConnMgrHigh  uint
+	// When not disabled (default), lotus asks NAT devices (e.g., routers), to
+	// open up an external port and forward it to the port lotus is running on.
+	// When this works (i.e., when your router supports NAT port forwarding),
+	// it makes the local lotus node accessible from the public internet
+	DisableNatPortMap bool
+
+	// ConnMgrLow is the number of connections that the basic connection manager
+	// will trim down to.
+	ConnMgrLow uint
+	// ConnMgrHigh is the number of connections that, when exceeded, will trigger
+	// a connection GC operation. Note: protected/recently formed connections don't
+	// count towards this limit.
+	ConnMgrHigh uint
+	// ConnMgrGrace is a time duration that new connections are immune from being
+	// closed by the connection manager.
 	ConnMgrGrace Duration
 }
 
@@ -356,8 +379,11 @@ type Client struct {
 	IpfsMAddr           string
 	IpfsUseForRetrieval bool
 	// The maximum number of simultaneous data transfers between the client
-	// and storage providers
-	SimultaneousTransfers uint64
+	// and storage providers for storage deals
+	SimultaneousTransfersForStorage uint64
+	// The maximum number of simultaneous data transfers between the client
+	// and storage providers for retrieval deals
+	SimultaneousTransfersForRetrieval uint64
 }
 
 type Wallet struct {

node/hello/hello.go

@@ -7,6 +7,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"golang.org/x/xerrors"
 
+	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
@@ -15,9 +16,9 @@ import (
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/protocol"
 
-	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
+	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/peermgr"
@@ -48,10 +49,11 @@ type Service struct {
 
 	cs     *store.ChainStore
 	syncer *chain.Syncer
+	cons   consensus.Consensus
 	pmgr   *peermgr.PeerMgr
 }
 
-func NewHelloService(h host.Host, cs *store.ChainStore, syncer *chain.Syncer, pmgr peermgr.MaybePeerMgr) *Service {
+func NewHelloService(h host.Host, cs *store.ChainStore, syncer *chain.Syncer, cons consensus.Consensus, pmgr peermgr.MaybePeerMgr) *Service {
 	if pmgr.Mgr == nil {
 		log.Warn("running without peer manager")
 	}
@@ -61,6 +63,7 @@ func NewHelloService(h host.Host, cs *store.ChainStore, syncer *chain.Syncer, pm
 
 		cs:     cs,
 		syncer: syncer,
+		cons:   cons,
 		pmgr:   pmgr.Mgr,
 	}
 }

node/impl/client/client.go

@@ -14,7 +14,7 @@ import (
 	unixfile "github.com/ipfs/go-unixfs/file"
 	"github.com/ipld/go-car"
 	carv2 "github.com/ipld/go-car/v2"
-	"github.com/ipld/go-car/v2/blockstore"
+	carv2bs "github.com/ipld/go-car/v2/blockstore"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-padreader"
@@ -24,10 +24,16 @@ import (
 	"github.com/ipfs/go-cid"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
 	files "github.com/ipfs/go-ipfs-files"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipfs/go-merkledag"
+	"github.com/ipld/go-ipld-prime"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	"github.com/ipld/go-ipld-prime/traversal"
 	"github.com/ipld/go-ipld-prime/traversal/selector"
 	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/multiformats/go-multibase"
@@ -41,9 +47,7 @@ import (
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 
 	"github.com/filecoin-project/go-fil-markets/discovery"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	rm "github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/shared"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket/network"
 	"github.com/filecoin-project/go-fil-markets/stores"
@@ -55,6 +59,7 @@ import (
 	"github.com/filecoin-project/specs-actors/v3/actors/builtin/market"
 
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
+	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 
 	"github.com/filecoin-project/lotus/api"
@@ -69,6 +74,8 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
+var log = logging.Logger("client")
+
 var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
 
 // 8 days ~=  SealDuration + PreCommit + MaxProveCommitDuration + 8 hour buffer
@@ -91,7 +98,7 @@ type API struct {
 	// accessors for imports and retrievals.
 	Imports                   dtypes.ClientImportMgr
 	StorageBlockstoreAccessor storagemarket.BlockstoreAccessor
-	RtvlBlockstoreAccessor    retrievalmarket.BlockstoreAccessor
+	RtvlBlockstoreAccessor    rm.BlockstoreAccessor
 
 	DataTransfer dtypes.ClientDataTransfer
 	Host         host.Host
@@ -501,7 +508,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
 	}
 
 	if ref.IsCAR {
-		// user gave us a CAR fil, use it as-is
+		// user gave us a CAR file, use it as-is
 		// validate that it's either a carv1 or carv2, and has one root.
 		f, err := os.Open(ref.Path)
 		if err != nil {
@@ -619,7 +626,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro
 		return cid.Undef, xerrors.Errorf("failed to calculate placeholder root: %w", err)
 	}
 
-	bs, err := blockstore.OpenReadWrite(path, []cid.Cid{placeholderRoot}, blockstore.UseWholeCIDs(true))
+	bs, err := carv2bs.OpenReadWrite(path, []cid.Cid{placeholderRoot}, carv2bs.UseWholeCIDs(true))
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("failed to create carv2 read/write blockstore: %w", err)
 	}
@@ -730,7 +737,7 @@ func (a *API) ClientListImports(_ context.Context) ([]api.Import, error) {
 	return out, nil
 }
 
-func (a *API) ClientCancelRetrievalDeal(ctx context.Context, dealID retrievalmarket.DealID) error {
+func (a *API) ClientCancelRetrievalDeal(ctx context.Context, dealID rm.DealID) error {
 	cerr := make(chan error)
 	go func() {
 		err := a.Retrieval.CancelDeal(dealID)
@@ -784,7 +791,7 @@ type retrievalSubscribeEvent struct {
 	state rm.ClientDealState
 }
 
-func consumeAllEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error {
+func consumeAllEvents(ctx context.Context, dealID rm.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error {
 	for {
 		var subscribeEvent retrievalSubscribeEvent
 		select {
@@ -835,24 +842,52 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		}
 	}
 
+	sel := selectorparse.CommonSelector_ExploreAllRecursively
+	if order.DatamodelPathSelector != nil {
+
+		ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+
+		selspec, err := textselector.SelectorSpecFromPath(
+
+			*order.DatamodelPathSelector,
+
+			// URGH - this is a direct copy from https://github.com/filecoin-project/go-fil-markets/blob/v1.12.0/shared/selectors.go#L10-L16
+			// Unable to use it because we need the SelectorSpec, and markets exposes just a reified node
+			ssb.ExploreRecursive(
+				selector.RecursionLimitNone(),
+				ssb.ExploreAll(ssb.ExploreRecursiveEdge()),
+			),
+		)
+		if err != nil {
+			finish(xerrors.Errorf("failed to parse text-selector '%s': %w", *order.DatamodelPathSelector, err))
+			return
+		}
+
+		sel = selspec.Node()
+		log.Infof("partial retrieval of datamodel-path-selector %s/*", *order.DatamodelPathSelector)
+	}
+
 	// summary:
-	// 1. if we're retrieving from an import, FromLocalCAR will be informed.
-	//    Open as a Filestore and populate the target CAR or UnixFS export from it.
-	//    (cannot use ExtractV1File because user wants a dense CAR, not a ref CAR/filestore)
+	// 1. if we're retrieving from an import, FromLocalCAR will be set.
+	//    Skip the retrieval itself, and use the provided car as a blockstore further down
+	//    to extract a CAR or UnixFS export from.
 	// 2. if we're using an IPFS blockstore for retrieval, retrieve into it,
-	//    then extract the CAR or UnixFS export from it.
-	// 3. if we have to retrieve, perform a CARv2 retrieval, then extract
-	//    the CARv1 (with ExtractV1File) or UnixFS export from it.
+	//    then use the virtual blockstore to extract a CAR or UnixFS export from it.
+	// 3. if we have to retrieve, perform a CARv2 retrieval, then either
+	//    extract the CARv1 (with ExtractV1File) or use it as a blockstore further down.
 
 	// this indicates we're proxying to IPFS.
 	proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor)
+
 	carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor)
 
 	carPath := order.FromLocalCAR
+
+	// we actually need to retrieve from the network
 	if carPath == "" {
+
 		if !retrieveIntoIPFS && !retrieveIntoCAR {
-			// we actually need to retrieve from the network, but we don't
-			// recognize the blockstore accessor.
+			// we don't recognize the blockstore accessor.
 			finish(xerrors.Errorf("unsupported retrieval blockstore accessor"))
 			return
 		}
@@ -864,7 +899,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 				return
 			}
 
-			order.MinerPeer = &retrievalmarket.RetrievalPeer{
+			order.MinerPeer = &rm.RetrievalPeer{
 				ID:      *mi.PeerId,
 				Address: order.Miner,
 			}
@@ -882,7 +917,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 
 		ppb := types.BigDiv(order.Total, types.NewInt(order.Size))
 
-		params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, shared.AllSelector(), order.Piece, order.UnsealPrice)
+		params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, sel, order.Piece, order.UnsealPrice)
 		if err != nil {
 			finish(xerrors.Errorf("Error in retrieval params: %s", err))
 			return
@@ -940,37 +975,10 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		return
 	}
 
-	// Are we outputting a CAR?
-	if ref.IsCAR {
-		if retrieveIntoIPFS {
-			// generating a CARv1 from IPFS.
-			f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644)
-			if err != nil {
-				finish(err)
-				return
-			}
-
-			bs := proxyBss.Blockstore
-			dags := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
-			err = car.WriteCar(ctx, dags, []cid.Cid{order.Root}, f)
-			if err != nil {
-				finish(err)
-				return
-			}
-			finish(f.Close())
-			return
-		}
-
-		// generating a CARv1 from the CARv2 where we stored the retrieval.
-		err := carv2.ExtractV1File(carPath, ref.Path)
-		finish(err)
-		return
-	}
-
-	// we are extracting a UnixFS file.
-	var bs bstore.Blockstore
+	// determine where did the retrieval go
+	var retrievalBs bstore.Blockstore
 	if retrieveIntoIPFS {
-		bs = proxyBss.Blockstore
+		retrievalBs = proxyBss.Blockstore
 	} else {
 		cbs, err := stores.ReadOnlyFilestore(carPath)
 		if err != nil {
@@ -978,18 +986,93 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 			return
 		}
 		defer cbs.Close() //nolint:errcheck
-		bs = cbs
+		retrievalBs = cbs
 	}
 
-	bsvc := blockservice.New(bs, offline.Exchange(bs))
-	dag := merkledag.NewDAGService(bsvc)
+	// Are we outputting a CAR?
+	if ref.IsCAR {
 
-	nd, err := dag.Get(ctx, order.Root)
+		// not IPFS and we do full selection - just extract the CARv1 from the CARv2 we stored the retrieval in
+		if !retrieveIntoIPFS && order.DatamodelPathSelector == nil {
+			finish(carv2.ExtractV1File(carPath, ref.Path))
+			return
+		}
+
+		// generating a CARv1 from the configured blockstore
+		f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			finish(err)
+			return
+		}
+
+		err = car.NewSelectiveCar(
+			ctx,
+			retrievalBs,
+			[]car.Dag{{
+				Root:     order.Root,
+				Selector: sel,
+			}},
+			car.MaxTraversalLinks(config.MaxTraversalLinks),
+		).Write(f)
+		if err != nil {
+			finish(err)
+			return
+		}
+
+		finish(f.Close())
+		return
+	}
+
+	// we are extracting a UnixFS file.
+	ds := merkledag.NewDAGService(blockservice.New(retrievalBs, offline.Exchange(retrievalBs)))
+	root := order.Root
+
+	// if we used a selector - need to find the sub-root the user actually wanted to retrieve
+	if order.DatamodelPathSelector != nil {
+
+		var subRootFound bool
+
+		// no err check - we just compiled this before starting, but now we do not wrap a `*`
+		selspec, _ := textselector.SelectorSpecFromPath(*order.DatamodelPathSelector, nil) //nolint:errcheck
+		if err := utils.TraverseDag(
+			ctx,
+			ds,
+			root,
+			selspec.Node(),
+			func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
+				if r == traversal.VisitReason_SelectionMatch {
+
+					if p.LastBlock.Path.String() != p.Path.String() {
+						return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String())
+					}
+
+					cidLnk, castOK := p.LastBlock.Link.(cidlink.Link)
+					if !castOK {
+						return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link.String())
+					}
+
+					root = cidLnk.Cid
+					subRootFound = true
+				}
+				return nil
+			},
+		); err != nil {
+			finish(xerrors.Errorf("error while locating partial retrieval sub-root: %w", err))
+			return
+		}
+
+		if !subRootFound {
+			finish(xerrors.Errorf("path selection '%s' does not match a node within %s", *order.DatamodelPathSelector, root))
+			return
+		}
+	}
+
+	nd, err := ds.Get(ctx, root)
 	if err != nil {
 		finish(xerrors.Errorf("ClientRetrieve: %w", err))
 		return
 	}
-	file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
+	file, err := unixfile.NewUnixfsFile(ctx, ds, nd)
 	if err != nil {
 		finish(xerrors.Errorf("ClientRetrieve: %w", err))
 		return
@@ -1216,17 +1299,23 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
 	}
 	defer fs.Close() //nolint:errcheck
 
-	// build a dense deterministic CAR (dense = containing filled leaves)
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
-	allSelector := ssb.ExploreRecursive(
-		selector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-	sc := car.NewSelectiveCar(ctx, fs, []car.Dag{{Root: root, Selector: allSelector}})
 	f, err := os.Create(outputPath)
 	if err != nil {
 		return err
 	}
-	if err = sc.Write(f); err != nil {
+
+	// build a dense deterministic CAR (dense = containing filled leaves)
+	if err := car.NewSelectiveCar(
+		ctx,
+		fs,
+		[]car.Dag{{
+			Root:     root,
+			Selector: selectorparse.CommonSelector_ExploreAllRecursively,
+		}},
+		car.MaxTraversalLinks(config.MaxTraversalLinks),
+	).Write(
+		f,
+	); err != nil {
 		return xerrors.Errorf("failed to write CAR to output file: %w", err)
 	}
 

node/impl/common/common.go

@@ -10,9 +10,11 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
+
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/journal/alerting"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
@@ -21,6 +23,7 @@ var session = uuid.New()
 type CommonAPI struct {
 	fx.In
 
+	Alerting     *alerting.Alerting
 	APISecret    *dtypes.APIAlg
 	ShutdownChan dtypes.ShutdownChan
 }
@@ -72,6 +75,10 @@ func (a *CommonAPI) LogSetLevel(ctx context.Context, subsystem, level string) er
 	return logging.SetLogLevel(subsystem, level)
 }
 
+func (a *CommonAPI) LogAlerts(ctx context.Context) ([]alerting.Alert, error) {
+	return a.Alerting.GetAlerts(), nil
+}
+
 func (a *CommonAPI) Shutdown(ctx context.Context) error {
 	a.ShutdownChan <- struct{}{}
 	return nil

node/impl/full/chain.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/stmgr"
 
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
@@ -29,7 +29,6 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/api"
@@ -52,6 +51,7 @@ type ChainModuleAPI interface {
 	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
 	ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error)
 }
 
 var _ ChainModuleAPI = *new(api.FullNode)
@@ -78,7 +78,8 @@ type ChainAPI struct {
 	WalletAPI
 	ChainModuleAPI
 
-	Chain *store.ChainStore
+	Chain  *store.ChainStore
+	TsExec stmgr.Executor
 
 	// ExposedBlockstore is the global monolith blockstore that is safe to
 	// expose externally. In the future, this will be segregated into two
@@ -97,34 +98,6 @@ func (m *ChainModule) ChainHead(context.Context) (*types.TipSet, error) {
 	return m.Chain.GetHeaviestTipSet(), nil
 }
 
-func (a *ChainAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
-	pts, err := a.Chain.LoadTipSet(tsk)
-	if err != nil {
-		return nil, xerrors.Errorf("loading tipset key: %w", err)
-	}
-
-	// Doing this here is slightly nicer than doing it in the chainstore directly, but it's still bad for ChainAPI to reason about network upgrades
-	if randEpoch > build.UpgradeHyperdriveHeight {
-		return a.Chain.GetChainRandomnessLookingForward(ctx, pts.Cids(), personalization, randEpoch, entropy)
-	}
-
-	return a.Chain.GetChainRandomnessLookingBack(ctx, pts.Cids(), personalization, randEpoch, entropy)
-}
-
-func (a *ChainAPI) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
-	pts, err := a.Chain.LoadTipSet(tsk)
-	if err != nil {
-		return nil, xerrors.Errorf("loading tipset key: %w", err)
-	}
-
-	// Doing this here is slightly nicer than doing it in the chainstore directly, but it's still bad for ChainAPI to reason about network upgrades
-	if randEpoch > build.UpgradeHyperdriveHeight {
-		return a.Chain.GetBeaconRandomnessLookingForward(ctx, pts.Cids(), personalization, randEpoch, entropy)
-	}
-
-	return a.Chain.GetBeaconRandomnessLookingBack(ctx, pts.Cids(), personalization, randEpoch, entropy)
-}
-
 func (a *ChainAPI) ChainGetBlock(ctx context.Context, msg cid.Cid) (*types.BlockHeader, error) {
 	return a.Chain.GetBlock(msg)
 }
@@ -133,6 +106,10 @@ func (m *ChainModule) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (
 	return m.Chain.LoadTipSet(key)
 }
 
+func (m *ChainModule) ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) {
+	return m.Chain.GetPath(ctx, from, to)
+}
+
 func (m *ChainModule) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) {
 	b, err := m.Chain.GetBlock(msg)
 	if err != nil {
@@ -394,7 +371,7 @@ func (s stringKey) Key() string {
 
 // TODO: ActorUpgrade: this entire function is a problem (in theory) as we don't know the HAMT version.
 // In practice, hamt v0 should work "just fine" for reading.
-func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) {
+func resolveOnce(bs blockstore.Blockstore, tse stmgr.Executor) func(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) {
 	return func(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) {
 		store := adt.WrapStore(ctx, cbor.NewCborStore(bs))
 
@@ -468,7 +445,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod
 				}, nil, nil
 			}
 
-			return resolveOnce(bs)(ctx, ds, n, names[1:])
+			return resolveOnce(bs, tse)(ctx, ds, n, names[1:])
 		}
 
 		if strings.HasPrefix(names[0], "@A:") {
@@ -517,7 +494,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod
 				}, nil, nil
 			}
 
-			return resolveOnce(bs)(ctx, ds, n, names[1:])
+			return resolveOnce(bs, tse)(ctx, ds, n, names[1:])
 		}
 
 		if names[0] == "@state" {
@@ -531,7 +508,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod
 				return nil, nil, xerrors.Errorf("getting actor head for @state: %w", err)
 			}
 
-			m, err := vm.DumpActorState(&act, head.RawData())
+			m, err := vm.DumpActorState(tse.NewActorRegistry(), &act, head.RawData())
 			if err != nil {
 				return nil, nil, err
 			}
@@ -565,7 +542,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod
 				}, nil, nil
 			}
 
-			return resolveOnce(bs)(ctx, ds, n, names[1:])
+			return resolveOnce(bs, tse)(ctx, ds, n, names[1:])
 		}
 
 		return nd.ResolveLink(names)
@@ -585,7 +562,7 @@ func (a *ChainAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject,
 
 	r := &resolver.Resolver{
 		DAG:         dag,
-		ResolveOnce: resolveOnce(bs),
+		ResolveOnce: resolveOnce(bs, a.TsExec),
 	}
 
 	node, err := r.ResolvePath(ctx, ip)

node/impl/full/state.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"strconv"
 
+	"github.com/filecoin-project/go-state-types/crypto"
+
 	"github.com/filecoin-project/go-state-types/cbor"
 	cid "github.com/ipfs/go-cid"
 	"go.uber.org/fx"
@@ -29,7 +31,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/beacon"
-	"github.com/filecoin-project/lotus/chain/gen"
+	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -88,6 +90,8 @@ type StateAPI struct {
 	StateManager  *stmgr.StateManager
 	Chain         *store.ChainStore
 	Beacon        beacon.Schedule
+	Consensus     consensus.Consensus
+	TsExec        stmgr.Executor
 }
 
 func (a *StateAPI) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
@@ -469,7 +473,7 @@ func (a *StateAPI) StateReadState(ctx context.Context, actor address.Address, ts
 		return nil, xerrors.Errorf("getting actor head: %w", err)
 	}
 
-	oif, err := vm.DumpActorState(act, blk.RawData())
+	oif, err := vm.DumpActorState(a.TsExec.NewActorRegistry(), act, blk.RawData())
 	if err != nil {
 		return nil, xerrors.Errorf("dumping actor state (a:%s): %w", actor, err)
 	}
@@ -487,7 +491,7 @@ func (a *StateAPI) StateDecodeParams(ctx context.Context, toAddr address.Address
 		return nil, xerrors.Errorf("getting actor: %w", err)
 	}
 
-	paramType, err := stmgr.GetParamType(act.Code, method)
+	paramType, err := stmgr.GetParamType(a.TsExec.NewActorRegistry(), act.Code, method)
 	if err != nil {
 		return nil, xerrors.Errorf("getting params type: %w", err)
 	}
@@ -500,7 +504,7 @@ func (a *StateAPI) StateDecodeParams(ctx context.Context, toAddr address.Address
 }
 
 func (a *StateAPI) StateEncodeParams(ctx context.Context, toActCode cid.Cid, method abi.MethodNum, params json.RawMessage) ([]byte, error) {
-	paramType, err := stmgr.GetParamType(toActCode, method)
+	paramType, err := stmgr.GetParamType(a.TsExec.NewActorRegistry(), toActCode, method)
 	if err != nil {
 		return nil, xerrors.Errorf("getting params type: %w", err)
 	}
@@ -524,10 +528,13 @@ func (a *StateAPI) MinerGetBaseInfo(ctx context.Context, maddr address.Address,
 }
 
 func (a *StateAPI) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) (*types.BlockMsg, error) {
-	fblk, err := gen.MinerCreateBlock(ctx, a.StateManager, a.Wallet, bt)
+	fblk, err := a.Consensus.CreateBlock(ctx, a.Wallet, bt)
 	if err != nil {
 		return nil, err
 	}
+	if fblk == nil {
+		return nil, nil
+	}
 
 	var out types.BlockMsg
 	out.Header = fblk.Header
@@ -1413,3 +1420,12 @@ func (m *StateModule) StateNetworkVersion(ctx context.Context, tsk types.TipSetK
 	// But that's likely going to break a bunch of stuff.
 	return m.StateManager.GetNtwkVersion(ctx, ts.Height()), nil
 }
+
+func (a *StateAPI) StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) {
+	return a.StateManager.GetRandomnessFromTickets(ctx, personalization, randEpoch, entropy, tsk)
+}
+
+func (a *StateAPI) StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) {
+	return a.StateManager.GetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk)
+
+}

node/impl/full/sync.go

@@ -21,7 +21,7 @@ import (
 type SyncAPI struct {
 	fx.In
 
-	SlashFilter *slashfilter.SlashFilter
+	SlashFilter *slashfilter.SlashFilter `optional:"true"`
 	Syncer      *chain.Syncer
 	PubSub      *pubsub.PubSub
 	NetName     dtypes.NetworkName
@@ -56,9 +56,11 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro
 		return xerrors.Errorf("loading parent block: %w", err)
 	}
 
-	if err := a.SlashFilter.MinedBlock(blk.Header, parent.Height); err != nil {
-		log.Errorf("<!!> SLASH FILTER ERROR: %s", err)
-		return xerrors.Errorf("<!!> SLASH FILTER ERROR: %w", err)
+	if a.SlashFilter != nil {
+		if err := a.SlashFilter.MinedBlock(blk.Header, parent.Height); err != nil {
+			log.Errorf("<!!> SLASH FILTER ERROR: %s", err)
+			return xerrors.Errorf("<!!> SLASH FILTER ERROR: %w", err)
+		}
 	}
 
 	// TODO: should we have some sort of fast path to adding a local block?

node/impl/storminer.go

@@ -557,6 +557,10 @@ func (sm *StorageMinerAPI) MarketPendingDeals(ctx context.Context) (api.PendingD
 	return sm.DealPublisher.PendingDeals(), nil
 }
 
+func (sm *StorageMinerAPI) MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error {
+	return sm.StorageProvider.RetryDealPublishing(propcid)
+}
+
 func (sm *StorageMinerAPI) MarketPublishPendingDeals(ctx context.Context) error {
 	sm.DealPublisher.ForcePublishPendingDeals()
 	return nil
@@ -748,7 +752,10 @@ func (sm *StorageMinerAPI) DagstoreInitializeAll(ctx context.Context, params api
 				}
 
 				err := sm.DagstoreInitializeShard(ctx, k)
-				throttle <- struct{}{}
+
+				if throttle != nil {
+					throttle <- struct{}{}
+				}
 
 				r.Event = "end"
 				if err == nil {

node/modules/alerts.go

@@ -0,0 +1,44 @@
+package modules
+
+import (
+	"github.com/filecoin-project/lotus/journal/alerting"
+	"github.com/filecoin-project/lotus/lib/ulimit"
+)
+
+func CheckFdLimit(min uint64) func(al *alerting.Alerting) {
+	return func(al *alerting.Alerting) {
+		soft, _, err := ulimit.GetLimit()
+
+		if err == ulimit.ErrUnsupported {
+			log.Warn("FD limit monitoring not available")
+			return
+		}
+
+		alert := al.AddAlertType("process", "fd-limit")
+		if err != nil {
+			al.Raise(alert, map[string]string{
+				"message": "failed to get FD limit",
+				"error":   err.Error(),
+			})
+		}
+
+		if soft < min {
+			al.Raise(alert, map[string]interface{}{
+				"message":         "soft FD limit is low",
+				"soft_limit":      soft,
+				"recommended_min": min,
+			})
+		}
+	}
+}
+
+// TODO: More things:
+//  * Space in repo dirs (taking into account mounts)
+//  * Miner
+//    * Faulted partitions
+//    * Low balances
+//  * Market provider
+//    * Reachability
+//    * on-chain config
+//  * Low memory (maybe)
+//  * Network / sync issues

node/modules/chain.go

@@ -17,13 +17,13 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/beacon"
+	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/exchange"
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/vm"
-	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/journal"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
@@ -58,8 +58,8 @@ func ChainBlockService(bs dtypes.ExposedBlockstore, rem dtypes.ChainBitswap) dty
 	return blockservice.New(bs, rem)
 }
 
-func MessagePool(lc fx.Lifecycle, mpp messagepool.Provider, ds dtypes.MetadataDS, nn dtypes.NetworkName, j journal.Journal, protector dtypes.GCReferenceProtector) (*messagepool.MessagePool, error) {
-	mp, err := messagepool.New(mpp, ds, nn, j)
+func MessagePool(lc fx.Lifecycle, us stmgr.UpgradeSchedule, mpp messagepool.Provider, ds dtypes.MetadataDS, nn dtypes.NetworkName, j journal.Journal, protector dtypes.GCReferenceProtector) (*messagepool.MessagePool, error) {
+	mp, err := messagepool.New(mpp, ds, us, nn, j)
 	if err != nil {
 		return nil, xerrors.Errorf("constructing mpool: %w", err)
 	}
@@ -72,8 +72,15 @@ func MessagePool(lc fx.Lifecycle, mpp messagepool.Provider, ds dtypes.MetadataDS
 	return mp, nil
 }
 
-func ChainStore(lc fx.Lifecycle, cbs dtypes.ChainBlockstore, sbs dtypes.StateBlockstore, ds dtypes.MetadataDS, basebs dtypes.BaseBlockstore, j journal.Journal) *store.ChainStore {
-	chain := store.NewChainStore(cbs, sbs, ds, j)
+func ChainStore(lc fx.Lifecycle,
+	cbs dtypes.ChainBlockstore,
+	sbs dtypes.StateBlockstore,
+	ds dtypes.MetadataDS,
+	basebs dtypes.BaseBlockstore,
+	weight store.WeightFunc,
+	j journal.Journal) *store.ChainStore {
+
+	chain := store.NewChainStore(cbs, sbs, ds, weight, j)
 
 	if err := chain.Load(); err != nil {
 		log.Warnf("loading chain state from disk: %s", err)
@@ -100,14 +107,20 @@ func ChainStore(lc fx.Lifecycle, cbs dtypes.ChainBlockstore, sbs dtypes.StateBlo
 	return chain
 }
 
-func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore, syscalls vm.SyscallBuilder, us stmgr.UpgradeSchedule, _ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) {
+func NetworkName(mctx helpers.MetricsCtx,
+	lc fx.Lifecycle,
+	cs *store.ChainStore,
+	tsexec stmgr.Executor,
+	syscalls vm.SyscallBuilder,
+	us stmgr.UpgradeSchedule,
+	_ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) {
 	if !build.Devnet {
 		return "testnetnet", nil
 	}
 
 	ctx := helpers.LifecycleCtx(mctx, lc)
 
-	sm, err := stmgr.NewStateManagerWithUpgradeSchedule(cs, syscalls, us)
+	sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil)
 	if err != nil {
 		return "", err
 	}
@@ -126,7 +139,8 @@ type SyncerParams struct {
 	SyncMgrCtor  chain.SyncManagerCtor
 	Host         host.Host
 	Beacon       beacon.Schedule
-	Verifier     ffiwrapper.Verifier
+	Gent         chain.Genesis
+	Consensus    consensus.Consensus
 }
 
 func NewSyncer(params SyncerParams) (*chain.Syncer, error) {
@@ -138,9 +152,8 @@ func NewSyncer(params SyncerParams) (*chain.Syncer, error) {
 		smCtor = params.SyncMgrCtor
 		h      = params.Host
 		b      = params.Beacon
-		v      = params.Verifier
 	)
-	syncer, err := chain.NewSyncer(ds, sm, ex, smCtor, h.ConnManager(), h.ID(), b, v)
+	syncer, err := chain.NewSyncer(ds, sm, ex, smCtor, h.ConnManager(), h.ID(), b, params.Gent, params.Consensus)
 	if err != nil {
 		return nil, err
 	}

node/modules/client.go

@@ -36,6 +36,7 @@ import (
 	"github.com/filecoin-project/lotus/markets"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/markets/retrievaladapter"
+	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/impl/full"
 	payapi "github.com/filecoin-project/lotus/node/impl/paych"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -112,7 +113,7 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap
 	net := dtnet.NewFromLibp2pHost(h, dtRetryParams)
 
 	dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/client/transfers"))
-	transport := dtgstransport.NewTransport(h.ID(), gs)
+	transport := dtgstransport.NewTransport(h.ID(), gs, net)
 	err := os.MkdirAll(filepath.Join(r.Path(), "data-transfer"), 0755) //nolint: gosec
 	if err != nil && !os.IsExist(err) {
 		return nil, err
@@ -182,7 +183,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT
 	marketsRetryParams := smnet.RetryParameters(time.Second, 5*time.Minute, 15, 5)
 	net := smnet.NewFromLibp2pHost(h, marketsRetryParams)
 
-	c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, accessor, storageimpl.DealPollingInterval(time.Second))
+	c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, accessor, storageimpl.DealPollingInterval(time.Second), storageimpl.MaxTraversalLinks(config.MaxTraversalLinks))
 	if err != nil {
 		return nil, err
 	}

node/modules/graphsync.go

@@ -1,30 +1,41 @@
 package modules
 
 import (
+	"context"
+	"time"
+
 	"github.com/ipfs/go-graphsync"
 	graphsyncimpl "github.com/ipfs/go-graphsync/impl"
 	gsnet "github.com/ipfs/go-graphsync/network"
 	"github.com/ipfs/go-graphsync/storeutil"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
+	"go.opencensus.io/stats"
 	"go.uber.org/fx"
 
+	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
 // Graphsync creates a graphsync instance from the given loader and storer
-func Graphsync(parallelTransfers uint64) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, clientBs dtypes.ClientBlockstore, chainBs dtypes.ExposedBlockstore, h host.Host) (dtypes.Graphsync, error) {
+func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForRetrieval uint64) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, clientBs dtypes.ClientBlockstore, chainBs dtypes.ExposedBlockstore, h host.Host) (dtypes.Graphsync, error) {
 	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, clientBs dtypes.ClientBlockstore, chainBs dtypes.ExposedBlockstore, h host.Host) (dtypes.Graphsync, error) {
 		graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
-		loader := storeutil.LoaderForBlockstore(clientBs)
-		storer := storeutil.StorerForBlockstore(clientBs)
+		lsys := storeutil.LinkSystemForBlockstore(clientBs)
 
-		gs := graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, loader, storer, graphsyncimpl.RejectAllRequestsByDefault(), graphsyncimpl.MaxInProgressRequests(parallelTransfers))
-		chainLoader := storeutil.LoaderForBlockstore(chainBs)
-		chainStorer := storeutil.StorerForBlockstore(chainBs)
-		err := gs.RegisterPersistenceOption("chainstore", chainLoader, chainStorer)
+		gs := graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc),
+			graphsyncNetwork,
+			lsys,
+			graphsyncimpl.RejectAllRequestsByDefault(),
+			graphsyncimpl.MaxInProgressIncomingRequests(parallelTransfersForStorage),
+			graphsyncimpl.MaxInProgressOutgoingRequests(parallelTransfersForRetrieval),
+			graphsyncimpl.MaxLinksPerIncomingRequests(config.MaxTraversalLinks),
+			graphsyncimpl.MaxLinksPerOutgoingRequests(config.MaxTraversalLinks))
+		chainLinkSystem := storeutil.LinkSystemForBlockstore(chainBs)
+		err := gs.RegisterPersistenceOption("chainstore", chainLinkSystem)
 		if err != nil {
 			return nil, err
 		}
@@ -43,6 +54,48 @@ func Graphsync(parallelTransfers uint64) func(mctx helpers.MetricsCtx, lc fx.Lif
 				hookActions.UsePersistenceOption("chainstore")
 			}
 		})
+
+		graphsyncStats(mctx, lc, gs)
+
 		return gs, nil
 	}
 }
+
+func graphsyncStats(mctx helpers.MetricsCtx, lc fx.Lifecycle, gs dtypes.Graphsync) {
+	stopStats := make(chan struct{})
+	lc.Append(fx.Hook{
+		OnStart: func(context.Context) error {
+			go func() {
+				t := time.NewTicker(10 * time.Second)
+				for {
+					select {
+					case <-t.C:
+
+						st := gs.Stats()
+						stats.Record(mctx, metrics.GraphsyncReceivingPeersCount.M(int64(st.OutgoingRequests.TotalPeers)))
+						stats.Record(mctx, metrics.GraphsyncReceivingActiveCount.M(int64(st.OutgoingRequests.Active)))
+						stats.Record(mctx, metrics.GraphsyncReceivingCountCount.M(int64(st.OutgoingRequests.Pending)))
+						stats.Record(mctx, metrics.GraphsyncReceivingTotalMemoryAllocated.M(int64(st.IncomingResponses.TotalAllocatedAllPeers)))
+						stats.Record(mctx, metrics.GraphsyncReceivingTotalPendingAllocations.M(int64(st.IncomingResponses.TotalPendingAllocations)))
+						stats.Record(mctx, metrics.GraphsyncReceivingPeersPending.M(int64(st.IncomingResponses.NumPeersWithPendingAllocations)))
+						stats.Record(mctx, metrics.GraphsyncSendingPeersCount.M(int64(st.IncomingRequests.TotalPeers)))
+						stats.Record(mctx, metrics.GraphsyncSendingActiveCount.M(int64(st.IncomingRequests.Active)))
+						stats.Record(mctx, metrics.GraphsyncSendingCountCount.M(int64(st.IncomingRequests.Pending)))
+						stats.Record(mctx, metrics.GraphsyncSendingTotalMemoryAllocated.M(int64(st.OutgoingResponses.TotalAllocatedAllPeers)))
+						stats.Record(mctx, metrics.GraphsyncSendingTotalPendingAllocations.M(int64(st.OutgoingResponses.TotalPendingAllocations)))
+						stats.Record(mctx, metrics.GraphsyncSendingPeersPending.M(int64(st.OutgoingResponses.NumPeersWithPendingAllocations)))
+
+					case <-stopStats:
+						return
+					}
+				}
+			}()
+
+			return nil
+		},
+		OnStop: func(ctx context.Context) error {
+			close(stopStats)
+			return nil
+		},
+	})
+}

node/modules/lp2p/addrs.go

@@ -11,19 +11,6 @@ import (
 	mamask "github.com/whyrusleeping/multiaddr-filter"
 )
 
-func AddrFilters(filters []string) func() (opts Libp2pOpts, err error) {
-	return func() (opts Libp2pOpts, err error) {
-		for _, s := range filters {
-			f, err := mamask.NewMask(s)
-			if err != nil {
-				return opts, fmt.Errorf("incorrectly formatted address filter in config: %s", s)
-			}
-			opts.Opts = append(opts.Opts, libp2p.FilterAddresses(f)) //nolint:staticcheck
-		}
-		return opts, nil
-	}
-}
-
 func makeAddrsFactory(announce []string, noAnnounce []string) (p2pbhost.AddrsFactory, error) {
 	var annAddrs []ma.Multiaddr
 	for _, addr := range announce {

node/modules/lp2p/libp2p.go

@@ -42,7 +42,7 @@ func PrivKey(ks types.KeyStore) (crypto.PrivKey, error) {
 	if err != nil {
 		return nil, err
 	}
-	kbytes, err := pk.Bytes()
+	kbytes, err := crypto.MarshalPrivateKey(pk)
 	if err != nil {
 		return nil, err
 	}

node/modules/services.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	eventbus "github.com/libp2p/go-eventbus"
@@ -30,6 +31,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/sub"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/journal/fsjournal"
 	"github.com/filecoin-project/lotus/lib/peermgr"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/hello"
@@ -135,11 +137,19 @@ func waitForSync(stmgr *stmgr.StateManager, epochs int, subscribe func()) {
 	})
 }
 
-func HandleIncomingBlocks(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.PubSub, s *chain.Syncer, bserv dtypes.ChainBlockService, chain *store.ChainStore, stmgr *stmgr.StateManager, h host.Host, nn dtypes.NetworkName) {
+func HandleIncomingBlocks(mctx helpers.MetricsCtx,
+	lc fx.Lifecycle,
+	ps *pubsub.PubSub,
+	s *chain.Syncer,
+	bserv dtypes.ChainBlockService,
+	chain *store.ChainStore,
+	cns consensus.Consensus,
+	h host.Host,
+	nn dtypes.NetworkName) {
 	ctx := helpers.LifecycleCtx(mctx, lc)
 
 	v := sub.NewBlockValidator(
-		h.ID(), chain, stmgr,
+		h.ID(), chain, cns,
 		func(p peer.ID) {
 			ps.BlacklistPeer(p)
 			h.ConnManager().TagPeer(p, "badblock", -1000)
@@ -237,7 +247,7 @@ func RandomSchedule(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Sche
 }
 
 func OpenFilesystemJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled journal.DisabledEvents) (journal.Journal, error) {
-	jrnl, err := journal.OpenFSJournal(lr, disabled)
+	jrnl, err := fsjournal.OpenFSJournal(lr, disabled)
 	if err != nil {
 		return nil, err
 	}

node/modules/stmgr.go

@@ -1,6 +1,7 @@
 package modules
 
 import (
+	"github.com/filecoin-project/lotus/chain/beacon"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"go.uber.org/fx"
 
@@ -8,8 +9,8 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 )
 
-func StateManager(lc fx.Lifecycle, cs *store.ChainStore, sys vm.SyscallBuilder, us stmgr.UpgradeSchedule) (*stmgr.StateManager, error) {
-	sm, err := stmgr.NewStateManagerWithUpgradeSchedule(cs, sys, us)
+func StateManager(lc fx.Lifecycle, cs *store.ChainStore, exec stmgr.Executor, sys vm.SyscallBuilder, us stmgr.UpgradeSchedule, b beacon.Schedule) (*stmgr.StateManager, error) {
+	sm, err := stmgr.NewStateManager(cs, exec, sys, us, b)
 	if err != nil {
 		return nil, err
 	}

node/modules/storageminer.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	graphsync "github.com/ipfs/go-graphsync/impl"
+	graphsyncimpl "github.com/ipfs/go-graphsync/impl"
 	gsnet "github.com/ipfs/go-graphsync/network"
 	"github.com/ipfs/go-graphsync/storeutil"
 	"github.com/libp2p/go-libp2p-core/host"
@@ -70,7 +71,10 @@ import (
 	"github.com/filecoin-project/lotus/storage"
 )
 
-var StorageCounterDSPrefix = "/storage/nextid"
+var (
+	StorageCounterDSPrefix = "/storage/nextid"
+	StagingAreaDirName     = "deal-staging"
+)
 
 func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
 	maddrb, err := ds.Get(datastore.NewKey("miner-address"))
@@ -337,7 +341,7 @@ func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.S
 	net := dtnet.NewFromLibp2pHost(h)
 
 	dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/provider/transfers"))
-	transport := dtgstransport.NewTransport(h.ID(), gs)
+	transport := dtgstransport.NewTransport(h.ID(), gs, net)
 	err := os.MkdirAll(filepath.Join(r.Path(), "data-transfer"), 0755) //nolint: gosec
 	if err != nil && !os.IsExist(err) {
 		return nil, err
@@ -391,12 +395,20 @@ func StagingBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRe
 
 // StagingGraphsync creates a graphsync instance which reads and writes blocks
 // to the StagingBlockstore
-func StagingGraphsync(parallelTransfers uint64) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync {
+func StagingGraphsync(parallelTransfersForStorage uint64, parallelTransfersForRetrieval uint64) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync {
 	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync {
 		graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
-		loader := storeutil.LoaderForBlockstore(ibs)
-		storer := storeutil.StorerForBlockstore(ibs)
-		gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, loader, storer, graphsync.RejectAllRequestsByDefault(), graphsync.MaxInProgressRequests(parallelTransfers))
+		lsys := storeutil.LinkSystemForBlockstore(ibs)
+		gs := graphsync.New(helpers.LifecycleCtx(mctx, lc),
+			graphsyncNetwork,
+			lsys,
+			graphsync.RejectAllRequestsByDefault(),
+			graphsync.MaxInProgressIncomingRequests(parallelTransfersForRetrieval),
+			graphsync.MaxInProgressOutgoingRequests(parallelTransfersForStorage),
+			graphsyncimpl.MaxLinksPerIncomingRequests(config.MaxTraversalLinks),
+			graphsyncimpl.MaxLinksPerOutgoingRequests(config.MaxTraversalLinks))
+
+		graphsyncStats(mctx, lc, gs)
 
 		return gs
 	}
@@ -442,14 +454,16 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi v1api.FullNode, ds dtypes.Metada
 		storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize)))
 }
 
-func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
+func BasicDealFilter(cfg config.DealmakingConfig, user dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
 	offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc,
 	verifiedOk dtypes.ConsiderVerifiedStorageDealsConfigFunc,
 	unverifiedOk dtypes.ConsiderUnverifiedStorageDealsConfigFunc,
 	blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc,
 	expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc,
 	startDelay dtypes.GetMaxDealStartDelayFunc,
-	spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter {
+	spn storagemarket.StorageProviderNode,
+	r repo.LockedRepo,
+) dtypes.StorageDealFilter {
 	return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc,
 		offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc,
 		verifiedOk dtypes.ConsiderVerifiedStorageDealsConfigFunc,
@@ -457,7 +471,9 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside
 		blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc,
 		expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc,
 		startDelay dtypes.GetMaxDealStartDelayFunc,
-		spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter {
+		spn storagemarket.StorageProviderNode,
+		r repo.LockedRepo,
+	) dtypes.StorageDealFilter {
 
 		return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) {
 			b, err := onlineOk()
@@ -533,6 +549,17 @@ func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.Conside
 				return false, "miner error", err
 			}
 
+			dir := filepath.Join(r.Path(), StagingAreaDirName)
+			diskUsageBytes, err := r.DiskUsage(dir)
+			if err != nil {
+				return false, "miner error", err
+			}
+
+			if cfg.MaxStagingDealsBytes != 0 && diskUsageBytes >= cfg.MaxStagingDealsBytes {
+				log.Errorw("proposed deal rejected because there are too many deals in the staging area at the moment", "MaxStagingDealsBytes", cfg.MaxStagingDealsBytes, "DiskUsageBytes", diskUsageBytes)
+				return false, "cannot accept deal as miner is overloaded at the moment - there are too many staging deals being processed", nil
+			}
+
 			// Reject if it's more than 7 days in the future
 			// TODO: read from cfg
 			maxStartEpoch := earliest + abi.ChainEpoch(uint64(sd.Seconds())/build.BlockDelaySecs)
@@ -561,7 +588,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
 ) (storagemarket.StorageProvider, error) {
 	net := smnet.NewFromLibp2pHost(h)
 
-	dir := filepath.Join(r.Path(), "deal-staging")
+	dir := filepath.Join(r.Path(), StagingAreaDirName)
 
 	// migrate temporary files that were created directly under the repo, by
 	// moving them to the new directory and symlinking them.
@@ -853,12 +880,13 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 	return func(cfg sealiface.Config) (err error) {
 		err = mutateCfg(r, func(c *config.StorageMiner) {
 			c.Sealing = config.SealingConfig{
-				MaxWaitDealsSectors:       cfg.MaxWaitDealsSectors,
-				MaxSealingSectors:         cfg.MaxSealingSectors,
-				MaxSealingSectorsForDeals: cfg.MaxSealingSectorsForDeals,
-				WaitDealsDelay:            config.Duration(cfg.WaitDealsDelay),
-				AlwaysKeepUnsealedCopy:    cfg.AlwaysKeepUnsealedCopy,
-				FinalizeEarly:             cfg.FinalizeEarly,
+				MaxWaitDealsSectors:             cfg.MaxWaitDealsSectors,
+				MaxSealingSectors:               cfg.MaxSealingSectors,
+				MaxSealingSectorsForDeals:       cfg.MaxSealingSectorsForDeals,
+				CommittedCapacitySectorLifetime: config.Duration(cfg.CommittedCapacitySectorLifetime),
+				WaitDealsDelay:                  config.Duration(cfg.WaitDealsDelay),
+				AlwaysKeepUnsealedCopy:          cfg.AlwaysKeepUnsealedCopy,
+				FinalizeEarly:                   cfg.FinalizeEarly,
 
 				CollateralFromMinerBalance: cfg.CollateralFromMinerBalance,
 				AvailableBalanceBuffer:     types.FIL(cfg.AvailableBalanceBuffer),
@@ -869,12 +897,13 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 				PreCommitBatchWait:  config.Duration(cfg.PreCommitBatchWait),
 				PreCommitBatchSlack: config.Duration(cfg.PreCommitBatchSlack),
 
-				AggregateCommits:      cfg.AggregateCommits,
-				MinCommitBatch:        cfg.MinCommitBatch,
-				MaxCommitBatch:        cfg.MaxCommitBatch,
-				CommitBatchWait:       config.Duration(cfg.CommitBatchWait),
-				CommitBatchSlack:      config.Duration(cfg.CommitBatchSlack),
-				AggregateAboveBaseFee: types.FIL(cfg.AggregateAboveBaseFee),
+				AggregateCommits:           cfg.AggregateCommits,
+				MinCommitBatch:             cfg.MinCommitBatch,
+				MaxCommitBatch:             cfg.MaxCommitBatch,
+				CommitBatchWait:            config.Duration(cfg.CommitBatchWait),
+				CommitBatchSlack:           config.Duration(cfg.CommitBatchSlack),
+				AggregateAboveBaseFee:      types.FIL(cfg.AggregateAboveBaseFee),
+				BatchPreCommitAboveBaseFee: types.FIL(cfg.BatchPreCommitAboveBaseFee),
 
 				TerminateBatchMax:  cfg.TerminateBatchMax,
 				TerminateBatchMin:  cfg.TerminateBatchMin,
@@ -887,12 +916,13 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 
 func ToSealingConfig(cfg *config.StorageMiner) sealiface.Config {
 	return sealiface.Config{
-		MaxWaitDealsSectors:       cfg.Sealing.MaxWaitDealsSectors,
-		MaxSealingSectors:         cfg.Sealing.MaxSealingSectors,
-		MaxSealingSectorsForDeals: cfg.Sealing.MaxSealingSectorsForDeals,
-		WaitDealsDelay:            time.Duration(cfg.Sealing.WaitDealsDelay),
-		AlwaysKeepUnsealedCopy:    cfg.Sealing.AlwaysKeepUnsealedCopy,
-		FinalizeEarly:             cfg.Sealing.FinalizeEarly,
+		MaxWaitDealsSectors:             cfg.Sealing.MaxWaitDealsSectors,
+		MaxSealingSectors:               cfg.Sealing.MaxSealingSectors,
+		MaxSealingSectorsForDeals:       cfg.Sealing.MaxSealingSectorsForDeals,
+		CommittedCapacitySectorLifetime: time.Duration(cfg.Sealing.CommittedCapacitySectorLifetime),
+		WaitDealsDelay:                  time.Duration(cfg.Sealing.WaitDealsDelay),
+		AlwaysKeepUnsealedCopy:          cfg.Sealing.AlwaysKeepUnsealedCopy,
+		FinalizeEarly:                   cfg.Sealing.FinalizeEarly,
 
 		CollateralFromMinerBalance: cfg.Sealing.CollateralFromMinerBalance,
 		AvailableBalanceBuffer:     types.BigInt(cfg.Sealing.AvailableBalanceBuffer),
@@ -903,16 +933,19 @@ func ToSealingConfig(cfg *config.StorageMiner) sealiface.Config {
 		PreCommitBatchWait:  time.Duration(cfg.Sealing.PreCommitBatchWait),
 		PreCommitBatchSlack: time.Duration(cfg.Sealing.PreCommitBatchSlack),
 
-		AggregateCommits:      cfg.Sealing.AggregateCommits,
-		MinCommitBatch:        cfg.Sealing.MinCommitBatch,
-		MaxCommitBatch:        cfg.Sealing.MaxCommitBatch,
-		CommitBatchWait:       time.Duration(cfg.Sealing.CommitBatchWait),
-		CommitBatchSlack:      time.Duration(cfg.Sealing.CommitBatchSlack),
-		AggregateAboveBaseFee: types.BigInt(cfg.Sealing.AggregateAboveBaseFee),
+		AggregateCommits:           cfg.Sealing.AggregateCommits,
+		MinCommitBatch:             cfg.Sealing.MinCommitBatch,
+		MaxCommitBatch:             cfg.Sealing.MaxCommitBatch,
+		CommitBatchWait:            time.Duration(cfg.Sealing.CommitBatchWait),
+		CommitBatchSlack:           time.Duration(cfg.Sealing.CommitBatchSlack),
+		AggregateAboveBaseFee:      types.BigInt(cfg.Sealing.AggregateAboveBaseFee),
+		BatchPreCommitAboveBaseFee: types.BigInt(cfg.Sealing.BatchPreCommitAboveBaseFee),
 
 		TerminateBatchMax:  cfg.Sealing.TerminateBatchMax,
 		TerminateBatchMin:  cfg.Sealing.TerminateBatchMin,
 		TerminateBatchWait: time.Duration(cfg.Sealing.TerminateBatchWait),
+
+		StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer),
 	}
 }
 

node/options.go

@@ -110,7 +110,9 @@ func as(in interface{}, as interface{}) interface{} {
 		panic("outType is not a pointer")
 	}
 
-	if reflect.TypeOf(in).Kind() != reflect.Func {
+	inType := reflect.TypeOf(in)
+
+	if inType.Kind() != reflect.Func || inType.AssignableTo(outType.Elem()) {
 		ctype := reflect.FuncOf(nil, []reflect.Type{outType.Elem()}, false)
 
 		return reflect.MakeFunc(ctype, func(args []reflect.Value) (results []reflect.Value) {
@@ -121,8 +123,6 @@ func as(in interface{}, as interface{}) interface{} {
 		}).Interface()
 	}
 
-	inType := reflect.TypeOf(in)
-
 	ins := make([]reflect.Type, inType.NumIn())
 	outs := make([]reflect.Type, inType.NumOut())
 

node/rpc.go

@@ -25,6 +25,7 @@ import (
 	"github.com/filecoin-project/lotus/api/v1api"
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/metrics/proxy"
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
@@ -78,7 +79,7 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
 		m.Handle(path, handler)
 	}
 
-	fnapi := metrics.MetricedFullAPI(a)
+	fnapi := proxy.MetricedFullAPI(a)
 	if permissioned {
 		fnapi = api.PermissionedFullAPI(fnapi)
 	}
@@ -113,7 +114,7 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
 func MinerHandler(a api.StorageMiner, permissioned bool) (http.Handler, error) {
 	m := mux.NewRouter()
 
-	mapi := metrics.MetricedStorMinerAPI(a)
+	mapi := proxy.MetricedStorMinerAPI(a)
 	if permissioned {
 		mapi = api.PermissionedStorMinerAPI(mapi)
 	}