Merge branch 'master' into adlrocha/consistent-bcast

node/builder.go

@@ -121,6 +121,7 @@ const (
 	SettlePaymentChannelsKey
 	RunPeerTaggerKey
 	SetupFallbackBlockstoresKey
+	GoRPCServer
 
 	SetApiEndpointKey
 
@@ -155,6 +156,7 @@ func defaults() []Option {
 		Override(new(journal.DisabledEvents), journal.EnvDisabledEvents),
 		Override(new(journal.Journal), modules.OpenFilesystemJournal),
 		Override(new(*alerting.Alerting), alerting.NewAlertingSystem),
+		Override(new(dtypes.NodeStartTime), FromVal(dtypes.NodeStartTime(time.Now()))),
 
 		Override(CheckFDLimit, modules.CheckFdLimit(build.DefaultFDLimit)),
 

node/builder_chain.go

@@ -3,6 +3,7 @@ package node
 import (
 	"os"
 
+	gorpc "github.com/libp2p/go-libp2p-gorpc"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
@@ -28,7 +29,9 @@ import (
 	"github.com/filecoin-project/lotus/chain/wallet"
 	ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger"
 	"github.com/filecoin-project/lotus/chain/wallet/remotewallet"
+	raftcns "github.com/filecoin-project/lotus/lib/consensus/raft"
 	"github.com/filecoin-project/lotus/lib/peermgr"
+	"github.com/filecoin-project/lotus/markets/retrievaladapter"
 	"github.com/filecoin-project/lotus/markets/storageadapter"
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/hello"
@@ -105,6 +108,7 @@ var ChainNode = Options(
 
 	// Service: Wallet
 	Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
+	Override(new(messagesigner.MsgSigner), func(ms *messagesigner.MessageSigner) *messagesigner.MessageSigner { return ms }),
 	Override(new(*wallet.LocalWallet), wallet.NewWallet),
 	Override(new(wallet.Default), From(new(*wallet.LocalWallet))),
 	Override(new(api.Wallet), From(new(wallet.MultiWallet))),
@@ -129,6 +133,7 @@ var ChainNode = Options(
 	Override(new(*market.FundManager), market.NewFundManager),
 	Override(new(dtypes.ClientDatastore), modules.NewClientDatastore),
 	Override(new(storagemarket.BlockstoreAccessor), modules.StorageBlockstoreAccessor),
+	Override(new(*retrievaladapter.APIBlockstoreAccessor), retrievaladapter.NewAPIBlockstoreAdapter),
 	Override(new(storagemarket.StorageClient), modules.StorageClient),
 	Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
 	Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds),
@@ -140,7 +145,7 @@ var ChainNode = Options(
 	// Lite node API
 	ApplyIf(isLiteNode,
 		Override(new(messagepool.Provider), messagepool.NewProviderLite),
-		Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))),
+		Override(new(messagepool.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))),
 		Override(new(full.ChainModuleAPI), From(new(api.Gateway))),
 		Override(new(full.GasModuleAPI), From(new(api.Gateway))),
 		Override(new(full.MpoolModuleAPI), From(new(api.Gateway))),
@@ -151,7 +156,7 @@ var ChainNode = Options(
 	// Full node API / service startup
 	ApplyIf(isFullNode,
 		Override(new(messagepool.Provider), messagepool.NewProvider),
-		Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))),
+		Override(new(messagepool.MpoolNonceAPI), From(new(*messagepool.MessagePool))),
 		Override(new(full.ChainModuleAPI), From(new(full.ChainModule))),
 		Override(new(full.GasModuleAPI), From(new(full.GasModule))),
 		Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))),
@@ -181,7 +186,7 @@ func ConfigFullNode(c interface{}) Option {
 		Override(new(dtypes.UniversalBlockstore), modules.UniversalBlockstore),
 
 		If(cfg.Chainstore.EnableSplitstore,
-			If(cfg.Chainstore.Splitstore.ColdStoreType == "universal",
+			If(cfg.Chainstore.Splitstore.ColdStoreType == "universal" || cfg.Chainstore.Splitstore.ColdStoreType == "messages",
 				Override(new(dtypes.ColdBlockstore), From(new(dtypes.UniversalBlockstore)))),
 			If(cfg.Chainstore.Splitstore.ColdStoreType == "discard",
 				Override(new(dtypes.ColdBlockstore), modules.DiscardColdBlockstore)),
@@ -236,6 +241,16 @@ func ConfigFullNode(c interface{}) Option {
 			Unset(new(*wallet.LocalWallet)),
 			Override(new(wallet.Default), wallet.NilDefault),
 		),
+		// Chain node cluster enabled
+		If(cfg.Cluster.ClusterModeEnabled,
+			Override(new(*gorpc.Client), modules.NewRPCClient),
+			Override(new(*raftcns.ClusterRaftConfig), raftcns.NewClusterRaftConfig(&cfg.Cluster)),
+			Override(new(*raftcns.Consensus), raftcns.NewConsensusWithRPCClient(false)),
+			Override(new(*messagesigner.MessageSignerConsensus), messagesigner.NewMessageSignerConsensus),
+			Override(new(messagesigner.MsgSigner), From(new(*messagesigner.MessageSignerConsensus))),
+			Override(new(*modules.RPCHandler), modules.NewRPCHandler),
+			Override(GoRPCServer, modules.NewRPCServer),
+		),
 	)
 }
 

node/builder_miner.go

@@ -94,6 +94,10 @@ func ConfigStorageMiner(c interface{}) Option {
 		Override(new(paths.Store), From(new(*paths.Remote))),
 		Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)),
 
+		If(cfg.Subsystems.EnableMining || cfg.Subsystems.EnableSealing,
+			Override(GetParamsKey, modules.GetParams(!cfg.Proving.DisableBuiltinWindowPoSt || !cfg.Proving.DisableBuiltinWinningPoSt || cfg.Storage.AllowCommit || cfg.Storage.AllowProveReplicaUpdate2)),
+		),
+
 		If(!cfg.Subsystems.EnableMining,
 			If(cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))),
 			If(cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))),
@@ -107,9 +111,6 @@ func ConfigStorageMiner(c interface{}) Option {
 			Override(new(storiface.Prover), ffiwrapper.ProofProver),
 			Override(new(storiface.ProverPoSt), From(new(sectorstorage.SectorManager))),
 
-			// Sealing (todo should be under EnableSealing, but storagefsm is currently bundled with storage.Miner)
-			Override(GetParamsKey, modules.GetParams),
-
 			Override(new(dtypes.SetSealingConfigFunc), modules.NewSetSealConfigFunc),
 			Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc),
 
@@ -223,7 +224,8 @@ func ConfigStorageMiner(c interface{}) Option {
 			Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)),
 		),
 
-		Override(new(sectorstorage.Config), cfg.StorageManager()),
+		Override(new(config.SealerConfig), cfg.Storage),
+		Override(new(config.ProvingConfig), cfg.Proving),
 		Override(new(*ctladdr.AddressSelector), modules.AddressSelector(&cfg.Addresses)),
 	)
 }

node/bundle/bundle.go

@@ -10,6 +10,8 @@ import (
 	"github.com/ipld/go-car"
 	"golang.org/x/xerrors"
 
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
+
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
@@ -39,10 +41,10 @@ func LoadBundle(ctx context.Context, bs blockstore.Blockstore, r io.Reader) (cid
 
 // LoadBundles loads the bundles for the specified actor versions into the passed blockstore, if and
 // only if the bundle's manifest is not already present in the blockstore.
-func LoadBundles(ctx context.Context, bs blockstore.Blockstore, versions ...actors.Version) error {
+func LoadBundles(ctx context.Context, bs blockstore.Blockstore, versions ...actorstypes.Version) error {
 	for _, av := range versions {
 		// No bundles before version 8.
-		if av < actors.Version8 {
+		if av < actorstypes.Version8 {
 			continue
 		}
 

node/config/def.go

@@ -15,7 +15,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/storage/sealer"
 )
 
 const (
@@ -91,14 +90,14 @@ func DefaultFullNode() *FullNode {
 		Chainstore: Chainstore{
 			EnableSplitstore: false,
 			Splitstore: Splitstore{
-				ColdStoreType: "universal",
+				ColdStoreType: "messages",
 				HotStoreType:  "badger",
 				MarkSetType:   "badger",
 
-				HotStoreFullGCFrequency:  20,
-				ColdStoreFullGCFrequency: 7,
+				HotStoreFullGCFrequency: 20,
 			},
 		},
+		Cluster: *DefaultUserRaftConfig(),
 	}
 }
 
@@ -142,10 +141,13 @@ func DefaultStorageMiner() *StorageMiner {
 		},
 
 		Proving: ProvingConfig{
-			ParallelCheckLimit: 128,
+			ParallelCheckLimit:    128,
+			PartitionCheckTimeout: Duration(20 * time.Minute),
+			SingleCheckTimeout:    Duration(10 * time.Minute),
 		},
 
 		Storage: SealerConfig{
+			AllowSectorDownload:      true,
 			AllowAddPiece:            true,
 			AllowPreCommit1:          true,
 			AllowPreCommit2:          true,
@@ -162,7 +164,7 @@ func DefaultStorageMiner() *StorageMiner {
 			Assigner: "utilization",
 
 			// By default use the hardware resource filtering strategy.
-			ResourceFiltering: sealer.ResourceFilteringHardware,
+			ResourceFiltering: ResourceFilteringHardware,
 		},
 
 		Dealmaking: DealmakingConfig{
@@ -274,3 +276,39 @@ func (dur Duration) MarshalText() ([]byte, error) {
 	d := time.Duration(dur)
 	return []byte(d.String()), nil
 }
+
+// ResourceFilteringStrategy is an enum indicating the kinds of resource
+// filtering strategies that can be configured for workers.
+type ResourceFilteringStrategy string
+
+const (
+	// ResourceFilteringHardware specifies that available hardware resources
+	// should be evaluated when scheduling a task against the worker.
+	ResourceFilteringHardware = ResourceFilteringStrategy("hardware")
+
+	// ResourceFilteringDisabled disables resource filtering against this
+	// worker. The scheduler may assign any task to this worker.
+	ResourceFilteringDisabled = ResourceFilteringStrategy("disabled")
+)
+
+var (
+	DefaultDataSubFolder        = "raft"
+	DefaultWaitForLeaderTimeout = 15 * time.Second
+	DefaultCommitRetries        = 1
+	DefaultNetworkTimeout       = 100 * time.Second
+	DefaultCommitRetryDelay     = 200 * time.Millisecond
+	DefaultBackupsRotate        = 6
+)
+
+func DefaultUserRaftConfig() *UserRaftConfig {
+	var cfg UserRaftConfig
+	cfg.DataFolder = "" // empty so it gets omitted
+	cfg.InitPeersetMultiAddr = []string{}
+	cfg.WaitForLeaderTimeout = Duration(DefaultWaitForLeaderTimeout)
+	cfg.NetworkTimeout = Duration(DefaultNetworkTimeout)
+	cfg.CommitRetries = DefaultCommitRetries
+	cfg.CommitRetryDelay = Duration(DefaultCommitRetryDelay)
+	cfg.BackupsRotate = DefaultBackupsRotate
+
+	return &cfg
+}

node/config/dep_test.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func goCmd() string {
+	var exeSuffix string
+	if runtime.GOOS == "windows" {
+		exeSuffix = ".exe"
+	}
+	path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix)
+	if _, err := os.Stat(path); err == nil {
+		return path
+	}
+	return "go"
+}
+
+func TestDoesntDependOnFFI(t *testing.T) {
+	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/node/config").Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range strings.Fields(string(deps)) {
+		if pkg == "github.com/filecoin-project/filecoin-ffi" {
+			t.Fatal("config depends on filecoin-ffi")
+		}
+	}
+}

node/config/doc_gen.go

@@ -325,14 +325,14 @@ regardless of this number.`,
 			Type: "string",
 
 			Comment: `A command used for fine-grained evaluation of storage deals
-see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
+see https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
 		},
 		{
 			Name: "RetrievalFilter",
 			Type: "string",
 
 			Comment: `A command used for fine-grained evaluation of retrieval deals
-see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
+see https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
 		},
 		{
 			Name: "RetrievalPricing",
@@ -372,6 +372,12 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f
 			Name: "Chainstore",
 			Type: "Chainstore",
 
+			Comment: ``,
+		},
+		{
+			Name: "Cluster",
+			Type: "UserRaftConfig",
+
 			Comment: ``,
 		},
 	},
@@ -638,6 +644,29 @@ to late submission.
 
 After changing this option, confirm that the new value works in your setup by invoking
 'lotus-miner proving compute window-post 0'`,
+		},
+		{
+			Name: "SingleCheckTimeout",
+			Type: "Duration",
+
+			Comment: `Maximum amount of time a proving pre-check can take for a sector. If the check times out the sector will be skipped
+
+WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+test challenge took longer than this timeout
+WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
+blocked (e.g. in case of disconnected NFS mount)`,
+		},
+		{
+			Name: "PartitionCheckTimeout",
+			Type: "Duration",
+
+			Comment: `Maximum amount of time a proving pre-check can take for an entire partition. If the check times out, sectors in
+the partition which didn't get checked on time will be skipped
+
+WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+test challenge took longer than this timeout
+WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
+blocked or slow`,
 		},
 		{
 			Name: "DisableBuiltinWindowPoSt",
@@ -698,11 +727,9 @@ After changing this option, confirm that the new value works in your setup by in
 A single partition may contain up to 2349 32GiB sectors, or 2300 64GiB sectors.
 
 The maximum number of sectors which can be proven in a single PoSt message is 25000 in network version 16, which
-means that a single message can prove at most 10 partinions
+means that a single message can prove at most 10 partitions
 
-In some cases when submitting PoSt messages which are recovering sectors, the default network limit may still be
-too high to fit in the block gas limit; In those cases it may be necessary to set this value to something lower
-than 10; Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+Note that setting this value lower may result in less efficient gas use - more messages will be sent,
 to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
 
 Setting this value above the network limit has no effect`,
@@ -717,6 +744,19 @@ In those cases it may be necessary to set this value to something low (eg 1);
 Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed,
 resulting in more total gas use (but each message will have lower gas limit)`,
 		},
+		{
+			Name: "SingleRecoveringPartitionPerPostMessage",
+			Type: "bool",
+
+			Comment: `Enable single partition per PoSt Message for partitions containing recovery sectors
+
+In cases when submitting PoSt messages which contain recovering sectors, the default network limit may still be
+too high to fit in the block gas limit. In those cases, it becomes useful to only house the single partition
+with recovering sectors in the post message
+
+Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)`,
+		},
 	},
 	"Pubsub": []DocField{
 		{
@@ -748,6 +788,35 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 
 			Comment: ``,
 		},
+		{
+			Name: "JsonTracer",
+			Type: "string",
+
+			Comment: `Path to file that will be used to output tracer content in JSON format.
+If present tracer will save data to defined file.
+Format: file path`,
+		},
+		{
+			Name: "ElasticSearchTracer",
+			Type: "string",
+
+			Comment: `Connection string for elasticsearch instance.
+If present tracer will save data to elasticsearch.
+Format: https://<username>:<password>@<elasticsearch_url>:<port>/`,
+		},
+		{
+			Name: "ElasticSearchIndex",
+			Type: "string",
+
+			Comment: `Name of elasticsearch index that will be used to save tracer data.
+This property is used only if ElasticSearchTracer propery is set.`,
+		},
+		{
+			Name: "TracerSourceAuth",
+			Type: "string",
+
+			Comment: `Auth token that will be passed with logs to elasticsearch - used for weighted peers score.`,
+		},
 	},
 	"RetrievalPricing": []DocField{
 		{
@@ -796,11 +865,17 @@ This parameter is ONLY applicable if the retrieval pricing policy strategy has b
 
 			Comment: ``,
 		},
+		{
+			Name: "AllowSectorDownload",
+			Type: "bool",
+
+			Comment: ``,
+		},
 		{
 			Name: "AllowAddPiece",
 			Type: "bool",
 
-			Comment: `Local worker config`,
+			Comment: ``,
 		},
 		{
 			Name: "AllowPreCommit1",
@@ -877,7 +952,7 @@ If you see stuck Finalize tasks after enabling this setting, check
 		},
 		{
 			Name: "ResourceFiltering",
-			Type: "sealer.ResourceFilteringStrategy",
+			Type: "ResourceFilteringStrategy",
 
 			Comment: `ResourceFiltering instructs the system which resource filtering strategy
 to use when evaluating tasks against this worker. An empty value defaults
@@ -922,6 +997,30 @@ flow when the volume of storage deals is lower.`,
 
 			Comment: `Upper bound on how many sectors can be sealing+upgrading at the same time when upgrading CC sectors with deals (0 = MaxSealingSectorsForDeals)`,
 		},
+		{
+			Name: "MinUpgradeSectorExpiration",
+			Type: "uint64",
+
+			Comment: `When set to a non-zero value, minimum number of epochs until sector expiration required for sectors to be considered
+for upgrades (0 = DealMinDuration = 180 days = 518400 epochs)
+
+Note that if all deals waiting in the input queue have lifetimes longer than this value, upgrade sectors will be
+required to have expiration of at least the soonest-ending deal`,
+		},
+		{
+			Name: "MinTargetUpgradeSectorExpiration",
+			Type: "uint64",
+
+			Comment: `When set to a non-zero value, minimum number of epochs until sector expiration above which upgrade candidates will
+be selected based on lowest initial pledge.
+
+Target sector expiration is calculated by looking at the input deal queue, sorting it by deal expiration, and
+selecting N deals from the queue up to sector size. The target expiration will be Nth deal end epoch, or in case
+where there weren't enough deals to fill a sector, DealMaxDuration (540 days = 1555200 epochs)
+
+Setting this to a high value (for example to maximum deal duration - 1555200) will disable selection based on
+initial pledge - upgrade sectors will always be chosen based on longest expiration`,
+		},
 		{
 			Name: "CommittedCapacitySectorLifetime",
 			Type: "Duration",
@@ -1075,7 +1174,7 @@ submitting proofs to the chain individually`,
 			Type: "string",
 
 			Comment: `ColdStoreType specifies the type of the coldstore.
-It can be "universal" (default) or "discard" for discarding cold blocks.`,
+It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.`,
 		},
 		{
 			Name: "HotStoreType",
@@ -1106,30 +1205,6 @@ the compaction boundary; default is 0.`,
 A value of 0 disables, while a value 1 will do full GC in every compaction.
 Default is 20 (about once a week).`,
 		},
-		{
-			Name: "EnableColdStoreAutoPrune",
-			Type: "bool",
-
-			Comment: `EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning
-where hotstore compaction occurs every finality epochs pruning happens every 3 finalities
-Default is false`,
-		},
-		{
-			Name: "ColdStoreFullGCFrequency",
-			Type: "uint64",
-
-			Comment: `ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore.
-Only applies if auto prune is enabled.  A value of 0 disables while a value of 1 will do
-full GC in every prune.
-Default is 7 (about once every a week)`,
-		},
-		{
-			Name: "ColdStoreRetention",
-			Type: "int64",
-
-			Comment: `ColdStoreRetention specifies the retention policy for data reachable from the chain, in
-finalities beyond the compaction boundary, default is 0, -1 retains everything`,
-		},
 	},
 	"StorageMiner": []DocField{
 		{
@@ -1187,6 +1262,68 @@ finalities beyond the compaction boundary, default is 0, -1 retains everything`,
 			Comment: ``,
 		},
 	},
+	"UserRaftConfig": []DocField{
+		{
+			Name: "ClusterModeEnabled",
+			Type: "bool",
+
+			Comment: `EXPERIMENTAL. config to enabled node cluster with raft consensus`,
+		},
+		{
+			Name: "DataFolder",
+			Type: "string",
+
+			Comment: `A folder to store Raft's data.`,
+		},
+		{
+			Name: "InitPeersetMultiAddr",
+			Type: "[]string",
+
+			Comment: `InitPeersetMultiAddr provides the list of initial cluster peers for new Raft
+peers (with no prior state). It is ignored when Raft was already
+initialized or when starting in staging mode.`,
+		},
+		{
+			Name: "WaitForLeaderTimeout",
+			Type: "Duration",
+
+			Comment: `LeaderTimeout specifies how long to wait for a leader before
+failing an operation.`,
+		},
+		{
+			Name: "NetworkTimeout",
+			Type: "Duration",
+
+			Comment: `NetworkTimeout specifies how long before a Raft network
+operation is timed out`,
+		},
+		{
+			Name: "CommitRetries",
+			Type: "int",
+
+			Comment: `CommitRetries specifies how many times we retry a failed commit until
+we give up.`,
+		},
+		{
+			Name: "CommitRetryDelay",
+			Type: "Duration",
+
+			Comment: `How long to wait between retries`,
+		},
+		{
+			Name: "BackupsRotate",
+			Type: "int",
+
+			Comment: `BackupsRotate specifies the maximum number of Raft's DataFolder
+copies that we keep as backups (renaming) after cleanup.`,
+		},
+		{
+			Name: "Tracing",
+			Type: "bool",
+
+			Comment: `Tracing enables propagation of contexts across binary boundaries.`,
+		},
+	},
 	"Wallet": []DocField{
 		{
 			Name: "RemoteBackend",

node/config/storage.go

@@ -8,11 +8,10 @@ import (
 
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/storage/paths"
-	"github.com/filecoin-project/lotus/storage/sealer"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
-func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfig, error) {
+func StorageFromFile(path string, def *storiface.StorageConfig) (*storiface.StorageConfig, error) {
 	file, err := os.Open(path)
 	switch {
 	case os.IsNotExist(err):
@@ -28,8 +27,8 @@ func StorageFromFile(path string, def *paths.StorageConfig) (*paths.StorageConfi
 	return StorageFromReader(file)
 }
 
-func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) {
-	var cfg paths.StorageConfig
+func StorageFromReader(reader io.Reader) (*storiface.StorageConfig, error) {
+	var cfg storiface.StorageConfig
 	err := json.NewDecoder(reader).Decode(&cfg)
 	if err != nil {
 		return nil, err
@@ -38,7 +37,7 @@ func StorageFromReader(reader io.Reader) (*paths.StorageConfig, error) {
 	return &cfg, nil
 }
 
-func WriteStorageFile(path string, config paths.StorageConfig) error {
+func WriteStorageFile(path string, config storiface.StorageConfig) error {
 	b, err := json.MarshalIndent(config, "", "  ")
 	if err != nil {
 		return xerrors.Errorf("marshaling storage config: %w", err)
@@ -50,27 +49,3 @@ func WriteStorageFile(path string, config paths.StorageConfig) error {
 
 	return nil
 }
-
-func (c *StorageMiner) StorageManager() sealer.Config {
-	return sealer.Config{
-		ParallelFetchLimit:       c.Storage.ParallelFetchLimit,
-		AllowAddPiece:            c.Storage.AllowAddPiece,
-		AllowPreCommit1:          c.Storage.AllowPreCommit1,
-		AllowPreCommit2:          c.Storage.AllowPreCommit2,
-		AllowCommit:              c.Storage.AllowCommit,
-		AllowUnseal:              c.Storage.AllowUnseal,
-		AllowReplicaUpdate:       c.Storage.AllowReplicaUpdate,
-		AllowProveReplicaUpdate2: c.Storage.AllowProveReplicaUpdate2,
-		AllowRegenSectorKey:      c.Storage.AllowRegenSectorKey,
-		ResourceFiltering:        c.Storage.ResourceFiltering,
-		DisallowRemoteFinalize:   c.Storage.DisallowRemoteFinalize,
-
-		LocalWorkerName: c.Storage.LocalWorkerName,
-
-		Assigner: c.Storage.Assigner,
-
-		ParallelCheckLimit:        c.Proving.ParallelCheckLimit,
-		DisableBuiltinWindowPoSt:  c.Proving.DisableBuiltinWindowPoSt,
-		DisableBuiltinWinningPoSt: c.Proving.DisableBuiltinWinningPoSt,
-	}
-}

node/config/types.go

@@ -4,7 +4,6 @@ import (
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/storage/sealer"
 )
 
 // // NOTE: ONLY PUT STRUCT DEFINITIONS IN THIS FILE
@@ -27,6 +26,7 @@ type FullNode struct {
 	Wallet     Wallet
 	Fees       FeeConfig
 	Chainstore Chainstore
+	Cluster    UserRaftConfig
 }
 
 // // Common
@@ -158,10 +158,10 @@ type DealmakingConfig struct {
 	StartEpochSealingBuffer uint64
 
 	// A command used for fine-grained evaluation of storage deals
-	// see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
+	// see https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
 	Filter string
 	// A command used for fine-grained evaluation of retrieval deals
-	// see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
+	// see https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details
 	RetrievalFilter string
 
 	RetrievalPricing *RetrievalPricing
@@ -230,6 +230,23 @@ type ProvingConfig struct {
 	// 'lotus-miner proving compute window-post 0'
 	ParallelCheckLimit int
 
+	// Maximum amount of time a proving pre-check can take for a sector. If the check times out the sector will be skipped
+	//
+	// WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+	// test challenge took longer than this timeout
+	// WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
+	// blocked (e.g. in case of disconnected NFS mount)
+	SingleCheckTimeout Duration
+
+	// Maximum amount of time a proving pre-check can take for an entire partition. If the check times out, sectors in
+	// the partition which didn't get checked on time will be skipped
+	//
+	// WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+	// test challenge took longer than this timeout
+	// WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
+	// blocked or slow
+	PartitionCheckTimeout Duration
+
 	// Disable Window PoSt computation on the lotus-miner process even if no window PoSt workers are present.
 	//
 	// WARNING: If no windowPoSt workers are connected, window PoSt WILL FAIL resulting in faulty sectors which will need
@@ -276,11 +293,9 @@ type ProvingConfig struct {
 	// A single partition may contain up to 2349 32GiB sectors, or 2300 64GiB sectors.
 	//
 	// The maximum number of sectors which can be proven in a single PoSt message is 25000 in network version 16, which
-	// means that a single message can prove at most 10 partinions
+	// means that a single message can prove at most 10 partitions
 	//
-	// In some cases when submitting PoSt messages which are recovering sectors, the default network limit may still be
-	// too high to fit in the block gas limit; In those cases it may be necessary to set this value to something lower
-	// than 10; Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+	// Note that setting this value lower may result in less efficient gas use - more messages will be sent,
 	// to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
 	//
 	// Setting this value above the network limit has no effect
@@ -294,6 +309,16 @@ type ProvingConfig struct {
 	// Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed,
 	// resulting in more total gas use (but each message will have lower gas limit)
 	MaxPartitionsPerRecoveryMessage int
+
+	// Enable single partition per PoSt Message for partitions containing recovery sectors
+	//
+	// In cases when submitting PoSt messages which contain recovering sectors, the default network limit may still be
+	// too high to fit in the block gas limit. In those cases, it becomes useful to only house the single partition
+	// with recovering sectors in the post message
+	//
+	// Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+	// to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
+	SingleRecoveringPartitionPerPostMessage bool
 }
 
 type SealingConfig struct {
@@ -319,6 +344,24 @@ type SealingConfig struct {
 	// Upper bound on how many sectors can be sealing+upgrading at the same time when upgrading CC sectors with deals (0 = MaxSealingSectorsForDeals)
 	MaxUpgradingSectors uint64
 
+	// When set to a non-zero value, minimum number of epochs until sector expiration required for sectors to be considered
+	// for upgrades (0 = DealMinDuration = 180 days = 518400 epochs)
+	//
+	// Note that if all deals waiting in the input queue have lifetimes longer than this value, upgrade sectors will be
+	// required to have expiration of at least the soonest-ending deal
+	MinUpgradeSectorExpiration uint64
+
+	// When set to a non-zero value, minimum number of epochs until sector expiration above which upgrade candidates will
+	// be selected based on lowest initial pledge.
+	//
+	// Target sector expiration is calculated by looking at the input deal queue, sorting it by deal expiration, and
+	// selecting N deals from the queue up to sector size. The target expiration will be Nth deal end epoch, or in case
+	// where there weren't enough deals to fill a sector, DealMaxDuration (540 days = 1555200 epochs)
+	//
+	// Setting this to a high value (for example to maximum deal duration - 1555200) will disable selection based on
+	// initial pledge - upgrade sectors will always be chosen based on longest expiration
+	MinTargetUpgradeSectorExpiration uint64
+
 	// CommittedCapacitySectorLifetime is the duration a Committed Capacity (CC) sector will
 	// live before it must be extended or converted into sector containing deals before it is
 	// terminated. Value must be between 180-540 days inclusive
@@ -391,7 +434,7 @@ type SealingConfig struct {
 type SealerConfig struct {
 	ParallelFetchLimit int
 
-	// Local worker config
+	AllowSectorDownload      bool
 	AllowAddPiece            bool
 	AllowPreCommit1          bool
 	AllowPreCommit2          bool
@@ -426,7 +469,7 @@ type SealerConfig struct {
 	// ResourceFiltering instructs the system which resource filtering strategy
 	// to use when evaluating tasks against this worker. An empty value defaults
 	// to "hardware".
-	ResourceFiltering sealer.ResourceFilteringStrategy
+	ResourceFiltering ResourceFilteringStrategy
 }
 
 type BatchFeeConfig struct {
@@ -520,6 +563,19 @@ type Pubsub struct {
 	DirectPeers           []string
 	IPColocationWhitelist []string
 	RemoteTracer          string
+	// Path to file that will be used to output tracer content in JSON format.
+	// If present tracer will save data to defined file.
+	// Format: file path
+	JsonTracer string
+	// Connection string for elasticsearch instance.
+	// If present tracer will save data to elasticsearch.
+	// Format: https://<username>:<password>@<elasticsearch_url>:<port>/
+	ElasticSearchTracer string
+	// Name of elasticsearch index that will be used to save tracer data.
+	// This property is used only if ElasticSearchTracer propery is set.
+	ElasticSearchIndex string
+	// Auth token that will be passed with logs to elasticsearch - used for weighted peers score.
+	TracerSourceAuth string
 }
 
 type Chainstore struct {
@@ -529,7 +585,7 @@ type Chainstore struct {
 
 type Splitstore struct {
 	// ColdStoreType specifies the type of the coldstore.
-	// It can be "universal" (default) or "discard" for discarding cold blocks.
+	// It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.
 	ColdStoreType string
 	// HotStoreType specifies the type of the hotstore.
 	// Only currently supported value is "badger".
@@ -545,21 +601,6 @@ type Splitstore struct {
 	// A value of 0 disables, while a value 1 will do full GC in every compaction.
 	// Default is 20 (about once a week).
 	HotStoreFullGCFrequency uint64
-
-	// EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning
-	// where hotstore compaction occurs every finality epochs pruning happens every 3 finalities
-	// Default is false
-	EnableColdStoreAutoPrune bool
-
-	// ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore.
-	// Only applies if auto prune is enabled.  A value of 0 disables while a value of 1 will do
-	// full GC in every prune.
-	// Default is 7 (about once every a week)
-	ColdStoreFullGCFrequency uint64
-
-	// ColdStoreRetention specifies the retention policy for data reachable from the chain, in
-	// finalities beyond the compaction boundary, default is 0, -1 retains everything
-	ColdStoreRetention int64
 }
 
 // // Full Node
@@ -590,3 +631,30 @@ type Wallet struct {
 type FeeConfig struct {
 	DefaultMaxFee types.FIL
 }
+
+type UserRaftConfig struct {
+	// EXPERIMENTAL. config to enabled node cluster with raft consensus
+	ClusterModeEnabled bool
+	// A folder to store Raft's data.
+	DataFolder string
+	// InitPeersetMultiAddr provides the list of initial cluster peers for new Raft
+	// peers (with no prior state). It is ignored when Raft was already
+	// initialized or when starting in staging mode.
+	InitPeersetMultiAddr []string
+	// LeaderTimeout specifies how long to wait for a leader before
+	// failing an operation.
+	WaitForLeaderTimeout Duration
+	// NetworkTimeout specifies how long before a Raft network
+	// operation is timed out
+	NetworkTimeout Duration
+	// CommitRetries specifies how many times we retry a failed commit until
+	// we give up.
+	CommitRetries int
+	// How long to wait between retries
+	CommitRetryDelay Duration
+	// BackupsRotate specifies the maximum number of Raft's DataFolder
+	// copies that we keep as backups (renaming) after cleanup.
+	BackupsRotate int
+	// Tracing enables propagation of contexts across binary boundaries.
+	Tracing bool
+}

node/impl/client/client.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/ipfs/go-blockservice"
@@ -52,7 +53,7 @@ import (
 	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	market8 "github.com/filecoin-project/go-state-types/builtin/v8/market"
+	markettypes "github.com/filecoin-project/go-state-types/builtin/v9/market"
 	"github.com/filecoin-project/go-state-types/dline"
 
 	"github.com/filecoin-project/lotus/api"
@@ -97,6 +98,7 @@ type API struct {
 	Imports                   dtypes.ClientImportMgr
 	StorageBlockstoreAccessor storagemarket.BlockstoreAccessor
 	RtvlBlockstoreAccessor    rm.BlockstoreAccessor
+	ApiBlockstoreAccessor     *retrievaladapter.APIBlockstoreAccessor
 
 	DataTransfer dtypes.ClientDataTransfer
 	Host         host.Host
@@ -228,12 +230,12 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
 	// stateless flow from here to the end
 	//
 
-	label, err := market8.NewLabelFromString(params.Data.Root.Encode(multibase.MustNewEncoder('u')))
+	label, err := markettypes.NewLabelFromString(params.Data.Root.Encode(multibase.MustNewEncoder('u')))
 	if err != nil {
 		return nil, xerrors.Errorf("failed to encode label: %w", err)
 	}
 
-	dealProposal := &market8.DealProposal{
+	dealProposal := &markettypes.DealProposal{
 		PieceCID:             *params.Data.PieceCid,
 		PieceSize:            params.Data.PieceSize.Padded(),
 		Client:               walletKey,
@@ -265,7 +267,7 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt
 		return nil, xerrors.Errorf("failed to sign proposal : %w", err)
 	}
 
-	dealProposalSigned := &market8.ClientDealProposal{
+	dealProposalSigned := &markettypes.ClientDealProposal{
 		Proposal:        *dealProposal,
 		ClientSignature: *dealProposalSig,
 	}
@@ -837,7 +839,7 @@ func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel dat
 		return 0, xerrors.Errorf("cannot make retrieval deal for zero bytes")
 	}
 
-	ppb := types.BigDiv(order.Total, types.NewInt(order.Size))
+	ppb := types.BigDiv(big.Sub(order.Total, order.UnsealPrice), types.NewInt(order.Size))
 
 	params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, sel, order.Piece, order.UnsealPrice)
 	if err != nil {
@@ -845,6 +847,13 @@ func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel dat
 	}
 
 	id := a.Retrieval.NextID()
+
+	if order.RemoteStore != nil {
+		if err := a.ApiBlockstoreAccessor.RegisterDealToRetrievalStore(id, *order.RemoteStore); err != nil {
+			return 0, xerrors.Errorf("registering api store: %w", err)
+		}
+	}
+
 	id, err = a.Retrieval.Retrieve(
 		ctx,
 		id,
@@ -999,6 +1008,8 @@ func (a *API) outputCAR(ctx context.Context, ds format.DAGService, bs bstore.Blo
 		roots[i] = dag.root
 	}
 
+	var lk sync.Mutex
+
 	return dest.doWrite(func(w io.Writer) error {
 
 		if err := car.WriteHeader(&car.CarHeader{
@@ -1011,13 +1022,29 @@ func (a *API) outputCAR(ctx context.Context, ds format.DAGService, bs bstore.Blo
 		cs := cid.NewSet()
 
 		for _, dagSpec := range dags {
+			dagSpec := dagSpec
+
 			if err := utils.TraverseDag(
 				ctx,
 				ds,
 				root,
 				dagSpec.selector,
+				func(node format.Node) error {
+					// if we're exporting merkle proofs for this dag, export all nodes read by the traversal
+					if dagSpec.exportAll {
+						lk.Lock()
+						defer lk.Unlock()
+						if cs.Visit(node.Cid()) {
+							err := util.LdWrite(w, node.Cid().Bytes(), node.RawData())
+							if err != nil {
+								return xerrors.Errorf("writing block data: %w", err)
+							}
+						}
+					}
+					return nil
+				},
 				func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
-					if r == traversal.VisitReason_SelectionMatch {
+					if !dagSpec.exportAll && r == traversal.VisitReason_SelectionMatch {
 						var c cid.Cid
 						if p.LastBlock.Link == nil {
 							c = root
@@ -1082,8 +1109,9 @@ func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, ds format.DAGServi
 }
 
 type dagSpec struct {
-	root     cid.Cid
-	selector ipld.Node
+	root      cid.Cid
+	selector  ipld.Node
+	exportAll bool
 }
 
 func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds format.DAGService, car bool) ([]dagSpec, error) {
@@ -1098,6 +1126,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma
 
 	out := make([]dagSpec, len(dsp))
 	for i, spec := range dsp {
+		out[i].exportAll = spec.ExportMerkleProof
 
 		if spec.DataSelector == nil {
 			return nil, xerrors.Errorf("invalid DagSpec at position %d: `DataSelector` can not be nil", i)
@@ -1131,6 +1160,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma
 			ds,
 			root,
 			rsn,
+			nil,
 			func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
 				if r == traversal.VisitReason_SelectionMatch {
 					if !car && p.LastBlock.Path.String() != p.Path.String() {

node/impl/common/common.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"context"
+	"time"
 
 	"github.com/gbrlsnchs/jwt/v3"
 	"github.com/google/uuid"
@@ -26,6 +27,8 @@ type CommonAPI struct {
 	Alerting     *alerting.Alerting
 	APISecret    *dtypes.APIAlg
 	ShutdownChan dtypes.ShutdownChan
+
+	Start dtypes.NodeStartTime
 }
 
 type jwtPayload struct {
@@ -91,3 +94,7 @@ func (a *CommonAPI) Session(ctx context.Context) (uuid.UUID, error) {
 func (a *CommonAPI) Closing(ctx context.Context) (<-chan struct{}, error) {
 	return make(chan struct{}), nil // relies on jsonrpc closing
 }
+
+func (a *CommonAPI) StartTime(context.Context) (time.Time, error) {
+	return time.Time(a.Start), nil
+}

node/impl/full.go

@@ -34,6 +34,7 @@ type FullNodeAPI struct {
 	full.MsigAPI
 	full.WalletAPI
 	full.SyncAPI
+	full.RaftAPI
 
 	DS          dtypes.MetadataDS
 	NetworkName dtypes.NetworkName
@@ -117,4 +118,12 @@ func (n *FullNodeAPI) NodeStatus(ctx context.Context, inclChainStatus bool) (sta
 	return status, nil
 }
 
+func (n *FullNodeAPI) RaftState(ctx context.Context) (*api.RaftStateData, error) {
+	return n.RaftAPI.GetRaftState(ctx)
+}
+
+func (n *FullNodeAPI) RaftLeader(ctx context.Context) (peer.ID, error) {
+	return n.RaftAPI.Leader(ctx)
+}
+
 var _ api.FullNode = &FullNodeAPI{}

node/impl/full/gas.go

@@ -290,6 +290,10 @@ func gasEstimateGasLimit(
 	if err != nil {
 		return -1, xerrors.Errorf("CallWithGas failed: %w", err)
 	}
+	if res.MsgRct.ExitCode == exitcode.SysErrOutOfGas {
+		return -1, &api.ErrOutOfGas{}
+	}
+
 	if res.MsgRct.ExitCode != exitcode.Ok {
 		return -1, xerrors.Errorf("message execution failed: exit %s, reason: %s", res.MsgRct.ExitCode, res.Error)
 	}
@@ -356,7 +360,7 @@ func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Messag
 	if msg.GasLimit == 0 {
 		gasLimit, err := m.GasEstimateGasLimit(ctx, msg, types.EmptyTSK)
 		if err != nil {
-			return nil, xerrors.Errorf("estimating gas used: %w", err)
+			return nil, err
 		}
 		msg.GasLimit = int64(float64(gasLimit) * m.Mpool.GetConfig().GasLimitOverestimation)
 	}

node/impl/full/mpool.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
@@ -43,7 +44,9 @@ type MpoolAPI struct {
 	WalletAPI
 	GasAPI
 
-	MessageSigner *messagesigner.MessageSigner
+	RaftAPI
+
+	MessageSigner messagesigner.MsgSigner
 
 	PushLocks *dtypes.MpoolLocker
 }
@@ -130,7 +133,7 @@ func (a *MpoolAPI) MpoolClear(ctx context.Context, local bool) error {
 }
 
 func (m *MpoolModule) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
-	return m.Mpool.Push(ctx, smsg)
+	return m.Mpool.Push(ctx, smsg, true)
 }
 
 func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
@@ -142,8 +145,29 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 	msg = &cp
 	inMsg := *msg
 
-	// Check if this uuid has already been processed
-	if spec != nil {
+	// Redirect to leader if current node is not leader. A single non raft based node is always the leader
+	if !a.RaftAPI.IsLeader(ctx) {
+		var signedMsg types.SignedMessage
+		redirected, err := a.RaftAPI.RedirectToLeader(ctx, "MpoolPushMessage", api.MpoolMessageWhole{Msg: msg, Spec: spec}, &signedMsg)
+		if err != nil {
+			return nil, err
+		}
+		// It's possible that the current node became the leader between the check and the redirect
+		// In that case, continue with rest of execution and only return signedMsg if something was redirected
+		if redirected {
+			return &signedMsg, nil
+		}
+	}
+
+	// Generate spec and uuid if not available in the message
+	if spec == nil {
+		spec = &api.MessageSendSpec{
+			MsgUuid: uuid.New(),
+		}
+	} else if (spec.MsgUuid == uuid.UUID{}) {
+		spec.MsgUuid = uuid.New()
+	} else {
+		// Check if this uuid has already been processed. Ignore if uuid is not populated
 		signedMessage, err := a.MessageSigner.GetSignedMessage(ctx, spec.MsgUuid)
 		if err == nil {
 			log.Warnf("Message already processed. cid=%s", signedMessage.Cid())
@@ -195,7 +219,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 	}
 
 	// Sign and push the message
-	signedMsg, err := a.MessageSigner.SignMessage(ctx, msg, func(smsg *types.SignedMessage) error {
+	signedMsg, err := a.MessageSigner.SignMessage(ctx, msg, spec, func(smsg *types.SignedMessage) error {
 		if _, err := a.MpoolModuleAPI.MpoolPush(ctx, smsg); err != nil {
 			return xerrors.Errorf("mpool push: failed to push message: %w", err)
 		}
@@ -206,11 +230,9 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 	}
 
 	// Store uuid->signed message in datastore
-	if spec != nil {
-		err = a.MessageSigner.StoreSignedMessage(ctx, spec.MsgUuid, signedMsg)
-		if err != nil {
-			return nil, err
-		}
+	err = a.MessageSigner.StoreSignedMessage(ctx, spec.MsgUuid, signedMsg)
+	if err != nil {
+		return nil, err
 	}
 
 	return signedMsg, nil
@@ -219,7 +241,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 func (a *MpoolAPI) MpoolBatchPush(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) {
 	var messageCids []cid.Cid
 	for _, smsg := range smsgs {
-		smsgCid, err := a.Mpool.Push(ctx, smsg)
+		smsgCid, err := a.Mpool.Push(ctx, smsg, true)
 		if err != nil {
 			return messageCids, err
 		}

node/impl/full/multisig.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/big"
 	multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
 
@@ -29,7 +30,7 @@ func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (mul
 	if err != nil {
 		return nil, err
 	}
-	av, err := actors.VersionForNetwork(nver)
+	av, err := actorstypes.VersionForNetwork(nver)
 	if err != nil {
 		return nil, err
 	}

node/impl/full/raft.go

@@ -0,0 +1,50 @@
+package full
+
+import (
+	"context"
+
+	"github.com/libp2p/go-libp2p/core/peer"
+	"go.uber.org/fx"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/messagesigner"
+)
+
+type RaftAPI struct {
+	fx.In
+
+	MessageSigner *messagesigner.MessageSignerConsensus `optional:"true"`
+}
+
+func (r *RaftAPI) GetRaftState(ctx context.Context) (*api.RaftStateData, error) {
+	if r.MessageSigner == nil {
+		return nil, xerrors.Errorf("raft consensus not enabled. Please check your configuration")
+	}
+	raftState, err := r.MessageSigner.GetRaftState(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &api.RaftStateData{NonceMap: raftState.NonceMap, MsgUuids: raftState.MsgUuids}, nil
+}
+
+func (r *RaftAPI) Leader(ctx context.Context) (peer.ID, error) {
+	if r.MessageSigner == nil {
+		return "", xerrors.Errorf("raft consensus not enabled. Please check your configuration")
+	}
+	return r.MessageSigner.Leader(ctx)
+}
+
+func (r *RaftAPI) IsLeader(ctx context.Context) bool {
+	if r.MessageSigner == nil {
+		return true
+	}
+	return r.MessageSigner.IsLeader(ctx)
+}
+
+func (r *RaftAPI) RedirectToLeader(ctx context.Context, method string, arg interface{}, ret interface{}) (bool, error) {
+	if r.MessageSigner == nil {
+		return false, xerrors.Errorf("raft consensus not enabled. Please check your configuration")
+	}
+	return r.MessageSigner.RedirectToLeader(ctx, method, arg, ret)
+}

node/impl/full/state.go

@@ -16,8 +16,10 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/big"
-	minertypes "github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	minertypes "github.com/filecoin-project/go-state-types/builtin/v9/miner"
+	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/cbor"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
@@ -29,6 +31,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/datacap"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
@@ -176,6 +179,9 @@ func (m *StateModule) StateMinerInfo(ctx context.Context, actor address.Address,
 		SectorSize:                 info.SectorSize,
 		WindowPoStPartitionSectors: info.WindowPoStPartitionSectors,
 		ConsensusFaultElapsed:      info.ConsensusFaultElapsed,
+		Beneficiary:                info.Beneficiary,
+		BeneficiaryTerm:            &info.BeneficiaryTerm,
+		PendingBeneficiaryTerm:     info.PendingBeneficiaryTerm,
 	}
 
 	if info.PendingWorkerKey != nil {
@@ -478,7 +484,12 @@ func (m *StateModule) StateLookupID(ctx context.Context, addr address.Address, t
 		return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err)
 	}
 
-	return m.StateManager.LookupID(ctx, addr, ts)
+	ret, err := m.StateManager.LookupID(ctx, addr, ts)
+	if err != nil && xerrors.Is(err, types.ErrActorNotFound) {
+		return address.Undef, &api.ErrActorNotFound{}
+	}
+
+	return ret, err
 }
 
 func (a *StateAPI) StateLookupRobustAddress(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
@@ -760,6 +771,135 @@ func (m *StateModule) StateMarketStorageDeal(ctx context.Context, dealId abi.Dea
 	return stmgr.GetStorageDeal(ctx, m.StateManager, dealId, ts)
 }
 
+func (a *StateAPI) StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifreg.Allocation, error) {
+	ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+
+	st, err := a.StateManager.GetMarketState(ctx, ts)
+	if err != nil {
+		return nil, err
+	}
+
+	allocationId, err := st.GetAllocationIdForPendingDeal(dealId)
+	if err != nil {
+		return nil, err
+	}
+	if allocationId == verifregtypes.NoAllocationID {
+		return nil, nil
+	}
+
+	dealState, err := a.StateMarketStorageDeal(ctx, dealId, tsk)
+	if err != nil {
+		return nil, err
+	}
+
+	return a.StateGetAllocation(ctx, dealState.Proposal.Client, allocationId, tsk)
+}
+
+func (a *StateAPI) StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifreg.AllocationId, tsk types.TipSetKey) (*verifreg.Allocation, error) {
+	idAddr, err := a.StateLookupID(ctx, clientAddr, tsk)
+	if err != nil {
+		return nil, err
+	}
+
+	ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+
+	st, err := a.StateManager.GetVerifregState(ctx, ts)
+	if err != nil {
+		return nil, err
+	}
+
+	allocation, found, err := st.GetAllocation(idAddr, allocationId)
+	if err != nil {
+		return nil, xerrors.Errorf("getting allocation: %w", err)
+	}
+	if !found {
+		return nil, nil
+	}
+
+	return allocation, nil
+}
+
+func (a *StateAPI) StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifreg.AllocationId]verifreg.Allocation, error) {
+	idAddr, err := a.StateLookupID(ctx, clientAddr, tsk)
+	if err != nil {
+		return nil, err
+	}
+
+	ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+
+	st, err := a.StateManager.GetVerifregState(ctx, ts)
+	if err != nil {
+		return nil, xerrors.Errorf("loading verifreg state: %w", err)
+	}
+
+	allocations, err := st.GetAllocations(idAddr)
+	if err != nil {
+		return nil, xerrors.Errorf("getting allocations: %w", err)
+	}
+
+	return allocations, nil
+}
+
+func (a *StateAPI) StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifreg.ClaimId, tsk types.TipSetKey) (*verifreg.Claim, error) {
+	idAddr, err := a.StateLookupID(ctx, providerAddr, tsk)
+	if err != nil {
+		return nil, err
+	}
+
+	ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+
+	st, err := a.StateManager.GetVerifregState(ctx, ts)
+	if err != nil {
+		return nil, err
+	}
+
+	claim, found, err := st.GetClaim(idAddr, claimId)
+	if err != nil {
+		return nil, xerrors.Errorf("getting claim: %w", err)
+	}
+	if !found {
+		return nil, nil
+	}
+
+	return claim, nil
+}
+
+func (a *StateAPI) StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifreg.ClaimId]verifreg.Claim, error) {
+	idAddr, err := a.StateLookupID(ctx, providerAddr, tsk)
+	if err != nil {
+		return nil, err
+	}
+
+	ts, err := a.Chain.GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+
+	st, err := a.StateManager.GetVerifregState(ctx, ts)
+	if err != nil {
+		return nil, xerrors.Errorf("loading verifreg state: %w", err)
+	}
+
+	claims, err := st.GetClaims(idAddr)
+	if err != nil {
+		return nil, xerrors.Errorf("getting claims: %w", err)
+	}
+
+	return claims, nil
+}
+
 func (a *StateAPI) StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) {
 	nv, err := a.StateNetworkVersion(ctx, tsk)
 	if err != nil {
@@ -906,6 +1046,7 @@ func (a *StateAPI) StateSectorPreCommitInfo(ctx context.Context, maddr address.A
 	return pci, err
 }
 
+// Returns nil, nil if sector is not found
 func (m *StateModule) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) {
 	ts, err := m.Chain.GetTipSetFromKey(ctx, tsk)
 	if err != nil {
@@ -953,7 +1094,7 @@ func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatc
 		_, err := a.StateLookupID(ctx, match.To, tsk)
 
 		// if the recipient doesn't exist at the start point, we're not gonna find any matches
-		if xerrors.Is(err, types.ErrActorNotFound) {
+		if xerrors.Is(err, &api.ErrActorNotFound{}) {
 			return nil, nil
 		}
 
@@ -964,7 +1105,7 @@ func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatc
 		_, err := a.StateLookupID(ctx, match.From, tsk)
 
 		// if the sender doesn't exist at the start point, we're not gonna find any matches
-		if xerrors.Is(err, types.ErrActorNotFound) {
+		if xerrors.Is(err, &api.ErrActorNotFound{}) {
 			return nil, nil
 		}
 
@@ -1185,16 +1326,20 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr
 	store := a.Chain.ActorStore(ctx)
 
 	var sectorWeight abi.StoragePower
-	if act, err := state.GetActor(market.Address); err != nil {
-		return types.EmptyInt, xerrors.Errorf("loading market actor %s: %w", maddr, err)
-	} else if s, err := market.Load(store, act); err != nil {
-		return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err)
-	} else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil {
-		return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err)
+	if a.StateManager.GetNetworkVersion(ctx, ts.Height()) <= network.Version16 {
+		if act, err := state.GetActor(market.Address); err != nil {
+			return types.EmptyInt, xerrors.Errorf("loading market actor %s: %w", maddr, err)
+		} else if s, err := market.Load(store, act); err != nil {
+			return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err)
+		} else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil {
+			return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err)
+		} else {
+			// NB: not exactly accurate, but should always lead us to *over* estimate, not under
+			duration := pci.Expiration - ts.Height()
+			sectorWeight = builtin.QAPowerForWeight(ssize, duration, w, vw)
+		}
 	} else {
-		// NB: not exactly accurate, but should always lead us to *over* estimate, not under
-		duration := pci.Expiration - ts.Height()
-		sectorWeight = builtin.QAPowerForWeight(ssize, duration, w, vw)
+		sectorWeight = minertypes.QAPowerMax(ssize)
 	}
 
 	var powerSmoothed builtin.FilterEstimate
@@ -1386,26 +1531,56 @@ func (a *StateAPI) StateVerifierStatus(ctx context.Context, addr address.Address
 // Returns zero if there is no entry in the data cap table for the
 // address.
 func (m *StateModule) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
-	act, err := m.StateGetActor(ctx, verifreg.Address, tsk)
-	if err != nil {
-		return nil, err
-	}
-
 	aid, err := m.StateLookupID(ctx, addr, tsk)
 	if err != nil {
 		log.Warnf("lookup failure %v", err)
 		return nil, err
 	}
 
-	vrs, err := verifreg.Load(m.StateManager.ChainStore().ActorStore(ctx), act)
+	nv, err := m.StateNetworkVersion(ctx, tsk)
 	if err != nil {
-		return nil, xerrors.Errorf("failed to load verified registry state: %w", err)
+		return nil, err
 	}
 
-	verified, dcap, err := vrs.VerifiedClientDataCap(aid)
+	av, err := actorstypes.VersionForNetwork(nv)
 	if err != nil {
-		return nil, xerrors.Errorf("looking up verified client: %w", err)
+		return nil, err
 	}
+
+	var dcap abi.StoragePower
+	var verified bool
+	if av <= 8 {
+		act, err := m.StateGetActor(ctx, verifreg.Address, tsk)
+		if err != nil {
+			return nil, err
+		}
+
+		vrs, err := verifreg.Load(m.StateManager.ChainStore().ActorStore(ctx), act)
+		if err != nil {
+			return nil, xerrors.Errorf("failed to load verified registry state: %w", err)
+		}
+
+		verified, dcap, err = vrs.VerifiedClientDataCap(aid)
+		if err != nil {
+			return nil, xerrors.Errorf("looking up verified client: %w", err)
+		}
+	} else {
+		act, err := m.StateGetActor(ctx, datacap.Address, tsk)
+		if err != nil {
+			return nil, err
+		}
+
+		dcs, err := datacap.Load(m.StateManager.ChainStore().ActorStore(ctx), act)
+		if err != nil {
+			return nil, xerrors.Errorf("failed to load datacap actor state: %w", err)
+		}
+
+		verified, dcap, err = dcs.VerifiedClientDataCap(aid)
+		if err != nil {
+			return nil, xerrors.Errorf("looking up verified client: %w", err)
+		}
+	}
+
 	if !verified {
 		return nil, nil
 	}
@@ -1536,7 +1711,7 @@ func (m *StateModule) StateNetworkVersion(ctx context.Context, tsk types.TipSetK
 }
 
 func (a *StateAPI) StateActorCodeCIDs(ctx context.Context, nv network.Version) (map[string]cid.Cid, error) {
-	actorVersion, err := actors.VersionForNetwork(nv)
+	actorVersion, err := actorstypes.VersionForNetwork(nv)
 	if err != nil {
 		return nil, xerrors.Errorf("invalid network version %d: %w", nv, err)
 	}
@@ -1550,7 +1725,7 @@ func (a *StateAPI) StateActorCodeCIDs(ctx context.Context, nv network.Version) (
 }
 
 func (a *StateAPI) StateActorManifestCID(ctx context.Context, nv network.Version) (cid.Cid, error) {
-	actorVersion, err := actors.VersionForNetwork(nv)
+	actorVersion, err := actorstypes.VersionForNetwork(nv)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("invalid network version")
 	}
@@ -1623,6 +1798,8 @@ func (a *StateAPI) StateGetNetworkParams(ctx context.Context) (*api.NetworkParam
 			UpgradeHyperdriveHeight:  build.UpgradeHyperdriveHeight,
 			UpgradeChocolateHeight:   build.UpgradeChocolateHeight,
 			UpgradeOhSnapHeight:      build.UpgradeOhSnapHeight,
+			UpgradeSkyrHeight:        build.UpgradeSkyrHeight,
+			UpgradeSharkHeight:       build.UpgradeSharkHeight,
 		},
 	}, nil
 }

node/impl/full/sync.go

@@ -2,6 +2,7 @@ package full
 
 import (
 	"context"
+	"os"
 	"sync/atomic"
 
 	"github.com/ipfs/go-cid"
@@ -56,7 +57,7 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro
 		return xerrors.Errorf("loading parent block: %w", err)
 	}
 
-	if a.SlashFilter != nil {
+	if a.SlashFilter != nil && os.Getenv("LOTUS_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" {
 		if err := a.SlashFilter.MinedBlock(ctx, blk.Header, parent.Height); err != nil {
 			log.Errorf("<!!> SLASH FILTER ERROR: %s", err)
 			return xerrors.Errorf("<!!> SLASH FILTER ERROR: %w", err)

node/impl/storminer.go

@@ -35,7 +35,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	builtintypes "github.com/filecoin-project/go-state-types/builtin"
-	minertypes "github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	minertypes "github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
@@ -182,16 +182,20 @@ func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) (abi.SectorID, erro
 		return abi.SectorID{}, err
 	}
 
+	return sm.waitSectorStarted(ctx, sr.ID)
+}
+
+func (sm *StorageMinerAPI) waitSectorStarted(ctx context.Context, si abi.SectorID) (abi.SectorID, error) {
 	// wait for the sector to enter the Packing state
 	// TODO: instead of polling implement some pubsub-type thing in storagefsm
 	for {
-		info, err := sm.Miner.SectorsStatus(ctx, sr.ID.Number, false)
+		info, err := sm.Miner.SectorsStatus(ctx, si.Number, false)
 		if err != nil {
 			return abi.SectorID{}, xerrors.Errorf("getting pledged sector info: %w", err)
 		}
 
 		if info.State != api.SectorState(sealing.UndefinedSectorState) {
-			return sr.ID, nil
+			return si, nil
 		}
 
 		select {
@@ -448,6 +452,15 @@ func (sm *StorageMinerAPI) SectorNumFree(ctx context.Context, name string) error
 	return sm.Miner.NumFree(ctx, name)
 }
 
+func (sm *StorageMinerAPI) SectorReceive(ctx context.Context, meta api.RemoteSectorMeta) error {
+	if err := sm.Miner.Receive(ctx, meta); err != nil {
+		return err
+	}
+
+	_, err := sm.waitSectorStarted(ctx, meta.Sector)
+	return err
+}
+
 func (sm *StorageMinerAPI) ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]minertypes.SubmitWindowedPoStParams, error) {
 	var ts *types.TipSet
 	var err error
@@ -1282,20 +1295,17 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error
 	return backup(ctx, sm.DS, fpath)
 }
 
-func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) {
-	var rg storiface.RGetter
-	if expensive {
-		rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, bool, error) {
-			si, err := sm.Miner.SectorsStatus(ctx, id.Number, false)
-			if err != nil {
-				return cid.Undef, false, err
-			}
-			if si.CommR == nil {
-				return cid.Undef, false, xerrors.Errorf("commr is nil")
-			}
-
-			return *si.CommR, si.ReplicaUpdateMessage != nil, nil
+func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef) (map[abi.SectorNumber]string, error) {
+	rg := func(ctx context.Context, id abi.SectorID) (cid.Cid, bool, error) {
+		si, err := sm.Miner.SectorsStatus(ctx, id.Number, false)
+		if err != nil {
+			return cid.Undef, false, err
 		}
+		if si.CommR == nil {
+			return cid.Undef, false, xerrors.Errorf("commr is nil")
+		}
+
+		return *si.CommR, si.ReplicaUpdateMessage != nil, nil
 	}
 
 	bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors, rg)
@@ -1349,6 +1359,14 @@ func (sm *StorageMinerAPI) RuntimeSubsystems(context.Context) (res api.MinerSubs
 }
 
 func (sm *StorageMinerAPI) ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) {
+	return sm.withdrawBalance(ctx, amount, true)
+}
+
+func (sm *StorageMinerAPI) BeneficiaryWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) {
+	return sm.withdrawBalance(ctx, amount, false)
+}
+
+func (sm *StorageMinerAPI) withdrawBalance(ctx context.Context, amount abi.TokenAmount, fromOwner bool) (cid.Cid, error) {
 	available, err := sm.Full.StateMinerAvailableBalance(ctx, sm.Miner.Address(), types.EmptyTSK)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("Error getting miner balance: %w", err)
@@ -1374,9 +1392,16 @@ func (sm *StorageMinerAPI) ActorWithdrawBalance(ctx context.Context, amount abi.
 		return cid.Undef, xerrors.Errorf("Error getting miner's owner address: %w", err)
 	}
 
+	var sender address.Address
+	if fromOwner {
+		sender = mi.Owner
+	} else {
+		sender = mi.Beneficiary
+	}
+
 	smsg, err := sm.Full.MpoolPushMessage(ctx, &types.Message{
 		To:     sm.Miner.Address(),
-		From:   mi.Owner,
+		From:   sender,
 		Value:  types.NewInt(0),
 		Method: builtintypes.MethodsMiner.WithdrawBalance,
 		Params: params,

node/modules/blockstore.go

@@ -84,11 +84,9 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked
 		cfg := &splitstore.Config{
 			MarkSetType:              cfg.Splitstore.MarkSetType,
 			DiscardColdBlocks:        cfg.Splitstore.ColdStoreType == "discard",
+			UniversalColdBlocks:      cfg.Splitstore.ColdStoreType == "universal",
 			HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention,
 			HotStoreFullGCFrequency:  cfg.Splitstore.HotStoreFullGCFrequency,
-			EnableColdStoreAutoPrune: cfg.Splitstore.EnableColdStoreAutoPrune,
-			ColdStoreFullGCFrequency: cfg.Splitstore.ColdStoreFullGCFrequency,
-			ColdStoreRetention:       cfg.Splitstore.ColdStoreRetention,
 		}
 		ss, err := splitstore.Open(path, ds, hot, cold, cfg)
 		if err != nil {

node/modules/client.go

@@ -202,9 +202,9 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT
 
 // RetrievalClient creates a new retrieval client attached to the client blockstore
 func RetrievalClient(forceOffChain bool) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
-	ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor retrievalmarket.BlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
+	ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor *retrievaladapter.APIBlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
 	return func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver,
-		ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor retrievalmarket.BlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
+		ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor *retrievaladapter.APIBlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) {
 		adapter := retrievaladapter.NewRetrievalClientNode(forceOffChain, payAPI, chainAPI, stateAPI)
 		network := rmnet.NewFromLibp2pHost(h)
 		ds = namespace.Wrap(ds, datastore.NewKey("/retrievals/client"))

node/modules/dtypes/api.go

@@ -1,6 +1,8 @@
 package dtypes
 
 import (
+	"time"
+
 	"github.com/gbrlsnchs/jwt/v3"
 	"github.com/multiformats/go-multiaddr"
 )
@@ -8,3 +10,5 @@ import (
 type APIAlg jwt.HMACSHA
 
 type APIEndpoint multiaddr.Multiaddr
+
+type NodeStartTime time.Time

node/modules/lp2p/host.go

@@ -7,11 +7,11 @@ import (
 	nilrouting "github.com/ipfs/go-ipfs-routing/none"
 	"github.com/libp2p/go-libp2p"
 	dht "github.com/libp2p/go-libp2p-kad-dht"
-	"github.com/libp2p/go-libp2p-peerstore/pstoremem"
 	record "github.com/libp2p/go-libp2p-record"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/peerstore"
+	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem"
 	routedhost "github.com/libp2p/go-libp2p/p2p/host/routed"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"go.uber.org/fx"

node/modules/lp2p/pubsub.go

@@ -21,6 +21,7 @@ import (
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
+	"github.com/filecoin-project/lotus/node/modules/tracer"
 )
 
 func init() {
@@ -49,6 +50,30 @@ func ScoreKeeper() *dtypes.ScoreKeeper {
 	return new(dtypes.ScoreKeeper)
 }
 
+type PeerScoreTracker interface {
+	UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
+}
+
+type peerScoreTracker struct {
+	sk *dtypes.ScoreKeeper
+	lt tracer.LotusTracer
+}
+
+func newPeerScoreTracker(lt tracer.LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
+	return &peerScoreTracker{
+		sk: sk,
+		lt: lt,
+	}
+}
+
+func (pst *peerScoreTracker) UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	if pst.lt != nil {
+		pst.lt.PeerScores(scores)
+	}
+
+	pst.sk.Update(scores)
+}
+
 type GossipIn struct {
 	fx.In
 	Mctx helpers.MetricsCtx
@@ -291,7 +316,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				OpportunisticGraftThreshold: OpportunisticGraftScoreThreshold,
 			},
 		),
-		pubsub.WithPeerScoreInspect(in.Sk.Update, 10*time.Second),
 	}
 
 	// enable Peer eXchange on bootstrappers
@@ -361,6 +385,27 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
 				100)))
 
+	var transports []tracer.TracerTransport
+	if in.Cfg.JsonTracer != "" {
+		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracer)
+		if err != nil {
+			return nil, err
+		}
+
+		transports = append(transports, jsonTransport)
+	}
+	if in.Cfg.ElasticSearchTracer != "" {
+		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
+			in.Cfg.ElasticSearchTracer,
+			in.Cfg.ElasticSearchIndex,
+		)
+		if err != nil {
+			return nil, err
+		}
+		transports = append(transports, elasticSearchTransport)
+	}
+	lt := tracer.NewLotusTracer(transports, in.Host.ID(), in.Cfg.TracerSourceAuth)
+
 	// tracer
 	if in.Cfg.RemoteTracer != "" {
 		a, err := ma.NewMultiaddr(in.Cfg.RemoteTracer)
@@ -378,12 +423,18 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 			return nil, err
 		}
 
-		trw := newTracerWrapper(tr, build.BlocksTopic(in.Nn))
+		pst := newPeerScoreTracker(lt, in.Sk)
+		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
+
 		options = append(options, pubsub.WithEventTracer(trw))
+		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	} else {
 		// still instantiate a tracer for collecting metrics
-		trw := newTracerWrapper(nil)
+		trw := newTracerWrapper(nil, lt)
 		options = append(options, pubsub.WithEventTracer(trw))
+
+		pst := newPeerScoreTracker(lt, in.Sk)
+		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	}
 
 	return pubsub.NewGossipSub(helpers.LifecycleCtx(in.Mctx, in.Lc), in.Host, options...)
@@ -394,7 +445,11 @@ func HashMsgId(m *pubsub_pb.Message) string {
 	return string(hash[:])
 }
 
-func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTracer {
+func newTracerWrapper(
+	lp2pTracer pubsub.EventTracer,
+	lotusTracer pubsub.EventTracer,
+	topics ...string,
+) pubsub.EventTracer {
 	var topicsMap map[string]struct{}
 	if len(topics) > 0 {
 		topicsMap = make(map[string]struct{})
@@ -403,12 +458,13 @@ func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTrace
 		}
 	}
 
-	return &tracerWrapper{tr: tr, topics: topicsMap}
+	return &tracerWrapper{lp2pTracer: lp2pTracer, lotusTracer: lotusTracer, topics: topicsMap}
 }
 
 type tracerWrapper struct {
-	tr     pubsub.EventTracer
-	topics map[string]struct{}
+	lp2pTracer  pubsub.EventTracer
+	lotusTracer pubsub.EventTracer
+	topics      map[string]struct{}
 }
 
 func (trw *tracerWrapper) traceMessage(topic string) bool {
@@ -426,33 +482,70 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 	switch evt.GetType() {
 	case pubsub_pb.TraceEvent_PUBLISH_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1))
-		if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) {
-			trw.tr.Trace(evt)
+		if trw.traceMessage(evt.GetPublishMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
 		}
 	case pubsub_pb.TraceEvent_DELIVER_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1))
-		if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
-			trw.tr.Trace(evt)
+		if trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
 		}
 	case pubsub_pb.TraceEvent_REJECT_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1))
+		if trw.traceMessage(evt.GetRejectMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
 	case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1))
 	case pubsub_pb.TraceEvent_JOIN:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_LEAVE:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_GRAFT:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_PRUNE:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_RECV_RPC:
 		stats.Record(context.TODO(), metrics.PubsubRecvRPC.M(1))

node/modules/mpoolnonceapi.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 
-	"github.com/filecoin-project/lotus/chain/messagesigner"
+	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/impl/full"
 )
@@ -104,4 +104,4 @@ func (a *MpoolNonceAPI) GetActor(ctx context.Context, addr address.Address, tsk
 	return act, nil
 }
 
-var _ messagesigner.MpoolNonceAPI = (*MpoolNonceAPI)(nil)
+var _ messagepool.MpoolNonceAPI = (*MpoolNonceAPI)(nil)

node/modules/rpc.go

@@ -0,0 +1,55 @@
+package modules
+
+import (
+	"context"
+
+	rpc "github.com/libp2p/go-libp2p-gorpc"
+	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/protocol"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	consensus "github.com/filecoin-project/lotus/lib/consensus/raft"
+	"github.com/filecoin-project/lotus/node/impl/full"
+)
+
+type RPCHandler struct {
+	mpoolAPI full.MpoolAPI
+	cons     *consensus.Consensus
+}
+
+func NewRPCHandler(mpoolAPI full.MpoolAPI, cons *consensus.Consensus) *RPCHandler {
+	return &RPCHandler{mpoolAPI, cons}
+}
+
+func (h *RPCHandler) MpoolPushMessage(ctx context.Context, msgWhole *api.MpoolMessageWhole, ret *types.SignedMessage) error {
+	signedMsg, err := h.mpoolAPI.MpoolPushMessage(ctx, msgWhole.Msg, msgWhole.Spec)
+	if err != nil {
+		return err
+	}
+	*ret = *signedMsg
+	return nil
+}
+
+func (h *RPCHandler) AddPeer(ctx context.Context, pid peer.ID, ret *struct{}) error {
+	return h.cons.AddPeer(ctx, pid)
+}
+
+// Add other consensus RPC calls here
+
+func NewRPCClient(host host.Host) *rpc.Client {
+	protocolID := protocol.ID("/rpc/lotus-chain/v0")
+	return rpc.NewClient(host, protocolID)
+}
+
+func NewRPCServer(ctx context.Context, host host.Host, rpcHandler *RPCHandler) error {
+
+	authF := func(pid peer.ID, svc, method string) bool {
+		return rpcHandler.cons.IsTrustedPeer(ctx, pid)
+	}
+
+	protocolID := protocol.ID("/rpc/lotus-chain/v0")
+	rpcServer := rpc.NewServer(host, protocolID, rpc.WithAuthorizeFunc(authF))
+	return rpcServer.RegisterName("Consensus", rpcHandler)
+}

node/modules/storageminer.go

@@ -37,7 +37,6 @@ import (
 	storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
 	smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
-	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-paramfetch"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -56,7 +55,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/lib/retry"
 	"github.com/filecoin-project/lotus/markets"
 	"github.com/filecoin-project/lotus/markets/dagstore"
 	"github.com/filecoin-project/lotus/markets/idxprov"
@@ -89,12 +87,7 @@ func (a *UuidWrapper) MpoolPushMessage(ctx context.Context, msg *types.Message,
 		spec = new(api.MessageSendSpec)
 	}
 	spec.MsgUuid = uuid.New()
-	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
-	initialBackoff, err := time.ParseDuration("1s")
-	if err != nil {
-		return nil, err
-	}
-	return retry.Retry(5, initialBackoff, errorsToRetry, func() (*types.SignedMessage, error) { return a.FullNode.MpoolPushMessage(ctx, msg, spec) })
+	return a.FullNode.MpoolPushMessage(ctx, msg, spec)
 }
 
 func MakeUuidWrapper(a v1api.RawFullNodeAPI) v1api.FullNode {
@@ -110,24 +103,31 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
 	return address.NewFromBytes(maddrb)
 }
 
-func GetParams(spt abi.RegisteredSealProof) error {
-	ssize, err := spt.SectorSize()
-	if err != nil {
-		return err
-	}
+func GetParams(prover bool) func(spt abi.RegisteredSealProof) error {
+	return func(spt abi.RegisteredSealProof) error {
+		ssize, err := spt.SectorSize()
+		if err != nil {
+			return err
+		}
+
+		// If built-in assets are disabled, we expect the user to have placed the right
+		// parameters in the right location on the filesystem (/var/tmp/filecoin-proof-parameters).
+		if build.DisableBuiltinAssets {
+			return nil
+		}
+
+		var provingSize uint64
+		if prover {
+			provingSize = uint64(ssize)
+		}
+
+		// TODO: We should fetch the params for the actual proof type, not just based on the size.
+		if err := paramfetch.GetParams(context.TODO(), build.ParametersJSON(), build.SrsJSON(), provingSize); err != nil {
+			return xerrors.Errorf("fetching proof parameters: %w", err)
+		}
 
-	// If built-in assets are disabled, we expect the user to have placed the right
-	// parameters in the right location on the filesystem (/var/tmp/filecoin-proof-parameters).
-	if build.DisableBuiltinAssets {
 		return nil
 	}
-
-	// TODO: We should fetch the params for the actual proof type, not just based on the size.
-	if err := paramfetch.GetParams(context.TODO(), build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil {
-		return xerrors.Errorf("fetching proof parameters: %w", err)
-	}
-
-	return nil
 }
 
 func MinerAddress(ds dtypes.MetadataDS) (dtypes.MinerAddress, error) {
@@ -233,7 +233,7 @@ func PreflightChecks(mctx helpers.MetricsCtx, lc fx.Lifecycle, api v1api.FullNod
 			return xerrors.New("key for worker not found in local wallet")
 		}
 
-		log.Infof("starting up miner %s, worker addr %s", maddr, workerKey)
+		log.Infof("starting up miner %s, worker addr %s", address.Address(maddr), workerKey)
 		return nil
 	}})
 
@@ -787,17 +787,17 @@ func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls paths.LocalStorag
 	return paths.NewLocal(ctx, ls, si, urls)
 }
 
-func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc sealer.Config) *paths.Remote {
+func RemoteStorage(lstor *paths.Local, si paths.SectorIndex, sa sealer.StorageAuth, sc config.SealerConfig) *paths.Remote {
 	return paths.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit, &paths.DefaultPartialFileHandler{})
 }
 
-func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc sealer.Config, ds dtypes.MetadataDS) (*sealer.Manager, error) {
+func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *paths.Local, stor paths.Store, ls paths.LocalStorage, si paths.SectorIndex, sc config.SealerConfig, pc config.ProvingConfig, ds dtypes.MetadataDS) (*sealer.Manager, error) {
 	ctx := helpers.LifecycleCtx(mctx, lc)
 
 	wsts := statestore.New(namespace.Wrap(ds, WorkerCallsPrefix))
 	smsts := statestore.New(namespace.Wrap(ds, ManagerWorkPrefix))
 
-	sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, wsts, smsts)
+	sst, err := sealer.New(ctx, lstor, stor, ls, si, sc, pc, wsts, smsts)
 	if err != nil {
 		return nil, err
 	}
@@ -983,17 +983,19 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 	return func(cfg sealiface.Config) (err error) {
 		err = mutateSealingCfg(r, func(c config.SealingConfiger) {
 			newCfg := config.SealingConfig{
-				MaxWaitDealsSectors:             cfg.MaxWaitDealsSectors,
-				MaxSealingSectors:               cfg.MaxSealingSectors,
-				MaxSealingSectorsForDeals:       cfg.MaxSealingSectorsForDeals,
-				PreferNewSectorsForDeals:        cfg.PreferNewSectorsForDeals,
-				MaxUpgradingSectors:             cfg.MaxUpgradingSectors,
-				CommittedCapacitySectorLifetime: config.Duration(cfg.CommittedCapacitySectorLifetime),
-				WaitDealsDelay:                  config.Duration(cfg.WaitDealsDelay),
-				MakeNewSectorForDeals:           cfg.MakeNewSectorForDeals,
-				MakeCCSectorsAvailable:          cfg.MakeCCSectorsAvailable,
-				AlwaysKeepUnsealedCopy:          cfg.AlwaysKeepUnsealedCopy,
-				FinalizeEarly:                   cfg.FinalizeEarly,
+				MaxWaitDealsSectors:              cfg.MaxWaitDealsSectors,
+				MaxSealingSectors:                cfg.MaxSealingSectors,
+				MaxSealingSectorsForDeals:        cfg.MaxSealingSectorsForDeals,
+				PreferNewSectorsForDeals:         cfg.PreferNewSectorsForDeals,
+				MaxUpgradingSectors:              cfg.MaxUpgradingSectors,
+				CommittedCapacitySectorLifetime:  config.Duration(cfg.CommittedCapacitySectorLifetime),
+				WaitDealsDelay:                   config.Duration(cfg.WaitDealsDelay),
+				MakeNewSectorForDeals:            cfg.MakeNewSectorForDeals,
+				MinUpgradeSectorExpiration:       cfg.MinUpgradeSectorExpiration,
+				MinTargetUpgradeSectorExpiration: cfg.MinTargetUpgradeSectorExpiration,
+				MakeCCSectorsAvailable:           cfg.MakeCCSectorsAvailable,
+				AlwaysKeepUnsealedCopy:           cfg.AlwaysKeepUnsealedCopy,
+				FinalizeEarly:                    cfg.FinalizeEarly,
 
 				CollateralFromMinerBalance: cfg.CollateralFromMinerBalance,
 				AvailableBalanceBuffer:     types.FIL(cfg.AvailableBalanceBuffer),
@@ -1024,11 +1026,14 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 
 func ToSealingConfig(dealmakingCfg config.DealmakingConfig, sealingCfg config.SealingConfig) sealiface.Config {
 	return sealiface.Config{
-		MaxWaitDealsSectors:             sealingCfg.MaxWaitDealsSectors,
-		MaxSealingSectors:               sealingCfg.MaxSealingSectors,
-		MaxSealingSectorsForDeals:       sealingCfg.MaxSealingSectorsForDeals,
-		PreferNewSectorsForDeals:        sealingCfg.PreferNewSectorsForDeals,
-		MaxUpgradingSectors:             sealingCfg.MaxUpgradingSectors,
+		MaxWaitDealsSectors:              sealingCfg.MaxWaitDealsSectors,
+		MaxSealingSectors:                sealingCfg.MaxSealingSectors,
+		MaxSealingSectorsForDeals:        sealingCfg.MaxSealingSectorsForDeals,
+		PreferNewSectorsForDeals:         sealingCfg.PreferNewSectorsForDeals,
+		MinUpgradeSectorExpiration:       sealingCfg.MinUpgradeSectorExpiration,
+		MinTargetUpgradeSectorExpiration: sealingCfg.MinTargetUpgradeSectorExpiration,
+		MaxUpgradingSectors:              sealingCfg.MaxUpgradingSectors,
+
 		StartEpochSealingBuffer:         abi.ChainEpoch(dealmakingCfg.StartEpochSealingBuffer),
 		MakeNewSectorForDeals:           sealingCfg.MakeNewSectorForDeals,
 		CommittedCapacitySectorLifetime: time.Duration(sealingCfg.CommittedCapacitySectorLifetime),

node/modules/storageminer_idxprov.go

@@ -46,7 +46,7 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo
 			engine.WithHost(marketHost),
 			engine.WithRetrievalAddrs(marketHost.Addrs()...),
 			engine.WithEntriesCacheCapacity(cfg.EntriesCacheCapacity),
-			engine.WithEntriesChunkSize(cfg.EntriesChunkSize),
+			engine.WithChainedEntries(cfg.EntriesChunkSize),
 			engine.WithTopicName(topicName),
 			engine.WithPurgeCacheOnStart(cfg.PurgeCacheOnStart),
 		}

node/modules/storageminer_idxprov_test.go

@@ -78,8 +78,9 @@ func Test_IndexProviderTopic(t *testing.T) {
 					func() *pubsub.PubSub { return ps },
 					func() dtypes.MetadataDS { return datastore.NewMapDatastore() },
 					modules.IndexProvider(config.IndexProviderConfig{
-						Enable:    true,
-						TopicName: test.givenConfiguredTopic,
+						Enable:           true,
+						TopicName:        test.givenConfiguredTopic,
+						EntriesChunkSize: 16384,
 					}),
 				),
 				fx.Invoke(func(p provider.Interface) {}),

node/modules/tracer/elasticsearch_transport.go

@@ -0,0 +1,97 @@
+package tracer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+)
+
+const (
+	ElasticSearchDefaultIndex = "lotus-pubsub"
+)
+
+func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
+	conUrl, err := url.Parse(connectionString)
+
+	if err != nil {
+		return nil, err
+	}
+
+	username := conUrl.User.Username()
+	password, _ := conUrl.User.Password()
+	cfg := elasticsearch.Config{
+		Addresses: []string{
+			conUrl.Scheme + "://" + conUrl.Host,
+		},
+		Username: username,
+		Password: password,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var esIndex string
+	if elasticsearchIndex != "" {
+		esIndex = elasticsearchIndex
+	} else {
+		esIndex = ElasticSearchDefaultIndex
+	}
+
+	return &elasticSearchTransport{
+		cl:      es,
+		esIndex: esIndex,
+	}, nil
+}
+
+type elasticSearchTransport struct {
+	cl      *elasticsearch.Client
+	esIndex string
+}
+
+func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
+	var e interface{}
+
+	if evt.lotusTraceEvent != nil {
+		e = *evt.lotusTraceEvent
+	} else if evt.pubsubTraceEvent != nil {
+		e = *evt.pubsubTraceEvent
+	} else {
+		return nil
+	}
+
+	jsonEvt, err := json.Marshal(e)
+	if err != nil {
+		return fmt.Errorf("error while marshaling event: %s", err)
+	}
+
+	req := esapi.IndexRequest{
+		Index:   est.esIndex,
+		Body:    strings.NewReader(string(jsonEvt)),
+		Refresh: "true",
+	}
+
+	// Perform the request with the client.
+	res, err := req.Do(context.Background(), est.cl)
+	if err != nil {
+		return err
+	}
+
+	err = res.Body.Close()
+	if err != nil {
+		return err
+	}
+
+	if res.IsError() {
+		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
+	}
+
+	return nil
+}

node/modules/tracer/json_transport.go

@@ -0,0 +1,41 @@
+package tracer
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+type jsonTracerTransport struct {
+	out *os.File
+}
+
+func NewJsonTracerTransport(file string) (TracerTransport, error) {
+	out, err := os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+	if err != nil {
+		return nil, err
+	}
+
+	return &jsonTracerTransport{
+		out: out,
+	}, nil
+}
+
+func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {
+	var e interface{}
+	if evt.lotusTraceEvent != nil {
+		e = *evt.lotusTraceEvent
+	} else if evt.pubsubTraceEvent != nil {
+		e = *evt.pubsubTraceEvent
+	} else {
+		return nil
+	}
+
+	jsonEvt, err := json.Marshal(e)
+	if err != nil {
+		return fmt.Errorf("error while marshaling event: %s", err)
+	}
+
+	_, err = jtt.out.WriteString(string(jsonEvt) + "\n")
+	return err
+}

node/modules/tracer/tracer.go

@@ -0,0 +1,120 @@
+package tracer
+
+import (
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+	"github.com/libp2p/go-libp2p/core/peer"
+)
+
+var log = logging.Logger("lotus-tracer")
+
+func NewLotusTracer(tt []TracerTransport, pid peer.ID, sourceAuth string) LotusTracer {
+	return &lotusTracer{
+		tt:  tt,
+		pid: pid,
+		sa:  sourceAuth,
+	}
+}
+
+type lotusTracer struct {
+	tt  []TracerTransport
+	pid peer.ID
+	sa  string
+}
+
+const (
+	TraceEventPeerScores pubsub_pb.TraceEvent_Type = 100
+)
+
+type LotusTraceEvent struct {
+	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
+	PeerID     string                    `json:"peerID,omitempty"`
+	Timestamp  *int64                    `json:"timestamp,omitempty"`
+	PeerScore  TraceEventPeerScore       `json:"peerScore,omitempty"`
+	SourceAuth string                    `json:"sourceAuth,omitempty"`
+}
+
+type TopicScore struct {
+	Topic                    string        `json:"topic"`
+	TimeInMesh               time.Duration `json:"timeInMesh"`
+	FirstMessageDeliveries   float64       `json:"firstMessageDeliveries"`
+	MeshMessageDeliveries    float64       `json:"meshMessageDeliveries"`
+	InvalidMessageDeliveries float64       `json:"invalidMessageDeliveries"`
+}
+
+type TraceEventPeerScore struct {
+	PeerID             string       `json:"peerID"`
+	Score              float64      `json:"score"`
+	AppSpecificScore   float64      `json:"appSpecificScore"`
+	IPColocationFactor float64      `json:"ipColocationFactor"`
+	BehaviourPenalty   float64      `json:"behaviourPenalty"`
+	Topics             []TopicScore `json:"topics"`
+}
+
+type LotusTracer interface {
+	Trace(evt *pubsub_pb.TraceEvent)
+	TraceLotusEvent(evt *LotusTraceEvent)
+
+	PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
+}
+
+func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	now := time.Now().UnixNano()
+	for pid, score := range scores {
+		var topics []TopicScore
+		for topic, snapshot := range score.Topics {
+			topics = append(topics, TopicScore{
+				Topic:                    topic,
+				TimeInMesh:               snapshot.TimeInMesh,
+				FirstMessageDeliveries:   snapshot.FirstMessageDeliveries,
+				MeshMessageDeliveries:    snapshot.MeshMessageDeliveries,
+				InvalidMessageDeliveries: snapshot.InvalidMessageDeliveries,
+			})
+		}
+
+		evt := &LotusTraceEvent{
+			Type:       *TraceEventPeerScores.Enum(),
+			PeerID:     lt.pid.Pretty(),
+			Timestamp:  &now,
+			SourceAuth: lt.sa,
+			PeerScore: TraceEventPeerScore{
+				PeerID:             pid.Pretty(),
+				Score:              score.Score,
+				AppSpecificScore:   score.AppSpecificScore,
+				IPColocationFactor: score.IPColocationFactor,
+				BehaviourPenalty:   score.BehaviourPenalty,
+				Topics:             topics,
+			},
+		}
+
+		lt.TraceLotusEvent(evt)
+	}
+}
+
+func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
+	for _, t := range lt.tt {
+		err := t.Transport(TracerTransportEvent{
+			lotusTraceEvent:  evt,
+			pubsubTraceEvent: nil,
+		})
+		if err != nil {
+			log.Errorf("error while transporting peer scores: %s", err)
+		}
+	}
+
+}
+
+func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
+	for _, t := range lt.tt {
+		err := t.Transport(TracerTransportEvent{
+			lotusTraceEvent:  nil,
+			pubsubTraceEvent: evt,
+		})
+		if err != nil {
+			log.Errorf("error while transporting trace event: %s", err)
+		}
+	}
+}

node/modules/tracer/tracer_test.go

@@ -0,0 +1,109 @@
+package tracer
+
+import (
+	"testing"
+	"time"
+
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/stretchr/testify/require"
+)
+
+type testTracerTransport struct {
+	t           *testing.T
+	executeTest func(t *testing.T, evt TracerTransportEvent)
+}
+
+const peerIDA peer.ID = "12D3KooWAbSVMgRejb6ECg6fRTkCPGCfu8396msZVryu8ivcz44G"
+
+func NewTestTraceTransport(t *testing.T, executeTest func(t *testing.T, evt TracerTransportEvent)) TracerTransport {
+	return &testTracerTransport{
+		t:           t,
+		executeTest: executeTest,
+	}
+}
+
+func (ttt *testTracerTransport) Transport(evt TracerTransportEvent) error {
+	ttt.executeTest(ttt.t, evt)
+	return nil
+}
+
+func TestTracer_PeerScores(t *testing.T) {
+
+	testTransport := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, peerIDA.Pretty(), evt.lotusTraceEvent.PeerID)
+		require.Equal(t, "source-auth-token-test", evt.lotusTraceEvent.SourceAuth)
+		require.Equal(t, float64(32), evt.lotusTraceEvent.PeerScore.Score)
+
+		n := time.Now().UnixNano()
+		require.LessOrEqual(t, *evt.lotusTraceEvent.Timestamp, n)
+
+		require.Equal(t, peerIDA.Pretty(), evt.lotusTraceEvent.PeerScore.PeerID)
+		require.Equal(t, 1, len(evt.lotusTraceEvent.PeerScore.Topics))
+
+		topic := evt.lotusTraceEvent.PeerScore.Topics[0]
+		require.Equal(t, "topicA", topic.Topic)
+		require.Equal(t, float64(100), topic.FirstMessageDeliveries)
+	})
+
+	lt := NewLotusTracer(
+		[]TracerTransport{testTransport},
+		peerIDA,
+		"source-auth-token-test",
+	)
+
+	topics := make(map[string]*pubsub.TopicScoreSnapshot)
+	topics["topicA"] = &pubsub.TopicScoreSnapshot{
+		FirstMessageDeliveries: float64(100),
+	}
+
+	m := make(map[peer.ID]*pubsub.PeerScoreSnapshot)
+	m[peerIDA] = &pubsub.PeerScoreSnapshot{
+		Score:  float64(32),
+		Topics: topics,
+	}
+
+	lt.PeerScores(m)
+}
+
+func TestTracer_PubSubTrace(t *testing.T) {
+	n := time.Now().Unix()
+
+	testTransport := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, []byte(peerIDA), evt.pubsubTraceEvent.PeerID)
+		require.Equal(t, &n, evt.pubsubTraceEvent.Timestamp)
+	})
+
+	lt := NewLotusTracer(
+		[]TracerTransport{testTransport},
+		"pid",
+		"source-auth",
+	)
+
+	lt.Trace(&pubsub_pb.TraceEvent{
+		PeerID:    []byte(peerIDA),
+		Timestamp: &n,
+	})
+
+}
+
+func TestTracer_MultipleTransports(t *testing.T) {
+	testTransportA := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, []byte(peerIDA), evt.pubsubTraceEvent.PeerID)
+	})
+
+	testTransportB := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, []byte(peerIDA), evt.pubsubTraceEvent.PeerID)
+	})
+
+	executeTest := NewLotusTracer(
+		[]TracerTransport{testTransportA, testTransportB},
+		"pid",
+		"source-auth",
+	)
+
+	executeTest.Trace(&pubsub_pb.TraceEvent{
+		PeerID: []byte(peerIDA),
+	})
+}

node/modules/tracer/transport.go

@@ -0,0 +1,12 @@
+package tracer
+
+import pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+
+type TracerTransport interface {
+	Transport(evt TracerTransportEvent) error
+}
+
+type TracerTransportEvent struct {
+	lotusTraceEvent  *LotusTraceEvent
+	pubsubTraceEvent *pubsub_pb.TraceEvent
+}

node/options.go

@@ -93,6 +93,12 @@ func From(typ interface{}) interface{} {
 	}).Interface()
 }
 
+func FromVal[T any](v T) func() T {
+	return func() T {
+		return v
+	}
+}
+
 // from go-ipfs
 // as casts input constructor to a given interface (if a value is given, it
 // wraps it into a constructor).

node/repo/fsrepo.go

@@ -25,8 +25,8 @@ import (
 	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/storage/paths"
 	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 const (
@@ -572,26 +572,26 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
 	return nil
 }
 
-func (fsr *fsLockedRepo) GetStorage() (paths.StorageConfig, error) {
+func (fsr *fsLockedRepo) GetStorage() (storiface.StorageConfig, error) {
 	fsr.storageLk.Lock()
 	defer fsr.storageLk.Unlock()
 
 	return fsr.getStorage(nil)
 }
 
-func (fsr *fsLockedRepo) getStorage(def *paths.StorageConfig) (paths.StorageConfig, error) {
+func (fsr *fsLockedRepo) getStorage(def *storiface.StorageConfig) (storiface.StorageConfig, error) {
 	c, err := config.StorageFromFile(fsr.join(fsStorageConfig), def)
 	if err != nil {
-		return paths.StorageConfig{}, err
+		return storiface.StorageConfig{}, err
 	}
 	return *c, nil
 }
 
-func (fsr *fsLockedRepo) SetStorage(c func(*paths.StorageConfig)) error {
+func (fsr *fsLockedRepo) SetStorage(c func(*storiface.StorageConfig)) error {
 	fsr.storageLk.Lock()
 	defer fsr.storageLk.Unlock()
 
-	sc, err := fsr.getStorage(&paths.StorageConfig{})
+	sc, err := fsr.getStorage(&storiface.StorageConfig{})
 	if err != nil {
 		return xerrors.Errorf("get storage: %w", err)
 	}

node/repo/interface.go

@@ -9,8 +9,8 @@ import (
 
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/storage/paths"
 	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 // BlockstoreDomain represents the domain of a blockstore.
@@ -73,8 +73,8 @@ type LockedRepo interface {
 	Config() (interface{}, error)
 	SetConfig(func(interface{})) error
 
-	GetStorage() (paths.StorageConfig, error)
-	SetStorage(func(*paths.StorageConfig)) error
+	GetStorage() (storiface.StorageConfig, error)
+	SetStorage(func(*storiface.StorageConfig)) error
 	Stat(path string) (fsutil.FsStat, error)
 	DiskUsage(path string) (int64, error)
 

node/repo/memrepo.go

@@ -18,7 +18,6 @@ import (
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/storage/paths"
 	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
 	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
@@ -37,7 +36,7 @@ type MemRepo struct {
 	keystore   map[string]types.KeyInfo
 	blockstore blockstore.Blockstore
 
-	sc      *paths.StorageConfig
+	sc      *storiface.StorageConfig
 	tempDir string
 
 	// holds the current config value
@@ -59,13 +58,13 @@ func (lmem *lockedMemRepo) RepoType() RepoType {
 	return lmem.t
 }
 
-func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) {
+func (lmem *lockedMemRepo) GetStorage() (storiface.StorageConfig, error) {
 	if err := lmem.checkToken(); err != nil {
-		return paths.StorageConfig{}, err
+		return storiface.StorageConfig{}, err
 	}
 
 	if lmem.mem.sc == nil {
-		lmem.mem.sc = &paths.StorageConfig{StoragePaths: []paths.LocalPath{
+		lmem.mem.sc = &storiface.StorageConfig{StoragePaths: []storiface.LocalPath{
 			{Path: lmem.Path()},
 		}}
 	}
@@ -73,7 +72,7 @@ func (lmem *lockedMemRepo) GetStorage() (paths.StorageConfig, error) {
 	return *lmem.mem.sc, nil
 }
 
-func (lmem *lockedMemRepo) SetStorage(c func(*paths.StorageConfig)) error {
+func (lmem *lockedMemRepo) SetStorage(c func(*storiface.StorageConfig)) error {
 	if err := lmem.checkToken(); err != nil {
 		return err
 	}
@@ -126,14 +125,14 @@ func (lmem *lockedMemRepo) Path() string {
 }
 
 func (lmem *lockedMemRepo) initSectorStore(t string) {
-	if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), paths.StorageConfig{
-		StoragePaths: []paths.LocalPath{
+	if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), storiface.StorageConfig{
+		StoragePaths: []storiface.LocalPath{
 			{Path: t},
 		}}); err != nil {
 		panic(err)
 	}
 
-	b, err := json.MarshalIndent(&paths.LocalStorageMeta{
+	b, err := json.MarshalIndent(&storiface.LocalStorageMeta{
 		ID:       storiface.ID(uuid.New().String()),
 		Weight:   10,
 		CanSeal:  true,

node/rpc.go

@@ -3,13 +3,17 @@ package node
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"runtime"
 	"strconv"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
+	"github.com/gorilla/websocket"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/multiformats/go-multiaddr"
@@ -23,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/api/v1api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/metrics/proxy"
@@ -47,7 +52,8 @@ func ServeRPC(h http.Handler, id string, addr multiaddr.Multiaddr) (StopFunc, er
 
 	// Instantiate the server and start listening.
 	srv := &http.Server{
-		Handler: h,
+		Handler:           h,
+		ReadHeaderTimeout: 30 * time.Second,
 		BaseContext: func(listener net.Listener) context.Context {
 			ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, id))
 			return ctx
@@ -69,7 +75,7 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
 	m := mux.NewRouter()
 
 	serveRpc := func(path string, hnd interface{}) {
-		rpcServer := jsonrpc.NewServer(opts...)
+		rpcServer := jsonrpc.NewServer(append(opts, jsonrpc.WithServerErrors(api.RPCErrors))...)
 		rpcServer.Register("Filecoin", hnd)
 		rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover")
 
@@ -92,6 +98,7 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
 	// Import handler
 	handleImportFunc := handleImport(a.(*impl.FullNodeAPI))
 	handleExportFunc := handleExport(a.(*impl.FullNodeAPI))
+	handleRemoteStoreFunc := handleRemoteStore(a.(*impl.FullNodeAPI))
 	if permissioned {
 		importAH := &auth.Handler{
 			Verify: a.AuthVerify,
@@ -104,9 +111,16 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server
 			Next:   handleExportFunc,
 		}
 		m.Handle("/rest/v0/export", exportAH)
+
+		storeAH := &auth.Handler{
+			Verify: a.AuthVerify,
+			Next:   handleRemoteStoreFunc,
+		}
+		m.Handle("/rest/v0/store/{uuid}", storeAH)
 	} else {
 		m.HandleFunc("/rest/v0/import", handleImportFunc)
 		m.HandleFunc("/rest/v0/export", handleExportFunc)
+		m.HandleFunc("/rest/v0/store/{uuid}", handleRemoteStoreFunc)
 	}
 
 	// debugging
@@ -130,7 +144,7 @@ func MinerHandler(a api.StorageMiner, permissioned bool) (http.Handler, error) {
 	}
 
 	readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder()
-	rpcServer := jsonrpc.NewServer(readerServerOpt)
+	rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt)
 	rpcServer.Register("Filecoin", mapi)
 	rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover")
 
@@ -256,3 +270,34 @@ func handleFractionOpt(name string, setter func(int)) http.HandlerFunc {
 		setter(fr)
 	}
 }
+
+var upgrader = websocket.Upgrader{
+	CheckOrigin: func(r *http.Request) bool {
+		return true
+	},
+}
+
+func handleRemoteStore(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		id, err := uuid.Parse(vars["uuid"])
+		if err != nil {
+			http.Error(w, fmt.Sprintf("parse uuid: %s", err), http.StatusBadRequest)
+			return
+		}
+
+		c, err := upgrader.Upgrade(w, r, nil)
+		if err != nil {
+			log.Error(err)
+			w.WriteHeader(500)
+			return
+		}
+
+		nstore := bstore.NewNetworkStoreWS(c)
+		if err := a.ApiBlockstoreAccessor.RegisterApiStore(id, nstore); err != nil {
+			log.Errorw("registering api bstore", "error", err)
+			_ = c.Close()
+			return
+		}
+	}
+}