remove debug

chore: releases: merge releases branch to master

.circleci/config.yml

@@ -879,12 +879,6 @@ workflows:
             - build
           suite: itest-pending_deal_allocation
           target: "./itests/pending_deal_allocation_test.go"
-      - test:
-          name: test-itest-raft_messagesigner
-          requires:
-            - build
-          suite: itest-raft_messagesigner
-          target: "./itests/raft_messagesigner_test.go"
       - test:
           name: test-itest-remove_verifreg_datacap
           requires:

CHANGELOG.md

@@ -196,8 +196,6 @@ This patch release allows for up to 10k messages per block. Additionally, it int
 ## Improvements
 - fix: exchange: allow up to 10k messages per block ([filecoin-project/lotus#11506](https://github.com/filecoin-project/lotus/pull/11506))
 
->>>>>>> releases
-
 # v 1.25.0 / 2023-11-22
 
 This is a highly recommended feature release of Lotus. This optional release supports the Filecoin network version 21 upgrade, codenamed Watermelon 🍉, in addition to the numerous improvements and enhancements for node operators, ETH RPC-providers and storage providers.

api/api_full.go

@@ -879,9 +879,6 @@ type FullNode interface {
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
-
-	RaftState(ctx context.Context) (*RaftStateData, error) //perm:read
-	RaftLeader(ctx context.Context) (peer.ID, error)       //perm:read
 }
 
 // reverse interface to the client, called after EthSubscribe

api/docgen/docgen.go

@@ -355,10 +355,6 @@ func init() {
 	addExample(map[string]bitfield.BitField{
 		"": bitfield.NewFromSet([]uint64{5, 6, 7, 10}),
 	})
-	addExample(&api.RaftStateData{
-		NonceMap: make(map[address.Address]uint64),
-		MsgUuids: make(map[uuid.UUID]*types.SignedMessage),
-	})
 
 	addExample(http.Header{
 		"Authorization": []string{"Bearer ey.."},

api/mocks/mock_full.go

@@ -2919,36 +2919,6 @@ func (mr *MockFullNodeMockRecorder) PaychVoucherSubmit(arg0, arg1, arg2, arg3, a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychVoucherSubmit", reflect.TypeOf((*MockFullNode)(nil).PaychVoucherSubmit), arg0, arg1, arg2, arg3, arg4)
 }
 
-// RaftLeader mocks base method.
-func (m *MockFullNode) RaftLeader(arg0 context.Context) (peer.ID, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "RaftLeader", arg0)
-	ret0, _ := ret[0].(peer.ID)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// RaftLeader indicates an expected call of RaftLeader.
-func (mr *MockFullNodeMockRecorder) RaftLeader(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RaftLeader", reflect.TypeOf((*MockFullNode)(nil).RaftLeader), arg0)
-}
-
-// RaftState mocks base method.
-func (m *MockFullNode) RaftState(arg0 context.Context) (*api.RaftStateData, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "RaftState", arg0)
-	ret0, _ := ret[0].(*api.RaftStateData)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// RaftState indicates an expected call of RaftState.
-func (mr *MockFullNodeMockRecorder) RaftState(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RaftState", reflect.TypeOf((*MockFullNode)(nil).RaftState), arg0)
-}
-
 // Session mocks base method.
 func (m *MockFullNode) Session(arg0 context.Context) (uuid.UUID, error) {
 	m.ctrl.T.Helper()

api/proxy_gen.go

@@ -453,10 +453,6 @@ type FullNodeMethods struct {
 
 	PaychVoucherSubmit func(p0 context.Context, p1 address.Address, p2 *paych.SignedVoucher, p3 []byte, p4 []byte) (cid.Cid, error) `perm:"sign"`
 
-	RaftLeader func(p0 context.Context) (peer.ID, error) `perm:"read"`
-
-	RaftState func(p0 context.Context) (*RaftStateData, error) `perm:"read"`
-
 	StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `perm:"read"`
 
 	StateActorCodeCIDs func(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) `perm:"read"`
@@ -3236,28 +3232,6 @@ func (s *FullNodeStub) PaychVoucherSubmit(p0 context.Context, p1 address.Address
 	return *new(cid.Cid), ErrNotSupported
 }
 
-func (s *FullNodeStruct) RaftLeader(p0 context.Context) (peer.ID, error) {
-	if s.Internal.RaftLeader == nil {
-		return *new(peer.ID), ErrNotSupported
-	}
-	return s.Internal.RaftLeader(p0)
-}
-
-func (s *FullNodeStub) RaftLeader(p0 context.Context) (peer.ID, error) {
-	return *new(peer.ID), ErrNotSupported
-}
-
-func (s *FullNodeStruct) RaftState(p0 context.Context) (*RaftStateData, error) {
-	if s.Internal.RaftState == nil {
-		return nil, ErrNotSupported
-	}
-	return s.Internal.RaftState(p0)
-}
-
-func (s *FullNodeStub) RaftState(p0 context.Context) (*RaftStateData, error) {
-	return nil, ErrNotSupported
-}
-
 func (s *FullNodeStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
 	if s.Internal.StateAccountKey == nil {
 		return *new(address.Address), ErrNotSupported

api/types.go

@@ -69,11 +69,6 @@ type MessageSendSpec struct {
 	MaximizeFeeCap bool
 }
 
-type MpoolMessageWhole struct {
-	Msg  *types.Message
-	Spec *MessageSendSpec
-}
-
 // GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
 type GraphSyncDataTransfer struct {
 	// GraphSync request id for this transfer
@@ -351,64 +346,6 @@ type ForkUpgradeParams struct {
 	UpgradeWatermelonHeight  abi.ChainEpoch
 }
 
-type NonceMapType map[address.Address]uint64
-type MsgUuidMapType map[uuid.UUID]*types.SignedMessage
-
-type RaftStateData struct {
-	NonceMap NonceMapType
-	MsgUuids MsgUuidMapType
-}
-
-func (n *NonceMapType) MarshalJSON() ([]byte, error) {
-	marshalled := make(map[string]uint64)
-	for a, n := range *n {
-		marshalled[a.String()] = n
-	}
-	return json.Marshal(marshalled)
-}
-
-func (n *NonceMapType) UnmarshalJSON(b []byte) error {
-	unmarshalled := make(map[string]uint64)
-	err := json.Unmarshal(b, &unmarshalled)
-	if err != nil {
-		return err
-	}
-	*n = make(map[address.Address]uint64)
-	for saddr, nonce := range unmarshalled {
-		a, err := address.NewFromString(saddr)
-		if err != nil {
-			return err
-		}
-		(*n)[a] = nonce
-	}
-	return nil
-}
-
-func (m *MsgUuidMapType) MarshalJSON() ([]byte, error) {
-	marshalled := make(map[string]*types.SignedMessage)
-	for u, msg := range *m {
-		marshalled[u.String()] = msg
-	}
-	return json.Marshal(marshalled)
-}
-
-func (m *MsgUuidMapType) UnmarshalJSON(b []byte) error {
-	unmarshalled := make(map[string]*types.SignedMessage)
-	err := json.Unmarshal(b, &unmarshalled)
-	if err != nil {
-		return err
-	}
-	*m = make(map[uuid.UUID]*types.SignedMessage)
-	for suid, msg := range unmarshalled {
-		u, err := uuid.Parse(suid)
-		if err != nil {
-			return err
-		}
-		(*m)[u] = msg
-	}
-	return nil
-}
-
 // ChainExportConfig holds configuration for chain ranged exports.
 type ChainExportConfig struct {
 	WriteBufferSize   int

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.25.2"
+const BuildVersion = "1.25.3-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/messagesigner/messagesigner_consensus.go

@@ -1,98 +0,0 @@
-package messagesigner
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagepool"
-	"github.com/filecoin-project/lotus/chain/types"
-	consensus "github.com/filecoin-project/lotus/lib/consensus/raft"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-type MessageSignerConsensus struct {
-	MsgSigner
-	Consensus *consensus.Consensus
-}
-
-func NewMessageSignerConsensus(
-	wallet api.Wallet,
-	mpool messagepool.MpoolNonceAPI,
-	ds dtypes.MetadataDS,
-	consensus *consensus.Consensus) *MessageSignerConsensus {
-
-	ds = namespace.Wrap(ds, datastore.NewKey("/message-signer-consensus/"))
-	return &MessageSignerConsensus{
-		MsgSigner: &MessageSigner{
-			wallet: wallet,
-			mpool:  mpool,
-			ds:     ds,
-		},
-		Consensus: consensus,
-	}
-}
-
-func (ms *MessageSignerConsensus) IsLeader(ctx context.Context) bool {
-	return ms.Consensus.IsLeader(ctx)
-}
-
-func (ms *MessageSignerConsensus) RedirectToLeader(ctx context.Context, method string, arg interface{}, ret interface{}) (bool, error) {
-	ok, err := ms.Consensus.RedirectToLeader(method, arg, ret.(*types.SignedMessage))
-	if err != nil {
-		return ok, err
-	}
-	return ok, nil
-}
-
-func (ms *MessageSignerConsensus) SignMessage(
-	ctx context.Context,
-	msg *types.Message,
-	spec *api.MessageSendSpec,
-	cb func(*types.SignedMessage) error) (*types.SignedMessage, error) {
-
-	signedMsg, err := ms.MsgSigner.SignMessage(ctx, msg, spec, cb)
-	if err != nil {
-		return nil, err
-	}
-
-	op := &consensus.ConsensusOp{
-		Nonce:     signedMsg.Message.Nonce,
-		Uuid:      spec.MsgUuid,
-		Addr:      signedMsg.Message.From,
-		SignedMsg: signedMsg,
-	}
-	err = ms.Consensus.Commit(ctx, op)
-	if err != nil {
-		return nil, err
-	}
-
-	return signedMsg, nil
-}
-
-func (ms *MessageSignerConsensus) GetSignedMessage(ctx context.Context, uuid uuid.UUID) (*types.SignedMessage, error) {
-	cstate, err := ms.Consensus.State(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	//cstate := state.(Consensus.RaftState)
-	msg, ok := cstate.MsgUuids[uuid]
-	if !ok {
-		return nil, xerrors.Errorf("Msg with Uuid %s not available", uuid)
-	}
-	return msg, nil
-}
-
-func (ms *MessageSignerConsensus) GetRaftState(ctx context.Context) (*consensus.RaftState, error) {
-	return ms.Consensus.State(ctx)
-}
-
-func (ms *MessageSignerConsensus) Leader(ctx context.Context) (peer.ID, error) {
-	return ms.Consensus.Leader(ctx)
-}

chain/types/ethtypes/eth_types.go

@@ -229,13 +229,25 @@ type EthCall struct {
 }
 
 func (c *EthCall) UnmarshalJSON(b []byte) error {
-	type TempEthCall EthCall
-	var params TempEthCall
+	type EthCallRaw EthCall // Avoid a recursive call.
+	type EthCallDecode struct {
+		// The field should be "input" by spec, but many clients use "data" so we support
+		// both, but prefer "input".
+		Input *EthBytes `json:"input"`
+		EthCallRaw
+	}
 
+	var params EthCallDecode
 	if err := json.Unmarshal(b, &params); err != nil {
 		return err
 	}
-	*c = EthCall(params)
+
+	// If input is specified, prefer it.
+	if params.Input != nil {
+		params.Data = *params.Input
+	}
+
+	*c = EthCall(params.EthCallRaw)
 	return nil
 }
 

chain/types/ethtypes/eth_types_test.go

@@ -194,11 +194,40 @@ func TestMaskedIDInF4(t *testing.T) {
 }
 
 func TestUnmarshalEthCall(t *testing.T) {
-	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":""}`
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFF"}`
 
 	var c EthCall
 	err := c.UnmarshalJSON([]byte(data))
 	require.Nil(t, err)
+	require.EqualValues(t, []byte{0xff}, c.Data)
+}
+
+func TestUnmarshalEthCallInput(t *testing.T) {
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","input":"0xFF"}`
+
+	var c EthCall
+	err := c.UnmarshalJSON([]byte(data))
+	require.Nil(t, err)
+	require.EqualValues(t, []byte{0xff}, c.Data)
+}
+
+func TestUnmarshalEthCallInputAndData(t *testing.T) {
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFE","input":"0xFF"}`
+
+	var c EthCall
+	err := c.UnmarshalJSON([]byte(data))
+	require.Nil(t, err)
+	require.EqualValues(t, []byte{0xff}, c.Data)
+}
+
+func TestUnmarshalEthCallInputAndDataEmpty(t *testing.T) {
+	// Even if the input is empty, it should be used when specified.
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFE","input":""}`
+
+	var c EthCall
+	err := c.UnmarshalJSON([]byte(data))
+	require.Nil(t, err)
+	require.EqualValues(t, []byte{}, c.Data)
 }
 
 func TestUnmarshalEthBytes(t *testing.T) {

cli/util/api.go

@@ -445,7 +445,7 @@ func GetFullNodeAPIV1LotusProvider(ctx *cli.Context, ainfoCfg []string, opts ...
 	for _, head := range heads {
 		v1api, closer, err := client.NewFullNodeRPCV1(ctx.Context, head.addr, head.header, rpcOpts...)
 		if err != nil {
-			log.Warnf("Not able to establish connection to node with addr: %s", head.addr)
+			log.Warnf("Not able to establish connection to node with addr: %s, Reason: %s", head.addr, err.Error())
 			continue
 		}
 		fullNodes = append(fullNodes, v1api)

cmd/lotus-miner/init.go

@@ -120,6 +120,11 @@ var initCmd = &cli.Command{
 			Name:  "from",
 			Usage: "select which address to send actor creation message from",
 		},
+		&cli.Uint64Flag{
+			Name:  "confidence",
+			Usage: "number of block confirmations to wait for",
+			Value: build.MessageConfidence,
+		},
 	},
 	Subcommands: []*cli.Command{
 		restoreCmd,
@@ -146,6 +151,8 @@ var initCmd = &cli.Command{
 			return xerrors.Errorf("failed to parse gas-price flag: %s", err)
 		}
 
+		confidence := cctx.Uint64("confidence")
+
 		symlink := cctx.Bool("symlink-imported-sectors")
 		if symlink {
 			log.Info("will attempt to symlink to imported sectors")
@@ -265,7 +272,7 @@ var initCmd = &cli.Command{
 			}
 		}
 
-		if err := storageMinerInit(ctx, cctx, api, r, ssize, gasPrice); err != nil {
+		if err := storageMinerInit(ctx, cctx, api, r, ssize, gasPrice, confidence); err != nil {
 			log.Errorf("Failed to initialize lotus-miner: %+v", err)
 			path, err := homedir.Expand(repoPath)
 			if err != nil {
@@ -414,7 +421,7 @@ func findMarketDealID(ctx context.Context, api v1api.FullNode, deal markettypes.
 	return 0, xerrors.New("deal not found")
 }
 
-func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt) error {
+func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt, confidence uint64) error {
 	lr, err := r.Lock(repo.StorageMiner)
 	if err != nil {
 		return err
@@ -501,7 +508,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 					return xerrors.Errorf("failed to start up genesis miner: %w", err)
 				}
 
-				cerr := configureStorageMiner(ctx, api, a, peerid, gasPrice)
+				cerr := configureStorageMiner(ctx, api, a, peerid, gasPrice, confidence)
 
 				if err := m.Stop(ctx); err != nil {
 					log.Error("failed to shut down miner: ", err)
@@ -541,13 +548,13 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			}
 		}
 
-		if err := configureStorageMiner(ctx, api, a, peerid, gasPrice); err != nil {
+		if err := configureStorageMiner(ctx, api, a, peerid, gasPrice, confidence); err != nil {
 			return xerrors.Errorf("failed to configure miner: %w", err)
 		}
 
 		addr = a
 	} else {
-		a, err := createStorageMiner(ctx, api, ssize, peerid, gasPrice, cctx)
+		a, err := createStorageMiner(ctx, api, ssize, peerid, gasPrice, confidence, cctx)
 		if err != nil {
 			return xerrors.Errorf("creating miner failed: %w", err)
 		}
@@ -589,7 +596,7 @@ func makeHostKey(lr repo.LockedRepo) (crypto.PrivKey, error) {
 	return pk, nil
 }
 
-func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address.Address, peerid peer.ID, gasPrice types.BigInt) error {
+func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address.Address, peerid peer.ID, gasPrice types.BigInt, confidence uint64) error {
 	mi, err := api.StateMinerInfo(ctx, addr, types.EmptyTSK)
 	if err != nil {
 		return xerrors.Errorf("getWorkerAddr returned bad address: %w", err)
@@ -615,7 +622,7 @@ func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address
 	}
 
 	log.Info("Waiting for message: ", smsg.Cid())
-	ret, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+	ret, err := api.StateWaitMsg(ctx, smsg.Cid(), confidence, lapi.LookbackNoLimit, true)
 	if err != nil {
 		return err
 	}
@@ -627,7 +634,7 @@ func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address
 	return nil
 }
 
-func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.SectorSize, peerid peer.ID, gasPrice types.BigInt, cctx *cli.Context) (address.Address, error) {
+func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.SectorSize, peerid peer.ID, gasPrice types.BigInt, confidence uint64, cctx *cli.Context) (address.Address, error) {
 	var err error
 	var owner address.Address
 	if cctx.String("owner") != "" {
@@ -679,7 +686,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto
 		log.Infof("Initializing worker account %s, message: %s", worker, signed.Cid())
 		log.Infof("Waiting for confirmation")
 
-		mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+		mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true)
 		if err != nil {
 			return address.Undef, xerrors.Errorf("waiting for worker init: %w", err)
 		}
@@ -703,7 +710,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto
 		log.Infof("Initializing owner account %s, message: %s", worker, signed.Cid())
 		log.Infof("Waiting for confirmation")
 
-		mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+		mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true)
 		if err != nil {
 			return address.Undef, xerrors.Errorf("waiting for owner init: %w", err)
 		}
@@ -752,7 +759,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto
 	log.Infof("Pushed CreateMiner message: %s", signed.Cid())
 	log.Infof("Waiting for confirmation")
 
-	mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+	mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true)
 	if err != nil {
 		return address.Undef, xerrors.Errorf("waiting for createMiner message: %w", err)
 	}

cmd/lotus-miner/init_restore.go

@@ -80,8 +80,7 @@ var restoreCmd = &cli.Command{
 			}
 
 			log.Info("Configuring miner actor")
-
-			if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
+			if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero(), cctx.Uint64("confidence")); err != nil {
 				return err
 			}
 

cmd/lotus-miner/init_service.go

@@ -105,7 +105,7 @@ var serviceCmd = &cli.Command{
 			if es.Contains(MarketsService) {
 				log.Info("Configuring miner actor")
 
-				if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
+				if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero(), cctx.Uint64("confidence")); err != nil {
 					return err
 				}
 			}

cmd/lotus-miner/sectors.go

@@ -2290,7 +2290,7 @@ var sectorsCompactPartitionsCmd = &cli.Command{
 		if len(parts) <= 0 {
 			return fmt.Errorf("must include at least one partition to compact")
 		}
-		fmt.Printf("compacting %d paritions\n", len(parts))
+		fmt.Printf("compacting %d partitions\n", len(parts))
 
 		var makeMsgForPartitions func(partitionsBf bitfield.BitField) ([]*types.Message, error)
 		makeMsgForPartitions = func(partitionsBf bitfield.BitField) ([]*types.Message, error) {

cmd/lotus-provider/config.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"fmt"
 	"io"
@@ -14,6 +13,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/node/config"
 )
@@ -77,7 +77,7 @@ var configSetCmd = &cli.Command{
 	Action: func(cctx *cli.Context) error {
 		args := cctx.Args()
 
-		db, err := makeDB(cctx)
+		db, err := deps.MakeDB(cctx)
 		if err != nil {
 			return err
 		}
@@ -109,9 +109,8 @@ var configSetCmd = &cli.Command{
 		}
 		_ = lp
 
-		_, err = db.Exec(context.Background(),
-			`INSERT INTO harmony_config (title, config) VALUES ($1, $2) 
-			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, string(bytes))
+		err = setConfig(db, name, string(bytes))
+
 		if err != nil {
 			return fmt.Errorf("unable to save config layer: %w", err)
 		}
@@ -121,6 +120,13 @@ var configSetCmd = &cli.Command{
 	},
 }
 
+func setConfig(db *harmonydb.DB, name, config string) error {
+	_, err := db.Exec(context.Background(),
+		`INSERT INTO harmony_config (title, config) VALUES ($1, $2) 
+			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, config)
+	return err
+}
+
 var configGetCmd = &cli.Command{
 	Name:      "get",
 	Aliases:   []string{"cat", "show"},
@@ -131,13 +137,12 @@ var configGetCmd = &cli.Command{
 		if args.Len() != 1 {
 			return fmt.Errorf("want 1 layer arg, got %d", args.Len())
 		}
-		db, err := makeDB(cctx)
+		db, err := deps.MakeDB(cctx)
 		if err != nil {
 			return err
 		}
 
-		var cfg string
-		err = db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, args.First()).Scan(&cfg)
+		cfg, err := getConfig(db, args.First())
 		if err != nil {
 			return err
 		}
@@ -147,13 +152,22 @@ var configGetCmd = &cli.Command{
 	},
 }
 
+func getConfig(db *harmonydb.DB, layer string) (string, error) {
+	var cfg string
+	err := db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&cfg)
+	if err != nil {
+		return "", err
+	}
+	return cfg, nil
+}
+
 var configListCmd = &cli.Command{
 	Name:    "list",
 	Aliases: []string{"ls"},
 	Usage:   "List config layers you can get.",
 	Flags:   []cli.Flag{},
 	Action: func(cctx *cli.Context) error {
-		db, err := makeDB(cctx)
+		db, err := deps.MakeDB(cctx)
 		if err != nil {
 			return err
 		}
@@ -180,7 +194,7 @@ var configRmCmd = &cli.Command{
 		if args.Len() != 1 {
 			return errors.New("must have exactly 1 arg for the layer name")
 		}
-		db, err := makeDB(cctx)
+		db, err := deps.MakeDB(cctx)
 		if err != nil {
 			return err
 		}
@@ -209,11 +223,11 @@ var configViewCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		db, err := makeDB(cctx)
+		db, err := deps.MakeDB(cctx)
 		if err != nil {
 			return err
 		}
-		lp, err := getConfig(cctx, db)
+		lp, err := deps.GetConfig(cctx, db)
 		if err != nil {
 			return err
 		}
@@ -225,35 +239,3 @@ var configViewCmd = &cli.Command{
 		return nil
 	},
 }
-
-func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
-	lp := config.DefaultLotusProvider()
-	have := []string{}
-	layers := cctx.StringSlice("layers")
-	for _, layer := range layers {
-		text := ""
-		err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
-		if err != nil {
-			if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
-				return nil, fmt.Errorf("missing layer '%s' ", layer)
-			}
-			if layer == "base" {
-				return nil, errors.New(`lotus-provider defaults to a layer named 'base'. 
-				Either use 'migrate' command or edit a base.toml and upload it with: lotus-provider config set base.toml`)
-			}
-			return nil, fmt.Errorf("could not read layer '%s': %w", layer, err)
-		}
-		meta, err := toml.Decode(text, &lp)
-		if err != nil {
-			return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
-		}
-		for _, k := range meta.Keys() {
-			have = append(have, strings.Join(k, " "))
-		}
-	}
-	_ = have // FUTURE: verify that required fields are here.
-	// If config includes 3rd-party config, consider JSONSchema as a way that
-	// 3rd-parties can dynamically include config requirements and we can
-	// validate the config. Because of layering, we must validate @ startup.
-	return lp, nil
-}

cmd/lotus-provider/deps/deps.go

@@ -0,0 +1,282 @@
+// Package deps provides the dependencies for the lotus provider node.
+package deps
+
+import (
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/gbrlsnchs/jwt/v3"
+	ds "github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-statestore"
+
+	"github.com/filecoin-project/lotus/api"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/journal/alerting"
+	"github.com/filecoin-project/lotus/journal/fsjournal"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/lotus/provider"
+	"github.com/filecoin-project/lotus/storage/ctladdr"
+	"github.com/filecoin-project/lotus/storage/paths"
+	"github.com/filecoin-project/lotus/storage/sealer"
+	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
+)
+
+var log = logging.Logger("lotus-provider/deps")
+
+func MakeDB(cctx *cli.Context) (*harmonydb.DB, error) {
+	dbConfig := config.HarmonyDB{
+		Username: cctx.String("db-user"),
+		Password: cctx.String("db-password"),
+		Hosts:    strings.Split(cctx.String("db-host"), ","),
+		Database: cctx.String("db-name"),
+		Port:     cctx.String("db-port"),
+	}
+	return harmonydb.NewFromConfig(dbConfig)
+}
+
+type JwtPayload struct {
+	Allow []auth.Permission
+}
+
+func StorageAuth(apiKey string) (sealer.StorageAuth, error) {
+	if apiKey == "" {
+		return nil, xerrors.Errorf("no api key provided")
+	}
+
+	rawKey, err := base64.StdEncoding.DecodeString(apiKey)
+	if err != nil {
+		return nil, xerrors.Errorf("decoding api key: %w", err)
+	}
+
+	key := jwt.NewHS256(rawKey)
+
+	p := JwtPayload{
+		Allow: []auth.Permission{"admin"},
+	}
+
+	token, err := jwt.Sign(&p, key)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := http.Header{}
+	headers.Add("Authorization", "Bearer "+string(token))
+	return sealer.StorageAuth(headers), nil
+}
+
+func GetDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) {
+	var deps Deps
+	return &deps, deps.PopulateRemainingDeps(ctx, cctx, true)
+}
+
+type Deps struct {
+	Cfg        *config.LotusProviderConfig
+	DB         *harmonydb.DB
+	Full       api.FullNode
+	Verif      storiface.Verifier
+	LW         *sealer.LocalWorker
+	As         *ctladdr.AddressSelector
+	Maddrs     []dtypes.MinerAddress
+	Stor       *paths.Remote
+	Si         *paths.DBIndex
+	LocalStore *paths.Local
+	ListenAddr string
+}
+
+const (
+	FlagRepoPath = "repo-path"
+)
+
+func (deps *Deps) PopulateRemainingDeps(ctx context.Context, cctx *cli.Context, makeRepo bool) error {
+	var err error
+	if makeRepo {
+		// Open repo
+		repoPath := cctx.String(FlagRepoPath)
+		fmt.Println("repopath", repoPath)
+		r, err := repo.NewFS(repoPath)
+		if err != nil {
+			return err
+		}
+
+		ok, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !ok {
+			if err := r.Init(repo.Provider); err != nil {
+				return err
+			}
+		}
+	}
+
+	if deps.Cfg == nil {
+		deps.DB, err = MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	if deps.Cfg == nil {
+		// The config feeds into task runners & their helpers
+		deps.Cfg, err = GetConfig(cctx, deps.DB)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Debugw("config", "config", deps.Cfg)
+
+	if deps.Verif == nil {
+		deps.Verif = ffiwrapper.ProofVerifier
+	}
+
+	if deps.As == nil {
+		deps.As, err = provider.AddressSelector(&deps.Cfg.Addresses)()
+		if err != nil {
+			return err
+		}
+	}
+
+	if deps.Si == nil {
+		de, err := journal.ParseDisabledEvents(deps.Cfg.Journal.DisabledEvents)
+		if err != nil {
+			return err
+		}
+		j, err := fsjournal.OpenFSJournalPath(cctx.String("journal"), de)
+		if err != nil {
+			return err
+		}
+		go func() {
+			<-ctx.Done()
+			_ = j.Close()
+		}()
+
+		al := alerting.NewAlertingSystem(j)
+		deps.Si = paths.NewDBIndex(al, deps.DB)
+	}
+
+	if deps.Full == nil {
+		var fullCloser func()
+		cfgApiInfo := deps.Cfg.Apis.ChainApiInfo
+		if v := os.Getenv("FULLNODE_API_INFO"); v != "" {
+			cfgApiInfo = []string{v}
+		}
+		deps.Full, fullCloser, err = cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfgApiInfo)
+		if err != nil {
+			return err
+		}
+
+		go func() {
+			<-ctx.Done()
+			fullCloser()
+		}()
+	}
+
+	bls := &paths.BasicLocalStorage{
+		PathToJSON: cctx.String("storage-json"),
+	}
+
+	if deps.ListenAddr == "" {
+		listenAddr := cctx.String("listen")
+		const unspecifiedAddress = "0.0.0.0"
+		addressSlice := strings.Split(listenAddr, ":")
+		if ip := net.ParseIP(addressSlice[0]); ip != nil {
+			if ip.String() == unspecifiedAddress {
+				rip, err := deps.DB.GetRoutableIP()
+				if err != nil {
+					return err
+				}
+				deps.ListenAddr = rip + ":" + addressSlice[1]
+			}
+		}
+	}
+	if deps.LocalStore == nil {
+		deps.LocalStore, err = paths.NewLocal(ctx, bls, deps.Si, []string{"http://" + deps.ListenAddr + "/remote"})
+		if err != nil {
+			return err
+		}
+	}
+
+	sa, err := StorageAuth(deps.Cfg.Apis.StorageRPCSecret)
+	if err != nil {
+		return xerrors.Errorf(`'%w' while parsing the config toml's 
+	[Apis]
+	StorageRPCSecret=%v
+Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, err, deps.Cfg.Apis.StorageRPCSecret)
+	}
+	if deps.Stor == nil {
+		deps.Stor = paths.NewRemote(deps.LocalStore, deps.Si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{})
+	}
+	if deps.LW == nil {
+		wstates := statestore.New(dssync.MutexWrap(ds.NewMapDatastore()))
+
+		// todo localWorker isn't the abstraction layer we want to use here, we probably want to go straight to ffiwrapper
+		//  maybe with a lotus-provider specific abstraction. LocalWorker does persistent call tracking which we probably
+		//  don't need (ehh.. maybe we do, the async callback system may actually work decently well with harmonytask)
+		deps.LW = sealer.NewLocalWorker(sealer.WorkerConfig{}, deps.Stor, deps.LocalStore, deps.Si, nil, wstates)
+	}
+	if len(deps.Maddrs) == 0 {
+		for _, s := range deps.Cfg.Addresses.MinerAddresses {
+			addr, err := address.NewFromString(s)
+			if err != nil {
+				return err
+			}
+			deps.Maddrs = append(deps.Maddrs, dtypes.MinerAddress(addr))
+		}
+	}
+	fmt.Println("last line of populate")
+	return nil
+}
+
+func GetConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
+	lp := config.DefaultLotusProvider()
+	have := []string{}
+	layers := cctx.StringSlice("layers")
+	for _, layer := range layers {
+		text := ""
+		err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
+		if err != nil {
+			if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
+				return nil, fmt.Errorf("missing layer '%s' ", layer)
+			}
+			if layer == "base" {
+				return nil, errors.New(`lotus-provider defaults to a layer named 'base'. 
+				Either use 'migrate' command or edit a base.toml and upload it with: lotus-provider config set base.toml`)
+			}
+			return nil, fmt.Errorf("could not read layer '%s': %w", layer, err)
+		}
+		meta, err := toml.Decode(text, &lp)
+		if err != nil {
+			return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
+		}
+		for _, k := range meta.Keys() {
+			have = append(have, strings.Join(k, " "))
+		}
+		log.Infow("Using layer", "layer", layer, "config", lp)
+	}
+	_ = have // FUTURE: verify that required fields are here.
+	// If config includes 3rd-party config, consider JSONSchema as a way that
+	// 3rd-parties can dynamically include config requirements and we can
+	// validate the config. Because of layering, we must validate @ startup.
+	return lp, nil
+}

cmd/lotus-provider/main.go

@@ -5,16 +5,18 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
-	"runtime/debug"
+	"runtime/pprof"
 	"syscall"
 
 	"github.com/fatih/color"
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/lib/lotuslog"
 	"github.com/filecoin-project/lotus/lib/tracing"
 	"github.com/filecoin-project/lotus/node/repo"
@@ -28,8 +30,8 @@ func SetupCloseHandler() {
 	go func() {
 		<-c
 		fmt.Println("\r- Ctrl+C pressed in Terminal")
-		debug.PrintStack()
-		os.Exit(1)
+		_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		panic(1)
 	}()
 }
 
@@ -44,6 +46,7 @@ func main() {
 		stopCmd,
 		configCmd,
 		testCmd,
+		webCmd,
 		//backupCmd,
 		//lcli.WithCategory("chain", actorCmd),
 		//lcli.WithCategory("storage", sectorsCmd),
@@ -131,7 +134,7 @@ func main() {
 				Value:   "base",
 			},
 			&cli.StringFlag{
-				Name:    FlagRepoPath,
+				Name:    deps.FlagRepoPath,
 				EnvVars: []string{"LOTUS_REPO_PATH"},
 				Value:   "~/.lotusprovider",
 			},
@@ -143,8 +146,14 @@ func main() {
 		},
 		After: func(c *cli.Context) error {
 			if r := recover(); r != nil {
+				p, err := homedir.Expand(c.String(FlagMinerRepo))
+				if err != nil {
+					log.Errorw("could not expand repo path for panic report", "error", err)
+					panic(r)
+				}
+
 				// Generate report in LOTUS_PATH and re-raise panic
-				build.GeneratePanicReport(c.String("panic-reports"), c.String(FlagRepoPath), c.App.Name)
+				build.GeneratePanicReport(c.String("panic-reports"), p, c.App.Name)
 				panic(r)
 			}
 			return nil
@@ -154,7 +163,3 @@ func main() {
 	app.Metadata["repoType"] = repo.Provider
 	lcli.RunApp(app)
 }
-
-const (
-	FlagRepoPath = "repo-path"
-)

cmd/lotus-provider/proving.go

@@ -15,6 +15,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/dline"
 
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/provider"
 )
@@ -62,18 +63,18 @@ var wdPostTaskCmd = &cli.Command{
 	Action: func(cctx *cli.Context) error {
 		ctx := context.Background()
 
-		deps, err := getDeps(ctx, cctx)
+		deps, err := deps.GetDeps(ctx, cctx)
 		if err != nil {
 			return err
 		}
 
-		ts, err := deps.full.ChainHead(ctx)
+		ts, err := deps.Full.ChainHead(ctx)
 		if err != nil {
 			return xerrors.Errorf("cannot get chainhead %w", err)
 		}
 		ht := ts.Height()
 
-		addr, err := address.NewFromString(deps.cfg.Addresses.MinerAddresses[0])
+		addr, err := address.NewFromString(deps.Cfg.Addresses.MinerAddresses[0])
 		if err != nil {
 			return xerrors.Errorf("cannot get miner address %w", err)
 		}
@@ -82,7 +83,10 @@ var wdPostTaskCmd = &cli.Command{
 			return xerrors.Errorf("cannot get miner id %w", err)
 		}
 		var id int64
-		_, err = deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
+
+		retryDelay := time.Millisecond * 10
+	retryAddTask:
+		_, err = deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
 			err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id)
 			if err != nil {
 				log.Error("inserting harmony_task: ", err)
@@ -102,13 +106,18 @@ var wdPostTaskCmd = &cli.Command{
 			return true, nil
 		})
 		if err != nil {
+			if harmonydb.IsErrSerialization(err) {
+				time.Sleep(retryDelay)
+				retryDelay *= 2
+				goto retryAddTask
+			}
 			return xerrors.Errorf("writing SQL transaction: %w", err)
 		}
 		fmt.Printf("Inserted task %v. Waiting for success ", id)
 		var result sql.NullString
 		for {
 			time.Sleep(time.Second)
-			err = deps.db.QueryRow(ctx, `SELECT result FROM harmony_test WHERE task_id=$1`, id).Scan(&result)
+			err = deps.DB.QueryRow(ctx, `SELECT result FROM harmony_test WHERE task_id=$1`, id).Scan(&result)
 			if err != nil {
 				return xerrors.Errorf("reading result from harmony_test: %w", err)
 			}
@@ -157,29 +166,29 @@ It will not send any messages to the chain. Since it can compute any deadline, o
 	Action: func(cctx *cli.Context) error {
 
 		ctx := context.Background()
-		deps, err := getDeps(ctx, cctx)
+		deps, err := deps.GetDeps(ctx, cctx)
 		if err != nil {
 			return err
 		}
 
-		wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, deps.cfg.Fees, deps.cfg.Proving, deps.full, deps.verif, deps.lw, nil,
-			deps.as, deps.maddrs, deps.db, deps.stor, deps.si, deps.cfg.Subsystems.WindowPostMaxTasks)
+		wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, deps.Cfg.Fees, deps.Cfg.Proving, deps.Full, deps.Verif, deps.LW, nil,
+			deps.As, deps.Maddrs, deps.DB, deps.Stor, deps.Si, deps.Cfg.Subsystems.WindowPostMaxTasks)
 		if err != nil {
 			return err
 		}
 		_, _ = wdPoStSubmitTask, derlareRecoverTask
 
-		if len(deps.maddrs) == 0 {
+		if len(deps.Maddrs) == 0 {
 			return errors.New("no miners to compute WindowPoSt for")
 		}
-		head, err := deps.full.ChainHead(ctx)
+		head, err := deps.Full.ChainHead(ctx)
 		if err != nil {
 			return xerrors.Errorf("failed to get chain head: %w", err)
 		}
 
 		di := dline.NewInfo(head.Height(), cctx.Uint64("deadline"), 0, 0, 0, 10 /*challenge window*/, 0, 0)
 
-		for _, maddr := range deps.maddrs {
+		for _, maddr := range deps.Maddrs {
 			out, err := wdPostTask.DoPartition(ctx, head, address.Address(maddr), di, cctx.Uint64("partition"))
 			if err != nil {
 				fmt.Println("Error computing WindowPoSt for miner", maddr, err)

cmd/lotus-provider/rpc/rpc.go

@@ -1,21 +1,34 @@
+// Package rpc provides all direct access to this node.
 package rpc
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net"
 	"net/http"
+	"time"
 
+	"github.com/gbrlsnchs/jwt/v3"
 	"github.com/gorilla/mux"
+	logging "github.com/ipfs/go-log/v2"
+	"go.opencensus.io/tag"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
 
-	// logging "github.com/ipfs/go-log/v2"
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/lib/rpcenc"
+	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/metrics/proxy"
+	"github.com/filecoin-project/lotus/provider/lpweb"
+	"github.com/filecoin-project/lotus/storage/paths"
 )
 
-//var log = logging.Logger("lp/rpc")
+var log = logging.Logger("lp/rpc")
 
 func LotusProviderHandler(
 	authv func(ctx context.Context, token string) ([]auth.Permission, error),
@@ -49,3 +62,94 @@ func LotusProviderHandler(
 	}
 	return ah
 }
+
+type ProviderAPI struct {
+	*deps.Deps
+	ShutdownChan chan struct{}
+}
+
+func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
+	return api.ProviderAPIVersion0, nil
+}
+
+// Trigger shutdown
+func (p *ProviderAPI) Shutdown(context.Context) error {
+	close(p.ShutdownChan)
+	return nil
+}
+
+func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) error {
+	fh := &paths.FetchHandler{Local: dependencies.LocalStore, PfHandler: &paths.DefaultPartialFileHandler{}}
+	remoteHandler := func(w http.ResponseWriter, r *http.Request) {
+		if !auth.HasPerm(r.Context(), nil, api.PermAdmin) {
+			w.WriteHeader(401)
+			_ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"})
+			return
+		}
+
+		fh.ServeHTTP(w, r)
+	}
+	// local APIs
+	{
+		// debugging
+		mux := mux.NewRouter()
+		mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
+		mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
+	}
+
+	var authVerify func(context.Context, string) ([]auth.Permission, error)
+	{
+		privateKey, err := base64.StdEncoding.DecodeString(dependencies.Cfg.Apis.StorageRPCSecret)
+		if err != nil {
+			return xerrors.Errorf("decoding storage rpc secret: %w", err)
+		}
+		authVerify = func(ctx context.Context, token string) ([]auth.Permission, error) {
+			var payload deps.JwtPayload
+			if _, err := jwt.Verify([]byte(token), jwt.NewHS256(privateKey), &payload); err != nil {
+				return nil, xerrors.Errorf("JWT Verification failed: %w", err)
+			}
+
+			return payload.Allow, nil
+		}
+	}
+	// Serve the RPC.
+	srv := &http.Server{
+		Handler: LotusProviderHandler(
+			authVerify,
+			remoteHandler,
+			&ProviderAPI{dependencies, shutdownChan},
+			true),
+		ReadHeaderTimeout: time.Minute * 3,
+		BaseContext: func(listener net.Listener) context.Context {
+			ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker"))
+			return ctx
+		},
+		Addr: dependencies.ListenAddr,
+	}
+
+	log.Infof("Setting up RPC server at %s", dependencies.ListenAddr)
+	eg := errgroup.Group{}
+	eg.Go(srv.ListenAndServe)
+
+	if dependencies.Cfg.Subsystems.EnableWebGui {
+		web, err := lpweb.GetSrv(ctx, dependencies)
+		if err != nil {
+			return err
+		}
+
+		go func() {
+			<-ctx.Done()
+			log.Warn("Shutting down...")
+			if err := srv.Shutdown(context.TODO()); err != nil {
+				log.Errorf("shutting down RPC server failed: %s", err)
+			}
+			if err := web.Shutdown(context.Background()); err != nil {
+				log.Errorf("shutting down web server failed: %s", err)
+			}
+			log.Warn("Graceful shutdown successful")
+		}()
+		log.Infof("Setting up web server at %s", dependencies.Cfg.Subsystems.GuiAddress)
+		eg.Go(web.ListenAndServe)
+	}
+	return eg.Wait()
+}

cmd/lotus-provider/run.go

@@ -2,54 +2,24 @@ package main
 
 import (
 	"context"
-	"encoding/base64"
-	"encoding/json"
 	"fmt"
-	"net"
-	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/gbrlsnchs/jwt/v3"
-	"github.com/gorilla/mux"
-	ds "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
 	"github.com/pkg/errors"
-	"github.com/samber/lo"
 	"github.com/urfave/cli/v2"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/tag"
-	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/go-statestore"
-
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
-	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/cmd/lotus-provider/rpc"
-	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/journal/alerting"
-	"github.com/filecoin-project/lotus/journal/fsjournal"
-	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
-	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/tasks"
 	"github.com/filecoin-project/lotus/lib/ulimit"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/node/repo"
-	"github.com/filecoin-project/lotus/provider"
-	"github.com/filecoin-project/lotus/provider/lpmessage"
-	"github.com/filecoin-project/lotus/provider/lpwinning"
-	"github.com/filecoin-project/lotus/storage/ctladdr"
-	"github.com/filecoin-project/lotus/storage/paths"
-	"github.com/filecoin-project/lotus/storage/sealer"
-	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 type stackTracer interface {
@@ -144,112 +114,26 @@ var runCmd = &cli.Command{
 			}
 		}
 
-		deps, err := getDeps(ctx, cctx)
-
+		dependencies := &deps.Deps{}
+		err = dependencies.PopulateRemainingDeps(ctx, cctx, true)
 		if err != nil {
+			fmt.Println("err", err)
 			return err
 		}
-		cfg, db, full, verif, lw, as, maddrs, stor, si, localStore := deps.cfg, deps.db, deps.full, deps.verif, deps.lw, deps.as, deps.maddrs, deps.stor, deps.si, deps.localStore
+		fmt.Println("ef")
 
-		var activeTasks []harmonytask.TaskInterface
+		taskEngine, err := tasks.StartTasks(ctx, dependencies)
+		fmt.Println("gh")
 
-		sender, sendTask := lpmessage.NewSender(full, full, db)
-		activeTasks = append(activeTasks, sendTask)
-
-		///////////////////////////////////////////////////////////////////////
-		///// Task Selection
-		///////////////////////////////////////////////////////////////////////
-		{
-
-			if cfg.Subsystems.EnableWindowPost {
-				wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, cfg.Fees, cfg.Proving, full, verif, lw, sender,
-					as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks)
-				if err != nil {
-					return err
-				}
-				activeTasks = append(activeTasks, wdPostTask, wdPoStSubmitTask, derlareRecoverTask)
-			}
-
-			if cfg.Subsystems.EnableWinningPost {
-				winPoStTask := lpwinning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, lw, verif, full, maddrs)
-				activeTasks = append(activeTasks, winPoStTask)
-			}
-		}
-		log.Infow("This lotus_provider instance handles",
-			"miner_addresses", minerAddressesToStrings(maddrs),
-			"tasks", lo.Map(activeTasks, func(t harmonytask.TaskInterface, _ int) string { return t.TypeDetails().Name }))
-
-		taskEngine, err := harmonytask.New(db, activeTasks, deps.listenAddr)
 		if err != nil {
-			return err
+			return nil
 		}
-
 		defer taskEngine.GracefullyTerminate(time.Hour)
 
-		fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}}
-		remoteHandler := func(w http.ResponseWriter, r *http.Request) {
-			if !auth.HasPerm(r.Context(), nil, api.PermAdmin) {
-				w.WriteHeader(401)
-				_ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"})
-				return
-			}
-
-			fh.ServeHTTP(w, r)
+		err = rpc.ListenAndServe(ctx, dependencies, shutdownChan) // Monitor for shutdown.
+		if err != nil {
+			return err
 		}
-		// local APIs
-		{
-			// debugging
-			mux := mux.NewRouter()
-			mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
-			mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
-
-			/*ah := &auth.Handler{
-				Verify: authv,
-				Next:   mux.ServeHTTP,
-			}*/ // todo
-
-		}
-
-		var authVerify func(context.Context, string) ([]auth.Permission, error)
-		{
-			privateKey, err := base64.StdEncoding.DecodeString(deps.cfg.Apis.StorageRPCSecret)
-			if err != nil {
-				return xerrors.Errorf("decoding storage rpc secret: %w", err)
-			}
-			authVerify = func(ctx context.Context, token string) ([]auth.Permission, error) {
-				var payload jwtPayload
-				if _, err := jwt.Verify([]byte(token), jwt.NewHS256(privateKey), &payload); err != nil {
-					return nil, xerrors.Errorf("JWT Verification failed: %w", err)
-				}
-
-				return payload.Allow, nil
-			}
-		}
-		// Serve the RPC.
-		srv := &http.Server{
-			Handler: rpc.LotusProviderHandler(
-				authVerify,
-				remoteHandler,
-				&ProviderAPI{deps, shutdownChan},
-				true),
-			ReadHeaderTimeout: time.Minute * 3,
-			BaseContext: func(listener net.Listener) context.Context {
-				ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker"))
-				return ctx
-			},
-		}
-
-		go func() {
-			<-ctx.Done()
-			log.Warn("Shutting down...")
-			if err := srv.Shutdown(context.TODO()); err != nil {
-				log.Errorf("shutting down RPC server failed: %s", err)
-			}
-			log.Warn("Graceful shutdown successful")
-		}()
-
-		// Monitor for shutdown.
-		// TODO provide a graceful shutdown API on shutdownChan
 		finishCh := node.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
 		//node.ShutdownHandler{Component: "provider", StopFunc: stop},
 
@@ -258,210 +142,48 @@ var runCmd = &cli.Command{
 	},
 }
 
-func makeDB(cctx *cli.Context) (*harmonydb.DB, error) {
-	dbConfig := config.HarmonyDB{
-		Username: cctx.String("db-user"),
-		Password: cctx.String("db-password"),
-		Hosts:    strings.Split(cctx.String("db-host"), ","),
-		Database: cctx.String("db-name"),
-		Port:     cctx.String("db-port"),
-	}
-	return harmonydb.NewFromConfig(dbConfig)
-}
-
-type jwtPayload struct {
-	Allow []auth.Permission
-}
-
-func StorageAuth(apiKey string) (sealer.StorageAuth, error) {
-	if apiKey == "" {
-		return nil, xerrors.Errorf("no api key provided")
-	}
-
-	rawKey, err := base64.StdEncoding.DecodeString(apiKey)
-	if err != nil {
-		return nil, xerrors.Errorf("decoding api key: %w", err)
-	}
-
-	key := jwt.NewHS256(rawKey)
-
-	p := jwtPayload{
-		Allow: []auth.Permission{"admin"},
-	}
-
-	token, err := jwt.Sign(&p, key)
-	if err != nil {
-		return nil, err
-	}
-
-	headers := http.Header{}
-	headers.Add("Authorization", "Bearer "+string(token))
-	return sealer.StorageAuth(headers), nil
-}
-
-type Deps struct {
-	cfg        *config.LotusProviderConfig
-	db         *harmonydb.DB
-	full       api.FullNode
-	verif      storiface.Verifier
-	lw         *sealer.LocalWorker
-	as         *ctladdr.AddressSelector
-	maddrs     []dtypes.MinerAddress
-	stor       *paths.Remote
-	si         *paths.DBIndex
-	localStore *paths.Local
-	listenAddr string
-}
-
-func getDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) {
-	// Open repo
-
-	repoPath := cctx.String(FlagRepoPath)
-	fmt.Println("repopath", repoPath)
-	r, err := repo.NewFS(repoPath)
-	if err != nil {
-		return nil, err
-	}
-
-	ok, err := r.Exists()
-	if err != nil {
-		return nil, err
-	}
-	if !ok {
-		if err := r.Init(repo.Provider); err != nil {
-			return nil, err
-		}
-	}
-
-	db, err := makeDB(cctx)
-	if err != nil {
-		return nil, err
-	}
-
-	///////////////////////////////////////////////////////////////////////
-	///// Dependency Setup
-	///////////////////////////////////////////////////////////////////////
-
-	// The config feeds into task runners & their helpers
-	cfg, err := getConfig(cctx, db)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Debugw("config", "config", cfg)
-
-	var verif storiface.Verifier = ffiwrapper.ProofVerifier
-
-	as, err := provider.AddressSelector(&cfg.Addresses)()
-	if err != nil {
-		return nil, err
-	}
-
-	de, err := journal.ParseDisabledEvents(cfg.Journal.DisabledEvents)
-	if err != nil {
-		return nil, err
-	}
-	j, err := fsjournal.OpenFSJournalPath(cctx.String("journal"), de)
-	if err != nil {
-		return nil, err
-	}
-
-	full, fullCloser, err := cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfg.Apis.ChainApiInfo)
-	if err != nil {
-		return nil, err
-	}
-
-	go func() {
-		select {
-		case <-ctx.Done():
-			fullCloser()
-			_ = j.Close()
-		}
-	}()
-	sa, err := StorageAuth(cfg.Apis.StorageRPCSecret)
-	if err != nil {
-		return nil, xerrors.Errorf(`'%w' while parsing the config toml's 
-	[Apis]
-	StorageRPCSecret=%v
-Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, err, cfg.Apis.StorageRPCSecret)
-	}
-
-	al := alerting.NewAlertingSystem(j)
-	si := paths.NewDBIndex(al, db)
-	bls := &paths.BasicLocalStorage{
-		PathToJSON: cctx.String("storage-json"),
-	}
-
-	listenAddr := cctx.String("listen")
-	const unspecifiedAddress = "0.0.0.0"
-	addressSlice := strings.Split(listenAddr, ":")
-	if ip := net.ParseIP(addressSlice[0]); ip != nil {
-		if ip.String() == unspecifiedAddress {
-			rip, err := db.GetRoutableIP()
-			if err != nil {
-				return nil, err
-			}
-			listenAddr = rip + ":" + addressSlice[1]
-		}
-	}
-	localStore, err := paths.NewLocal(ctx, bls, si, []string{"http://" + listenAddr + "/remote"})
-	if err != nil {
-		return nil, err
-	}
-
-	stor := paths.NewRemote(localStore, si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{})
-
-	wstates := statestore.New(dssync.MutexWrap(ds.NewMapDatastore()))
-
-	// todo localWorker isn't the abstraction layer we want to use here, we probably want to go straight to ffiwrapper
-	//  maybe with a lotus-provider specific abstraction. LocalWorker does persistent call tracking which we probably
-	//  don't need (ehh.. maybe we do, the async callback system may actually work decently well with harmonytask)
-	lw := sealer.NewLocalWorker(sealer.WorkerConfig{}, stor, localStore, si, nil, wstates)
-
-	var maddrs []dtypes.MinerAddress
-	for _, s := range cfg.Addresses.MinerAddresses {
-		addr, err := address.NewFromString(s)
+var webCmd = &cli.Command{
+	Name:  "web",
+	Usage: "Start lotus provider web interface",
+	Description: `Start an instance of lotus provider web interface. 
+	This creates the 'web' layer if it does not exist, then calls run with that layer.`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "listen",
+			Usage: "Address to listen on",
+			Value: "127.0.0.1:4701",
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base. Web will be added",
+			Value: cli.NewStringSlice("base"),
+		},
+		&cli.BoolFlag{
+			Name:  "nosync",
+			Usage: "don't check full-node sync status",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		db, err := deps.MakeDB(cctx)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		maddrs = append(maddrs, dtypes.MinerAddress(addr))
-	}
 
-	return &Deps{ // lint: intentionally not-named so it will fail if one is forgotten
-		cfg,
-		db,
-		full,
-		verif,
-		lw,
-		as,
-		maddrs,
-		stor,
-		si,
-		localStore,
-		listenAddr,
-	}, nil
+		webtxt, err := getConfig(db, "web")
+		if err != nil || webtxt == "" {
 
-}
-
-type ProviderAPI struct {
-	*Deps
-	ShutdownChan chan struct{}
-}
-
-func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
-	return api.ProviderAPIVersion0, nil
-}
-
-// Trigger shutdown
-func (p *ProviderAPI) Shutdown(context.Context) error {
-	close(p.ShutdownChan)
-	return nil
-}
-
-func minerAddressesToStrings(maddrs []dtypes.MinerAddress) []string {
-	strs := make([]string, len(maddrs))
-	for i, addr := range maddrs {
-		strs[i] = address.Address(addr).String()
-	}
-	return strs
+			s := `[Susbystems]
+			EnableWebGui = true
+			`
+			if err = setConfig(db, "web", s); err != nil {
+				return err
+			}
+		}
+		layers := append([]string{"web"}, cctx.StringSlice("layers")...)
+		err = cctx.Set("layers", strings.Join(layers, ","))
+		if err != nil {
+			return err
+		}
+		return runCmd.Action(cctx)
+	},
 }

cmd/lotus-provider/tasks/tasks.go

@@ -0,0 +1,58 @@
+// Package tasks contains tasks that can be run by the lotus-provider command.
+package tasks
+
+import (
+	"context"
+
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/samber/lo"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
+	"github.com/filecoin-project/lotus/provider"
+	"github.com/filecoin-project/lotus/provider/lpmessage"
+	"github.com/filecoin-project/lotus/provider/lpwinning"
+)
+
+var log = logging.Logger("lotus-provider/deps")
+
+func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.TaskEngine, error) {
+	cfg := dependencies.Cfg
+	db := dependencies.DB
+	full := dependencies.Full
+	verif := dependencies.Verif
+	lw := dependencies.LW
+	as := dependencies.As
+	maddrs := dependencies.Maddrs
+	stor := dependencies.Stor
+	si := dependencies.Si
+	var activeTasks []harmonytask.TaskInterface
+
+	sender, sendTask := lpmessage.NewSender(full, full, db)
+	activeTasks = append(activeTasks, sendTask)
+
+	///////////////////////////////////////////////////////////////////////
+	///// Task Selection
+	///////////////////////////////////////////////////////////////////////
+	{
+
+		if cfg.Subsystems.EnableWindowPost {
+			wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, cfg.Fees, cfg.Proving, full, verif, lw, sender,
+				as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks)
+			if err != nil {
+				return nil, err
+			}
+			activeTasks = append(activeTasks, wdPostTask, wdPoStSubmitTask, derlareRecoverTask)
+		}
+
+		if cfg.Subsystems.EnableWinningPost {
+			winPoStTask := lpwinning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, lw, verif, full, maddrs)
+			activeTasks = append(activeTasks, winPoStTask)
+		}
+	}
+	log.Infow("This lotus_provider instance handles",
+		"miner_addresses", maddrs,
+		"tasks", lo.Map(activeTasks, func(t harmonytask.TaskInterface, _ int) string { return t.TypeDetails().Name }))
+
+	return harmonytask.New(db, activeTasks, dependencies.ListenAddr)
+}

cmd/lotus-shed/miner.go

@@ -553,7 +553,7 @@ var sendInvalidWindowPoStCmd = &cli.Command{
 			return xerrors.Errorf("serializing params: %w", err)
 		}
 
-		fmt.Printf("submitting bad PoST for %d paritions\n", len(partitionIndices))
+		fmt.Printf("submitting bad PoST for %d partitions\n", len(partitionIndices))
 		smsg, err := api.MpoolPushMessage(ctx, &types.Message{
 			From:   minfo.Worker,
 			To:     maddr,

documentation/en/api-v1-unstable-methods.md

@@ -211,9 +211,6 @@
   * [PaychVoucherCreate](#PaychVoucherCreate)
   * [PaychVoucherList](#PaychVoucherList)
   * [PaychVoucherSubmit](#PaychVoucherSubmit)
-* [Raft](#Raft)
-  * [RaftLeader](#RaftLeader)
-  * [RaftState](#RaftState)
 * [Start](#Start)
   * [StartTime](#StartTime)
 * [State](#State)
@@ -6182,33 +6179,6 @@ Response:
 }
 ```
 
-## Raft
-
-
-### RaftLeader
-
-
-Perms: read
-
-Inputs: `null`
-
-Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"`
-
-### RaftState
-
-
-Perms: read
-
-Inputs: `null`
-
-Response:
-```json
-{
-  "NonceMap": {},
-  "MsgUuids": {}
-}
-```
-
 ## Start
 
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.2
+   1.25.3-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -66,6 +66,7 @@ OPTIONS:
    --no-local-storage                                         don't use storageminer repo for sector storage (default: false)
    --gas-premium value                                        set gas premium for initialization messages in AttoFIL (default: "0")
    --from value                                               select which address to send actor creation message from
+   --confidence value                                         number of block confirmations to wait for (default: 5)
    --help, -h                                                 show help
 ```
 

documentation/en/cli-lotus-provider.md

@@ -7,13 +7,14 @@ USAGE:
    lotus-provider [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.2
+   1.25.3-dev
 
 COMMANDS:
    run      Start a lotus provider process
    stop     Stop a running lotus provider
    config   Manage node config by layers. The layer 'base' will always be applied. 
    test     Utility functions for testing
+   web      Start lotus provider web interface
    version  Print version
    help, h  Shows a list of commands or help for one command
    DEVELOPER:
@@ -247,6 +248,25 @@ OPTIONS:
    --help, -h                         show help
 ```
 
+## lotus-provider web
+```
+NAME:
+   lotus-provider web - Start lotus provider web interface
+
+USAGE:
+   lotus-provider web [command options] [arguments...]
+
+DESCRIPTION:
+   Start an instance of lotus provider web interface. 
+     This creates the 'web' layer if it does not exist, then calls run with that layer.
+
+OPTIONS:
+   --listen value                     Address to listen on (default: "127.0.0.1:4701")
+   --layers value [ --layers value ]  list of layers to be interpreted (atop defaults). Default: base. Web will be added (default: "base")
+   --nosync                           don't check full-node sync status (default: false)
+   --help, -h                         show help
+```
+
 ## lotus-provider version
 ```
 NAME:

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.2
+   1.25.3-dev
 
 COMMANDS:
    run        Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.2
+   1.25.3-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process

documentation/en/default-lotus-config.toml

@@ -260,68 +260,6 @@
     #HotstoreMaxSpaceSafetyBuffer = 50000000000
 
 
-[Cluster]
-  # EXPERIMENTAL. config to enabled node cluster with raft consensus
-  #
-  # type: bool
-  # env var: LOTUS_CLUSTER_CLUSTERMODEENABLED
-  #ClusterModeEnabled = false
-
-  # A folder to store Raft's data.
-  #
-  # type: string
-  # env var: LOTUS_CLUSTER_DATAFOLDER
-  #DataFolder = ""
-
-  # InitPeersetMultiAddr provides the list of initial cluster peers for new Raft
-  # peers (with no prior state). It is ignored when Raft was already
-  # initialized or when starting in staging mode.
-  #
-  # type: []string
-  # env var: LOTUS_CLUSTER_INITPEERSETMULTIADDR
-  #InitPeersetMultiAddr = []
-
-  # LeaderTimeout specifies how long to wait for a leader before
-  # failing an operation.
-  #
-  # type: Duration
-  # env var: LOTUS_CLUSTER_WAITFORLEADERTIMEOUT
-  #WaitForLeaderTimeout = "15s"
-
-  # NetworkTimeout specifies how long before a Raft network
-  # operation is timed out
-  #
-  # type: Duration
-  # env var: LOTUS_CLUSTER_NETWORKTIMEOUT
-  #NetworkTimeout = "1m40s"
-
-  # CommitRetries specifies how many times we retry a failed commit until
-  # we give up.
-  #
-  # type: int
-  # env var: LOTUS_CLUSTER_COMMITRETRIES
-  #CommitRetries = 1
-
-  # How long to wait between retries
-  #
-  # type: Duration
-  # env var: LOTUS_CLUSTER_COMMITRETRYDELAY
-  #CommitRetryDelay = "200ms"
-
-  # BackupsRotate specifies the maximum number of Raft's DataFolder
-  # copies that we keep as backups (renaming) after cleanup.
-  #
-  # type: int
-  # env var: LOTUS_CLUSTER_BACKUPSROTATE
-  #BackupsRotate = 6
-
-  # Tracing enables propagation of contexts across binary boundaries.
-  #
-  # type: bool
-  # env var: LOTUS_CLUSTER_TRACING
-  #Tracing = false
-
-
 [Fevm]
   # EnableEthRPC enables eth_ rpc, and enables storing a mapping of eth transaction hashes to filecoin message Cids.
   # This will also enable the RealTimeFilterAPI and HistoricFilterAPI by default, but they can be disabled by config options above.

documentation/en/default-lotus-provider-config.toml

@@ -11,6 +11,14 @@
   # type: int
   #WinningPostMaxTasks = 0
 
+  # type: bool
+  #EnableWebGui = false
+
+  # The address that should listen for Web GUI requests.
+  #
+  # type: string
+  #GuiAddress = ":4701"
+
 
 [Fees]
   # type: types.FIL

go.mod

@@ -71,12 +71,9 @@ require (
 	github.com/gregdhill/go-openrpc v0.0.0-20220114144539-ae6f44720487
 	github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
-	github.com/hashicorp/go-hclog v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.5
 	github.com/hashicorp/golang-lru/v2 v2.0.5
-	github.com/hashicorp/raft v1.3.10
-	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
 	github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94
 	github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab
 	github.com/ipfs/bbloom v0.0.4
@@ -111,11 +108,8 @@ require (
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
 	github.com/libp2p/go-libp2p v0.31.1
-	github.com/libp2p/go-libp2p-consensus v0.0.1
-	github.com/libp2p/go-libp2p-gorpc v0.5.0
 	github.com/libp2p/go-libp2p-kad-dht v0.24.0
 	github.com/libp2p/go-libp2p-pubsub v0.9.3
-	github.com/libp2p/go-libp2p-raft v0.4.0
 	github.com/libp2p/go-libp2p-record v0.2.0
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.0
 	github.com/libp2p/go-maddr-filter v0.1.0
@@ -142,7 +136,7 @@ require (
 	github.com/samber/lo v1.38.1
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
-	github.com/triplewz/poseidon v0.0.0-20220525065023-a7cdb0e183e7
+	github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed
 	github.com/urfave/cli/v2 v2.25.5
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230923211252-36a87e1ba72f
@@ -161,7 +155,6 @@ require (
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.25.0
 	golang.org/x/crypto v0.17.0
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	golang.org/x/net v0.14.0
 	golang.org/x/sync v0.3.0
 	golang.org/x/sys v0.15.0
@@ -180,11 +173,9 @@ require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/akavel/rsrc v0.8.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
-	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
@@ -231,8 +222,6 @@ require (
 	github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/huin/goupnp v1.2.0 // indirect
 	github.com/iancoleman/orderedmap v0.1.0 // indirect
@@ -315,7 +304,6 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.0.1 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
@@ -331,6 +319,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect
@@ -346,5 +335,3 @@ require (
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
-
-replace github.com/triplewz/poseidon => github.com/magik6k/poseidon v0.0.0-neptune

go.sum

@@ -48,8 +48,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -99,9 +97,6 @@ github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
-github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18=
-github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
@@ -118,8 +113,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY=
 github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
 github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@@ -164,8 +157,6 @@ github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4=
 github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
-github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -592,26 +583,16 @@ github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-hclog v1.3.0 h1:G0ACM8Z2WilWgPv3Vdzwm3V0BQu/kSmrkVtpe1fy9do=
-github.com/hashicorp/go-hclog v1.3.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
-github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
-github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
@@ -628,10 +609,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/raft v1.3.10 h1:LR5QZX1VQd0DFWZfeCwWawyeKfpS/Tm1yjnJIY5X4Tw=
-github.com/hashicorp/raft v1.3.10/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4=
-github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4=
-github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
@@ -902,7 +879,6 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -1001,8 +977,6 @@ github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFk
 github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
 github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo=
 github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA=
-github.com/libp2p/go-libp2p-consensus v0.0.1 h1:jcVbHRZLwTXU9iT/mPi+Lx4/OrIzq3bU1TbZNhYFCV8=
-github.com/libp2p/go-libp2p-consensus v0.0.1/go.mod h1:+9Wrfhc5QOqWB0gXI0m6ARlkHfdJpcFXmRU0WoHz4Mo=
 github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
 github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
 github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE=
@@ -1031,8 +1005,6 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT
 github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg=
 github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw=
 github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug=
-github.com/libp2p/go-libp2p-gorpc v0.5.0 h1:mmxxAPdP3JzpYH4KcDf4csXnqtd1HazLPfdyB2MBRb8=
-github.com/libp2p/go-libp2p-gorpc v0.5.0/go.mod h1:GpHuvY3m0YFkd0+inOGo4HDtc4up9OS/mBPXvEpNuRY=
 github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=
 github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA=
 github.com/libp2p/go-libp2p-kad-dht v0.24.0 h1:nZnFDQEFU4N8GzclnR+IGxIgR7k4PPCDk/GK9A28onk=
@@ -1063,8 +1035,6 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc
 github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo=
 github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc=
 github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA=
-github.com/libp2p/go-libp2p-raft v0.4.0 h1:2atEs7/FWH35bRiLh8gTejGh5NA9u4eG7BXjpf/B+Z4=
-github.com/libp2p/go-libp2p-raft v0.4.0/go.mod h1:qJCYtFBTbip2wngLxFeAb9o52XmAPi2vSIQ4hV7IpSA=
 github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
@@ -1178,8 +1148,6 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
 github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE=
 github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magik6k/poseidon v0.0.0-neptune h1:Dfz15iiYGGE9Esvn8pZFlbiiCrHuyZDxm6LGXQfaf9c=
-github.com/magik6k/poseidon v0.0.0-neptune/go.mod h1:QYG1d0B4YZD7TgF6qZndTTu4rxUGFCCZAQRDanDj+9c=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -1198,7 +1166,6 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -1391,8 +1358,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
-github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -1423,7 +1388,6 @@ github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
-github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
@@ -1445,7 +1409,6 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7q
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
@@ -1608,14 +1571,13 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed h1:C8H2ql+vCBhEi7d3vMBBbdCAKv9s/thfPyLEuSvFpMU=
+github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed/go.mod h1:QYG1d0B4YZD7TgF6qZndTTu4rxUGFCCZAQRDanDj+9c=
 github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
 github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -2038,11 +2000,9 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

itests/kit/ensemble.go

@@ -20,6 +20,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/go-address"
 	cborutil "github.com/filecoin-project/go-cbor-util"
@@ -45,6 +46,9 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet/key"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/rpc"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/tasks"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker"
 	"github.com/filecoin-project/lotus/gateway"
@@ -120,15 +124,17 @@ type Ensemble struct {
 	options      *ensembleOpts
 
 	inactive struct {
-		fullnodes []*TestFullNode
-		miners    []*TestMiner
-		workers   []*TestWorker
+		fullnodes     []*TestFullNode
+		providernodes []*TestProviderNode
+		miners        []*TestMiner
+		workers       []*TestWorker
 	}
 	active struct {
-		fullnodes []*TestFullNode
-		miners    []*TestMiner
-		workers   []*TestWorker
-		bms       map[*TestMiner]*BlockMiner
+		fullnodes     []*TestFullNode
+		providernodes []*TestProviderNode
+		miners        []*TestMiner
+		workers       []*TestWorker
+		bms           map[*TestMiner]*BlockMiner
 	}
 	genesis struct {
 		version  network.Version
@@ -221,6 +227,20 @@ func (n *Ensemble) FullNode(full *TestFullNode, opts ...NodeOpt) *Ensemble {
 	return n
 }
 
+// FullNode enrolls a new Provider node.
+func (n *Ensemble) Provider(lp *TestProviderNode, opts ...NodeOpt) *Ensemble {
+	options := DefaultNodeOpts
+	for _, o := range opts {
+		err := o(&options)
+		require.NoError(n.t, err)
+	}
+
+	*lp = TestProviderNode{t: n.t, options: options, Deps: &deps.Deps{}}
+
+	n.inactive.providernodes = append(n.inactive.providernodes, lp)
+	return n
+}
+
 // Miner enrolls a new miner, using the provided full node for chain
 // interactions.
 func (n *Ensemble) MinerEnroll(minerNode *TestMiner, full *TestFullNode, opts ...NodeOpt) *Ensemble {
@@ -693,8 +713,6 @@ func (n *Ensemble) Start() *Ensemble {
 		copy.FullNode = modules.MakeUuidWrapper(copy.FullNode)
 		m.FullNode = &copy
 
-		//m.FullNode.FullNode = modules.MakeUuidWrapper(fn.FullNode)
-
 		opts := []node.Option{
 			node.StorageMiner(&m.StorageMiner, cfg.Subsystems),
 			node.Base(),
@@ -702,8 +720,6 @@ func (n *Ensemble) Start() *Ensemble {
 			node.Test(),
 
 			node.If(m.options.disableLibp2p, node.MockHost(n.mn)),
-			//node.Override(new(v1api.RawFullNodeAPI), func() api.FullNode { return modules.MakeUuidWrapper(m.FullNode) }),
-			//node.Override(new(v1api.RawFullNodeAPI), modules.MakeUuidWrapper),
 			node.Override(new(v1api.RawFullNodeAPI), m.FullNode),
 			node.Override(new(*lotusminer.Miner), lotusminer.NewTestMiner(mineBlock, m.ActorAddr)),
 
@@ -886,6 +902,28 @@ func (n *Ensemble) Start() *Ensemble {
 	// to active, so clear the slice.
 	n.inactive.workers = n.inactive.workers[:0]
 
+	for _, p := range n.inactive.providernodes {
+
+		// TODO setup config with options
+		err := p.Deps.PopulateRemainingDeps(context.Background(), &cli.Context{}, false)
+		require.NoError(n.t, err)
+
+		shutdownChan := make(chan struct{})
+		taskEngine, err := tasks.StartTasks(ctx, p.Deps)
+		if err != nil {
+			return nil
+		}
+		defer taskEngine.GracefullyTerminate(time.Hour)
+
+		err = rpc.ListenAndServe(ctx, p.Deps, shutdownChan) // Monitor for shutdown.
+		require.NoError(n.t, err)
+		finishCh := node.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
+		//node.ShutdownHandler{Component: "provider", StopFunc: stop},
+
+		<-finishCh
+
+		n.active.providernodes = append(n.active.providernodes, p)
+	}
 	// ---------------------
 	//  MISC
 	// ---------------------

itests/kit/ensemble_presets.go

@@ -101,6 +101,21 @@ func EnsembleOneTwo(t *testing.T, opts ...interface{}) (*TestFullNode, *TestMine
 	return &full, &one, &two, ens
 }
 
+// EnsembleProvider creates and starts an Ensemble with a single full node and a single provider.
+// It does not interconnect nodes nor does it begin mining.
+func EnsembleProvider(t *testing.T, opts ...interface{}) (*TestFullNode, *TestProviderNode, *Ensemble) {
+	opts = append(opts, WithAllSubsystems())
+
+	eopts, nopts := siftOptions(t, opts)
+
+	var (
+		full     TestFullNode
+		provider TestProviderNode
+	)
+	ens := NewEnsemble(t, eopts...).FullNode(&full, nopts...).Provider(&provider, nopts...).Start()
+	return &full, &provider, ens
+}
+
 func siftOptions(t *testing.T, opts []interface{}) (eopts []EnsembleOpt, nopts []NodeOpt) {
 	for _, v := range opts {
 		switch o := v.(type) {

itests/kit/node_full.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet/key"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/gateway"
 	"github.com/filecoin-project/lotus/node"
 )
@@ -54,6 +55,17 @@ type TestFullNode struct {
 	options nodeOpts
 }
 
+// TestProviderNode represents a Provider node enrolled in an Ensemble.
+type TestProviderNode struct {
+	v1api.LotusProviderStruct
+
+	t *testing.T
+
+	*deps.Deps
+
+	options nodeOpts
+}
+
 func MergeFullNodes(fullNodes []*TestFullNode) *TestFullNode {
 	var wrappedFullNode TestFullNode
 	var fns api.FullNodeStruct

itests/raft_messagesigner_test.go

@@ -1,577 +0,0 @@
-package itests
-
-import (
-	"context"
-	"crypto/rand"
-	"fmt"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	gorpc "github.com/libp2p/go-libp2p-gorpc"
-	libp2pcrypto "github.com/libp2p/go-libp2p/core/crypto"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/exitcode"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagesigner"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/itests/kit"
-	consensus "github.com/filecoin-project/lotus/lib/consensus/raft"
-	"github.com/filecoin-project/lotus/node"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/impl"
-	"github.com/filecoin-project/lotus/node/modules"
-)
-
-func generatePrivKey() (*kit.Libp2p, error) {
-	privkey, _, err := libp2pcrypto.GenerateEd25519Key(rand.Reader)
-	if err != nil {
-		return nil, err
-	}
-
-	peerId, err := peer.IDFromPrivateKey(privkey)
-	if err != nil {
-		return nil, err
-	}
-
-	return &kit.Libp2p{PeerID: peerId, PrivKey: privkey}, nil
-}
-
-func getRaftState(ctx context.Context, t *testing.T, node *kit.TestFullNode) *api.RaftStateData {
-	raftState, err := node.RaftState(ctx)
-	require.NoError(t, err)
-	return raftState
-}
-
-func setup(ctx context.Context, t *testing.T, node0 *kit.TestFullNode, node1 *kit.TestFullNode, node2 *kit.TestFullNode, miner *kit.TestMiner) *kit.Ensemble {
-
-	blockTime := 1 * time.Second
-
-	pkey0, _ := generatePrivKey()
-	pkey1, _ := generatePrivKey()
-	pkey2, _ := generatePrivKey()
-
-	pkeys := []*kit.Libp2p{pkey0, pkey1, pkey2}
-	initPeerSet := []string{}
-	for _, pkey := range pkeys {
-		initPeerSet = append(initPeerSet, "/p2p/"+pkey.PeerID.String())
-	}
-
-	//initPeerSet := []peer.ID{pkey0.PeerID, pkey1.PeerID, pkey2.PeerID}
-
-	raftOps := kit.ConstructorOpts(
-		node.Override(new(*gorpc.Client), modules.NewRPCClient),
-		node.Override(new(*consensus.ClusterRaftConfig), func() *consensus.ClusterRaftConfig {
-			cfg := consensus.DefaultClusterRaftConfig()
-			cfg.InitPeerset = initPeerSet
-			return cfg
-		}),
-		node.Override(new(*consensus.Consensus), consensus.NewConsensusWithRPCClient(false)),
-		node.Override(new(*messagesigner.MessageSignerConsensus), messagesigner.NewMessageSignerConsensus),
-		node.Override(new(messagesigner.MsgSigner), func(ms *messagesigner.MessageSignerConsensus) *messagesigner.MessageSignerConsensus { return ms }),
-		node.Override(new(*modules.RPCHandler), modules.NewRPCHandler),
-		node.Override(node.GoRPCServer, modules.NewRPCServer),
-	)
-	//raftOps := kit.ConstructorOpts()
-
-	ens := kit.NewEnsemble(t).FullNode(node0, raftOps, kit.ThroughRPC()).FullNode(node1, raftOps, kit.ThroughRPC()).FullNode(node2, raftOps, kit.ThroughRPC())
-	node0.AssignPrivKey(pkey0)
-	node1.AssignPrivKey(pkey1)
-	node2.AssignPrivKey(pkey2)
-
-	nodes := []*kit.TestFullNode{node0, node1, node2}
-	wrappedFullNode := kit.MergeFullNodes(nodes)
-
-	ens.MinerEnroll(miner, wrappedFullNode, kit.WithAllSubsystems(), kit.ThroughRPC())
-	ens.Start()
-
-	// Import miner wallet to all nodes
-	addr0, err := node0.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr1, err := node1.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr2, err := node2.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-
-	fmt.Println(addr0, addr1, addr2)
-
-	ens.InterconnectAll()
-
-	ens.AddInactiveMiner(miner)
-	ens.Start()
-
-	ens.InterconnectAll().BeginMining(blockTime)
-
-	return ens
-}
-
-func TestRaftState(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	fmt.Println(node0.WalletList(context.Background()))
-	fmt.Println(node1.WalletList(context.Background()))
-	fmt.Println(node2.WalletList(context.Background()))
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-
-	mu := uuid.New()
-	smHalfBal, err := node0.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-	mLookup, err := node0.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate0 := getRaftState(ctx, t, &node0)
-	rstate1 := getRaftState(ctx, t, &node1)
-	rstate2 := getRaftState(ctx, t, &node2)
-
-	require.EqualValues(t, rstate0, rstate1)
-	require.EqualValues(t, rstate0, rstate2)
-}
-
-func TestRaftStateLeaderDisconnects(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := node0.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-	mLookup, err := node0.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate0 := getRaftState(ctx, t, &node0)
-	rstate1 := getRaftState(ctx, t, &node1)
-	rstate2 := getRaftState(ctx, t, &node2)
-
-	require.True(t, reflect.DeepEqual(rstate0, rstate1))
-	require.True(t, reflect.DeepEqual(rstate0, rstate2))
-
-	leader, err := node1.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	err = leaderNode.Stop(ctx)
-	require.NoError(t, err)
-	oldLeaderNode := leaderNode
-
-	time.Sleep(5 * time.Second)
-
-	newLeader := leader
-	for _, n := range nodes {
-		if n != leaderNode {
-			newLeader, err = n.RaftLeader(ctx)
-			require.NoError(t, err)
-			require.NotEqual(t, newLeader, leader)
-		}
-	}
-
-	require.NotEqual(t, newLeader, leader)
-	leaderNode = peerToNode[newLeader]
-
-	msg2 := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    leaderNode.DefaultKey.Address,
-		Value: big.NewInt(100000),
-	}
-	mu2 := uuid.New()
-	signedMsg2, err := leaderNode.MpoolPushMessage(ctx, msg2, &api.MessageSendSpec{
-		MsgUuid: mu2,
-	})
-	require.NoError(t, err)
-	mLookup, err = leaderNode.StateWaitMsg(ctx, signedMsg2.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate := getRaftState(ctx, t, leaderNode)
-
-	for _, n := range nodes {
-		if n != oldLeaderNode {
-			rs := getRaftState(ctx, t, n)
-			require.True(t, reflect.DeepEqual(rs, rstate))
-		}
-	}
-}
-
-func TestRaftStateMiner(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	fmt.Println(node0.WalletList(context.Background()))
-	fmt.Println(node1.WalletList(context.Background()))
-	fmt.Println(node2.WalletList(context.Background()))
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := miner.FullNode.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-	mLookup, err := node0.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate0 := getRaftState(ctx, t, &node0)
-	rstate1 := getRaftState(ctx, t, &node1)
-	rstate2 := getRaftState(ctx, t, &node2)
-
-	require.EqualValues(t, rstate0, rstate1)
-	require.EqualValues(t, rstate0, rstate2)
-}
-
-func TestRaftStateLeaderDisconnectsMiner(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	// Take leader node down
-	err = leaderNode.Stop(ctx)
-	require.NoError(t, err)
-	oldLeaderNode := leaderNode
-
-	time.Sleep(5 * time.Second)
-
-	newLeader := leader
-	for _, n := range nodes {
-		if n != leaderNode {
-			newLeader, err = n.RaftLeader(ctx)
-			require.NoError(t, err)
-			require.NotEqual(t, newLeader, leader)
-		}
-	}
-
-	require.NotEqual(t, newLeader, leader)
-	leaderNode = peerToNode[newLeader]
-
-	msg2 := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.NewInt(100000),
-	}
-	mu2 := uuid.New()
-
-	signedMsg2, err := miner.FullNode.MpoolPushMessage(ctx, msg2, &api.MessageSendSpec{
-		MaxFee:  abi.TokenAmount(config.DefaultDefaultMaxFee),
-		MsgUuid: mu2,
-	})
-	require.NoError(t, err)
-
-	mLookup, err := leaderNode.StateWaitMsg(ctx, signedMsg2.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate := getRaftState(ctx, t, leaderNode)
-
-	for _, n := range nodes {
-		if n != oldLeaderNode {
-			rs := getRaftState(ctx, t, n)
-			require.True(t, reflect.DeepEqual(rs, rstate))
-		}
-	}
-}
-
-// Miner sends message on leader
-// Leader disconnects
-// Call StateWaitMsg on new leader
-func TestLeaderDisconnectsCheckMsgStateOnNewLeader(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := miner.FullNode.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	// Take leader node down
-	err = leaderNode.Stop(ctx)
-	require.NoError(t, err)
-	oldLeaderNode := leaderNode
-
-	time.Sleep(5 * time.Second)
-
-	// Check if all active nodes update their leader
-	newLeader := leader
-	for _, n := range nodes {
-		if n != leaderNode {
-			newLeader, err = n.RaftLeader(ctx)
-			require.NoError(t, err)
-			require.NotEqual(t, newLeader, leader)
-		}
-	}
-
-	require.NotEqual(t, newLeader, leader)
-	leaderNode = peerToNode[newLeader]
-
-	mLookup, err := leaderNode.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate := getRaftState(ctx, t, leaderNode)
-
-	// Check if Raft state is consistent on all active nodes
-	for _, n := range nodes {
-		if n != oldLeaderNode {
-			rs := getRaftState(ctx, t, n)
-			require.True(t, reflect.DeepEqual(rs, rstate))
-		}
-	}
-}
-
-func TestChainStoreSync(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := miner.FullNode.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-
-	for _, n := range nodes {
-		fmt.Println(n != leaderNode)
-		if n != leaderNode {
-			mLookup, err := n.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-			require.NoError(t, err)
-			require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-			//break
-		}
-	}
-}
-
-func TestGoRPCAuth(t *testing.T) {
-	// TODO Fix Raft, then enable this test. https://github.com/filecoin-project/lotus/issues/9888
-	t.SkipNow()
-
-	blockTime := 1 * time.Second
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		node3 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	pkey0, _ := generatePrivKey()
-	pkey1, _ := generatePrivKey()
-	pkey2, _ := generatePrivKey()
-
-	pkeys := []*kit.Libp2p{pkey0, pkey1, pkey2}
-	initPeerSet := []string{}
-	for _, pkey := range pkeys {
-		initPeerSet = append(initPeerSet, "/p2p/"+pkey.PeerID.String())
-	}
-
-	raftOps := kit.ConstructorOpts(
-		node.Override(new(*gorpc.Client), modules.NewRPCClient),
-		node.Override(new(*consensus.ClusterRaftConfig), func() *consensus.ClusterRaftConfig {
-			cfg := consensus.DefaultClusterRaftConfig()
-			cfg.InitPeerset = initPeerSet
-			return cfg
-		}),
-		node.Override(new(*consensus.Consensus), consensus.NewConsensusWithRPCClient(false)),
-		node.Override(new(*messagesigner.MessageSignerConsensus), messagesigner.NewMessageSignerConsensus),
-		node.Override(new(messagesigner.MsgSigner), func(ms *messagesigner.MessageSignerConsensus) *messagesigner.MessageSignerConsensus { return ms }),
-		node.Override(new(*modules.RPCHandler), modules.NewRPCHandler),
-		node.Override(node.GoRPCServer, modules.NewRPCServer),
-	)
-	//raftOps := kit.ConstructorOpts()
-
-	ens := kit.NewEnsemble(t).FullNode(&node0, raftOps, kit.ThroughRPC()).FullNode(&node1, raftOps, kit.ThroughRPC()).FullNode(&node2, raftOps, kit.ThroughRPC()).FullNode(&node3, raftOps)
-	node0.AssignPrivKey(pkey0)
-	node1.AssignPrivKey(pkey1)
-	node2.AssignPrivKey(pkey2)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-	wrappedFullNode := kit.MergeFullNodes(nodes)
-
-	ens.MinerEnroll(&miner, wrappedFullNode, kit.WithAllSubsystems(), kit.ThroughRPC())
-	ens.Start()
-
-	// Import miner wallet to all nodes
-	addr0, err := node0.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr1, err := node1.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr2, err := node2.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-
-	fmt.Println(addr0, addr1, addr2)
-
-	ens.InterconnectAll()
-
-	ens.AddInactiveMiner(&miner)
-	ens.Start()
-
-	ens.InterconnectAll().BeginMining(blockTime)
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-
-	client := node3.FullNode.(*impl.FullNodeAPI).RaftAPI.MessageSigner.Consensus.RpcClient
-	method := "MpoolPushMessage"
-
-	msg := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.NewInt(100000),
-	}
-	msgWhole := &api.MpoolMessageWhole{Msg: msg}
-	var ret types.SignedMessage
-
-	err = client.CallContext(ctx, leader, "Consensus", method, msgWhole, &ret)
-	require.True(t, gorpc.IsAuthorizationError(err))
-
-}

journal/fsjournal/fs.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/build"
@@ -41,6 +42,11 @@ func OpenFSJournal(lr repo.LockedRepo, disabled journal.DisabledEvents) (journal
 }
 
 func OpenFSJournalPath(path string, disabled journal.DisabledEvents) (journal.Journal, error) {
+	path, err := homedir.Expand(path)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to expand repo path: %w", err)
+	}
+
 	dir := filepath.Join(path, "journal")
 	if err := os.MkdirAll(dir, 0755); err != nil {
 		return nil, fmt.Errorf("failed to mk directory %s for file journal: %w", dir, err)

lib/consensus/raft/config.go

@@ -1,135 +0,0 @@
-package consensus
-
-import (
-	"io"
-	"path/filepath"
-	"time"
-
-	hraft "github.com/hashicorp/raft"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/repo"
-)
-
-// Configuration defaults
-var (
-	DefaultDataSubFolder        = "raft-cluster"
-	DefaultWaitForLeaderTimeout = 15 * time.Second
-	DefaultCommitRetries        = 1
-	DefaultNetworkTimeout       = 100 * time.Second
-	DefaultCommitRetryDelay     = 200 * time.Millisecond
-	DefaultBackupsRotate        = 6
-)
-
-// ClusterRaftConfig allows to configure the Raft Consensus component for the node cluster.
-type ClusterRaftConfig struct {
-	// config to enabled node cluster with raft consensus
-	ClusterModeEnabled bool
-	// A folder to store Raft's data.
-	DataFolder string
-	// InitPeerset provides the list of initial cluster peers for new Raft
-	// peers (with no prior state). It is ignored when Raft was already
-	// initialized or when starting in staging mode.
-	InitPeerset []string
-	// LeaderTimeout specifies how long to wait for a leader before
-	// failing an operation.
-	WaitForLeaderTimeout time.Duration
-	// NetworkTimeout specifies how long before a Raft network
-	// operation is timed out
-	NetworkTimeout time.Duration
-	// CommitRetries specifies how many times we retry a failed commit until
-	// we give up.
-	CommitRetries int
-	// How long to wait between retries
-	CommitRetryDelay time.Duration
-	// BackupsRotate specifies the maximum number of Raft's DataFolder
-	// copies that we keep as backups (renaming) after cleanup.
-	BackupsRotate int
-	// A Hashicorp Raft's configuration object.
-	RaftConfig *hraft.Config
-
-	// Tracing enables propagation of contexts across binary boundaries.
-	Tracing bool
-}
-
-func DefaultClusterRaftConfig() *ClusterRaftConfig {
-	var cfg ClusterRaftConfig
-	cfg.DataFolder = "" // empty so it gets omitted
-	cfg.InitPeerset = []string{}
-	cfg.WaitForLeaderTimeout = DefaultWaitForLeaderTimeout
-	cfg.NetworkTimeout = DefaultNetworkTimeout
-	cfg.CommitRetries = DefaultCommitRetries
-	cfg.CommitRetryDelay = DefaultCommitRetryDelay
-	cfg.BackupsRotate = DefaultBackupsRotate
-	cfg.RaftConfig = hraft.DefaultConfig()
-
-	// These options are imposed over any Default Raft Config.
-	cfg.RaftConfig.ShutdownOnRemove = false
-	cfg.RaftConfig.LocalID = "will_be_set_automatically"
-
-	// Set up logging
-	cfg.RaftConfig.LogOutput = io.Discard
-	return &cfg
-}
-
-func NewClusterRaftConfig(userRaftConfig *config.UserRaftConfig) *ClusterRaftConfig {
-	var cfg ClusterRaftConfig
-	cfg.DataFolder = userRaftConfig.DataFolder
-	cfg.InitPeerset = userRaftConfig.InitPeersetMultiAddr
-	cfg.WaitForLeaderTimeout = time.Duration(userRaftConfig.WaitForLeaderTimeout)
-	cfg.NetworkTimeout = time.Duration(userRaftConfig.NetworkTimeout)
-	cfg.CommitRetries = userRaftConfig.CommitRetries
-	cfg.CommitRetryDelay = time.Duration(userRaftConfig.CommitRetryDelay)
-	cfg.BackupsRotate = userRaftConfig.BackupsRotate
-
-	// Keep this to be default hraft config for now
-	cfg.RaftConfig = hraft.DefaultConfig()
-
-	// These options are imposed over any Default Raft Config.
-	cfg.RaftConfig.ShutdownOnRemove = false
-	cfg.RaftConfig.LocalID = "will_be_set_automatically"
-
-	// Set up logging
-	cfg.RaftConfig.LogOutput = io.Discard
-
-	return &cfg
-
-}
-
-// Validate checks that this configuration has working values,
-// at least in appearance.
-func ValidateConfig(cfg *ClusterRaftConfig) error {
-	if cfg.RaftConfig == nil {
-		return xerrors.Errorf("no hashicorp/raft.Config")
-	}
-	if cfg.WaitForLeaderTimeout <= 0 {
-		return xerrors.Errorf("wait_for_leader_timeout <= 0")
-	}
-
-	if cfg.NetworkTimeout <= 0 {
-		return xerrors.Errorf("network_timeout <= 0")
-	}
-
-	if cfg.CommitRetries < 0 {
-		return xerrors.Errorf("commit_retries is invalid")
-	}
-
-	if cfg.CommitRetryDelay <= 0 {
-		return xerrors.Errorf("commit_retry_delay is invalid")
-	}
-
-	if cfg.BackupsRotate <= 0 {
-		return xerrors.Errorf("backups_rotate should be larger than 0")
-	}
-
-	return hraft.ValidateConfig(cfg.RaftConfig)
-}
-
-// GetDataFolder returns the Raft data folder that we are using.
-func (cfg *ClusterRaftConfig) GetDataFolder(repo repo.LockedRepo) string {
-	if cfg.DataFolder == "" {
-		return filepath.Join(repo.Path(), DefaultDataSubFolder)
-	}
-	return filepath.Join(repo.Path(), cfg.DataFolder)
-}

lib/consensus/raft/consensus.go

@@ -1,512 +0,0 @@
-// Package raft implements a Consensus component for IPFS Cluster which uses
-// Raft (go-libp2p-raft).
-package consensus
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"sort"
-	"time"
-
-	"github.com/google/uuid"
-	"golang.org/x/exp/slices"
-
-	addr "github.com/filecoin-project/go-address"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagepool"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/addrutil"
-	"github.com/filecoin-project/lotus/node/repo"
-
-	//ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-	consensus "github.com/libp2p/go-libp2p-consensus"
-	rpc "github.com/libp2p/go-libp2p-gorpc"
-	libp2praft "github.com/libp2p/go-libp2p-raft"
-	host "github.com/libp2p/go-libp2p/core/host"
-	peer "github.com/libp2p/go-libp2p/core/peer"
-)
-
-var logger = logging.Logger("raft")
-
-type RaftState struct {
-	NonceMap api.NonceMapType
-	MsgUuids api.MsgUuidMapType
-
-	// TODO: add comment explaining why this is needed
-	// We need a reference to the messagepool in the raft state in order to
-	// sync messages that have been sent by the leader node
-	// Miner calls StateWaitMsg after MpoolPushMessage to check if the message has
-	// landed on chain. This check requires the message be stored in the local chainstore
-	// If a leadernode goes down after sending a message to the chain and is replaced by
-	// another node, the other node needs to have this message in its chainstore for the
-	// above check to succeed.
-
-	// This is because the miner only stores signed CIDs but the message received from in a
-	// block will be unsigned (for BLS). Hence, the process relies on the node to store the
-	// signed message which holds a copy of the unsigned message to properly perform all the
-	// needed checks
-	Mpool *messagepool.MessagePool
-}
-
-func newRaftState(mpool *messagepool.MessagePool) *RaftState {
-	return &RaftState{
-		NonceMap: make(map[addr.Address]uint64),
-		MsgUuids: make(map[uuid.UUID]*types.SignedMessage),
-		Mpool:    mpool,
-	}
-}
-
-type ConsensusOp struct {
-	Nonce     uint64               `codec:"nonce,omitempty"`
-	Uuid      uuid.UUID            `codec:"uuid,omitempty"`
-	Addr      addr.Address         `codec:"addr,omitempty"`
-	SignedMsg *types.SignedMessage `codec:"signedMsg,omitempty"`
-}
-
-func (c ConsensusOp) ApplyTo(state consensus.State) (consensus.State, error) {
-	s := state.(*RaftState)
-	s.NonceMap[c.Addr] = c.Nonce
-	if c.SignedMsg != nil {
-
-		// Deep copy to tmp
-		var buffer bytes.Buffer
-		err := c.SignedMsg.MarshalCBOR(&buffer)
-		if err != nil {
-			return nil, err
-		}
-		tmp, err := types.DecodeSignedMessage(buffer.Bytes())
-		if err != nil {
-			return nil, err
-		}
-		s.MsgUuids[c.Uuid] = tmp
-
-		_, err = s.Mpool.Push(context.TODO(), tmp, false)
-		// Since this is only meant to keep messages in sync, ignore any error which
-		// shows the message already exists in the mpool
-		if err != nil && !api.ErrorIsIn(err, []error{messagepool.ErrExistingNonce}) {
-			return nil, err
-		}
-	}
-
-	return s, nil
-}
-
-var _ consensus.Op = &ConsensusOp{}
-
-// Consensus handles the work of keeping a shared-state between
-// the peers of a Lotus Cluster, as well as modifying that state and
-// applying any updates in a thread-safe manner.
-type Consensus struct {
-	ctx    context.Context
-	cancel func()
-	config *ClusterRaftConfig
-
-	host host.Host
-
-	consensus consensus.OpLogConsensus
-	actor     consensus.Actor
-	raft      *raftWrapper
-	state     *RaftState
-
-	RpcClient *rpc.Client
-	rpcReady  chan struct{}
-	readyCh   chan struct{}
-
-	peerSet []peer.ID
-	repo    repo.LockedRepo
-}
-
-// NewConsensus builds a new ClusterConsensus component using Raft.
-//
-// Raft saves state snapshots regularly and persists log data in a bolt
-// datastore. Therefore, unless memory usage is a concern, it is recommended
-// to use an in-memory go-datastore as store parameter.
-//
-// The staging parameter controls if the Raft peer should start in
-// staging mode (used when joining a new Raft peerset with other peers).
-func NewConsensus(host host.Host, cfg *ClusterRaftConfig, mpool *messagepool.MessagePool, repo repo.LockedRepo, staging bool) (*Consensus, error) {
-	err := ValidateConfig(cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	logger.Debug("starting Consensus and waiting for a leader...")
-	state := newRaftState(mpool)
-
-	consensus := libp2praft.NewOpLog(state, &ConsensusOp{})
-
-	raft, err := newRaftWrapper(host, cfg, consensus.FSM(), repo, staging)
-	if err != nil {
-		logger.Error("error creating raft: ", err)
-		cancel()
-		return nil, err
-	}
-	actor := libp2praft.NewActor(raft.raft)
-	consensus.SetActor(actor)
-
-	peers := []peer.ID{}
-	addrInfos, err := addrutil.ParseAddresses(ctx, cfg.InitPeerset)
-	if err != nil {
-		logger.Error("error parsing addresses: ", err)
-		cancel()
-		return nil, err
-	}
-
-	for _, addrInfo := range addrInfos {
-		peers = append(peers, addrInfo.ID)
-
-		// Add peer to address book
-		host.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, time.Hour*100)
-	}
-
-	cc := &Consensus{
-		ctx:       ctx,
-		cancel:    cancel,
-		config:    cfg,
-		host:      host,
-		consensus: consensus,
-		actor:     actor,
-		state:     state,
-		raft:      raft,
-		peerSet:   peers,
-		rpcReady:  make(chan struct{}, 1),
-		readyCh:   make(chan struct{}, 1),
-		repo:      repo,
-	}
-
-	go cc.finishBootstrap()
-	return cc, nil
-
-}
-
-// TODO: Merge with NewConsensus and remove the rpcReady chan
-func NewConsensusWithRPCClient(staging bool) func(host host.Host,
-	cfg *ClusterRaftConfig,
-	rpcClient *rpc.Client,
-	mpool *messagepool.MessagePool,
-	repo repo.LockedRepo,
-) (*Consensus, error) {
-
-	return func(host host.Host, cfg *ClusterRaftConfig, rpcClient *rpc.Client, mpool *messagepool.MessagePool, repo repo.LockedRepo) (*Consensus, error) {
-		cc, err := NewConsensus(host, cfg, mpool, repo, staging)
-		if err != nil {
-			return nil, err
-		}
-		cc.RpcClient = rpcClient
-		cc.rpcReady <- struct{}{}
-		return cc, nil
-	}
-}
-
-// WaitForSync waits for a leader and for the state to be up to date, then returns.
-func (cc *Consensus) WaitForSync(ctx context.Context) error {
-
-	leaderCtx, cancel := context.WithTimeout(ctx, cc.config.WaitForLeaderTimeout)
-	defer cancel()
-
-	// 1 - wait for leader
-	// 2 - wait until we are a Voter
-	// 3 - wait until last index is applied
-
-	// From raft docs:
-
-	// once a staging server receives enough log entries to be sufficiently
-	// caught up to the leader's log, the leader will invoke a  membership
-	// change to change the Staging server to a Voter
-
-	// Thus, waiting to be a Voter is a guarantee that we have a reasonable
-	// up to date state. Otherwise, we might return too early (see
-	// https://github.com/ipfs-cluster/ipfs-cluster/issues/378)
-
-	_, err := cc.raft.WaitForLeader(leaderCtx)
-	if err != nil {
-		return errors.New("error waiting for leader: " + err.Error())
-	}
-
-	err = cc.raft.WaitForVoter(ctx)
-	if err != nil {
-		return errors.New("error waiting to become a Voter: " + err.Error())
-	}
-
-	err = cc.raft.WaitForUpdates(ctx)
-	if err != nil {
-		return errors.New("error waiting for consensus updates: " + err.Error())
-	}
-	return nil
-}
-
-// waits until there is a consensus leader and syncs the state
-// to the tracker. If errors happen, this will return and never
-// signal the component as Ready.
-func (cc *Consensus) finishBootstrap() {
-	// wait until we have RPC to perform any actions.
-	select {
-	case <-cc.ctx.Done():
-		return
-	case <-cc.rpcReady:
-	}
-
-	// Sometimes bootstrap is a no-Op. It only applies when
-	// no state exists and staging=false.
-	_, err := cc.raft.Bootstrap()
-	if err != nil {
-		return
-	}
-
-	logger.Debugf("Bootstrap finished")
-	err = cc.WaitForSync(cc.ctx)
-	if err != nil {
-		return
-	}
-	logger.Debug("Raft state is now up to date")
-	logger.Debug("consensus ready")
-	cc.readyCh <- struct{}{}
-}
-
-// Shutdown stops the component so it will not process any
-// more updates. The underlying consensus is permanently
-// shutdown, along with the libp2p transport.
-func (cc *Consensus) Shutdown(ctx context.Context) error {
-
-	logger.Info("stopping Consensus component")
-
-	// Raft Shutdown
-	err := cc.raft.Shutdown(ctx)
-	if err != nil {
-		logger.Error(err)
-	}
-
-	cc.cancel()
-	close(cc.rpcReady)
-	return nil
-}
-
-// Ready returns a channel which is signaled when the Consensus
-// algorithm has finished bootstrapping and is ready to use
-func (cc *Consensus) Ready(ctx context.Context) <-chan struct{} {
-	return cc.readyCh
-}
-
-// IsTrustedPeer returns true. In Raft we trust all peers.
-func (cc *Consensus) IsTrustedPeer(ctx context.Context, p peer.ID) bool {
-	return slices.Contains(cc.peerSet, p)
-}
-
-// Trust is a no-Op.
-func (cc *Consensus) Trust(ctx context.Context, pid peer.ID) error { return nil }
-
-// Distrust is a no-Op.
-func (cc *Consensus) Distrust(ctx context.Context, pid peer.ID) error { return nil }
-
-// returns true if the operation was redirected to the leader
-// note that if the leader just dissappeared, the rpc call will
-// fail because we haven't heard that it's gone.
-func (cc *Consensus) RedirectToLeader(method string, arg interface{}, ret interface{}) (bool, error) {
-	ctx := cc.ctx
-
-	var finalErr error
-
-	// Retry redirects
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("redirect try %d", i)
-		leader, err := cc.Leader(ctx)
-
-		// No leader, wait for one
-		if err != nil {
-			logger.Warn("there seems to be no leader. Waiting for one")
-			rctx, cancel := context.WithTimeout(ctx, cc.config.WaitForLeaderTimeout)
-			defer cancel()
-			pidstr, err := cc.raft.WaitForLeader(rctx)
-
-			// means we timed out waiting for a leader
-			// we don't retry in this case
-			if err != nil {
-				return false, fmt.Errorf("timed out waiting for leader: %s", err)
-			}
-			leader, err = peer.Decode(pidstr)
-			if err != nil {
-				return false, err
-			}
-		}
-
-		logger.Infof("leader: %s, curr host: %s, peerSet: %s", leader, cc.host.ID(), cc.peerSet)
-
-		// We are the leader. Do not redirect
-		if leader == cc.host.ID() {
-			return false, nil
-		}
-
-		logger.Debugf("redirecting %s to leader: %s", method, leader)
-		finalErr = cc.RpcClient.CallContext(
-			ctx,
-			leader,
-			"Consensus",
-			method,
-			arg,
-			ret,
-		)
-		if finalErr != nil {
-			logger.Errorf("retrying to redirect request to leader: %s", finalErr)
-			time.Sleep(2 * cc.config.RaftConfig.HeartbeatTimeout)
-			continue
-		}
-		break
-	}
-
-	// We tried to redirect, but something happened
-	return true, finalErr
-}
-
-// commit submits a cc.consensus commit. It retries upon failures.
-func (cc *Consensus) Commit(ctx context.Context, op *ConsensusOp) error {
-
-	var finalErr error
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: committing %+v", i, op)
-
-		// this means we are retrying
-		if finalErr != nil {
-			logger.Errorf("retrying upon failed commit (retry %d): %s ",
-				i, finalErr)
-		}
-
-		// Being here means we are the LEADER. We can commit.
-		// now commit the changes to our state
-		_, finalErr = cc.consensus.CommitOp(op)
-		if finalErr != nil {
-			goto RETRY
-		}
-
-	RETRY:
-		time.Sleep(cc.config.CommitRetryDelay)
-	}
-	return finalErr
-}
-
-// AddPeer adds a new peer to participate in this consensus. It will
-// forward the operation to the leader if this is not it.
-func (cc *Consensus) AddPeer(ctx context.Context, pid peer.ID) error {
-	var finalErr error
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: AddPeer %s", i, pid)
-		if finalErr != nil {
-			logger.Errorf("retrying to add peer. Attempt #%d failed: %s", i, finalErr)
-		}
-		ok, err := cc.RedirectToLeader("AddPeer", pid, struct{}{})
-		if err != nil || ok {
-			return err
-		}
-		// Being here means we are the leader and can commit
-		finalErr = cc.raft.AddPeer(ctx, pid)
-		if finalErr != nil {
-			time.Sleep(cc.config.CommitRetryDelay)
-			continue
-		}
-		logger.Infof("peer added to Raft: %s", pid)
-		break
-	}
-	return finalErr
-}
-
-// RmPeer removes a peer from this consensus. It will
-// forward the operation to the leader if this is not it.
-func (cc *Consensus) RmPeer(ctx context.Context, pid peer.ID) error {
-	var finalErr error
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: RmPeer %s", i, pid)
-		if finalErr != nil {
-			logger.Errorf("retrying to remove peer. Attempt #%d failed: %s", i, finalErr)
-		}
-		ok, err := cc.RedirectToLeader("RmPeer", pid, struct{}{})
-		if err != nil || ok {
-			return err
-		}
-		// Being here means we are the leader and can commit
-		finalErr = cc.raft.RemovePeer(ctx, pid.String())
-		if finalErr != nil {
-			time.Sleep(cc.config.CommitRetryDelay)
-			continue
-		}
-		logger.Infof("peer removed from Raft: %s", pid)
-		break
-	}
-	return finalErr
-}
-
-// RaftState retrieves the current consensus RaftState. It may error if no RaftState has
-// been agreed upon or the state is not consistent. The returned RaftState is the
-// last agreed-upon RaftState known by this node. No writes are allowed, as all
-// writes to the shared state should happen through the Consensus component
-// methods.
-func (cc *Consensus) State(ctx context.Context) (*RaftState, error) {
-	st, err := cc.consensus.GetLogHead()
-	if err == libp2praft.ErrNoState {
-		return newRaftState(nil), nil
-	}
-
-	if err != nil {
-		return nil, err
-	}
-	state, ok := st.(*RaftState)
-	if !ok {
-		return nil, errors.New("wrong state type")
-	}
-	return state, nil
-}
-
-// Leader returns the peerID of the Leader of the
-// cluster. It returns an error when there is no leader.
-func (cc *Consensus) Leader(ctx context.Context) (peer.ID, error) {
-	// Note the hard-dependency on raft here...
-	raftactor := cc.actor.(*libp2praft.Actor)
-	return raftactor.Leader()
-}
-
-// Clean removes the Raft persisted state.
-func (cc *Consensus) Clean(ctx context.Context) error {
-	//return CleanupRaft(cc.config)
-	return nil
-}
-
-//Rollback replaces the current agreed-upon
-//state with the state provided. Only the consensus leader
-//can perform this operation.
-//func (cc *Consensus) Rollback(state RaftState) error {
-//	// This is unused. It *might* be used for upgrades.
-//	// There is rather untested magic in libp2p-raft's FSM()
-//	// to make this possible.
-//	return cc.consensus.Rollback(state)
-//}
-
-// Peers return the current list of peers in the consensus.
-// The list will be sorted alphabetically.
-func (cc *Consensus) Peers(ctx context.Context) ([]peer.ID, error) {
-
-	peers := []peer.ID{}
-	raftPeers, err := cc.raft.Peers(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("cannot retrieve list of peers: %s", err)
-	}
-
-	sort.Strings(raftPeers)
-
-	for _, p := range raftPeers {
-		id, err := peer.Decode(p)
-		if err != nil {
-			panic("could not decode peer")
-		}
-		peers = append(peers, id)
-	}
-	return peers, nil
-}
-
-func (cc *Consensus) IsLeader(ctx context.Context) bool {
-	leader, _ := cc.Leader(ctx)
-	return leader == cc.host.ID()
-}

lib/consensus/raft/interfaces.go

@@ -1,41 +0,0 @@
-package consensus
-
-import (
-	"context"
-
-	consensus "github.com/libp2p/go-libp2p-consensus"
-	"github.com/libp2p/go-libp2p/core/peer"
-)
-
-type ConsensusAPI interface {
-	// Returns a channel to signal that the consensus layer is ready
-	// allowing the main component to wait for it during start.
-	Ready(context.Context) <-chan struct{}
-
-	AddPeer(context.Context, peer.ID) error
-	RmPeer(context.Context, peer.ID) error
-	State(context.Context) (consensus.State, error)
-	// Provide a node which is responsible to perform
-	// specific tasks which must only run in 1 cluster peer.
-	Leader(context.Context) (peer.ID, error)
-	// Only returns when the consensus state has all log
-	// updates applied to it.
-	WaitForSync(context.Context) error
-	// Clean removes all consensus data.
-	Clean(context.Context) error
-	// Peers returns the peerset participating in the Consensus.
-	Peers(context.Context) ([]peer.ID, error)
-	// IsTrustedPeer returns true if the given peer is "trusted".
-	// This will grant access to more rpc endpoints and a
-	// non-trusted one. This should be fast as it will be
-	// called repeatedly for every remote RPC request.
-	IsTrustedPeer(context.Context, peer.ID) bool
-	// Trust marks a peer as "trusted".
-	Trust(context.Context, peer.ID) error
-	// Distrust removes a peer from the "trusted" set.
-	Distrust(context.Context, peer.ID) error
-	// Returns true if current node is the cluster leader
-	IsLeader(ctx context.Context) bool
-
-	Shutdown(context.Context) error
-}

lib/consensus/raft/raft.go

@@ -1,563 +0,0 @@
-package consensus
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/hashicorp/go-hclog"
-	hraft "github.com/hashicorp/raft"
-	raftboltdb "github.com/hashicorp/raft-boltdb"
-	"github.com/ipfs/go-log/v2"
-	p2praft "github.com/libp2p/go-libp2p-raft"
-	host "github.com/libp2p/go-libp2p/core/host"
-	peer "github.com/libp2p/go-libp2p/core/peer"
-	"go.uber.org/multierr"
-	"go.uber.org/zap"
-
-	"github.com/filecoin-project/lotus/lib/addrutil"
-	"github.com/filecoin-project/lotus/node/repo"
-)
-
-var raftLogger = log.Logger("raft-cluster")
-
-// ErrWaitingForSelf is returned when we are waiting for ourselves to depart
-// the peer set, which won't happen
-var errWaitingForSelf = errors.New("waiting for ourselves to depart")
-
-// RaftMaxSnapshots indicates how many snapshots to keep in the consensus data
-// folder.
-// TODO: Maybe include this in Config. Not sure how useful it is to touch
-// this anyways.
-var RaftMaxSnapshots = 5
-
-// RaftLogCacheSize is the maximum number of logs to cache in-memory.
-// This is used to reduce disk I/O for the recently committed entries.
-var RaftLogCacheSize = 512
-
-// How long we wait for updates during shutdown before snapshotting
-var waitForUpdatesShutdownTimeout = 5 * time.Second
-var waitForUpdatesInterval = 400 * time.Millisecond
-
-// How many times to retry snapshotting when shutting down
-var maxShutdownSnapshotRetries = 5
-
-// raftWrapper wraps the hraft.Raft object and related things like the
-// different stores used or the hraft.Configuration.
-// Its methods provide functionality for working with Raft.
-type raftWrapper struct {
-	ctx           context.Context
-	cancel        context.CancelFunc
-	raft          *hraft.Raft
-	config        *ClusterRaftConfig
-	host          host.Host
-	serverConfig  hraft.Configuration
-	transport     *hraft.NetworkTransport
-	snapshotStore hraft.SnapshotStore
-	logStore      hraft.LogStore
-	stableStore   hraft.StableStore
-	boltdb        *raftboltdb.BoltStore
-	repo          repo.LockedRepo
-	staging       bool
-}
-
-// newRaftWrapper creates a Raft instance and initializes
-// everything leaving it ready to use. Note, that Bootstrap() should be called
-// to make sure the raft instance is usable.
-func newRaftWrapper(
-	host host.Host,
-	cfg *ClusterRaftConfig,
-	fsm hraft.FSM,
-	repo repo.LockedRepo,
-	staging bool,
-) (*raftWrapper, error) {
-
-	raftW := &raftWrapper{}
-	raftW.config = cfg
-	raftW.host = host
-	raftW.staging = staging
-	raftW.repo = repo
-	// Set correct LocalID
-	cfg.RaftConfig.LocalID = hraft.ServerID(host.ID().String())
-
-	df := cfg.GetDataFolder(repo)
-	err := makeDataFolder(df)
-	if err != nil {
-		return nil, err
-	}
-
-	err = raftW.makeServerConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	err = raftW.makeTransport()
-	if err != nil {
-		return nil, err
-	}
-
-	err = raftW.makeStores()
-	if err != nil {
-		return nil, err
-	}
-
-	raftLogger.Debug("creating Raft")
-	raftW.raft, err = hraft.NewRaft(
-		cfg.RaftConfig,
-		fsm,
-		raftW.logStore,
-		raftW.stableStore,
-		raftW.snapshotStore,
-		raftW.transport,
-	)
-	if err != nil {
-		raftLogger.Error("initializing raft: ", err)
-		return nil, err
-	}
-
-	raftW.ctx, raftW.cancel = context.WithCancel(context.Background())
-
-	return raftW, nil
-}
-
-// makeDataFolder creates the folder that is meant to store Raft data. Ensures
-// we always set 0700 mode.
-func makeDataFolder(folder string) error {
-	return os.MkdirAll(folder, 0700)
-}
-
-func (rw *raftWrapper) makeTransport() (err error) {
-	raftLogger.Debug("creating libp2p Raft transport")
-	rw.transport, err = p2praft.NewLibp2pTransport(
-		rw.host,
-		rw.config.NetworkTimeout,
-	)
-	return err
-}
-
-func (rw *raftWrapper) makeStores() error {
-	raftLogger.Debug("creating BoltDB store")
-	df := rw.config.GetDataFolder(rw.repo)
-	store, err := raftboltdb.NewBoltStore(filepath.Join(df, "raft.db"))
-	if err != nil {
-		return err
-	}
-
-	// wraps the store in a LogCache to improve performance.
-	// See consul/agent/consul/server.go
-	cacheStore, err := hraft.NewLogCache(RaftLogCacheSize, store)
-	if err != nil {
-		return err
-	}
-
-	raftLogger.Debug("creating raft snapshot store")
-	snapstore, err := hraft.NewFileSnapshotStoreWithLogger(
-		df,
-		RaftMaxSnapshots,
-		hclog.FromStandardLogger(zap.NewStdLog(log.Logger("raft-snapshot").SugaredLogger.Desugar()), hclog.DefaultOptions),
-	)
-	if err != nil {
-		return err
-	}
-
-	rw.logStore = cacheStore
-	rw.stableStore = store
-	rw.snapshotStore = snapstore
-	rw.boltdb = store
-	return nil
-}
-
-// Bootstrap calls BootstrapCluster on the Raft instance with a valid
-// Configuration (generated from InitPeerset) when Raft has no state
-// and we are not setting up a staging peer. It returns if Raft
-// was boostrapped (true) and an error.
-func (rw *raftWrapper) Bootstrap() (bool, error) {
-	logger.Debug("checking for existing raft states")
-	hasState, err := hraft.HasExistingState(
-		rw.logStore,
-		rw.stableStore,
-		rw.snapshotStore,
-	)
-	if err != nil {
-		return false, err
-	}
-
-	if hasState {
-		logger.Debug("raft cluster is already initialized")
-
-		// Inform the user that we are working with a pre-existing peerset
-		logger.Info("existing Raft state found! raft.InitPeerset will be ignored")
-		cf := rw.raft.GetConfiguration()
-		if err := cf.Error(); err != nil {
-			logger.Debug(err)
-			return false, err
-		}
-		currentCfg := cf.Configuration()
-		srvs := ""
-		for _, s := range currentCfg.Servers {
-			srvs += fmt.Sprintf("        %s\n", s.ID)
-		}
-
-		logger.Debugf("Current Raft Peerset:\n%s\n", srvs)
-		return false, nil
-	}
-
-	if rw.staging {
-		logger.Debug("staging servers do not need initialization")
-		logger.Info("peer is ready to join a cluster")
-		return false, nil
-	}
-
-	voters := ""
-	for _, s := range rw.serverConfig.Servers {
-		voters += fmt.Sprintf("        %s\n", s.ID)
-	}
-
-	logger.Infof("initializing raft cluster with the following voters:\n%s\n", voters)
-
-	future := rw.raft.BootstrapCluster(rw.serverConfig)
-	if err := future.Error(); err != nil {
-		logger.Error("bootstrapping cluster: ", err)
-		return true, err
-	}
-	return true, nil
-}
-
-// create Raft servers configuration. The result is used
-// by Bootstrap() when it proceeds to Bootstrap.
-func (rw *raftWrapper) makeServerConfig() error {
-	peers := []peer.ID{}
-	addrInfos, err := addrutil.ParseAddresses(context.Background(), rw.config.InitPeerset)
-	if err != nil {
-		return err
-	}
-	for _, addrInfo := range addrInfos {
-		peers = append(peers, addrInfo.ID)
-	}
-	rw.serverConfig = makeServerConf(append(peers, rw.host.ID()))
-	return nil
-}
-
-// creates a server configuration with all peers as Voters.
-func makeServerConf(peers []peer.ID) hraft.Configuration {
-	sm := make(map[string]struct{})
-
-	servers := make([]hraft.Server, 0)
-
-	// Servers are peers + self. We avoid duplicate entries below
-	for _, pid := range peers {
-		p := pid.String()
-		_, ok := sm[p]
-		if !ok { // avoid dups
-			sm[p] = struct{}{}
-			servers = append(servers, hraft.Server{
-				Suffrage: hraft.Voter,
-				ID:       hraft.ServerID(p),
-				Address:  hraft.ServerAddress(p),
-			})
-		}
-	}
-	return hraft.Configuration{Servers: servers}
-}
-
-// WaitForLeader holds until Raft says we have a leader.
-// Returns if ctx is canceled.
-func (rw *raftWrapper) WaitForLeader(ctx context.Context) (string, error) {
-	ticker := time.NewTicker(time.Second / 2)
-	for {
-		select {
-		case <-ticker.C:
-			if l := rw.raft.Leader(); l != "" {
-				logger.Debug("waitForleaderTimer")
-				logger.Infof("Current Raft Leader: %s", l)
-				ticker.Stop()
-				return string(l), nil
-			}
-		case <-ctx.Done():
-			return "", ctx.Err()
-		}
-	}
-}
-
-func (rw *raftWrapper) WaitForVoter(ctx context.Context) error {
-	logger.Debug("waiting until we are promoted to a voter")
-
-	pid := hraft.ServerID(rw.host.ID().String())
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			logger.Debugf("%s: get configuration", pid)
-			configFuture := rw.raft.GetConfiguration()
-			if err := configFuture.Error(); err != nil {
-				return err
-			}
-
-			if isVoter(pid, configFuture.Configuration()) {
-				return nil
-			}
-			logger.Debugf("%s: not voter yet", pid)
-
-			time.Sleep(waitForUpdatesInterval)
-		}
-	}
-}
-
-func isVoter(srvID hraft.ServerID, cfg hraft.Configuration) bool {
-	for _, server := range cfg.Servers {
-		if server.ID == srvID && server.Suffrage == hraft.Voter {
-			return true
-		}
-	}
-	return false
-}
-
-// WaitForUpdates holds until Raft has synced to the last index in the log
-func (rw *raftWrapper) WaitForUpdates(ctx context.Context) error {
-
-	logger.Debug("Raft state is catching up to the latest known version. Please wait...")
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			lai := rw.raft.AppliedIndex()
-			li := rw.raft.LastIndex()
-			logger.Debugf("current Raft index: %d/%d",
-				lai, li)
-			if lai == li {
-				return nil
-			}
-			time.Sleep(waitForUpdatesInterval)
-		}
-	}
-}
-
-func (rw *raftWrapper) WaitForPeer(ctx context.Context, pid string, depart bool) error {
-
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			peers, err := rw.Peers(ctx)
-			if err != nil {
-				return err
-			}
-
-			if len(peers) == 1 && pid == peers[0] && depart {
-				return errWaitingForSelf
-			}
-
-			found := find(peers, pid)
-
-			// departing
-			if depart && !found {
-				return nil
-			}
-
-			// joining
-			if !depart && found {
-				return nil
-			}
-
-			time.Sleep(50 * time.Millisecond)
-		}
-	}
-}
-
-// Snapshot tells Raft to take a snapshot.
-func (rw *raftWrapper) Snapshot() error {
-	future := rw.raft.Snapshot()
-	err := future.Error()
-	if err != nil && err.Error() != hraft.ErrNothingNewToSnapshot.Error() {
-		return err
-	}
-	return nil
-}
-
-// snapshotOnShutdown attempts to take a snapshot before a shutdown.
-// Snapshotting might fail if the raft applied index is not the last index.
-// This waits for the updates and tries to take a snapshot when the
-// applied index is up to date.
-// It will retry if the snapshot still fails, in case more updates have arrived.
-// If waiting for updates times-out, it will not try anymore, since something
-// is wrong. This is a best-effort solution as there is no way to tell Raft
-// to stop processing entries because we want to take a snapshot before
-// shutting down.
-func (rw *raftWrapper) snapshotOnShutdown() error {
-	var err error
-	for i := 0; i < maxShutdownSnapshotRetries; i++ {
-		ctx, cancel := context.WithTimeout(context.Background(), waitForUpdatesShutdownTimeout)
-		err = rw.WaitForUpdates(ctx)
-		cancel()
-		if err != nil {
-			logger.Warn("timed out waiting for state updates before shutdown. Snapshotting may fail")
-			return rw.Snapshot()
-		}
-
-		err = rw.Snapshot()
-		if err == nil {
-			return nil // things worked
-		}
-
-		// There was an error
-		err = errors.New("could not snapshot raft: " + err.Error())
-		logger.Warnf("retrying to snapshot (%d/%d)...", i+1, maxShutdownSnapshotRetries)
-	}
-	return err
-}
-
-// Shutdown shutdown Raft and closes the BoltDB.
-func (rw *raftWrapper) Shutdown(ctx context.Context) error {
-
-	rw.cancel()
-
-	var finalErr error
-
-	err := rw.snapshotOnShutdown()
-	if err != nil {
-		finalErr = multierr.Append(finalErr, err)
-	}
-
-	future := rw.raft.Shutdown()
-	err = future.Error()
-	if err != nil {
-		finalErr = multierr.Append(finalErr, err)
-	}
-
-	err = rw.boltdb.Close() // important!
-	if err != nil {
-		finalErr = multierr.Append(finalErr, err)
-	}
-
-	return finalErr
-}
-
-// AddPeer adds a peer to Raft
-func (rw *raftWrapper) AddPeer(ctx context.Context, peerId peer.ID) error {
-
-	// Check that we don't have it to not waste
-	// log entries if so.
-	peers, err := rw.Peers(ctx)
-	if err != nil {
-		return err
-	}
-	if find(peers, peerId.String()) {
-		logger.Infof("%s is already a raft peerStr", peerId.String())
-		return nil
-	}
-
-	err = rw.host.Connect(ctx, peer.AddrInfo{ID: peerId})
-	if err != nil {
-		return err
-	}
-
-	future := rw.raft.AddVoter(
-		hraft.ServerID(peerId.String()),
-		hraft.ServerAddress(peerId.String()),
-		0,
-		0,
-	) // TODO: Extra cfg value?
-	err = future.Error()
-	if err != nil {
-		logger.Error("raft cannot add peer: ", err)
-	}
-	return err
-}
-
-// RemovePeer removes a peer from Raft
-func (rw *raftWrapper) RemovePeer(ctx context.Context, peer string) error {
-	// Check that we have it to not waste
-	// log entries if we don't.
-	peers, err := rw.Peers(ctx)
-	if err != nil {
-		return err
-	}
-	if !find(peers, peer) {
-		logger.Infof("%s is not among raft peers", peer)
-		return nil
-	}
-
-	if len(peers) == 1 && peers[0] == peer {
-		return errors.New("cannot remove ourselves from a 1-peer cluster")
-	}
-
-	rmFuture := rw.raft.RemoveServer(
-		hraft.ServerID(peer),
-		0,
-		0,
-	)
-	err = rmFuture.Error()
-	if err != nil {
-		logger.Error("raft cannot remove peer: ", err)
-		return err
-	}
-
-	return nil
-}
-
-// Leader returns Raft's leader. It may be an empty string if
-// there is no leader or it is unknown.
-func (rw *raftWrapper) Leader(ctx context.Context) string {
-	return string(rw.raft.Leader())
-}
-
-func (rw *raftWrapper) Peers(ctx context.Context) ([]string, error) {
-	ids := make([]string, 0)
-
-	configFuture := rw.raft.GetConfiguration()
-	if err := configFuture.Error(); err != nil {
-		return nil, err
-	}
-
-	for _, server := range configFuture.Configuration().Servers {
-		ids = append(ids, string(server.ID))
-	}
-
-	return ids, nil
-}
-
-// CleanupRaft moves the current data folder to a backup location
-//func CleanupRaft(cfg *Config) error {
-//	dataFolder := cfg.GetDataFolder()
-//	keep := cfg.BackupsRotate
-//
-//	meta, _, err := latestSnapshot(dataFolder)
-//	if meta == nil && err == nil {
-//		// no snapshots at all. Avoid creating backups
-//		// from empty state folders.
-//		logger.Infof("cleaning empty Raft data folder (%s)", dataFolder)
-//		os.RemoveAll(dataFolder)
-//		return nil
-//	}
-//
-//	logger.Infof("cleaning and backing up Raft data folder (%s)", dataFolder)
-//	dbh := newDataBackupHelper(dataFolder, keep)
-//	err = dbh.makeBackup()
-//	if err != nil {
-//		logger.Warn(err)
-//		logger.Warn("the state could not be cleaned properly")
-//		logger.Warn("manual intervention may be needed before starting cluster again")
-//	}
-//	return nil
-//}
-
-// only call when Raft is shutdown
-func (rw *raftWrapper) Clean() error {
-	//return CleanupRaft(rw.config)
-	return nil
-}
-
-func find(s []string, elem string) bool {
-	for _, selem := range s {
-		if selem == elem {
-			return true
-		}
-	}
-	return false
-}

lib/harmony/harmonydb/harmonydb.go

@@ -10,6 +10,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	logging "github.com/ipfs/go-log/v2"
@@ -33,6 +35,8 @@ type DB struct {
 	cfg       *pgxpool.Config
 	schema    string
 	hostnames []string
+	BTFPOnce  sync.Once
+	BTFP      atomic.Uintptr
 }
 
 var logger = logging.Logger("harmonydb")

lib/harmony/harmonydb/userfuncs.go

@@ -3,13 +3,17 @@ package harmonydb
 import (
 	"context"
 	"errors"
+	"runtime"
 
 	"github.com/georgysavva/scany/v2/pgxscan"
 	"github.com/jackc/pgerrcode"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/samber/lo"
 )
 
+var errTx = errors.New("Cannot use a non-transaction func in a transaction")
+
 // rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries.
 // In any package, raw strings will satisfy compilation.  Ex:
 //
@@ -22,6 +26,9 @@ type rawStringOnly string
 // Note, for CREATE & DROP please keep these permanent and express
 // them in the ./sql/ files (next number).
 func (db *DB) Exec(ctx context.Context, sql rawStringOnly, arguments ...any) (count int, err error) {
+	if db.usedInTransaction() {
+		return 0, errTx
+	}
 	res, err := db.pgx.Exec(ctx, string(sql), arguments...)
 	return int(res.RowsAffected()), err
 }
@@ -55,6 +62,9 @@ type Query struct {
 //	   fmt.Println(id, name)
 //	}
 func (db *DB) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (*Query, error) {
+	if db.usedInTransaction() {
+		return &Query{}, errTx
+	}
 	q, err := db.pgx.Query(ctx, string(sql), arguments...)
 	return &Query{q}, err
 }
@@ -66,6 +76,10 @@ type Row interface {
 	Scan(...any) error
 }
 
+type rowErr struct{}
+
+func (rowErr) Scan(_ ...any) error { return errTx }
+
 // QueryRow gets 1 row using column order matching.
 // This is a timesaver for the special case of wanting the first row returned only.
 // EX:
@@ -74,6 +88,9 @@ type Row interface {
 //	var ID = 123
 //	err := db.QueryRow(ctx, "SELECT name, pet FROM users WHERE ID=?", ID).Scan(&name, &pet)
 func (db *DB) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) Row {
+	if db.usedInTransaction() {
+		return rowErr{}
+	}
 	return db.pgx.QueryRow(ctx, string(sql), arguments...)
 }
 
@@ -92,6 +109,9 @@ Ex:
 	err := db.Select(ctx, &users, "SELECT name, id, tel_no FROM customers WHERE pet=?", pet)
 */
 func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error {
+	if db.usedInTransaction() {
+		return errTx
+	}
 	return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...)
 }
 
@@ -100,10 +120,32 @@ type Tx struct {
 	ctx context.Context
 }
 
+// usedInTransaction is a helper to prevent nesting transactions
+// & non-transaction calls in transactions. It only checks 20 frames.
+// Fast: This memory should all be in CPU Caches.
+func (db *DB) usedInTransaction() bool {
+	var framePtrs = (&[20]uintptr{})[:]                   // 20 can be stack-local (no alloc)
+	framePtrs = framePtrs[:runtime.Callers(3, framePtrs)] // skip past our caller.
+	return lo.Contains(framePtrs, db.BTFP.Load())         // Unsafe read @ beginTx overlap, but 'return false' is correct there.
+}
+
 // BeginTransaction is how you can access transactions using this library.
 // The entire transaction happens in the function passed in.
 // The return must be true or a rollback will occur.
+// Be sure to test the error for IsErrSerialization() if you want to retry
+//
+//	when there is a DB serialization error.
+//
+//go:noinline
 func (db *DB) BeginTransaction(ctx context.Context, f func(*Tx) (commit bool, err error)) (didCommit bool, retErr error) {
+	db.BTFPOnce.Do(func() {
+		fp := make([]uintptr, 20)
+		runtime.Callers(1, fp)
+		db.BTFP.Store(fp[0])
+	})
+	if db.usedInTransaction() {
+		return false, errTx
+	}
 	tx, err := db.pgx.BeginTx(ctx, pgx.TxOptions{})
 	if err != nil {
 		return false, err
@@ -156,3 +198,8 @@ func IsErrUniqueContraint(err error) bool {
 	var e2 *pgconn.PgError
 	return errors.As(err, &e2) && e2.Code == pgerrcode.UniqueViolation
 }
+
+func IsErrSerialization(err error) bool {
+	var e2 *pgconn.PgError
+	return errors.As(err, &e2) && e2.Code == pgerrcode.SerializationFailure
+}

lib/harmony/harmonytask/task_type_handler.go

@@ -25,6 +25,8 @@ type taskTypeHandler struct {
 
 func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error)) {
 	var tID TaskID
+	retryWait := time.Millisecond * 100
+retryAddTask:
 	_, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
 		// create taskID (from DB)
 		_, err := tx.Exec(`INSERT INTO harmony_task (name, added_by, posted_time) 
@@ -44,6 +46,11 @@ func (h *taskTypeHandler) AddTask(extra func(TaskID, *harmonydb.Tx) (bool, error
 			log.Debugf("addtask(%s) saw unique constraint, so it's added already.", h.Name)
 			return
 		}
+		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
+			goto retryAddTask
+		}
 		log.Error("Could not add task. AddTasFunc failed: %v", err)
 		return
 	}
@@ -161,7 +168,8 @@ top:
 
 func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done bool, doErr error) {
 	workEnd := time.Now()
-
+	retryWait := time.Millisecond * 100
+retryRecordCompletion:
 	cm, err := h.TaskEngine.db.BeginTransaction(h.TaskEngine.ctx, func(tx *harmonydb.Tx) (bool, error) {
 		var postedTime time.Time
 		err := tx.QueryRow(`SELECT posted_time FROM harmony_task WHERE id=$1`, tID).Scan(&postedTime)
@@ -176,6 +184,9 @@ func (h *taskTypeHandler) recordCompletion(tID TaskID, workStart time.Time, done
 				return false, fmt.Errorf("could not log completion: %w", err)
 			}
 			result = ""
+			if doErr != nil {
+				result = "non-failing error: " + doErr.Error()
+			}
 		} else {
 			if doErr != nil {
 				result = "error: " + doErr.Error()
@@ -214,6 +225,11 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, tID, h.Name, postedTime, workStart, wo
 		return true, nil
 	})
 	if err != nil {
+		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
+			goto retryRecordCompletion
+		}
 		log.Error("Could not record transaction: ", err)
 		return
 	}

node/builder.go

@@ -127,7 +127,6 @@ const (
 	SettlePaymentChannelsKey
 	RunPeerTaggerKey
 	SetupFallbackBlockstoresKey
-	GoRPCServer
 
 	ConsensusReporterKey
 

node/builder_chain.go

@@ -3,7 +3,6 @@ package node
 import (
 	"os"
 
-	gorpc "github.com/libp2p/go-libp2p-gorpc"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
@@ -31,7 +30,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/wallet"
 	ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger"
 	"github.com/filecoin-project/lotus/chain/wallet/remotewallet"
-	raftcns "github.com/filecoin-project/lotus/lib/consensus/raft"
 	"github.com/filecoin-project/lotus/lib/peermgr"
 	"github.com/filecoin-project/lotus/markets/retrievaladapter"
 	"github.com/filecoin-project/lotus/markets/storageadapter"
@@ -251,17 +249,6 @@ func ConfigFullNode(c interface{}) Option {
 			Override(new(wallet.Default), wallet.NilDefault),
 		),
 
-		// Chain node cluster enabled
-		If(cfg.Cluster.ClusterModeEnabled,
-			Override(new(*gorpc.Client), modules.NewRPCClient),
-			Override(new(*raftcns.ClusterRaftConfig), raftcns.NewClusterRaftConfig(&cfg.Cluster)),
-			Override(new(*raftcns.Consensus), raftcns.NewConsensusWithRPCClient(false)),
-			Override(new(*messagesigner.MessageSignerConsensus), messagesigner.NewMessageSignerConsensus),
-			Override(new(messagesigner.MsgSigner), From(new(*messagesigner.MessageSignerConsensus))),
-			Override(new(*modules.RPCHandler), modules.NewRPCHandler),
-			Override(GoRPCServer, modules.NewRPCServer),
-		),
-
 		// Actor event filtering support
 		Override(new(events.EventAPI), From(new(modules.EventAPI))),
 

node/config/def.go

@@ -106,7 +106,6 @@ func DefaultFullNode() *FullNode {
 				HotstoreMaxSpaceSafetyBuffer: 50_000_000_000,
 			},
 		},
-		Cluster: *DefaultUserRaftConfig(),
 		Fevm: FevmConfig{
 			EnableEthRPC:                 false,
 			EthTxHashMappingLifetimeDays: 0,
@@ -327,31 +326,11 @@ const (
 	ResourceFilteringDisabled = ResourceFilteringStrategy("disabled")
 )
 
-var (
-	DefaultDataSubFolder        = "raft"
-	DefaultWaitForLeaderTimeout = 15 * time.Second
-	DefaultCommitRetries        = 1
-	DefaultNetworkTimeout       = 100 * time.Second
-	DefaultCommitRetryDelay     = 200 * time.Millisecond
-	DefaultBackupsRotate        = 6
-)
-
-func DefaultUserRaftConfig() *UserRaftConfig {
-	var cfg UserRaftConfig
-	cfg.DataFolder = "" // empty so it gets omitted
-	cfg.InitPeersetMultiAddr = []string{}
-	cfg.WaitForLeaderTimeout = Duration(DefaultWaitForLeaderTimeout)
-	cfg.NetworkTimeout = Duration(DefaultNetworkTimeout)
-	cfg.CommitRetries = DefaultCommitRetries
-	cfg.CommitRetryDelay = Duration(DefaultCommitRetryDelay)
-	cfg.BackupsRotate = DefaultBackupsRotate
-
-	return &cfg
-}
-
 func DefaultLotusProvider() *LotusProviderConfig {
 	return &LotusProviderConfig{
-		Subsystems: ProviderSubsystemsConfig{},
+		Subsystems: ProviderSubsystemsConfig{
+			GuiAddress: ":4701",
+		},
 		Fees: LotusProviderFees{
 			DefaultMaxFee:      DefaultDefaultMaxFee,
 			MaxPreCommitGasFee: types.MustParseFIL("0.025"),

node/config/doc_gen.go

@@ -494,12 +494,6 @@ Set to 0 to keep all mappings`,
 
 			Comment: ``,
 		},
-		{
-			Name: "Cluster",
-			Type: "UserRaftConfig",
-
-			Comment: ``,
-		},
 		{
 			Name: "Fevm",
 			Type: "FevmConfig",
@@ -1013,6 +1007,18 @@ block rewards will be missed!`,
 
 			Comment: ``,
 		},
+		{
+			Name: "EnableWebGui",
+			Type: "bool",
+
+			Comment: ``,
+		},
+		{
+			Name: "GuiAddress",
+			Type: "string",
+
+			Comment: `The address that should listen for Web GUI requests.`,
+		},
 	},
 	"ProvingConfig": {
 		{
@@ -1680,68 +1686,6 @@ HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`,
 			Comment: ``,
 		},
 	},
-	"UserRaftConfig": {
-		{
-			Name: "ClusterModeEnabled",
-			Type: "bool",
-
-			Comment: `EXPERIMENTAL. config to enabled node cluster with raft consensus`,
-		},
-		{
-			Name: "DataFolder",
-			Type: "string",
-
-			Comment: `A folder to store Raft's data.`,
-		},
-		{
-			Name: "InitPeersetMultiAddr",
-			Type: "[]string",
-
-			Comment: `InitPeersetMultiAddr provides the list of initial cluster peers for new Raft
-peers (with no prior state). It is ignored when Raft was already
-initialized or when starting in staging mode.`,
-		},
-		{
-			Name: "WaitForLeaderTimeout",
-			Type: "Duration",
-
-			Comment: `LeaderTimeout specifies how long to wait for a leader before
-failing an operation.`,
-		},
-		{
-			Name: "NetworkTimeout",
-			Type: "Duration",
-
-			Comment: `NetworkTimeout specifies how long before a Raft network
-operation is timed out`,
-		},
-		{
-			Name: "CommitRetries",
-			Type: "int",
-
-			Comment: `CommitRetries specifies how many times we retry a failed commit until
-we give up.`,
-		},
-		{
-			Name: "CommitRetryDelay",
-			Type: "Duration",
-
-			Comment: `How long to wait between retries`,
-		},
-		{
-			Name: "BackupsRotate",
-			Type: "int",
-
-			Comment: `BackupsRotate specifies the maximum number of Raft's DataFolder
-copies that we keep as backups (renaming) after cleanup.`,
-		},
-		{
-			Name: "Tracing",
-			Type: "bool",
-
-			Comment: `Tracing enables propagation of contexts across binary boundaries.`,
-		},
-	},
 	"Wallet": {
 		{
 			Name: "RemoteBackend",

node/config/types.go

@@ -26,7 +26,6 @@ type FullNode struct {
 	Wallet        Wallet
 	Fees          FeeConfig
 	Chainstore    Chainstore
-	Cluster       UserRaftConfig
 	Fevm          FevmConfig
 	Index         IndexConfig
 	FaultReporter FaultReporterConfig
@@ -96,6 +95,10 @@ type ProviderSubsystemsConfig struct {
 	WindowPostMaxTasks  int
 	EnableWinningPost   bool
 	WinningPostMaxTasks int
+
+	EnableWebGui bool
+	// The address that should listen for Web GUI requests.
+	GuiAddress string
 }
 
 type DAGStoreConfig struct {
@@ -745,33 +748,6 @@ type FeeConfig struct {
 	DefaultMaxFee types.FIL
 }
 
-type UserRaftConfig struct {
-	// EXPERIMENTAL. config to enabled node cluster with raft consensus
-	ClusterModeEnabled bool
-	// A folder to store Raft's data.
-	DataFolder string
-	// InitPeersetMultiAddr provides the list of initial cluster peers for new Raft
-	// peers (with no prior state). It is ignored when Raft was already
-	// initialized or when starting in staging mode.
-	InitPeersetMultiAddr []string
-	// LeaderTimeout specifies how long to wait for a leader before
-	// failing an operation.
-	WaitForLeaderTimeout Duration
-	// NetworkTimeout specifies how long before a Raft network
-	// operation is timed out
-	NetworkTimeout Duration
-	// CommitRetries specifies how many times we retry a failed commit until
-	// we give up.
-	CommitRetries int
-	// How long to wait between retries
-	CommitRetryDelay Duration
-	// BackupsRotate specifies the maximum number of Raft's DataFolder
-	// copies that we keep as backups (renaming) after cleanup.
-	BackupsRotate int
-	// Tracing enables propagation of contexts across binary boundaries.
-	Tracing bool
-}
-
 type FevmConfig struct {
 	// EnableEthRPC enables eth_ rpc, and enables storing a mapping of eth transaction hashes to filecoin message Cids.
 	// This will also enable the RealTimeFilterAPI and HistoricFilterAPI by default, but they can be disabled by config options above.

node/impl/full.go

@@ -34,7 +34,6 @@ type FullNodeAPI struct {
 	full.MsigAPI
 	full.WalletAPI
 	full.SyncAPI
-	full.RaftAPI
 	full.EthAPI
 
 	DS          dtypes.MetadataDS
@@ -119,12 +118,4 @@ func (n *FullNodeAPI) NodeStatus(ctx context.Context, inclChainStatus bool) (sta
 	return status, nil
 }
 
-func (n *FullNodeAPI) RaftState(ctx context.Context) (*api.RaftStateData, error) {
-	return n.RaftAPI.GetRaftState(ctx)
-}
-
-func (n *FullNodeAPI) RaftLeader(ctx context.Context) (peer.ID, error) {
-	return n.RaftAPI.Leader(ctx)
-}
-
 var _ api.FullNode = &FullNodeAPI{}

node/impl/full/eth_utils.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -524,9 +525,6 @@ func ethTxFromNativeMessage(msg *types.Message, st *state.StateTree) (ethtypes.E
 		}
 		to = revertedEthAddress
 	}
-	toPtr := &to
-
-	// Finally, convert the input parameters to "solidity ABI".
 
 	// For empty, we use "0" as the codec. Otherwise, we use CBOR for message
 	// parameters.
@@ -535,31 +533,11 @@ func ethTxFromNativeMessage(msg *types.Message, st *state.StateTree) (ethtypes.E
 		codec = uint64(multicodec.Cbor)
 	}
 
-	// We try to decode the input as an EVM method invocation and/or a contract creation. If
-	// that fails, we encode the "native" parameters as Solidity ABI.
-	var input []byte
-	switch msg.Method {
-	case builtintypes.MethodsEVM.InvokeContract, builtintypes.MethodsEAM.CreateExternal:
-		inp, err := decodePayload(msg.Params, codec)
-		if err == nil {
-			// If this is a valid "create external", unset the "to" address.
-			if msg.Method == builtintypes.MethodsEAM.CreateExternal {
-				toPtr = nil
-			}
-			input = []byte(inp)
-			break
-		}
-		// Yeah, we're going to ignore errors here because the user can send whatever they
-		// want and may send garbage.
-		fallthrough
-	default:
-		input = encodeFilecoinParamsAsABI(msg.Method, codec, msg.Params)
-	}
-
-	return ethtypes.EthTx{
-		To:                   toPtr,
+	// We decode as a native call first.
+	ethTx := ethtypes.EthTx{
+		To:                   &to,
 		From:                 from,
-		Input:                input,
+		Input:                encodeFilecoinParamsAsABI(msg.Method, codec, msg.Params),
 		Nonce:                ethtypes.EthUint64(msg.Nonce),
 		ChainID:              ethtypes.EthUint64(build.Eip155ChainId),
 		Value:                ethtypes.EthBigInt(msg.Value),
@@ -568,7 +546,25 @@ func ethTxFromNativeMessage(msg *types.Message, st *state.StateTree) (ethtypes.E
 		MaxFeePerGas:         ethtypes.EthBigInt(msg.GasFeeCap),
 		MaxPriorityFeePerGas: ethtypes.EthBigInt(msg.GasPremium),
 		AccessList:           []ethtypes.EthHash{},
-	}, nil
+	}
+
+	// Then we try to see if it's "special". If we fail, we ignore the error and keep treating
+	// it as a native message. Unfortunately, the user is free to send garbage that may not
+	// properly decode.
+	if msg.Method == builtintypes.MethodsEVM.InvokeContract {
+		// try to decode it as a contract invocation first.
+		if inp, err := decodePayload(msg.Params, codec); err == nil {
+			ethTx.Input = []byte(inp)
+		}
+	} else if msg.To == builtin.EthereumAddressManagerActorAddr && msg.Method == builtintypes.MethodsEAM.CreateExternal {
+		// Then, try to decode it as a contract deployment from an EOA.
+		if inp, err := decodePayload(msg.Params, codec); err == nil {
+			ethTx.Input = []byte(inp)
+			ethTx.To = nil
+		}
+	}
+
+	return ethTx, nil
 }
 
 func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) (*types.SignedMessage, error) {

node/impl/full/mpool.go

@@ -44,8 +44,6 @@ type MpoolAPI struct {
 	WalletAPI
 	GasAPI
 
-	RaftAPI
-
 	MessageSigner messagesigner.MsgSigner
 
 	PushLocks *dtypes.MpoolLocker
@@ -145,20 +143,6 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 	msg = &cp
 	inMsg := *msg
 
-	// Redirect to leader if current node is not leader. A single non raft based node is always the leader
-	if !a.RaftAPI.IsLeader(ctx) {
-		var signedMsg types.SignedMessage
-		redirected, err := a.RaftAPI.RedirectToLeader(ctx, "MpoolPushMessage", api.MpoolMessageWhole{Msg: msg, Spec: spec}, &signedMsg)
-		if err != nil {
-			return nil, err
-		}
-		// It's possible that the current node became the leader between the check and the redirect
-		// In that case, continue with rest of execution and only return signedMsg if something was redirected
-		if redirected {
-			return &signedMsg, nil
-		}
-	}
-
 	// Generate spec and uuid if not available in the message
 	if spec == nil {
 		spec = &api.MessageSendSpec{

node/impl/full/raft.go

@@ -1,50 +0,0 @@
-package full
-
-import (
-	"context"
-
-	"github.com/libp2p/go-libp2p/core/peer"
-	"go.uber.org/fx"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagesigner"
-)
-
-type RaftAPI struct {
-	fx.In
-
-	MessageSigner *messagesigner.MessageSignerConsensus `optional:"true"`
-}
-
-func (r *RaftAPI) GetRaftState(ctx context.Context) (*api.RaftStateData, error) {
-	if r.MessageSigner == nil {
-		return nil, xerrors.Errorf("raft consensus not enabled. Please check your configuration")
-	}
-	raftState, err := r.MessageSigner.GetRaftState(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return &api.RaftStateData{NonceMap: raftState.NonceMap, MsgUuids: raftState.MsgUuids}, nil
-}
-
-func (r *RaftAPI) Leader(ctx context.Context) (peer.ID, error) {
-	if r.MessageSigner == nil {
-		return "", xerrors.Errorf("raft consensus not enabled. Please check your configuration")
-	}
-	return r.MessageSigner.Leader(ctx)
-}
-
-func (r *RaftAPI) IsLeader(ctx context.Context) bool {
-	if r.MessageSigner == nil {
-		return true
-	}
-	return r.MessageSigner.IsLeader(ctx)
-}
-
-func (r *RaftAPI) RedirectToLeader(ctx context.Context, method string, arg interface{}, ret interface{}) (bool, error) {
-	if r.MessageSigner == nil {
-		return false, xerrors.Errorf("raft consensus not enabled. Please check your configuration")
-	}
-	return r.MessageSigner.RedirectToLeader(ctx, method, arg, ret)
-}

node/modules/rpc.go

@@ -1,55 +0,0 @@
-package modules
-
-import (
-	"context"
-
-	rpc "github.com/libp2p/go-libp2p-gorpc"
-	"github.com/libp2p/go-libp2p/core/host"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/protocol"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/types"
-	consensus "github.com/filecoin-project/lotus/lib/consensus/raft"
-	"github.com/filecoin-project/lotus/node/impl/full"
-)
-
-type RPCHandler struct {
-	mpoolAPI full.MpoolAPI
-	cons     *consensus.Consensus
-}
-
-func NewRPCHandler(mpoolAPI full.MpoolAPI, cons *consensus.Consensus) *RPCHandler {
-	return &RPCHandler{mpoolAPI, cons}
-}
-
-func (h *RPCHandler) MpoolPushMessage(ctx context.Context, msgWhole *api.MpoolMessageWhole, ret *types.SignedMessage) error {
-	signedMsg, err := h.mpoolAPI.MpoolPushMessage(ctx, msgWhole.Msg, msgWhole.Spec)
-	if err != nil {
-		return err
-	}
-	*ret = *signedMsg
-	return nil
-}
-
-func (h *RPCHandler) AddPeer(ctx context.Context, pid peer.ID, ret *struct{}) error {
-	return h.cons.AddPeer(ctx, pid)
-}
-
-// Add other consensus RPC calls here
-
-func NewRPCClient(host host.Host) *rpc.Client {
-	protocolID := protocol.ID("/rpc/lotus-chain/v0")
-	return rpc.NewClient(host, protocolID)
-}
-
-func NewRPCServer(ctx context.Context, host host.Host, rpcHandler *RPCHandler) error {
-
-	authF := func(pid peer.ID, svc, method string) bool {
-		return rpcHandler.cons.IsTrustedPeer(ctx, pid)
-	}
-
-	protocolID := protocol.ID("/rpc/lotus-chain/v0")
-	rpcServer := rpc.NewServer(host, protocolID, rpc.WithAuthorizeFunc(authF))
-	return rpcServer.RegisterName("Consensus", rpcHandler)
-}

provider/lpweb/api/debug/debug.go

@@ -0,0 +1,229 @@
+// Package debug provides the API for various debug endpoints in lotus-provider.
+package debug
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/gorilla/mux"
+	logging "github.com/ipfs/go-log/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api/client"
+	"github.com/filecoin-project/lotus/build"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+)
+
+var log = logging.Logger("lp/web/debug")
+
+type debug struct {
+	*deps.Deps
+}
+
+func Routes(r *mux.Router, deps *deps.Deps) {
+	d := debug{deps}
+	r.HandleFunc("/chain-state-sse", d.chainStateSSE)
+}
+
+type rpcInfo struct {
+	Address   string
+	CLayers   []string
+	Reachable bool
+	SyncState string
+	Version   string
+}
+
+func (d *debug) chainStateSSE(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+
+	ctx := r.Context()
+
+	for {
+
+		type minimalApiInfo struct {
+			Apis struct {
+				ChainApiInfo []string
+			}
+		}
+
+		rpcInfos := map[string]minimalApiInfo{} // config name -> api info
+		confNameToAddr := map[string]string{}   // config name -> api address
+
+		err := forEachConfig[minimalApiInfo](d, func(name string, info minimalApiInfo) error {
+			if len(info.Apis.ChainApiInfo) == 0 {
+				return nil
+			}
+
+			rpcInfos[name] = info
+
+			for _, addr := range info.Apis.ChainApiInfo {
+				ai := cliutil.ParseApiInfo(addr)
+				confNameToAddr[name] = ai.Addr
+			}
+
+			return nil
+		})
+		if err != nil {
+			log.Errorw("getting api info", "error", err)
+			return
+		}
+
+		dedup := map[string]bool{} // for dedup by address
+
+		infos := map[string]rpcInfo{} // api address -> rpc info
+		var infosLk sync.Mutex
+
+		var wg sync.WaitGroup
+		for _, info := range rpcInfos {
+			ai := cliutil.ParseApiInfo(info.Apis.ChainApiInfo[0])
+			if dedup[ai.Addr] {
+				continue
+			}
+			dedup[ai.Addr] = true
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				var clayers []string
+				for layer, a := range confNameToAddr {
+					if a == ai.Addr {
+						clayers = append(clayers, layer)
+					}
+				}
+
+				myinfo := rpcInfo{
+					Address:   ai.Addr,
+					Reachable: false,
+					CLayers:   clayers,
+				}
+				defer func() {
+					infosLk.Lock()
+					defer infosLk.Unlock()
+					infos[ai.Addr] = myinfo
+				}()
+				da, err := ai.DialArgs("v1")
+				if err != nil {
+					log.Warnw("DialArgs", "error", err)
+					return
+				}
+
+				ah := ai.AuthHeader()
+
+				v1api, closer, err := client.NewFullNodeRPCV1(ctx, da, ah)
+				if err != nil {
+					log.Warnf("Not able to establish connection to node with addr: %s", ai.Addr)
+					return
+				}
+				defer closer()
+
+				ver, err := v1api.Version(ctx)
+				if err != nil {
+					log.Warnw("Version", "error", err)
+					return
+				}
+
+				head, err := v1api.ChainHead(ctx)
+				if err != nil {
+					log.Warnw("ChainHead", "error", err)
+					return
+				}
+
+				var syncState string
+				switch {
+				case time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs*3/2): // within 1.5 epochs
+					syncState = "ok"
+				case time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs*5): // within 5 epochs
+					syncState = fmt.Sprintf("slow (%s behind)", time.Since(time.Unix(int64(head.MinTimestamp()), 0)).Truncate(time.Second))
+				default:
+					syncState = fmt.Sprintf("behind (%s behind)", time.Since(time.Unix(int64(head.MinTimestamp()), 0)).Truncate(time.Second))
+				}
+
+				myinfo = rpcInfo{
+					Address:   ai.Addr,
+					CLayers:   clayers,
+					Reachable: true,
+					Version:   ver.Version,
+					SyncState: syncState,
+				}
+			}()
+		}
+		wg.Wait()
+
+		var infoList []rpcInfo
+		for _, i := range infos {
+			infoList = append(infoList, i)
+		}
+		sort.Slice(infoList, func(i, j int) bool {
+			return infoList[i].Address < infoList[j].Address
+		})
+
+		fmt.Fprintf(w, "data: ")
+		err = json.NewEncoder(w).Encode(&infoList)
+		if err != nil {
+			log.Warnw("json encode", "error", err)
+			return
+		}
+		fmt.Fprintf(w, "\n\n")
+		if f, ok := w.(http.Flusher); ok {
+			f.Flush()
+		}
+
+		time.Sleep(time.Duration(build.BlockDelaySecs) * time.Second)
+
+		select { // stop running if there is reader.
+		case <-ctx.Done():
+			return
+		default:
+		}
+	}
+}
+
+func forEachConfig[T any](a *debug, cb func(name string, v T) error) error {
+	confs, err := a.loadConfigs(context.Background())
+	if err != nil {
+		return err
+	}
+
+	for name, tomlStr := range confs { // todo for-each-config
+		var info T
+		if err := toml.Unmarshal([]byte(tomlStr), &info); err != nil {
+			return xerrors.Errorf("unmarshaling %s config: %w", name, err)
+		}
+
+		if err := cb(name, info); err != nil {
+			return xerrors.Errorf("cb: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (d *debug) loadConfigs(ctx context.Context) (map[string]string, error) {
+	//err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
+
+	rows, err := d.DB.Query(ctx, `SELECT title, config FROM harmony_config`)
+	if err != nil {
+		return nil, xerrors.Errorf("getting db configs: %w", err)
+	}
+
+	configs := make(map[string]string)
+	for rows.Next() {
+		var title, config string
+		if err := rows.Scan(&title, &config); err != nil {
+			return nil, xerrors.Errorf("scanning db configs: %w", err)
+		}
+		configs[title] = config
+	}
+
+	return configs, nil
+}

provider/lpweb/api/routes.go

@@ -0,0 +1,13 @@
+// Package api provides the HTTP API for the lotus provider web gui.
+package api
+
+import (
+	"github.com/gorilla/mux"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/provider/lpweb/api/debug"
+)
+
+func Routes(r *mux.Router, deps *deps.Deps) {
+	debug.Routes(r.PathPrefix("/debug").Subrouter(), deps)
+}

provider/lpweb/hapi/routes.go

@@ -0,0 +1,35 @@
+package hapi
+
+import (
+	"embed"
+	"html/template"
+
+	"github.com/gorilla/mux"
+	logging "github.com/ipfs/go-log/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+)
+
+//go:embed web/*
+var templateFS embed.FS
+
+func Routes(r *mux.Router, deps *deps.Deps) error {
+	t, err := template.ParseFS(templateFS, "web/*")
+	if err != nil {
+		return xerrors.Errorf("parse templates: %w", err)
+	}
+
+	a := &app{
+		db: deps.DB,
+		t:  t,
+	}
+
+	r.HandleFunc("/simpleinfo/actorsummary", a.actorSummary)
+	r.HandleFunc("/simpleinfo/machines", a.indexMachines)
+	r.HandleFunc("/simpleinfo/tasks", a.indexTasks)
+	r.HandleFunc("/simpleinfo/taskhistory", a.indexTasksHistory)
+	return nil
+}
+
+var log = logging.Logger("lpweb")

provider/lpweb/hapi/simpleinfo.go

@@ -0,0 +1,187 @@
+package hapi
+
+import (
+	"context"
+	"html/template"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+)
+
+type app struct {
+	db *harmonydb.DB
+	t  *template.Template
+
+	actorInfoLk sync.Mutex
+	actorInfos  []actorInfo
+}
+
+type actorInfo struct {
+	Address string
+	CLayers []string
+
+	QualityAdjustedPower string
+	RawBytePower         string
+
+	Deadlines []actorDeadline
+}
+
+type actorDeadline struct {
+	Empty      bool
+	Current    bool
+	Proven     bool
+	PartFaulty bool
+	Faulty     bool
+}
+
+func (a *app) actorSummary(w http.ResponseWriter, r *http.Request) {
+	a.actorInfoLk.Lock()
+	defer a.actorInfoLk.Unlock()
+
+	a.executeTemplate(w, "actor_summary", a.actorInfos)
+}
+
+func (a *app) indexMachines(w http.ResponseWriter, r *http.Request) {
+	s, err := a.clusterMachineSummary(r.Context())
+	if err != nil {
+		log.Errorf("cluster machine summary: %v", err)
+		http.Error(w, "internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	a.executeTemplate(w, "cluster_machines", s)
+}
+
+func (a *app) indexTasks(w http.ResponseWriter, r *http.Request) {
+	s, err := a.clusterTaskSummary(r.Context())
+	if err != nil {
+		log.Errorf("cluster task summary: %v", err)
+		http.Error(w, "internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	a.executeTemplate(w, "cluster_tasks", s)
+}
+
+func (a *app) indexTasksHistory(w http.ResponseWriter, r *http.Request) {
+	s, err := a.clusterTaskHistorySummary(r.Context())
+	if err != nil {
+		log.Errorf("cluster task history summary: %v", err)
+		http.Error(w, "internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	a.executeTemplate(w, "cluster_task_history", s)
+}
+
+var templateDev = os.Getenv("LOTUS_WEB_DEV") == "1"
+
+func (a *app) executeTemplate(w http.ResponseWriter, name string, data interface{}) {
+	if templateDev {
+		fs := os.DirFS("./cmd/lotus-provider/web/hapi/web")
+		a.t = template.Must(template.ParseFS(fs, "*"))
+	}
+	if err := a.t.ExecuteTemplate(w, name, data); err != nil {
+		log.Errorf("execute template %s: %v", name, err)
+		http.Error(w, "internal server error", http.StatusInternalServerError)
+	}
+}
+
+type machineSummary struct {
+	Address      string
+	ID           int64
+	SinceContact string
+}
+
+type taskSummary struct {
+	Name        string
+	SincePosted string
+	Owner       *string
+	ID          int64
+}
+
+type taskHistorySummary struct {
+	Name   string
+	TaskID int64
+
+	Posted, Start, End string
+
+	Result bool
+	Err    string
+
+	CompletedBy string
+}
+
+func (a *app) clusterMachineSummary(ctx context.Context) ([]machineSummary, error) {
+	rows, err := a.db.Query(ctx, "SELECT id, host_and_port, last_contact FROM harmony_machines")
+	if err != nil {
+		return nil, err // Handle error
+	}
+	defer rows.Close()
+
+	var summaries []machineSummary
+	for rows.Next() {
+		var m machineSummary
+		var lastContact time.Time
+
+		if err := rows.Scan(&m.ID, &m.Address, &lastContact); err != nil {
+			return nil, err // Handle error
+		}
+
+		m.SinceContact = time.Since(lastContact).Round(time.Second).String()
+
+		summaries = append(summaries, m)
+	}
+	return summaries, nil
+}
+
+func (a *app) clusterTaskSummary(ctx context.Context) ([]taskSummary, error) {
+	rows, err := a.db.Query(ctx, "SELECT id, name, update_time, owner_id FROM harmony_task")
+	if err != nil {
+		return nil, err // Handle error
+	}
+	defer rows.Close()
+
+	var summaries []taskSummary
+	for rows.Next() {
+		var t taskSummary
+		var posted time.Time
+
+		if err := rows.Scan(&t.ID, &t.Name, &posted, &t.Owner); err != nil {
+			return nil, err // Handle error
+		}
+
+		t.SincePosted = time.Since(posted).Round(time.Second).String()
+
+		summaries = append(summaries, t)
+	}
+	return summaries, nil
+}
+
+func (a *app) clusterTaskHistorySummary(ctx context.Context) ([]taskHistorySummary, error) {
+	rows, err := a.db.Query(ctx, "SELECT id, name, task_id, posted, work_start, work_end, result, err, completed_by_host_and_port FROM harmony_task_history ORDER BY work_end DESC LIMIT 15")
+	if err != nil {
+		return nil, err // Handle error
+	}
+	defer rows.Close()
+
+	var summaries []taskHistorySummary
+	for rows.Next() {
+		var t taskHistorySummary
+		var posted, start, end time.Time
+
+		if err := rows.Scan(&t.TaskID, &t.Name, &t.TaskID, &posted, &start, &end, &t.Result, &t.Err, &t.CompletedBy); err != nil {
+			return nil, err // Handle error
+		}
+
+		t.Posted = posted.Round(time.Second).Format("02 Jan 06 15:04")
+		t.Start = start.Round(time.Second).Format("02 Jan 06 15:04")
+		t.End = end.Round(time.Second).Format("02 Jan 06 15:04")
+
+		summaries = append(summaries, t)
+	}
+	return summaries, nil
+}

provider/lpweb/hapi/web/actor_summary.gohtml

@@ -0,0 +1,20 @@
+{{define "actor_summary"}}
+{{range .}}
+<tr>
+    <td>{{.Address}}</td>
+    <td>
+        {{range .CLayers}}
+            <span>{{.}} </span>
+        {{end}}
+    </td>
+    <td>{{.QualityAdjustedPower}}</td>
+    <td>
+        <div class="deadline-box">
+            {{range .Deadlines}}
+            <div class="deadline-entry{{if .Current}} deadline-entry-cur{{end}}{{if .Proven}} deadline-proven{{end}}{{if .PartFaulty}} deadline-partially-faulty{{end}}{{if .Faulty}} deadline-faulty{{end}}"></div>
+            {{end}}
+        </div>
+    </td>
+</tr>
+{{end}}
+{{end}}

provider/lpweb/hapi/web/chain_rpcs.gohtml

@@ -0,0 +1,15 @@
+{{define "chain_rpcs"}}
+{{range .}}
+<tr>
+    <td>{{.Address}}</td>
+    <td>
+        {{range .CLayers}}
+        <span>{{.}} </span>
+        {{end}}
+    </td>
+    <td>{{if .Reachable}}<span class="success">ok</span>{{else}}<span class="error">FAIL</span>{{end}}</td>
+    <td>{{if eq "ok" .SyncState}}<span class="success">ok</span>{{else}}<span class="warning">{{.SyncState}}</span>{{end}}</td>
+    <td>{{.Version}}</td>
+</tr>
+{{end}}
+{{end}}

provider/lpweb/hapi/web/cluster_machines.gohtml

@@ -0,0 +1,10 @@
+{{define "cluster_machines"}}
+{{range .}}
+    <tr>
+        <td>{{.Address}}</td>
+        <td>{{.ID}}</td>
+        <td>todo</td>
+        <td>{{.SinceContact}}</td>
+    </tr>
+{{end}}
+{{end}}

provider/lpweb/hapi/web/cluster_task_history.gohtml

@@ -0,0 +1,14 @@
+{{define "cluster_task_history"}}
+    {{range .}}
+        <tr>
+            <td>{{.Name}}</td>
+            <td>{{.TaskID}}</td>
+            <td>{{.CompletedBy}}</td>
+            <td>{{.Posted}}</td>
+            <td>{{.Start}}</td>
+            <td>{{.End}}</td>
+            <td>{{if .Result}}<span class="success">success</span>{{else}}<span class="error">error</span>{{end}}</td>
+            <td>{{.Err}}</td>
+        </tr>
+    {{end}}
+{{end}}

provider/lpweb/hapi/web/cluster_tasks.gohtml

@@ -0,0 +1,10 @@
+{{define "cluster_tasks"}}
+    {{range .}}
+        <tr>
+            <td>{{.Name}}</td>
+            <td>{{.ID}}</td>
+            <td>{{.SincePosted}}</td>
+            <td>{{.Owner}}</td>
+        </tr>
+    {{end}}
+{{end}}

provider/lpweb/srv.go

@@ -0,0 +1,84 @@
+// Package lpweb defines the HTTP web server for static files and endpoints.
+package lpweb
+
+import (
+	"context"
+	"embed"
+	"io"
+	"io/fs"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	"go.opencensus.io/tag"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/provider/lpweb/api"
+	"github.com/filecoin-project/lotus/provider/lpweb/hapi"
+)
+
+//go:embed static
+var static embed.FS
+
+var basePath = "/static/"
+
+// An dev mode hack for no-restart changes to static and templates.
+// You still need to recomplie the binary for changes to go code.
+var webDev = os.Getenv("LOTUS_WEB_DEV") == "1"
+
+func GetSrv(ctx context.Context, deps *deps.Deps) (*http.Server, error) {
+	mx := mux.NewRouter()
+	err := hapi.Routes(mx.PathPrefix("/hapi").Subrouter(), deps)
+	if err != nil {
+		return nil, err
+	}
+	api.Routes(mx.PathPrefix("/api").Subrouter(), deps)
+
+	basePath := basePath
+
+	var static fs.FS = static
+	if webDev {
+		basePath = "cmd/lotus-provider/web/static"
+		static = os.DirFS(basePath)
+	}
+
+	mx.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// If the request is for a directory, redirect to the index file.
+		if strings.HasSuffix(r.URL.Path, "/") {
+			r.URL.Path += "index.html"
+		}
+
+		file, err := static.Open(path.Join(basePath, r.URL.Path)[1:])
+		if err != nil {
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = w.Write([]byte("404 Not Found"))
+			return
+		}
+		defer func() { _ = file.Close() }()
+
+		fileInfo, err := file.Stat()
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte("500 Internal Server Error"))
+			return
+		}
+
+		http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file.(io.ReadSeeker))
+	})
+
+	return &http.Server{
+		Handler: http.HandlerFunc(mx.ServeHTTP),
+		BaseContext: func(listener net.Listener) context.Context {
+			ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-provider"))
+			return ctx
+		},
+		Addr:              deps.Cfg.Subsystems.GuiAddress,
+		ReadTimeout:       time.Minute * 3,
+		ReadHeaderTimeout: time.Minute * 3, // lint
+	}, nil
+}

provider/lpweb/static/chain-connectivity.js

@@ -0,0 +1,73 @@
+import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js';
+window.customElements.define('chain-connectivity', class MyElement extends LitElement {
+    constructor() {
+        super();
+        this.data = [];
+        this.loadData();
+    }
+    loadData() {
+        const eventSource = new EventSource('/api/debug/chain-state-sse');
+        eventSource.onmessage = (event) => {
+            this.data = JSON.parse(event.data);
+            super.requestUpdate();
+        };
+        eventSource.onerror = (error) => {
+            console.error('Error:', error);
+            loadData();
+        };
+    };
+
+    static get styles() {
+        return [css`
+        :host {
+            box-sizing: border-box; /* Don't forgert this to include padding/border inside width calculation */
+        }
+        table {
+            border-collapse: collapse;
+        }
+
+        table td, table th {
+            border-left: 1px solid #f0f0f0;
+            padding: 1px 5px;
+        }
+
+        table tr td:first-child, table tr th:first-child {
+            border-left: none;
+        }
+
+        .success {
+            color: green;
+        }
+        .warning {
+            color: yellow;
+        }
+        .error {
+            color: red;
+        }
+    `];
+    }
+    render = () => html`
+  <table>
+    <thead>
+        <tr>
+            <th>RPC Address</th>
+            <th>Reachability</th>
+            <th>Sync Status</th>
+            <th>Version</th>
+        </tr>
+    </thead>
+    <tbody>
+        ${this.data.map(item => html`
+        <tr>
+            <td>${item.Address}</td>
+            <td>${item.Reachable ? html`<span class="success">ok</span>` : html`<span class="error">FAIL</span>`}</td>
+            <td>${item.SyncState === "ok" ? html`<span class="success">ok</span>` : html`<span class="warning">${item.SyncState}</span>`}</td>
+            <td>${item.Version}</td>
+        </tr>
+        `)}
+        <tr>
+            <td colspan="4">Data incoming...</td>
+        </tr>
+    </tbody>
+  </table>`
+});

provider/lpweb/static/index.html

@@ -0,0 +1,193 @@
+<html>
+<head>
+    <title>Lotus Provider Cluster Overview</title>
+    <script src="https://unpkg.com/htmx.org@1.9.5" integrity="sha384-xcuj3WpfgjlKF+FXhSQFQ0ZNr39ln+hwjN3npfM9VBnUskLolQAcN80McRIVOPuO" crossorigin="anonymous"></script>
+    <script type="module" src="chain-connectivity.js"></script>
+    <style>
+        html, body {
+            background: #0f0f0f;
+            color: #ffffff;
+            padding: 0;
+            margin: 0;
+
+            font-family: monospace;
+        }
+
+        table td, table th {
+            font-size: 13px;
+        }
+
+        .app-head {
+            width: 100%;
+        }
+        .head-left {
+            display: inline-block;
+        }
+        .head-right {
+            display: inline-block;
+            float: right;
+        }
+
+        table {
+            border-collapse: collapse;
+        }
+
+        table td, table th {
+            border-left: 1px solid #f0f0f0;
+            padding: 1px 5px;
+        }
+
+        table tr td:first-child, table tr th:first-child {
+            border-left: none;
+        }
+
+        a:link {
+            color: #cfc;
+        }
+
+        a:visited {
+            color: #dfa;
+        }
+
+        a:hover {
+            color: #af7;
+        }
+
+        .success {
+            color: green;
+        }
+        .warning {
+            color: yellow;
+        }
+        .error {
+            color: red;
+        }
+
+        .dash-tile {
+            display: flex;
+            flex-direction: column;
+            padding: 0.75rem;
+            background: #3f3f3f;
+
+            & b {
+                padding-bottom: 0.5rem;
+                color: deeppink;
+            }
+        }
+
+        .deadline-box {
+            display: grid;
+            grid-template-columns: repeat(16, auto);
+            grid-template-rows: repeat(3, auto);
+            grid-gap: 1px;
+        }
+
+        .deadline-entry {
+            width: 10px;
+            height: 10px;
+            background-color: grey;
+            margin: 1px;
+        }
+
+        .deadline-entry-cur {
+            border-bottom: 3px solid deepskyblue;
+            height: 7px;
+        }
+
+        .deadline-proven {
+            background-color: green;
+        }
+        .deadline-partially-faulty {
+            background-color: yellow;
+        }
+        .deadline-faulty {
+            background-color: red;
+        }
+    </style>
+</head>
+<body>
+<div class="app-head">
+    <div class="head-left">
+        <h1>Lotus Provider Cluster</h1>
+    </div>
+    <div class="head-right">
+        version [todo]
+    </div>
+</div>
+<hr/>
+<div class="page">
+    <div class="info-block">
+        <h2>Chain Connectivity</h2>
+        <chain-connectivity></chain-connectivity>
+    </div>
+    <hr>
+    <div class="info-block">
+        <h2>Actor Summary</h2>
+        <table>
+            <thead>
+            <tr>
+                <th>Address</th>
+                <th>Config Layers</th>
+                <th>QaP</th>
+                <th>Deadlines</th>
+            </tr>
+            </thead>
+            <tbody hx-get="/hapi/simpleinfo/actorsummary" hx-trigger="load,every 5s">
+            </tbody>
+        </table>
+    </div>
+    <hr>
+    <div class="info-block">
+        <h2>Cluster Machines</h2>
+        <table>
+            <thead>
+            <tr>
+                <th>Host</th>
+                <th>ID</th>
+                <th>Config Layers</th>
+                <th>Last Contact</th>
+            </tr>
+            </thead>
+            <tbody hx-get="/hapi/simpleinfo/machines" hx-trigger="load,every 5s">
+            </tbody>
+        </table>
+    </div>
+    <hr>
+    <div class="info-block">
+        <h2>Recently Finished Tasks</h2>
+        <table>
+            <thead>
+            <tr>
+                <th>Name</th>
+                <th>ID</th>
+                <th>Executor</th>
+                <th>Posted</th>
+                <th>Start</th>
+                <th>End</th>
+                <th>Outcome</th>
+                <th>Message</th>
+            </tr>
+            </thead>
+            <tbody hx-get="/hapi/simpleinfo/taskhistory" hx-trigger="load, every 5s">
+            </tbody>
+        </table>
+    </div>
+    <hr>
+    <div class="info-block">
+        <h2>Cluster Tasks</h2>
+        <table>
+            <thead>
+            <tr>
+                <th>Task</th>
+                <th>ID</th>
+                <th>Posted</th>
+                <th>Owner</th>
+            </tr>
+            </thead>
+            <tbody hx-get="/hapi/simpleinfo/tasks" hx-trigger="load,every 5s">
+            </tbody>
+        </table>
+    </div>
+</div>
+</body>
+</html>

provider/lpwinning/winning_task.go

@@ -156,13 +156,18 @@ func (t *WinPostTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don
 		ComputeTime: details.CompTime,
 	}
 
-	persistNoWin := func() error {
-		_, err := t.db.Exec(ctx, `UPDATE mining_base_block SET no_win = true WHERE task_id = $1`, taskID)
+	persistNoWin := func() (bool, error) {
+		n, err := t.db.Exec(ctx, `UPDATE mining_base_block SET no_win = true WHERE task_id = $1`, taskID)
 		if err != nil {
-			return xerrors.Errorf("marking base as not-won: %w", err)
+			return false, xerrors.Errorf("marking base as not-won: %w", err)
+		}
+		log.Debugw("persisted no-win", "rows", n)
+
+		if n == 0 {
+			return false, xerrors.Errorf("persist no win: no rows updated")
 		}
 
-		return nil
+		return true, nil
 	}
 
 	// ensure we have a beacon entry for the epoch we're mining on
@@ -182,13 +187,13 @@ func (t *WinPostTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don
 	if mbi == nil {
 		// not eligible to mine on this base, we're done here
 		log.Debugw("WinPoSt not eligible to mine on this base", "tipset", types.LogCids(base.TipSet.Cids()))
-		return true, persistNoWin()
+		return persistNoWin()
 	}
 
 	if !mbi.EligibleForMining {
 		// slashed or just have no power yet, we're done here
 		log.Debugw("WinPoSt not eligible for mining", "tipset", types.LogCids(base.TipSet.Cids()))
-		return true, persistNoWin()
+		return persistNoWin()
 	}
 
 	if len(mbi.Sectors) == 0 {
@@ -217,7 +222,7 @@ func (t *WinPostTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don
 		if eproof == nil {
 			// not a winner, we're done here
 			log.Debugw("WinPoSt not a winner", "tipset", types.LogCids(base.TipSet.Cids()))
-			return true, persistNoWin()
+			return persistNoWin()
 		}
 	}
 

scripts/docker-lotus-entrypoint.sh

@@ -5,7 +5,7 @@ if [ ! -z $DOCKER_LOTUS_IMPORT_SNAPSHOT ]; then
 	# Don't init if already initialized.
 	if [ ! -f "$GATE" ]; then
 		echo importing minimal snapshot
-		/usr/local/bin/lotus daemon --import-snapshot "$DOCKER_LOTUS_IMPORT_SNAPSHOT" --halt-after-import
+		echo "yes" | /usr/local/bin/lotus daemon --import-snapshot "$DOCKER_LOTUS_IMPORT_SNAPSHOT" --halt-after-import
 		# Block future inits
 		date > "$GATE"
 	fi

storage/paths/db_index.go

@@ -180,12 +180,13 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
 		}
 	}
 
+	retryWait := time.Millisecond * 100
+retryAttachStorage:
 	// Single transaction to attach storage which is not present in the DB
 	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
-
 		var urls sql.NullString
 		var storageId sql.NullString
-		err = dbi.harmonyDB.QueryRow(ctx,
+		err = tx.QueryRow(
 			"Select storage_id, urls FROM storage_path WHERE storage_id = $1", string(si.ID)).Scan(&storageId, &urls)
 		if err != nil && !strings.Contains(err.Error(), "no rows in result set") {
 			return false, xerrors.Errorf("storage attach select fails: %v", err)
@@ -200,7 +201,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
 			}
 			currUrls = union(currUrls, si.URLs)
 
-			_, err = dbi.harmonyDB.Exec(ctx,
+			_, err = tx.Exec(
 				"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9 WHERE storage_id=$10",
 				strings.Join(currUrls, ","),
 				si.Weight,
@@ -220,7 +221,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
 		}
 
 		// Insert storage id
-		_, err = dbi.harmonyDB.Exec(ctx,
+		_, err = tx.Exec(
 			"INSERT INTO storage_path "+
 				"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)",
 			si.ID,
@@ -245,6 +246,11 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
 		return true, nil
 	})
 	if err != nil {
+		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
+			goto retryAttachStorage
+		}
 		return err
 	}
 
@@ -284,22 +290,29 @@ func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url stri
 
 		log.Warnw("Dropping sector path endpoint", "path", id, "url", url)
 	} else {
+		retryWait := time.Millisecond * 100
+	retryDropPath:
 		// Single transaction to drop storage path and sector decls which have this as a storage path
 		_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
 			// Drop storage path completely
-			_, err = dbi.harmonyDB.Exec(ctx, "DELETE FROM storage_path WHERE storage_id=$1", id)
+			_, err = tx.Exec("DELETE FROM storage_path WHERE storage_id=$1", id)
 			if err != nil {
 				return false, err
 			}
 
 			// Drop all sectors entries which use this storage path
-			_, err = dbi.harmonyDB.Exec(ctx, "DELETE FROM sector_location WHERE storage_id=$1", id)
+			_, err = tx.Exec("DELETE FROM sector_location WHERE storage_id=$1", id)
 			if err != nil {
 				return false, err
 			}
 			return true, nil
 		})
 		if err != nil {
+			if harmonydb.IsErrSerialization(err) {
+				time.Sleep(retryWait)
+				retryWait *= 2
+				goto retryDropPath
+			}
 			return err
 		}
 		log.Warnw("Dropping sector storage", "path", id)
@@ -373,9 +386,11 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
 		return xerrors.Errorf("invalid filetype")
 	}
 
+	retryWait := time.Millisecond * 100
+retryStorageDeclareSector:
 	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
 		var currPrimary sql.NullBool
-		err = dbi.harmonyDB.QueryRow(ctx,
+		err = tx.QueryRow(
 			"SELECT is_primary FROM sector_location WHERE miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
 			uint64(s.Miner), uint64(s.Number), int(ft), string(storageID)).Scan(&currPrimary)
 		if err != nil && !strings.Contains(err.Error(), "no rows in result set") {
@@ -385,7 +400,7 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
 		// If storage id already exists for this sector, update primary if need be
 		if currPrimary.Valid {
 			if !currPrimary.Bool && primary {
-				_, err = dbi.harmonyDB.Exec(ctx,
+				_, err = tx.Exec(
 					"UPDATE sector_location set is_primary = TRUE WHERE miner_id=$1 and sector_num=$2 and sector_filetype=$3 and storage_id=$4",
 					s.Miner, s.Number, ft, storageID)
 				if err != nil {
@@ -395,7 +410,7 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
 				log.Warnf("sector %v redeclared in %s", s, storageID)
 			}
 		} else {
-			_, err = dbi.harmonyDB.Exec(ctx,
+			_, err = tx.Exec(
 				"INSERT INTO sector_location "+
 					"values($1, $2, $3, $4, $5)",
 				s.Miner, s.Number, ft, storageID, primary)
@@ -407,6 +422,11 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
 		return true, nil
 	})
 	if err != nil {
+		if harmonydb.IsErrSerialization(err) {
+			time.Sleep(retryWait)
+			retryWait *= 2
+			goto retryStorageDeclareSector
+		}
 		return err
 	}
 
@@ -750,7 +770,7 @@ func (dbi *DBIndex) lock(ctx context.Context, sector abi.SectorID, read storifac
 	_, err := dbi.harmonyDB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
 
 		fts := (read | write).AllSet()
-		err = dbi.harmonyDB.Select(ctx, &rows,
+		err = tx.Select(&rows,
 			`SELECT sector_filetype, read_ts, read_refs, write_ts 
 								 FROM sector_location 
 								 WHERE miner_id=$1 
@@ -792,7 +812,7 @@ func (dbi *DBIndex) lock(ctx context.Context, sector abi.SectorID, read storifac
 		}
 
 		// Acquire write locks
-		_, err = dbi.harmonyDB.Exec(ctx,
+		_, err = tx.Exec(
 			`UPDATE sector_location
 							 	SET write_ts = NOW(), write_lock_owner = $1
 							 	WHERE miner_id=$2
@@ -807,7 +827,7 @@ func (dbi *DBIndex) lock(ctx context.Context, sector abi.SectorID, read storifac
 		}
 
 		// Acquire read locks
-		_, err = dbi.harmonyDB.Exec(ctx,
+		_, err = tx.Exec(
 			`UPDATE sector_location
 							 	SET read_ts = NOW(), read_refs = read_refs + 1
 							 	WHERE miner_id=$1

storage/pipeline/input.go

@@ -600,7 +600,6 @@ func (m *Sealing) updateInput(ctx context.Context, sp abi.RegisteredSealProof) e
 	log.Debugw("updateInput matching done", "matches", len(matches), "toAssign", len(toAssign), "assigned", assigned, "openSectors", len(m.openSectors), "pieces", len(m.pendingPieces))
 
 	if len(toAssign) > 0 {
-		log.Errorf("we are trying to create a new sector with open sectors %v", m.openSectors)
 		if err := m.tryGetDealSector(ctx, sp, getExpirationCached); err != nil {
 			log.Errorw("Failed to create a new sector for deals", "error", err)
 		}