diff --git a/api/api_full.go b/api/api_full.go index a90b0c89f..8631ec4b7 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -664,6 +664,11 @@ type FullNode interface { PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) //perm:write PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) //perm:sign + // MethodGroup: Node + // These methods are general node management and status commands + + NodeStatus(ctx context.Context, inclChainStatus bool) (NodeStatus, error) //perm:read + // CreateBackup creates node backup onder the specified file name. The // method requires that the lotus daemon is running with the // LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that diff --git a/api/mocks/mock_full.go b/api/mocks/mock_full.go index 4336a56f9..ede04fa20 100644 --- a/api/mocks/mock_full.go +++ b/api/mocks/mock_full.go @@ -1692,6 +1692,21 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0) } +// NodeStatus mocks base method +func (m *MockFullNode) NodeStatus(arg0 context.Context, arg1 bool) (api.NodeStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeStatus", arg0, arg1) + ret0, _ := ret[0].(api.NodeStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeStatus indicates an expected call of NodeStatus +func (mr *MockFullNodeMockRecorder) NodeStatus(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeStatus", reflect.TypeOf((*MockFullNode)(nil).NodeStatus), arg0, arg1) +} + // PaychAllocateLane mocks base method func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) { m.ctrl.T.Helper() diff --git a/api/proxy_gen.go b/api/proxy_gen.go index bfaaade94..b743a2ddb 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -287,6 +287,8 @@ type FullNodeStruct struct { MsigSwapPropose func(p0 context.Context, p1 address.Address, p2 address.Address, p3 address.Address, p4 address.Address) (cid.Cid, error) `perm:"sign"` + NodeStatus func(p0 context.Context, p1 bool) (NodeStatus, error) `perm:"read"` + PaychAllocateLane func(p0 context.Context, p1 address.Address) (uint64, error) `perm:"sign"` PaychAvailableFunds func(p0 context.Context, p1 address.Address) (*ChannelAvailableFunds, error) `perm:"sign"` @@ -1715,6 +1717,14 @@ func (s *FullNodeStub) MsigSwapPropose(p0 context.Context, p1 address.Address, p return *new(cid.Cid), xerrors.New("method not supported") } +func (s *FullNodeStruct) NodeStatus(p0 context.Context, p1 bool) (NodeStatus, error) { + return s.Internal.NodeStatus(p0, p1) +} + +func (s *FullNodeStub) NodeStatus(p0 context.Context, p1 bool) (NodeStatus, error) { + return *new(NodeStatus), xerrors.New("method not supported") +} + func (s *FullNodeStruct) PaychAllocateLane(p0 context.Context, p1 address.Address) (uint64, error) { return s.Internal.PaychAllocateLane(p0, p1) } diff --git a/api/types.go b/api/types.go index 6417ce756..bbcfa5c20 100644 --- a/api/types.go +++ b/api/types.go @@ -116,3 +116,24 @@ type ConnMgrInfo struct { Tags map[string]int Conns map[string]time.Time } + +type NodeStatus struct { + SyncStatus NodeSyncStatus + PeerStatus NodePeerStatus + ChainStatus NodeChainStatus +} + +type NodeSyncStatus struct { + Epoch uint64 + Behind uint64 +} + +type NodePeerStatus struct { + PeersToPublishMsgs int + PeersToPublishBlocks int +} + +type NodeChainStatus struct { + BlocksPerTipsetLast100 float64 + BlocksPerTipsetLastFinality float64 +} diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index df2b87f13..7763e4af8 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index a3407e7ca..501b23828 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 04d82b39a..dcb410ce0 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/cli/cmd.go b/cli/cmd.go index 6ecd236f4..09bf5c461 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -81,6 +81,7 @@ var Commands = []*cli.Command{ WithCategory("developer", FetchParamCmd), WithCategory("network", NetCmd), WithCategory("network", SyncCmd), + WithCategory("status", StatusCmd), PprofCmd, VersionCmd, } diff --git a/cli/status.go b/cli/status.go new file mode 100644 index 000000000..75f91196a --- /dev/null +++ b/cli/status.go @@ -0,0 +1,60 @@ +package cli + +import ( + "fmt" + + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/build" +) + +var StatusCmd = &cli.Command{ + Name: "status", + Usage: "Check node status", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "chain", + Usage: "include chain health status", + }, + }, + + Action: func(cctx *cli.Context) error { + apic, closer, err := GetFullNodeAPIV1(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + inclChainStatus := cctx.Bool("chain") + + status, err := apic.NodeStatus(ctx, inclChainStatus) + if err != nil { + return err + } + + fmt.Printf("Sync Epoch: %d\n", status.SyncStatus.Epoch) + fmt.Printf("Epochs Behind: %d\n", status.SyncStatus.Behind) + fmt.Printf("Peers to Publish Messages: %d\n", status.PeerStatus.PeersToPublishMsgs) + fmt.Printf("Peers to Publish Blocks: %d\n", status.PeerStatus.PeersToPublishBlocks) + + if inclChainStatus && status.SyncStatus.Epoch > uint64(build.Finality) { + var ok100, okFin string + if status.ChainStatus.BlocksPerTipsetLast100 >= 4.75 { + ok100 = "[OK]" + } else { + ok100 = "[UNHEALTHY]" + } + if status.ChainStatus.BlocksPerTipsetLastFinality >= 4.75 { + okFin = "[OK]" + } else { + okFin = "[UNHEALTHY]" + } + + fmt.Printf("Blocks per TipSet in last 100 epochs: %f %s\n", status.ChainStatus.BlocksPerTipsetLast100, ok100) + fmt.Printf("Blocks per TipSet in last finality: %f %s\n", status.ChainStatus.BlocksPerTipsetLastFinality, okFin) + } + + return nil + }, +} diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 261c0d51b..a0ee6fcf3 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -126,6 +126,8 @@ * [NetPeerInfo](#NetPeerInfo) * [NetPeers](#NetPeers) * [NetPubsubScores](#NetPubsubScores) +* [Node](#Node) + * [NodeStatus](#NodeStatus) * [Paych](#Paych) * [PaychAllocateLane](#PaychAllocateLane) * [PaychAvailableFunds](#PaychAvailableFunds) @@ -3000,6 +3002,40 @@ Inputs: `null` Response: `null` +## Node +These methods are general node management and status commands + + +### NodeStatus +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + true +] +``` + +Response: +```json +{ + "SyncStatus": { + "Epoch": 42, + "Behind": 42 + }, + "PeerStatus": { + "PeersToPublishMsgs": 123, + "PeersToPublishBlocks": 123 + }, + "ChainStatus": { + "BlocksPerTipsetLast100": 12.3, + "BlocksPerTipsetLastFinality": 12.3 + } +} +``` + ## Paych The Paych methods are for interacting with and managing payment channels diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index ebd3300f0..a3ac4d487 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -31,6 +31,8 @@ COMMANDS: NETWORK: net Manage P2P Network sync Inspect or interact with the chain syncer + STATUS: + status Check node status GLOBAL OPTIONS: --help, -h show help (default: false) @@ -2671,3 +2673,20 @@ OPTIONS: --help, -h show help (default: false) ``` + +## lotus status +``` +NAME: + lotus status - Check node status + +USAGE: + lotus status [command options] [arguments...] + +CATEGORY: + STATUS + +OPTIONS: + --chain include chain health status (default: false) + --help, -h show help (default: false) + +``` diff --git a/node/impl/full.go b/node/impl/full.go index add40917c..50fd09cdf 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -2,16 +2,21 @@ package impl import ( "context" + "time" + + "github.com/libp2p/go-libp2p-core/peer" logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/node/impl/client" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/market" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/modules/lp2p" ) var log = logging.Logger("node") @@ -30,11 +35,86 @@ type FullNodeAPI struct { full.SyncAPI full.BeaconAPI - DS dtypes.MetadataDS + DS dtypes.MetadataDS + NetworkName dtypes.NetworkName } func (n *FullNodeAPI) CreateBackup(ctx context.Context, fpath string) error { return backup(n.DS, fpath) } +func (n *FullNodeAPI) NodeStatus(ctx context.Context, inclChainStatus bool) (status api.NodeStatus, err error) { + curTs, err := n.ChainHead(ctx) + if err != nil { + return status, err + } + + status.SyncStatus.Epoch = uint64(curTs.Height()) + timestamp := time.Unix(int64(curTs.MinTimestamp()), 0) + delta := time.Since(timestamp).Seconds() + status.SyncStatus.Behind = uint64(delta / 30) + + // get peers in the messages and blocks topics + peersMsgs := make(map[peer.ID]struct{}) + peersBlocks := make(map[peer.ID]struct{}) + + for _, p := range n.PubSub.ListPeers(build.MessagesTopic(n.NetworkName)) { + peersMsgs[p] = struct{}{} + } + + for _, p := range n.PubSub.ListPeers(build.BlocksTopic(n.NetworkName)) { + peersBlocks[p] = struct{}{} + } + + // get scores for all connected and recent peers + scores, err := n.NetPubsubScores(ctx) + if err != nil { + return status, err + } + + for _, score := range scores { + if score.Score.Score > lp2p.PublishScoreThreshold { + _, inMsgs := peersMsgs[score.ID] + if inMsgs { + status.PeerStatus.PeersToPublishMsgs++ + } + + _, inBlocks := peersBlocks[score.ID] + if inBlocks { + status.PeerStatus.PeersToPublishBlocks++ + } + } + } + + if inclChainStatus && status.SyncStatus.Epoch > uint64(build.Finality) { + blockCnt := 0 + ts := curTs + + for i := 0; i < 100; i++ { + blockCnt += len(ts.Blocks()) + tsk := ts.Parents() + ts, err = n.ChainGetTipSet(ctx, tsk) + if err != nil { + return status, err + } + } + + status.ChainStatus.BlocksPerTipsetLast100 = float64(blockCnt) / 100 + + for i := 100; i < int(build.Finality); i++ { + blockCnt += len(ts.Blocks()) + tsk := ts.Parents() + ts, err = n.ChainGetTipSet(ctx, tsk) + if err != nil { + return status, err + } + } + + status.ChainStatus.BlocksPerTipsetLastFinality = float64(blockCnt) / float64(build.Finality) + + } + + return status, nil +} + var _ api.FullNode = &FullNodeAPI{} diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 748167d95..32b85daf3 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -36,6 +36,15 @@ func init() { pubsub.GossipSubHistoryLength = 10 pubsub.GossipSubGossipFactor = 0.1 } + +const ( + GossipScoreThreshold = -500 + PublishScoreThreshold = -1000 + GraylistScoreThreshold = -2500 + AcceptPXScoreThreshold = 1000 + OpportunisticGraftScoreThreshold = 3.5 +) + func ScoreKeeper() *dtypes.ScoreKeeper { return new(dtypes.ScoreKeeper) } @@ -256,11 +265,11 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { Topics: topicParams, }, &pubsub.PeerScoreThresholds{ - GossipThreshold: -500, - PublishThreshold: -1000, - GraylistThreshold: -2500, - AcceptPXThreshold: 1000, - OpportunisticGraftThreshold: 3.5, + GossipThreshold: GossipScoreThreshold, + PublishThreshold: PublishScoreThreshold, + GraylistThreshold: GraylistScoreThreshold, + AcceptPXThreshold: AcceptPXScoreThreshold, + OpportunisticGraftThreshold: OpportunisticGraftScoreThreshold, }, ), pubsub.WithPeerScoreInspect(in.Sk.Update, 10*time.Second),