Merge pull request #1294 from natewalck/more-opencensus-metrics

More opencensus metrics

.gitignore

@@ -3,6 +3,7 @@
 /lotus-seal-worker
 /lotus-seed
 /lotus-health
+/lotus-shed
 /pond
 /townhall
 /fountain

Makefile

@@ -77,8 +77,8 @@ lotus-shed: $(BUILD_DEPS)
 	rm -f lotus-shed
 	go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
 	go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build
-.PHONY: lotus-seal-worker
-BINS+=lotus-seal-worker
+.PHONY: lotus-shed
+BINS+=lotus-shed
 
 build: lotus lotus-storage-miner lotus-seal-worker
 	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \

api/api_common.go

@@ -23,6 +23,7 @@ type Common interface {
 	NetConnect(context.Context, peer.AddrInfo) error
 	NetAddrsListen(context.Context) (peer.AddrInfo, error)
 	NetDisconnect(context.Context, peer.ID) error
+	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
 
 	// ID returns peerID of libp2p node backing this API
 	ID(context.Context) (peer.ID, error)

api/api_full.go

@@ -71,6 +71,7 @@ type FullNode interface {
 	WalletBalance(context.Context, address.Address) (types.BigInt, error)
 	WalletSign(context.Context, address.Address, []byte) (*types.Signature, error)
 	WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error)
+	WalletVerify(context.Context, address.Address, []byte, *types.Signature) bool
 	WalletDefaultAddress(context.Context) (address.Address, error)
 	WalletSetDefault(context.Context, address.Address) error
 	WalletExport(context.Context, address.Address) (*types.KeyInfo, error)

api/apistruct/struct.go

@@ -29,6 +29,7 @@ type CommonStruct struct {
 		NetConnect       func(context.Context, peer.AddrInfo) error                    `perm:"write"`
 		NetAddrsListen   func(context.Context) (peer.AddrInfo, error)                  `perm:"read"`
 		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
+		NetFindPeer      func(context.Context, peer.ID) (peer.AddrInfo, error)         `perm:"read"`
 
 		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
 		Version func(context.Context) (api.Version, error) `perm:"read"`
@@ -82,6 +83,7 @@ type FullNodeStruct struct {
 		WalletBalance        func(context.Context, address.Address) (types.BigInt, error)                         `perm:"read"`
 		WalletSign           func(context.Context, address.Address, []byte) (*types.Signature, error)             `perm:"sign"`
 		WalletSignMessage    func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
+		WalletVerify         func(context.Context, address.Address, []byte, *types.Signature) bool                `perm:"read"`
 		WalletDefaultAddress func(context.Context) (address.Address, error)                                       `perm:"write"`
 		WalletSetDefault     func(context.Context, address.Address) error                                         `perm:"admin"`
 		WalletExport         func(context.Context, address.Address) (*types.KeyInfo, error)                       `perm:"admin"`
@@ -197,6 +199,10 @@ func (c *CommonStruct) NetDisconnect(ctx context.Context, p peer.ID) error {
 	return c.Internal.NetDisconnect(ctx, p)
 }
 
+func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
+	return c.Internal.NetFindPeer(ctx, p)
+}
+
 // ID implements API.ID
 func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 	return c.Internal.ID(ctx)
@@ -306,6 +312,10 @@ func (c *FullNodeStruct) WalletSignMessage(ctx context.Context, k address.Addres
 	return c.Internal.WalletSignMessage(ctx, k, msg)
 }
 
+func (c *FullNodeStruct) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *types.Signature) bool {
+	return c.Internal.WalletVerify(ctx, k, msg, sig)
+}
+
 func (c *FullNodeStruct) WalletDefaultAddress(ctx context.Context) (address.Address, error) {
 	return c.Internal.WalletDefaultAddress(ctx)
 }

chain/store/store.go

@@ -13,6 +13,8 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/metrics"
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 	"go.uber.org/multierr"
 
@@ -100,7 +102,15 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls *types.VMSy
 		return nil
 	}
 
-	cs.headChangeNotifs = append(cs.headChangeNotifs, hcnf)
+	hcmetric := func(rev, app []*types.TipSet) error {
+		ctx := context.Background()
+		for _, r := range app {
+			stats.Record(ctx, metrics.ChainNodeHeight.M(int64(r.Height())))
+		}
+		return nil
+	}
+
+	cs.headChangeNotifs = append(cs.headChangeNotifs, hcnf, hcmetric)
 
 	return cs
 }

chain/sub/incoming.go

@@ -2,6 +2,7 @@ package sub
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	lru "github.com/hashicorp/golang-lru"
@@ -10,11 +11,14 @@ import (
 	connmgr "github.com/libp2p/go-libp2p-core/connmgr"
 	peer "github.com/libp2p/go-libp2p-peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/metrics"
 )
 
 var log = logging.Logger("sub")
@@ -107,15 +111,25 @@ func (bv *BlockValidator) flagPeer(p peer.ID) {
 }
 
 func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) bool {
+	stats.Record(ctx, metrics.BlockReceived.M(1))
+	ctx, _ = tag.New(
+		ctx,
+		tag.Insert(metrics.PeerID, pid.String()),
+		tag.Insert(metrics.ReceivedFrom, msg.ReceivedFrom.String()),
+	)
 	blk, err := types.DecodeBlockMsg(msg.GetData())
 	if err != nil {
 		log.Error("got invalid block over pubsub: ", err)
+		ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, "invalid"))
+		stats.Record(ctx, metrics.BlockValidationFailure.M(1))
 		bv.flagPeer(pid)
 		return false
 	}
 
 	if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit {
 		log.Warnf("received block with too many messages over pubsub")
+		ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, "too_many_messages"))
+		stats.Record(ctx, metrics.BlockValidationFailure.M(1))
 		bv.flagPeer(pid)
 		return false
 	}
@@ -127,6 +141,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
 	}
 
 	msg.ValidatorData = blk
+	stats.Record(ctx, metrics.BlockValidationSuccess.M(1))
 	return true
 }
 
@@ -153,9 +168,44 @@ func (brc *blockReceiptCache) add(bcid cid.Cid) int {
 	return val.(int)
 }
 
+type MessageValidator struct {
+	mpool *messagepool.MessagePool
+}
+
+func NewMessageValidator(mp *messagepool.MessagePool) *MessageValidator {
+	return &MessageValidator{mp}
+}
+
+func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) bool {
+	stats.Record(ctx, metrics.MessageReceived.M(1))
+	ctx, _ = tag.New(ctx, tag.Insert(metrics.PeerID, pid.String()))
+	m, err := types.DecodeSignedMessage(msg.Message.GetData())
+	if err != nil {
+		log.Warnf("failed to decode incoming message: %s", err)
+		ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, "decode"))
+		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		return false
+	}
+
+	if err := mv.mpool.Add(m); err != nil {
+		log.Warnf("failed to add message from network to message pool (From: %s, To: %s, Nonce: %d, Value: %s): %s", m.Message.From, m.Message.To, m.Message.Nonce, types.FIL(m.Message.Value), err)
+		ctx, _ = tag.New(
+			ctx,
+			tag.Insert(metrics.MessageFrom, m.Message.From.String()),
+			tag.Insert(metrics.MessageTo, m.Message.To.String()),
+			tag.Insert(metrics.MessageNonce, fmt.Sprint(m.Message.Nonce)),
+			tag.Insert(metrics.FailureType, "add"),
+		)
+		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		return false
+	}
+	stats.Record(ctx, metrics.MessageValidationSuccess.M(1))
+	return true
+}
+
 func HandleIncomingMessages(ctx context.Context, mpool *messagepool.MessagePool, msub *pubsub.Subscription) {
 	for {
-		msg, err := msub.Next(ctx)
+		_, err := msub.Next(ctx)
 		if err != nil {
 			log.Warn("error from message subscription: ", err)
 			if ctx.Err() != nil {
@@ -165,15 +215,6 @@ func HandleIncomingMessages(ctx context.Context, mpool *messagepool.MessagePool,
 			continue
 		}
 
-		m, ok := msg.ValidatorData.(*types.SignedMessage)
-		if !ok {
-			log.Errorf("message validator func passed on wrong type: %#v", msg.ValidatorData)
-			continue
-		}
-
-		if err := mpool.Add(m); err != nil {
-			log.Warnf("failed to add message from network to message pool (From: %s, To: %s, Nonce: %d, Value: %s): %s", m.Message.From, m.Message.To, m.Message.Nonce, types.FIL(m.Message.Value), err)
-			continue
-		}
+		// Do nothing... everything happens in validate
 	}
 }

chain/sync.go

@@ -23,6 +23,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/peer"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"github.com/whyrusleeping/pubsub"
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
@@ -38,6 +39,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/sigs"
+	"github.com/filecoin-project/lotus/metrics"
 )
 
 var log = logging.Logger("chain")
@@ -1038,6 +1040,7 @@ func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []*
 			return xerrors.Errorf("message processing failed: %w", err)
 		}
 
+		stats.Record(ctx, metrics.ChainNodeWorkerHeight.M(int64(fts.TipSet().Height())))
 		ss.SetHeight(fts.TipSet().Height())
 
 		return nil

cli/cmd.go

@@ -28,6 +28,20 @@ const (
 	metadataTraceConetxt = "traceContext"
 )
 
+// custom CLI error
+
+type ErrCmdFailed struct {
+	msg string
+}
+
+func (e *ErrCmdFailed) Error() string {
+	return e.msg
+}
+
+func NewCliError(s string) error {
+	return &ErrCmdFailed{s}
+}
+
 // ApiConnector returns API instance
 type ApiConnector func() api.FullNode
 

cli/net.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"fmt"
+	"github.com/libp2p/go-libp2p-core/peer"
 	"sort"
 	"strings"
 
@@ -18,6 +19,7 @@ var netCmd = &cli.Command{
 		netConnect,
 		netListen,
 		netId,
+		netFindPeer,
 	},
 }
 
@@ -123,3 +125,37 @@ var netId = &cli.Command{
 		return nil
 	},
 }
+
+var netFindPeer = &cli.Command{
+	Name:      "findpeer",
+	Usage:     "Find the addresses of a given peerID",
+	ArgsUsage: "<peer ID>",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			fmt.Println("Usage: findpeer [peer ID]")
+			return nil
+		}
+
+		pid, err := peer.IDB58Decode(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		addrs, err := api.NetFindPeer(ctx, pid)
+
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(addrs)
+		return nil
+	},
+}

cli/state.go

@@ -17,6 +17,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/peer"
 	"golang.org/x/xerrors"
 
+	"github.com/docker/go-units"
 	"github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"gopkg.in/urfave/cli.v2"
@@ -48,6 +49,67 @@ var stateCmd = &cli.Command{
 		stateCallCmd,
 		stateGetDealSetCmd,
 		stateWaitMsgCmd,
+		stateMinerInfo,
+	},
+}
+
+var stateMinerInfo = &cli.Command{
+	Name:  "miner-info",
+	Usage: "Retrieve miner information",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must specify miner to get information for")
+		}
+
+		addr, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		ts, err := loadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		act, err := api.StateGetActor(ctx, addr, ts.Key())
+		if err != nil {
+			return err
+		}
+
+		aso, err := api.ChainReadObj(ctx, act.Head)
+		if err != nil {
+			return err
+		}
+
+		var mst actors.StorageMinerActorState
+		if err := mst.UnmarshalCBOR(bytes.NewReader(aso)); err != nil {
+			return err
+		}
+
+		mio, err := api.ChainReadObj(ctx, mst.Info)
+		if err != nil {
+			return err
+		}
+
+		var mi actors.MinerInfo
+		if err := mi.UnmarshalCBOR(bytes.NewReader(mio)); err != nil {
+			return err
+		}
+
+		fmt.Printf("Owner:\t%s\n", mi.Owner)
+		fmt.Printf("Worker:\t%s\n", mi.Worker)
+		fmt.Printf("PeerID:\t%s\n", mi.PeerID)
+		fmt.Printf("SectorSize:\t%s (%d)\n", units.BytesSize(float64(mi.SectorSize)), mi.SectorSize)
+
+		return nil
 	},
 }
 

cli/wallet.go

@@ -24,6 +24,8 @@ var walletCmd = &cli.Command{
 		walletImport,
 		walletGetDefault,
 		walletSetDefault,
+		walletSign,
+		walletVerify,
 	},
 }
 
@@ -235,3 +237,94 @@ var walletImport = &cli.Command{
 		return nil
 	},
 }
+
+var walletSign = &cli.Command{
+	Name:      "sign",
+	Usage:     "sign a message",
+	ArgsUsage: "<signing address> <hexMessage>",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() || cctx.NArg() != 2 {
+			return fmt.Errorf("must specify signing address and message to sign")
+		}
+
+		addr, err := address.NewFromString(cctx.Args().First())
+
+		if err != nil {
+			return err
+		}
+
+		msg, err := hex.DecodeString(cctx.Args().Get(1))
+
+		if err != nil {
+			return err
+		}
+
+		sig, err := api.WalletSign(ctx, addr, msg)
+
+		if err != nil {
+			return err
+		}
+
+		sigBytes := append([]byte{byte(sig.TypeCode())}, sig.Data...)
+
+		fmt.Println(hex.EncodeToString(sigBytes))
+		return nil
+	},
+}
+
+var walletVerify = &cli.Command{
+	Name:      "verify",
+	Usage:     "verify the signature of a message",
+	ArgsUsage: "<signing address> <hexMessage> <signature>",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() || cctx.NArg() != 3 {
+			return fmt.Errorf("must specify signing address, message, and signature to verify")
+		}
+
+		addr, err := address.NewFromString(cctx.Args().First())
+
+		if err != nil {
+			return err
+		}
+
+		msg, err := hex.DecodeString(cctx.Args().Get(1))
+
+		if err != nil {
+			return err
+		}
+
+		sigBytes, err := hex.DecodeString(cctx.Args().Get(2))
+
+		if err != nil {
+			return err
+		}
+
+		sig, err := types.SignatureFromBytes(sigBytes)
+
+		if err != nil {
+			return err
+		}
+
+		if api.WalletVerify(ctx, addr, msg, &sig) {
+			fmt.Println("valid")
+			return nil
+		} else {
+			fmt.Println("invalid")
+			return NewCliError("CLI Verify called with invalid signature")
+		}
+	},
+}

cmd/lotus-chainwatch/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	logging "github.com/ipfs/go-log/v2"
@@ -59,6 +60,10 @@ var runCmd = &cli.Command{
 			Name:  "front",
 			Value: "127.0.0.1:8418",
 		},
+		&cli.IntFlag{
+			Name:  "max-batch",
+			Value: 1000,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
@@ -75,13 +80,15 @@ var runCmd = &cli.Command{
 
 		log.Info("Remote version: %s", v.Version)
 
+		maxBatch := cctx.Int("max-batch")
+
 		st, err := openStorage(cctx.String("db"))
 		if err != nil {
 			return err
 		}
 		defer st.close()
 
-		runSyncer(ctx, api, st)
+		runSyncer(ctx, api, st, maxBatch)
 
 		h, err := newHandler(api, st)
 		if err != nil {

cmd/lotus-chainwatch/sync.go

@@ -18,9 +18,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-const maxBatch = 3000
-
-func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
+func runSyncer(ctx context.Context, api api.FullNode, st *storage, maxBatch int) {
 	notifs, err := api.ChainNotify(ctx)
 	if err != nil {
 		panic(err)
@@ -32,7 +30,7 @@ func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
 				case store.HCCurrent:
 					fallthrough
 				case store.HCApply:
-					syncHead(ctx, api, st, change.Val)
+					syncHead(ctx, api, st, change.Val, maxBatch)
 				case store.HCRevert:
 					log.Warnf("revert todo")
 				}
@@ -65,9 +63,7 @@ type actorInfo struct {
 	state     string
 }
 
-func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet) {
-	addresses := map[address.Address]address.Address{}
-	actors := map[address.Address]map[types.Actor]actorInfo{}
+func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet, maxBatch int) {
 	var alk sync.Mutex
 
 	log.Infof("Getting synced block list")
@@ -92,7 +88,6 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 		}
 
 		allToSync[bh.Cid()] = bh
-		addresses[bh.Miner] = address.Undef
 
 		if len(allToSync)%500 == 10 {
 			log.Infof("todo: (%d) %s @%d", len(allToSync), bh.Cid(), bh.Height)
@@ -114,6 +109,8 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 	}
 
 	for len(allToSync) > 0 {
+		actors := map[address.Address]map[types.Actor]actorInfo{}
+		addresses := map[address.Address]address.Address{}
 		minH := uint64(math.MaxUint64)
 
 		for _, header := range allToSync {
@@ -124,8 +121,9 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 
 		toSync := map[cid.Cid]*types.BlockHeader{}
 		for c, header := range allToSync {
-			if header.Height < minH+maxBatch {
+			if header.Height < minH+uint64(maxBatch) {
 				toSync[c] = header
+				addresses[header.Miner] = address.Undef
 			}
 		}
 		for c := range toSync {
@@ -200,7 +198,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 					log.Error(err)
 					return
 				}
-				ast, err := api.StateReadState(ctx, &act, ts.Key())
+				ast, err := api.StateReadState(ctx, &act, pts.Key())
 				if err != nil {
 					log.Error(err)
 					return

cmd/lotus/daemon.go

@@ -13,6 +13,9 @@ import (
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/mitchellh/go-homedir"
 	"github.com/multiformats/go-multiaddr"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 
@@ -21,6 +24,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/testing"
@@ -90,7 +94,7 @@ var DaemonCmd = &cli.Command{
 			defer pprof.StopCPUProfile()
 		}
 
-		ctx := context.Background()
+		ctx, _ := tag.New(context.Background(), tag.Insert(metrics.Version, build.BuildVersion), tag.Insert(metrics.Commit, build.CurrentCommit))
 		{
 			dir, err := homedir.Expand(cctx.String("repo"))
 			if err != nil {
@@ -171,6 +175,16 @@ var DaemonCmd = &cli.Command{
 			return xerrors.Errorf("initializing node: %w", err)
 		}
 
+		// Register all metric views
+		if err = view.Register(
+			metrics.DefaultViews...,
+		); err != nil {
+			log.Fatalf("Cannot register the view: %v", err)
+		}
+
+		// Set the metric to one so it is published to the exporter
+		stats.Record(ctx, metrics.LotusInfo.M(1))
+
 		endpoint, err := r.APIEndpoint()
 		if err != nil {
 			return xerrors.Errorf("getting api endpoint: %w", err)

cmd/lotus/main.go

@@ -73,7 +73,12 @@ func main() {
 			Code:    trace.StatusCodeFailedPrecondition,
 			Message: err.Error(),
 		})
-		log.Warnf("%+v", err)
+		_, ok := err.(*lcli.ErrCmdFailed)
+		if ok {
+			log.Debugf("%+v", err)
+		} else {
+			log.Warnf("%+v", err)
+		}
 		os.Exit(1)
 	}
 }

cmd/lotus/rpc.go

@@ -3,13 +3,14 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"github.com/filecoin-project/lotus/api/apistruct"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
 
+	"github.com/filecoin-project/lotus/api/apistruct"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/lib/auth"
 	"github.com/filecoin-project/lotus/lib/jsonrpc"
@@ -21,6 +22,8 @@ import (
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
 	"golang.org/x/xerrors"
+
+	"contrib.go.opencensus.io/exporter/prometheus"
 )
 
 var log = logging.Logger("main")
@@ -43,6 +46,15 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr) erro
 
 	http.Handle("/rest/v0/import", importAH)
 
+	exporter, err := prometheus.NewExporter(prometheus.Options{
+		Namespace: "lotus",
+	})
+	if err != nil {
+		log.Fatalf("could not create the prometheus stats exporter: %v", err)
+	}
+
+	http.Handle("/debug/metrics", exporter)
+
 	lst, err := manet.Listen(addr)
 	if err != nil {
 		return xerrors.Errorf("could not listen: %w", err)

documentation/en/.library.json

@@ -111,6 +111,13 @@
       "value": null,
       "posts": []
     },
+    {
+      "title": "Command Line Interface",
+      "slug": "en+cli",
+      "github": "en/cli.md",
+      "value": null,
+      "posts": []
+    },
     {
       "title": "API",
       "slug": "en+api",

documentation/en/cli.md

@@ -0,0 +1,108 @@
+# Lotus Command Line Interface
+
+The Command Line Interface (CLI) is a convenient way to interact with
+a Lotus node. You can use the CLI to operate your node,
+get information about the blockchain,
+manage your accounts and transfer funds,
+create storage deals, and much more! 
+
+The CLI is intended to be self-documenting, so when in doubt, simply add `--help`
+to whatever command you're trying to run! This will also display all of the 
+input parameters that can be provided to a command.
+
+We highlight some of the commonly
+used features of the CLI below.
+All CLI commands should be run from the home directory of the Lotus project.
+
+## Operating a Lotus node
+
+### Starting up a node
+
+```sh
+lotus daemon
+```
+This command will start up your Lotus node, with its API port open at 1234. 
+You can pass `--api=<number>` to use a different port.
+
+### Checking your sync progress
+
+```sh
+lotus sync status
+```
+This command will print your current tipset height under `Height`, and the target tipset height
+under `Taregt`. 
+
+You can also run `lotus sync wait` to get constant updates on your sync progress.
+
+### Getting the head tipset
+
+```sh
+lotus chain head
+```
+
+### Control the logging level
+
+```sh
+lotus log set-level
+```
+This command can be used to toggle the logging levels of the different
+systems of a Lotus node. In decreasing order
+of logging detail, the levels are `debug`, `info`, `warn`, and `error`. 
+
+As an example,
+to set the `chain` and `blocksync` to log at the `debug` level, run 
+`lotus log set-level --system chain --system blocksync debug`. 
+
+To see the various logging system, run `lotus log list`.
+
+### Find out what version of Lotus you're running
+
+```sh
+lotus version
+```
+
+## Managing your accounts
+
+### Listing accounts in your wallet
+
+```sh
+lo*tus wallet list
+``` 
+
+### Creating a new account
+
+```sh
+lotus wallet new bls
+```
+This command will create a new BLS account in your wallet; these
+addresses start with the prefix `t3`. Running `lotus wallet new secp256k1` 
+(or just `lotus wallet new`) will create
+a new Secp256k1 account, which begins with the prefix `t1`.
+
+### Getting an account's balance
+
+```sh
+lotus wallet balance <address>
+``` 
+
+### Transferring funds 
+
+```sh
+lotus send --source=<source address> <destination address> <amount>
+``` 
+This command will transfer `amount` (in attoFIL) from `source address` to `destination address`.
+
+### Importing an account into your wallet
+
+```sh
+lotus wallet import <path to private key>
+``` 
+This command will import an account whose private key is saved at the specified file.
+
+### Exporting an account from your wallet
+
+```sh
+lotus wallet export <address>
+``` 
+This command will print out the private key of the specified address
+if it is in your wallet. Always be careful with your private key!

documentation/en/join-testnet.md

@@ -36,13 +36,13 @@ In order to connect to the network, you need to be connected to at least 1 peer.
 
 ## Chain sync
 
-While the daemon is running, the next requirement is to sync the chain. Run the command below to start the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/).
+While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/).
 
 ```sh
 lotus sync wait
 ```
 
-- This step will take anywhere between 30 minutes to a few hours.
+- This step will take anywhere between a few hours to a couple of days.
 - You will be able to perform **Lotus Testnet** operations after it is finished.
 
 ## Create your first address

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.1.0
+	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	github.com/BurntSushi/toml v0.3.1
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee

go.sum

@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc=
 contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA=
+contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg=
+contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -33,6 +35,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
 github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@@ -562,6 +565,7 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -656,12 +660,18 @@ github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXx
 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M=
 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -813,6 +823,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

lib/jsonrpc/handler.go

@@ -9,6 +9,9 @@ import (
 	"io"
 	"reflect"
 
+	"github.com/filecoin-project/lotus/metrics"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
 	"go.opencensus.io/trace/propagation"
 	"golang.org/x/xerrors"
@@ -151,18 +154,22 @@ func (handlers) getSpan(ctx context.Context, req request) (context.Context, *tra
 }
 
 func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer)), rpcError rpcErrFunc, done func(keepCtx bool), chOut chanOut) {
+	// Not sure if we need to sanitize the incoming req.Method or not.
 	ctx, span := h.getSpan(ctx, req)
+	ctx, _ = tag.New(ctx, tag.Insert(metrics.RPCMethod, req.Method))
 	defer span.End()
 
 	handler, ok := h[req.Method]
 	if !ok {
 		rpcError(w, &req, rpcMethodNotFound, fmt.Errorf("method '%s' not found", req.Method))
+		stats.Record(ctx, metrics.RPCInvalidMethod.M(1))
 		done(false)
 		return
 	}
 
 	if len(req.Params) != handler.nParams {
 		rpcError(w, &req, rpcInvalidParams, fmt.Errorf("wrong param count"))
+		stats.Record(ctx, metrics.RPCRequestError.M(1))
 		done(false)
 		return
 	}
@@ -172,6 +179,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 
 	if chOut == nil && outCh {
 		rpcError(w, &req, rpcMethodNotFound, fmt.Errorf("method '%s' not supported in this mode (no out channel support)", req.Method))
+		stats.Record(ctx, metrics.RPCRequestError.M(1))
 		return
 	}
 
@@ -185,6 +193,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 		rp := reflect.New(handler.paramReceivers[i])
 		if err := json.NewDecoder(bytes.NewReader(req.Params[i].data)).Decode(rp.Interface()); err != nil {
 			rpcError(w, &req, rpcParseError, xerrors.Errorf("unmarshaling params for '%s': %w", handler.handlerFunc, err))
+			stats.Record(ctx, metrics.RPCRequestError.M(1))
 			return
 		}
 
@@ -196,6 +205,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 	callResult, err := doCall(req.Method, handler.handlerFunc, callParams)
 	if err != nil {
 		rpcError(w, &req, 0, xerrors.Errorf("fatal error calling '%s': %w", req.Method, err))
+		stats.Record(ctx, metrics.RPCRequestError.M(1))
 		return
 	}
 	if req.ID == nil {
@@ -213,6 +223,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 		err := callResult[handler.errOut].Interface()
 		if err != nil {
 			log.Warnf("error in RPC call to '%s': %+v", req.Method, err)
+			stats.Record(ctx, metrics.RPCResponseError.M(1))
 			resp.Error = &respError{
 				Code:    1,
 				Message: err.(error).Error(),
@@ -234,6 +245,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 		}
 
 		log.Warnf("failed to setup channel in RPC call to '%s': %+v", req.Method, err)
+		stats.Record(ctx, metrics.RPCResponseError.M(1))
 		resp.Error = &respError{
 			Code:    1,
 			Message: err.(error).Error(),
@@ -243,6 +255,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 	w(func(w io.Writer) {
 		if err := json.NewEncoder(w).Encode(resp); err != nil {
 			log.Error(err)
+			stats.Record(ctx, metrics.RPCResponseError.M(1))
 			return
 		}
 	})

metrics/metrics.go

@@ -0,0 +1,102 @@
+package metrics
+
+import (
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+// Global Tags
+var (
+	Version, _      = tag.NewKey("version")
+	Commit, _       = tag.NewKey("commit")
+	RPCMethod, _    = tag.NewKey("method")
+	PeerID, _       = tag.NewKey("peer_id")
+	FailureType, _  = tag.NewKey("failure_type")
+	MessageFrom, _  = tag.NewKey("message_from")
+	MessageTo, _    = tag.NewKey("message_to")
+	MessageNonce, _ = tag.NewKey("message_nonce")
+	ReceivedFrom, _ = tag.NewKey("received_from")
+)
+
+// Measures
+var (
+	LotusInfo                = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless)
+	ChainNodeHeight          = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless)
+	ChainNodeWorkerHeight    = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless)
+	MessageReceived          = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless)
+	MessageValidationFailure = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless)
+	MessageValidationSuccess = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless)
+	BlockReceived            = stats.Int64("block/received", "Counter for total received blocks", stats.UnitDimensionless)
+	BlockValidationFailure   = stats.Int64("block/failure", "Counter for block validation failures", stats.UnitDimensionless)
+	BlockValidationSuccess   = stats.Int64("block/success", "Counter for block validation successes", stats.UnitDimensionless)
+	PeerCount                = stats.Int64("peer/count", "Current number of FIL peers", stats.UnitDimensionless)
+	RPCInvalidMethod         = stats.Int64("rpc/invalid_method", "Total number of invalid RPC methods called", stats.UnitDimensionless)
+	RPCRequestError          = stats.Int64("rpc/request_error", "Total number of request errors handled", stats.UnitDimensionless)
+	RPCResponseError         = stats.Int64("rpc/response_error", "Total number of responses errors handled", stats.UnitDimensionless)
+)
+
+// DefaultViews is an array of Consensus views for metric gathering purposes
+var DefaultViews = []*view.View{
+	&view.View{
+		Name:        "info",
+		Description: "Lotus node information",
+		Measure:     LotusInfo,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{Version, Commit},
+	},
+	&view.View{
+		Measure:     ChainNodeHeight,
+		Aggregation: view.LastValue(),
+	},
+	&view.View{
+		Measure:     ChainNodeWorkerHeight,
+		Aggregation: view.LastValue(),
+	},
+	&view.View{
+		Measure:     BlockReceived,
+		Aggregation: view.Count(),
+	},
+	&view.View{
+		Measure:     BlockValidationFailure,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{FailureType, PeerID, ReceivedFrom},
+	},
+	&view.View{
+		Measure:     BlockValidationSuccess,
+		Aggregation: view.Count(),
+	},
+	&view.View{
+		Measure:     MessageReceived,
+		Aggregation: view.Count(),
+	},
+	&view.View{
+		Measure:     MessageValidationFailure,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{FailureType, MessageFrom, MessageTo, MessageNonce},
+	},
+	&view.View{
+		Measure:     MessageValidationSuccess,
+		Aggregation: view.Count(),
+	},
+	&view.View{
+		Measure:     PeerCount,
+		Aggregation: view.LastValue(),
+	},
+	// All RPC related metrics should at the very least tag the RPCMethod
+	&view.View{
+		Measure:     RPCInvalidMethod,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{RPCMethod},
+	},
+	&view.View{
+		Measure:     RPCRequestError,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{RPCMethod},
+	},
+	&view.View{
+		Measure:     RPCResponseError,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{RPCMethod},
+	},
+}

node/config/def.go

@@ -96,7 +96,7 @@ func DefaultStorageMiner() *StorageMiner {
 		Common: defCommon(),
 
 		SectorBuilder: SectorBuilder{
-			WorkerCount: 5,
+			WorkerCount: 2,
 		},
 	}
 	cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"

node/impl/common.go

@@ -2,6 +2,7 @@ package impl
 
 import (
 	"context"
+	"github.com/filecoin-project/lotus/node/modules/lp2p"
 
 	logging "github.com/ipfs/go-log/v2"
 
@@ -23,6 +24,7 @@ type CommonAPI struct {
 
 	APISecret *dtypes.APIAlg
 	Host      host.Host
+	Router    lp2p.BaseIpfsRouting
 }
 
 type jwtPayload struct {
@@ -81,6 +83,10 @@ func (a *CommonAPI) NetDisconnect(ctx context.Context, p peer.ID) error {
 	return a.Host.Network().ClosePeer(p)
 }
 
+func (a *CommonAPI) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
+	return a.Router.FindPeer(ctx, p)
+}
+
 func (a *CommonAPI) ID(context.Context) (peer.ID, error) {
 	return a.Host.ID(), nil
 }

node/impl/full/wallet.go

@@ -2,6 +2,7 @@ package full
 
 import (
 	"context"
+	"github.com/filecoin-project/lotus/lib/sigs"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/stmgr"
@@ -53,6 +54,10 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms
 	}, nil
 }
 
+func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *types.Signature) bool {
+	return sigs.Verify(sig, k, msg) == nil
+}
+
 func (a *WalletAPI) WalletDefaultAddress(ctx context.Context) (address.Address, error) {
 	return a.Wallet.GetDefault()
 }

node/modules/services.go

@@ -16,7 +16,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/blocksync"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/sub"
-	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/hello"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
@@ -78,18 +77,9 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub
 		panic(err)
 	}
 
-	v := func(ctx context.Context, pid peer.ID, msg *pubsub.Message) bool {
-		m, err := types.DecodeSignedMessage(msg.GetData())
-		if err != nil {
-			log.Errorf("got incorrectly formatted Message: %s", err)
-			return false
-		}
+	v := sub.NewMessageValidator(mpool)
 
-		msg.ValidatorData = m
-		return true
-	}
-
-	if err := ps.RegisterTopicValidator(MessagesTopic, v); err != nil {
+	if err := ps.RegisterTopicValidator(MessagesTopic, v.Validate); err != nil {
 		panic(err)
 	}
 

peermgr/peermgr.go

@@ -5,7 +5,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"go.opencensus.io/stats"
 	"go.uber.org/fx"
 
 	host "github.com/libp2p/go-libp2p-core/host"
@@ -115,6 +117,7 @@ func (pmgr *PeerMgr) Run(ctx context.Context) {
 			} else if pcount > pmgr.maxFilPeers {
 				log.Debug("peer count about threshold: %d > %d", pcount, pmgr.maxFilPeers)
 			}
+			stats.Record(ctx, metrics.PeerCount.M(int64(pmgr.getPeerCount())))
 		}
 	}
 }

tools/stats/head_buffer.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"container/list"
+
+	"github.com/filecoin-project/lotus/chain/store"
+)
+
+type headBuffer struct {
+	buffer *list.List
+	size   int
+}
+
+func NewHeadBuffer(size int) *headBuffer {
+	buffer := list.New()
+	buffer.Init()
+
+	return &headBuffer{
+		buffer: buffer,
+		size:   size,
+	}
+}
+
+func (h *headBuffer) Push(hc *store.HeadChange) (rethc *store.HeadChange) {
+	if h.buffer.Len() == h.size {
+		var ok bool
+
+		el := h.buffer.Front()
+		rethc, ok = el.Value.(*store.HeadChange)
+		if !ok {
+			panic("Value from list is not the correct type")
+		}
+
+		h.buffer.Remove(el)
+	}
+
+	h.buffer.PushBack(hc)
+
+	return
+}
+
+func (h *headBuffer) Pop() {
+	el := h.buffer.Back()
+	if el != nil {
+		h.buffer.Remove(el)
+	}
+}

tools/stats/head_buffer_test.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHeadBuffer(t *testing.T) {
+
+	t.Run("Straight push through", func(t *testing.T) {
+		hb := NewHeadBuffer(5)
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "1"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "2"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "3"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "4"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "5"}))
+
+		hc := hb.Push(&store.HeadChange{Type: "6"})
+		require.Equal(t, hc.Type, "1")
+	})
+
+	t.Run("Reverts", func(t *testing.T) {
+		hb := NewHeadBuffer(5)
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "1"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "2"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "3"}))
+		hb.Pop()
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "3a"}))
+		hb.Pop()
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "3b"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "4"}))
+		require.Nil(t, hb.Push(&store.HeadChange{Type: "5"}))
+
+		hc := hb.Push(&store.HeadChange{Type: "6"})
+		require.Equal(t, hc.Type, "1")
+		hc = hb.Push(&store.HeadChange{Type: "7"})
+		require.Equal(t, hc.Type, "2")
+		hc = hb.Push(&store.HeadChange{Type: "8"})
+		require.Equal(t, hc.Type, "3b")
+	})
+}

tools/stats/main.go

@@ -22,10 +22,12 @@ func main() {
 	var database string = "lotus"
 	var reset bool = false
 	var height int64 = 0
+	var headlag int = 3
 
 	flag.StringVar(&repo, "repo", repo, "lotus repo path")
 	flag.StringVar(&database, "database", database, "influx database")
 	flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)")
+	flag.IntVar(&headlag, "head-lag", headlag, "number of head events to hold to protect against small reorgs")
 	flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering")
 
 	flag.Parse()
@@ -66,7 +68,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	tipsetsCh, err := GetTips(ctx, api, uint64(height))
+	tipsetsCh, err := GetTips(ctx, api, uint64(height), headlag)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -75,6 +77,7 @@ func main() {
 	defer wq.Close()
 
 	for tipset := range tipsetsCh {
+		log.Infow("Collect stats", "height", tipset.Height())
 		pl := NewPointList()
 		height := tipset.Height()
 

tools/stats/rpc.go

@@ -120,9 +120,11 @@ sync_complete:
 	}
 }
 
-func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan *types.TipSet, error) {
+func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64, headlag int) (<-chan *types.TipSet, error) {
 	chmain := make(chan *types.TipSet)
 
+	hb := NewHeadBuffer(headlag)
+
 	notif, err := api.ChainNotify(ctx)
 	if err != nil {
 		return nil, err
@@ -151,7 +153,11 @@ func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan *
 							chmain <- tipset
 						}
 					case store.HCApply:
-						chmain <- change.Val
+						if out := hb.Push(change); out != nil {
+							chmain <- out.Val
+						}
+					case store.HCRevert:
+						hb.Pop()
 					}
 				}
 			case <-ping: