diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 486ac8ed7..e58523999 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -1,3 +1,4 @@ +//go:build !nodaemon // +build !nodaemon package main @@ -153,6 +154,18 @@ var DaemonCmd = &cli.Command{ Name: "restore-config", Usage: "config file to use when restoring from backup", }, + &cli.StringFlag{ + Name: "trace-to-json", + Usage: "starts tracer and outputs to json file defined with this flag", + }, + &cli.StringFlag{ + Name: "trace-to-elasticsearch", + Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch", + }, + &cli.StringFlag{ + Name: "trace-source-auth", + Usage: "auth token for trusted source of traces", + }, }, Action: func(cctx *cli.Context) error { isLite := cctx.Bool("lite") @@ -310,6 +323,10 @@ var DaemonCmd = &cli.Command{ log.Warnf("unable to inject prometheus ipfs/go-metrics exporter; some metrics will be unavailable; err: %s", err) } + traceToJsonFile := cctx.String("trace-to-json") + traceToElasticsearch := cctx.String("trace-to-elasticsearch") + traceSourceAuth := cctx.String("trace-source-auth") + var api api.FullNode stop, err := node.New(ctx, node.FullAPI(&api, node.Lite(isLite)), @@ -319,6 +336,9 @@ var DaemonCmd = &cli.Command{ node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), + node.Override(new(dtypes.JsonTracer), traceToJsonFile), + node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch), + node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth), genesis, liteModeDeps, diff --git a/go.mod b/go.mod index f73511574..a5adcca6f 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/drand/drand v1.2.1 github.com/drand/kyber v1.1.4 github.com/dustin/go-humanize v1.0.0 + github.com/elastic/go-elasticsearch/v7 v7.14.0 github.com/elastic/go-sysinfo v1.3.0 github.com/elastic/gosigar v0.12.0 github.com/etclabscore/go-openrpc-reflect v0.0.36 diff --git a/go.sum b/go.sum index f16a81a9f..6347ac4f7 100644 --- a/go.sum +++ b/go.sum @@ -235,6 +235,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/go-elasticsearch/v7 v7.14.0 h1:extp3jos/rwJn3J+lgbaGlwAgs0TVsIHme00GyNAyX4= +github.com/elastic/go-elasticsearch/v7 v7.14.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE= github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 9efe4e03c..21e9ccc6b 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -525,6 +525,24 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K... Name: "RemoteTracer", Type: "string", + Comment: ``, + }, + { + Name: "JsonTracer", + Type: "string", + + Comment: ``, + }, + { + Name: "ElasticSearchTracer", + Type: "string", + + Comment: ``, + }, + { + Name: "TracerSourceAuth", + Type: "string", + Comment: ``, }, }, diff --git a/node/config/types.go b/node/config/types.go index 1576169e6..a7de81328 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -309,6 +309,10 @@ type Pubsub struct { DirectPeers []string IPColocationWhitelist []string RemoteTracer string + JsonTracer string + ElasticSearchTracer string + ElasticSearchIndex string + TracerSourceAuth string } type Chainstore struct { diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go new file mode 100644 index 000000000..735ff05c3 --- /dev/null +++ b/node/modules/dtypes/tracer.go @@ -0,0 +1,5 @@ +package dtypes + +type JsonTracer string +type ElasticSearchTracer string +type TracerSourceAuth string diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 32b85daf3..9d9350074 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" + "github.com/filecoin-project/lotus/node/modules/tracer" ) func init() { @@ -49,6 +50,30 @@ func ScoreKeeper() *dtypes.ScoreKeeper { return new(dtypes.ScoreKeeper) } +type PeerScoreTracker interface { + UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) +} + +type peerScoreTracker struct { + sk *dtypes.ScoreKeeper + lt tracer.LotusTracer +} + +func newPeerScoreTracker(lt tracer.LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker { + return &peerScoreTracker{ + sk: sk, + lt: lt, + } +} + +func (pst *peerScoreTracker) UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) { + if pst.lt != nil { + pst.lt.PeerScores(scores) + } + + pst.sk.Update(scores) +} + type GossipIn struct { fx.In Mctx helpers.MetricsCtx @@ -272,7 +297,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { OpportunisticGraftThreshold: OpportunisticGraftScoreThreshold, }, ), - pubsub.WithPeerScoreInspect(in.Sk.Update, 10*time.Second), } // enable Peer eXchange on bootstrappers @@ -341,6 +365,27 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { pubsub.NewAllowlistSubscriptionFilter(allowTopics...), 100))) + var transports []tracer.TracerTransport + if in.Cfg.JsonTracer != "" { + jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracer) + if err != nil { + return nil, err + } + + transports = append(transports, jsonTransport) + } + if in.Cfg.ElasticSearchTracer != "" { + elasticSearchTransport, err := tracer.NewElasticSearchTransport( + in.Cfg.ElasticSearchTracer, + in.Cfg.ElasticSearchIndex, + ) + if err != nil { + return nil, err + } + transports = append(transports, elasticSearchTransport) + } + lt := tracer.NewLotusTracer(transports, in.Host.ID(), in.Cfg.TracerSourceAuth) + // tracer if in.Cfg.RemoteTracer != "" { a, err := ma.NewMultiaddr(in.Cfg.RemoteTracer) @@ -358,12 +403,18 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { return nil, err } - trw := newTracerWrapper(tr, build.BlocksTopic(in.Nn)) + pst := newPeerScoreTracker(lt, in.Sk) + trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn)) + options = append(options, pubsub.WithEventTracer(trw)) + options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second)) } else { // still instantiate a tracer for collecting metrics - trw := newTracerWrapper(nil) + trw := newTracerWrapper(nil, lt) options = append(options, pubsub.WithEventTracer(trw)) + + pst := newPeerScoreTracker(lt, in.Sk) + options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second)) } return pubsub.NewGossipSub(helpers.LifecycleCtx(in.Mctx, in.Lc), in.Host, options...) @@ -374,7 +425,11 @@ func HashMsgId(m *pubsub_pb.Message) string { return string(hash[:]) } -func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTracer { +func newTracerWrapper( + lp2pTracer pubsub.EventTracer, + lotusTracer pubsub.EventTracer, + topics ...string, +) pubsub.EventTracer { var topicsMap map[string]struct{} if len(topics) > 0 { topicsMap = make(map[string]struct{}) @@ -383,12 +438,13 @@ func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTrace } } - return &tracerWrapper{tr: tr, topics: topicsMap} + return &tracerWrapper{lp2pTracer: lp2pTracer, lotusTracer: lotusTracer, topics: topicsMap} } type tracerWrapper struct { - tr pubsub.EventTracer - topics map[string]struct{} + lp2pTracer pubsub.EventTracer + lotusTracer pubsub.EventTracer + topics map[string]struct{} } func (trw *tracerWrapper) traceMessage(topic string) bool { @@ -406,33 +462,61 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { switch evt.GetType() { case pubsub_pb.TraceEvent_PUBLISH_MESSAGE: stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) { - trw.tr.Trace(evt) + if trw.traceMessage(evt.GetPublishMessage().GetTopic()) { + if trw.lp2pTracer != nil { + trw.lp2pTracer.Trace(evt) + } + + if trw.lotusTracer != nil { + trw.lotusTracer.Trace(evt) + } } case pubsub_pb.TraceEvent_DELIVER_MESSAGE: stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) { - trw.tr.Trace(evt) + if trw.traceMessage(evt.GetDeliverMessage().GetTopic()) { + if trw.lp2pTracer != nil { + trw.lp2pTracer.Trace(evt) + } + + if trw.lotusTracer != nil { + trw.lotusTracer.Trace(evt) + } } case pubsub_pb.TraceEvent_REJECT_MESSAGE: stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1)) case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE: stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1)) case pubsub_pb.TraceEvent_JOIN: - if trw.tr != nil { - trw.tr.Trace(evt) + if trw.lp2pTracer != nil { + trw.lp2pTracer.Trace(evt) + } + + if trw.lotusTracer != nil { + trw.lotusTracer.Trace(evt) } case pubsub_pb.TraceEvent_LEAVE: - if trw.tr != nil { - trw.tr.Trace(evt) + if trw.lp2pTracer != nil { + trw.lp2pTracer.Trace(evt) + } + + if trw.lotusTracer != nil { + trw.lotusTracer.Trace(evt) } case pubsub_pb.TraceEvent_GRAFT: - if trw.tr != nil { - trw.tr.Trace(evt) + if trw.lp2pTracer != nil { + trw.lp2pTracer.Trace(evt) + } + + if trw.lotusTracer != nil { + trw.lotusTracer.Trace(evt) } case pubsub_pb.TraceEvent_PRUNE: - if trw.tr != nil { - trw.tr.Trace(evt) + if trw.lp2pTracer != nil { + trw.lp2pTracer.Trace(evt) + } + + if trw.lotusTracer != nil { + trw.lotusTracer.Trace(evt) } case pubsub_pb.TraceEvent_RECV_RPC: stats.Record(context.TODO(), metrics.PubsubRecvRPC.M(1)) diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go new file mode 100644 index 000000000..49060a13d --- /dev/null +++ b/node/modules/tracer/elasticsearch_transport.go @@ -0,0 +1,92 @@ +package tracer + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/elastic/go-elasticsearch/v7/esapi" +) + +const ( + ElasticSearch_INDEX_DEFAULT = "lotus-pubsub" +) + +func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) { + conUrl, err := url.Parse(connectionString) + + if err != nil { + return nil, err + } + + username := conUrl.User.Username() + password, _ := conUrl.User.Password() + cfg := elasticsearch.Config{ + Addresses: []string{ + conUrl.Scheme + "://" + conUrl.Host, + }, + Username: username, + Password: password, + } + + es, err := elasticsearch.NewClient(cfg) + + if err != nil { + return nil, err + } + + var esIndex string + if elasticsearchIndex != "" { + esIndex = elasticsearchIndex + } else { + esIndex = ElasticSearch_INDEX_DEFAULT + } + + return &elasticSearchTransport{ + cl: es, + esIndex: esIndex, + }, nil +} + +type elasticSearchTransport struct { + cl *elasticsearch.Client + esIndex string +} + +func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error { + var e interface{} + + if evt.lotusTraceEvent != nil { + e = *evt.lotusTraceEvent + } else if evt.pubsubTraceEvent != nil { + e = *evt.pubsubTraceEvent + } else { + return nil + } + + jsonEvt, err := json.Marshal(e) + if err != nil { + return fmt.Errorf("error while marshaling event: %s", err) + } + + req := esapi.IndexRequest{ + Index: est.esIndex, + Body: strings.NewReader(string(jsonEvt)), + Refresh: "true", + } + + // Perform the request with the client. + res, err := req.Do(context.Background(), est.cl) + if err != nil { + return err + } + defer res.Body.Close() + + if res.IsError() { + return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID) + } + return nil +} diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go new file mode 100644 index 000000000..ca8535f4b --- /dev/null +++ b/node/modules/tracer/json_transport.go @@ -0,0 +1,41 @@ +package tracer + +import ( + "encoding/json" + "fmt" + "os" +) + +type jsonTracerTransport struct { + out *os.File +} + +func NewJsonTracerTransport(file string) (TracerTransport, error) { + out, err := os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660) + if err != nil { + return nil, err + } + + return &jsonTracerTransport{ + out: out, + }, nil +} + +func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error { + var e interface{} + if evt.lotusTraceEvent != nil { + e = *evt.lotusTraceEvent + } else if evt.pubsubTraceEvent != nil { + e = *evt.pubsubTraceEvent + } else { + return nil + } + + jsonEvt, err := json.Marshal(e) + if err != nil { + return fmt.Errorf("error while marshaling event: %s", err) + } + + _, err = jtt.out.WriteString(string(jsonEvt) + "\n") + return err +} diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go new file mode 100644 index 000000000..9badfc9cb --- /dev/null +++ b/node/modules/tracer/tracer.go @@ -0,0 +1,90 @@ +package tracer + +import ( + "time" + + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p-core/peer" + pubsub "github.com/libp2p/go-libp2p-pubsub" + pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" +) + +var log = logging.Logger("lotus-tracer") + +func NewLotusTracer(tt []TracerTransport, pid peer.ID, sourceAuth string) LotusTracer { + return &lotusTracer{ + tt: tt, + pid: pid, + sa: sourceAuth, + } +} + +type lotusTracer struct { + tt []TracerTransport + pid peer.ID + sa string +} + +const ( + TraceEvent_PEER_SCORES pubsub_pb.TraceEvent_Type = 100 +) + +type LotusTraceEvent struct { + Type pubsub_pb.TraceEvent_Type `json:"type,omitempty"` + PeerID string `json:"peerID,omitempty"` + Timestamp *int64 `json:"timestamp,omitempty"` + PeerScore TraceEvent_PeerScore `json:"peerScore,omitempty"` + SourceAuth string `json:"sourceAuth,omitempty"` +} + +type TraceEvent_PeerScore struct { + PeerID string `json:"peerID"` + Score float32 `json:"score"` +} + +type LotusTracer interface { + Trace(evt *pubsub_pb.TraceEvent) + TraceLotusEvent(evt *LotusTraceEvent) + + PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) +} + +func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) { + now := time.Now().UnixNano() + for pid, score := range scores { + evt := &LotusTraceEvent{ + Type: *TraceEvent_PEER_SCORES.Enum(), + PeerID: lt.pid.Pretty(), + Timestamp: &now, + SourceAuth: lt.sa, + PeerScore: TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)}, + } + + lt.TraceLotusEvent(evt) + } +} + +func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) { + for _, t := range lt.tt { + err := t.Transport(TracerTransportEvent{ + lotusTraceEvent: evt, + pubsubTraceEvent: nil, + }) + if err != nil { + log.Errorf("error while transporting peer scores: %s", err) + } + } + +} + +func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) { + for _, t := range lt.tt { + err := t.Transport(TracerTransportEvent{ + lotusTraceEvent: nil, + pubsubTraceEvent: evt, + }) + if err != nil { + log.Errorf("error while transporting trace event: %s", err) + } + } +} diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go new file mode 100644 index 000000000..56d926afc --- /dev/null +++ b/node/modules/tracer/transport.go @@ -0,0 +1,12 @@ +package tracer + +import pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" + +type TracerTransport interface { + Transport(evt TracerTransportEvent) error +} + +type TracerTransportEvent struct { + lotusTraceEvent *LotusTraceEvent + pubsubTraceEvent *pubsub_pb.TraceEvent +} diff --git a/tools/kibana/README.md b/tools/kibana/README.md new file mode 100644 index 000000000..e4e451bbb --- /dev/null +++ b/tools/kibana/README.md @@ -0,0 +1,14 @@ +## Lotus Kibana Dashboard + +This folder contains configuration files to create Kibana dashboards to track peer scores and block propagation +throughout Filecoin network. + +### Importing dashboard + +The peer score and block propagation dashboard configuration is imported via Kibana import saved object [functionality](https://www.elastic.co/guide/en/kibana/current/managing-saved-objects.html#managing-saved-objects-export-objects). + +The index patterns will be created automatically when importing dashboards. + +#### Custom index + +By default, the dashboards target `lotus-pubsub` index which is the default one when running node. The index can be customised via edit on dashboard visualizations. diff --git a/tools/kibana/block-propagation-dashboard.ndjson b/tools/kibana/block-propagation-dashboard.ndjson new file mode 100644 index 000000000..dc5121183 --- /dev/null +++ b/tools/kibana/block-propagation-dashboard.ndjson @@ -0,0 +1,2 @@ +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":42,\"i\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\"},\"panelIndex\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"Block propagation\",\"description\":\"\",\"type\":\"vega\",\"params\":{\"spec\":\"{\\n $schema: https://vega.github.io/schema/vega/v5.json\\n title: Node block propagation\\n\\n\\n \\\"signals\\\": [\\n { \\\"name\\\": \\\"cx\\\", \\\"update\\\": \\\"width / 2\\\" },\\n { \\\"name\\\": \\\"cy\\\", \\\"update\\\": \\\"height / 2\\\" },\\n { \\\"name\\\": \\\"nodeRadius\\\", \\\"value\\\": 10,\\n \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 20, \\\"step\\\": 1} },\\n { \\\"name\\\": \\\"propagationMultiplier\\\", \\\"value\\\": 50,\\n \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 100, \\\"step\\\": 1} },\\n ],\\n\\n data: [\\n {\\n \\n name: \\\"node-data\\\",\\n url: {\\n index: lotus-pubsub\\n body: {\\n size: 0,\\n aggs: {\\n unique_peerID: {\\n terms: { \\n field: \\\"peerID\\\",\\n size: 10000\\n },\\n }\\n }\\n }\\n },\\n format: {\\\"property\\\": \\\"aggregations.unique_peerID.buckets\\\"},\\n transform: [\\n {\\n \\\"type\\\": \\\"project\\\",\\n \\\"fields\\\": [\\\"key\\\"],\\n \\\"as\\\": [\\\"peerID\\\"]\\n },\\n {\\n type: \\\"identifier\\\",\\n as: \\\"id\\\"\\n },\\n {\\n type: \\\"formula\\\",\\n expr: \\\"datum.id - 1\\\",\\n as: \\\"index\\\"\\n }\\n ]\\n },\\n {\\n name: \\\"published-message-data\\\",\\n url: {\\n index: lotus-pubsub\\n body: {\\n size: 10000,\\n query: {\\n bool: {\\n must: [\\n {\\n match: {\\n type: 0\\n }\\n }\\n ]\\n }\\n }\\n }\\n },\\n format: {\\\"property\\\": \\\"hits.hits\\\"},\\n },\\n {\\n name: \\\"link-data\\\",\\n url: {\\n index: lotus-pubsub\\n body: {\\n size: 10000,\\n query: {\\n bool: {\\n must: [\\n {\\n match: {\\n type: 1\\n }\\n }\\n ]\\n }\\n }\\n }\\n },\\n format: {\\\"property\\\": \\\"hits.hits\\\"},\\n transform: [\\n {\\n \\\"type\\\": \\\"lookup\\\",\\n \\\"from\\\": \\\"published-message-data\\\",\\n \\\"key\\\": \\\"_source.publishMessage.messageID\\\",\\n \\\"fields\\\": [\\\"_source.deliverMessage.messageID\\\"],\\n \\\"as\\\": [\\\"publishedMessage\\\"],\\n },\\n {\\n \\\"type\\\": \\\"lookup\\\",\\n \\\"from\\\": \\\"node-data\\\",\\n \\\"key\\\": \\\"peerID\\\",\\n \\\"fields\\\": [\\\"_source.peerID\\\"],\\n \\\"as\\\": [\\\"source\\\"],\\n },\\n {\\n \\\"type\\\": \\\"lookup\\\",\\n \\\"from\\\": \\\"node-data\\\",\\n \\\"key\\\": \\\"peerID\\\",\\n \\\"fields\\\": [\\\"publishedMessage._source.peerID\\\"],\\n \\\"as\\\": [\\\"target\\\"],\\n },\\n {\\n \\\"type\\\": \\\"formula\\\",\\n \\\"expr\\\": \\\"(datum._source.timestamp - datum.publishedMessage._source.timestamp) * propagationMultiplier\\\",\\n \\\"as\\\": \\\"distance\\\"\\n },\\n {\\n \\\"type\\\": \\\"project\\\",\\n \\\"fields\\\": [\\\"source.index\\\", \\\"target.index\\\", \\\"distance\\\"]\\n \\\"as\\\": [\\\"source\\\", \\\"target\\\", \\\"distance\\\"]\\n },\\n {\\n \\\"type\\\": \\\"aggregate\\\",\\n \\\"ops\\\": [\\\"average\\\"],\\n \\\"fields\\\": [\\\"distance\\\"],\\n \\\"groupby\\\": [\\\"source\\\", \\\"target\\\"],\\n \\\"as\\\": [\\\"distance\\\"]\\n }\\n ]\\n }\\n ]\\n\\n scales: [\\n {\\n \\\"name\\\": \\\"color\\\",\\n \\\"type\\\": \\\"ordinal\\\",\\n \\\"domain\\\": {\\\"data\\\": \\\"node-data\\\", \\\"field\\\": \\\"id\\\"},\\n \\\"range\\\": {\\\"scheme\\\": \\\"category20c\\\"}\\n }\\n ]\\n\\n marks: [\\n {\\n \\\"name\\\": \\\"nodes\\\",\\n \\\"type\\\": \\\"symbol\\\",\\n \\\"zindex\\\": 1,\\n \\n \\\"encode\\\": {\\n \\\"enter\\\": {\\n \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n \\\"stroke\\\": {\\\"value\\\": \\\"black\\\"},\\n \\\"tooltip\\\": {\\\"signal\\\": \\\"{'PeerID': datum.peerID}\\\"}\\n },\\n \\\"update\\\": {\\n \\\"size\\\": {\\\"signal\\\": \\\"2 * nodeRadius * nodeRadius\\\"},\\n \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n },\\n \\\"hover\\\": { \\\"fill\\\": {\\\"value\\\": \\\"red\\\"} },\\n },\\n\\n \\\"from\\\": {\\\"data\\\": \\\"node-data\\\"},\\n \\\"transform\\\": [\\n {\\n \\\"type\\\": \\\"force\\\",\\n \\\"restart\\\": false,\\n \\\"static\\\": true,\\n \\\"signal\\\": \\\"force\\\",\\n \\\"forces\\\": [\\n {\\\"force\\\": \\\"link\\\", \\\"links\\\": \\\"link-data\\\", distance: {field: \\\"distance\\\"}},\\n {\\\"force\\\": \\\"center\\\", \\\"x\\\": {\\\"signal\\\": \\\"cx\\\"}, \\\"y\\\": {\\\"signal\\\": \\\"cy\\\"}},\\n ]\\n }\\n ]\\n },\\n {\\n \\\"type\\\": \\\"path\\\",\\n \\\"from\\\": {\\\"data\\\": \\\"link-data\\\"},\\n \\\"encode\\\": {\\n \\\"enter\\\": {\\n \\\"tooltip\\\": {\\\"signal\\\": \\\"{'Block propagation': datum.distance}\\\"}\\n },\\n \\\"update\\\": {\\n \\\"stroke\\\": {\\\"value\\\": \\\"#ccc\\\"},\\n \\\"strokeWidth\\\": {\\\"value\\\": 1.5}\\n },\\n \\\"hover\\\": { \\\"stroke\\\": { \\\"value\\\": \\\"#0B33A0\\\" }}\\n },\\n \\\"transform\\\": [\\n {\\n \\\"type\\\": \\\"linkpath\\\",\\n \\\"require\\\": {\\\"signal\\\": \\\"force\\\"},\\n \\\"shape\\\": \\\"line\\\",\\n \\\"sourceX\\\": \\\"datum.source.x\\\", \\\"sourceY\\\": \\\"datum.source.y\\\",\\n \\\"targetX\\\": \\\"datum.target.x\\\", \\\"targetY\\\": \\\"datum.target.y\\\"\\n }\\n ]\\n }\\n ]\\n}\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}}},\"enhancements\":{}}}]","timeRestore":false,"title":"Block propagation","version":1},"coreMigrationVersion":"7.14.1","id":"eff2ade0-19fa-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[],"type":"dashboard","updated_at":"2021-09-24T09:46:01.693Z","version":"WzYzMjAsMV0="} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]} diff --git a/tools/kibana/peer-scores-dashboard.ndjson b/tools/kibana/peer-scores-dashboard.ndjson new file mode 100644 index 000000000..c4d491706 --- /dev/null +++ b/tools/kibana/peer-scores-dashboard.ndjson @@ -0,0 +1,3 @@ +{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{\"peerScore.weightedScore\":{\"type\":\"double\",\"script\":{\"source\":\"if (doc['type'].value == 100) {\\n def score = doc['peerScore.score'].value;\\n if (doc['sourceAuth'] == \\\"\\\") {\\n\\n emit(score * 1.2)\\n } else {\\n emit(score)\\n }\\n}\\n\\n\"}},\"sourceAuth\":{\"type\":\"keyword\"}}","title":"lotus-pubsub*","typeMeta":"{}"},"coreMigrationVersion":"7.14.1","id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-09-24T12:03:02.575Z","version":"WzcwMzksMV0="} +{"attributes":{"description":"Average peer score table per node peerID","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":43,\"i\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\"},\"panelIndex\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsDatatable\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"666e2f39-8868-45ad-b747-fe124830b0ae\":{\"columns\":{\"504c50bd-14c1-4119-820e-c961866fc3b4\":{\"label\":\"peerID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"peerScore.peerID.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\"},\"orderDirection\":\"desc\",\"otherBucket\":false,\"missingBucket\":false},\"customLabel\":true},\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\":{\"label\":\"Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.score\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\":{\"label\":\"Weighted Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.weightedScore\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true}},\"columnOrder\":[\"504c50bd-14c1-4119-820e-c961866fc3b4\",\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"504c50bd-14c1-4119-820e-c961866fc3b4\"},{\"isTransposed\":false,\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}},{\"columnId\":\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\",\"isTransposed\":false,\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}}],\"layerId\":\"666e2f39-8868-45ad-b747-fe124830b0ae\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"9890c040-17b7-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"2c407db0-1acb-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Peer Scores\"}]","timeRestore":false,"title":"Peer Scores","version":1},"coreMigrationVersion":"7.14.1","id":"e7e4fd70-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-09-24T11:59:46.187Z","version":"WzY5NjcsMV0="} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file