From 522858c8711e5f56f3b5d6d32ec6f75b5013527f Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 30 Aug 2021 14:35:15 +0200
Subject: [PATCH 01/51] Add lotus tracer interface

---
 node/modules/lp2p/tracer.go | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 node/modules/lp2p/tracer.go

diff --git a/node/modules/lp2p/tracer.go b/node/modules/lp2p/tracer.go
new file mode 100644
index 000000000..36dd6d456
--- /dev/null
+++ b/node/modules/lp2p/tracer.go
@@ -0,0 +1,26 @@
+package lp2p
+
+import (
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+)
+
+func newLotusTracer(tr pubsub.EventTracer) LotusTracer {
+	return &lotusTracer{
+		tr: tr,
+	}
+}
+
+type lotusTracer struct {
+	tr pubsub.EventTracer
+}
+
+type LotusTracer interface {
+	TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
+	Trace(evt *pubsub_pb.TraceEvent)
+}
+
+func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {}
+
+func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {}

From 99fbd7039e524daca65106ff1846d20178a726a1 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 30 Aug 2021 14:38:15 +0200
Subject: [PATCH 02/51] Update pubsub tracking to trace from lotus and lp2p
 tracers

---
 node/modules/lp2p/pubsub.go | 70 +++++++++++++++++++++++++++----------
 1 file changed, 52 insertions(+), 18 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 32b85daf3..5bee17298 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -358,11 +358,12 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 			return nil, err
 		}
 
-		trw := newTracerWrapper(tr, build.BlocksTopic(in.Nn))
+		lt := newLotusTracer(tr)
+		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
 		options = append(options, pubsub.WithEventTracer(trw))
 	} else {
 		// still instantiate a tracer for collecting metrics
-		trw := newTracerWrapper(nil)
+		trw := newTracerWrapper(nil, nil)
 		options = append(options, pubsub.WithEventTracer(trw))
 	}
 
@@ -374,7 +375,11 @@ func HashMsgId(m *pubsub_pb.Message) string {
 	return string(hash[:])
 }
 
-func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTracer {
+func newTracerWrapper(
+	lp2pTracer pubsub.EventTracer,
+	lotusTracer pubsub.EventTracer,
+	topics ...string,
+) pubsub.EventTracer {
 	var topicsMap map[string]struct{}
 	if len(topics) > 0 {
 		topicsMap = make(map[string]struct{})
@@ -383,12 +388,13 @@ func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTrace
 		}
 	}
 
-	return &tracerWrapper{tr: tr, topics: topicsMap}
+	return &tracerWrapper{lp2pTracer: lp2pTracer, lotusTracer: lotusTracer, topics: topicsMap}
 }
 
 type tracerWrapper struct {
-	tr     pubsub.EventTracer
-	topics map[string]struct{}
+	lp2pTracer  pubsub.EventTracer
+	lotusTracer pubsub.EventTracer
+	topics      map[string]struct{}
 }
 
 func (trw *tracerWrapper) traceMessage(topic string) bool {
@@ -406,33 +412,61 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 	switch evt.GetType() {
 	case pubsub_pb.TraceEvent_PUBLISH_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1))
-		if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) {
-			trw.tr.Trace(evt)
+		if trw.traceMessage(evt.GetPublishMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
 		}
 	case pubsub_pb.TraceEvent_DELIVER_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1))
-		if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
-			trw.tr.Trace(evt)
+		if trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
 		}
 	case pubsub_pb.TraceEvent_REJECT_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1))
 	case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1))
 	case pubsub_pb.TraceEvent_JOIN:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_LEAVE:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_GRAFT:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_PRUNE:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_RECV_RPC:
 		stats.Record(context.TODO(), metrics.PubsubRecvRPC.M(1))

From b1dafd81b84cbe298fc514e0d2840d0c4aeaa4d1 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 30 Aug 2021 14:49:10 +0200
Subject: [PATCH 03/51] Add peer score tracing on pubsub peer score inspect

---
 node/modules/lp2p/pubsub.go | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 5bee17298..321108e4a 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -49,6 +49,30 @@ func ScoreKeeper() *dtypes.ScoreKeeper {
 	return new(dtypes.ScoreKeeper)
 }
 
+type PeerScoreTracker interface {
+	UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
+}
+
+type peerScoreTracker struct {
+	sk *dtypes.ScoreKeeper
+	lt LotusTracer
+}
+
+func newPeerScoreTracker(lt LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
+	return &peerScoreTracker{
+		sk: sk,
+		lt: lt,
+	}
+}
+
+func (pst *peerScoreTracker) UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	if pst.lt != nil {
+		pst.lt.TracePeerScore(scores)
+	}
+
+	pst.sk.Update(scores)
+}
+
 type GossipIn struct {
 	fx.In
 	Mctx helpers.MetricsCtx
@@ -272,7 +296,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				OpportunisticGraftThreshold: OpportunisticGraftScoreThreshold,
 			},
 		),
-		pubsub.WithPeerScoreInspect(in.Sk.Update, 10*time.Second),
 	}
 
 	// enable Peer eXchange on bootstrappers
@@ -359,12 +382,18 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		}
 
 		lt := newLotusTracer(tr)
+		pst := newPeerScoreTracker(lt, in.Sk)
 		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
+
 		options = append(options, pubsub.WithEventTracer(trw))
+		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	} else {
 		// still instantiate a tracer for collecting metrics
 		trw := newTracerWrapper(nil, nil)
 		options = append(options, pubsub.WithEventTracer(trw))
+
+		pst := newPeerScoreTracker(nil, in.Sk)
+		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	}
 
 	return pubsub.NewGossipSub(helpers.LifecycleCtx(in.Mctx, in.Lc), in.Host, options...)

From c367533a3cd5f901ea474d17d7ea510331573439 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 10 Sep 2021 12:36:38 +0200
Subject: [PATCH 04/51] Add transport interface

---
 node/modules/{lp2p => tracer}/tracer.go | 10 ++++++----
 node/modules/tracer/transport.go        |  5 +++++
 2 files changed, 11 insertions(+), 4 deletions(-)
 rename node/modules/{lp2p => tracer}/tracer.go (75%)
 create mode 100644 node/modules/tracer/transport.go

diff --git a/node/modules/lp2p/tracer.go b/node/modules/tracer/tracer.go
similarity index 75%
rename from node/modules/lp2p/tracer.go
rename to node/modules/tracer/tracer.go
index 36dd6d456..a19ae065d 100644
--- a/node/modules/lp2p/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -1,4 +1,4 @@
-package lp2p
+package tracer
 
 import (
 	peer "github.com/libp2p/go-libp2p-core/peer"
@@ -6,14 +6,16 @@ import (
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
-func newLotusTracer(tr pubsub.EventTracer) LotusTracer {
+func newLotusTracer(et pubsub.EventTracer, tt TracerTransport) LotusTracer {
 	return &lotusTracer{
-		tr: tr,
+		et: et,
+		tt: tt,
 	}
 }
 
 type lotusTracer struct {
-	tr pubsub.EventTracer
+	et pubsub.EventTracer
+	tt TracerTransport
 }
 
 type LotusTracer interface {
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
new file mode 100644
index 000000000..73ef80690
--- /dev/null
+++ b/node/modules/tracer/transport.go
@@ -0,0 +1,5 @@
+package tracer
+
+type TracerTransport interface {
+	Transport(jsonEvent []byte)
+}

From d5a4b0637abf50b418756d02c796c15230febc0c Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 10 Sep 2021 15:37:38 +0200
Subject: [PATCH 05/51] Add lotus tracer transport calls

---
 node/modules/tracer/tracer.go    | 31 +++++++++++++++++++++++++++++--
 node/modules/tracer/transport.go |  2 +-
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index a19ae065d..6d60352e7 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -1,11 +1,16 @@
 package tracer
 
 import (
+	"encoding/json"
+
+	logging "github.com/ipfs/go-log/v2"
 	peer "github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
+var log = logging.Logger("lotus-tracer")
+
 func newLotusTracer(et pubsub.EventTracer, tt TracerTransport) LotusTracer {
 	return &lotusTracer{
 		et: et,
@@ -23,6 +28,28 @@ type LotusTracer interface {
 	Trace(evt *pubsub_pb.TraceEvent)
 }
 
-func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {}
+func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	jsonEvent, err := json.Marshal(scores)
+	if err != nil {
+		log.Errorf("error while marshaling peer score: %s", err)
+		return
+	}
 
-func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {}
+	err = lt.tt.Transport(jsonEvent)
+	if err != nil {
+		log.Errorf("error while transporting peer scores: %s", err)
+	}
+}
+
+func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
+	jsonEvent, err := json.Marshal(evt)
+	if err != nil {
+		log.Errorf("error while marshaling tracer event: %s", err)
+		return
+	}
+
+	err = lt.tt.Transport(jsonEvent)
+	if err != nil {
+		log.Errorf("error while transporting trace event: %s", err)
+	}
+}
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
index 73ef80690..c8495821e 100644
--- a/node/modules/tracer/transport.go
+++ b/node/modules/tracer/transport.go
@@ -1,5 +1,5 @@
 package tracer
 
 type TracerTransport interface {
-	Transport(jsonEvent []byte)
+	Transport(jsonEvent []byte) error
 }

From 8e6a01de6ddd156f721c9039db0407a8277874eb Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 13:40:54 +0200
Subject: [PATCH 06/51] Add configuration in pubsub to trace locally to json
 file

---
 node/config/types.go        |  1 +
 node/modules/lp2p/pubsub.go | 20 ++++++++++++++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/node/config/types.go b/node/config/types.go
index 1576169e6..05ea472f3 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -309,6 +309,7 @@ type Pubsub struct {
 	DirectPeers           []string
 	IPColocationWhitelist []string
 	RemoteTracer          string
+	JsonTracerFile        string
 }
 
 type Chainstore struct {
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 321108e4a..6cf52fdf0 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"net"
+	"os"
 	"time"
 
 	host "github.com/libp2p/go-libp2p-core/host"
@@ -21,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
+	"github.com/filecoin-project/lotus/node/modules/tracer"
 )
 
 func init() {
@@ -55,10 +57,10 @@ type PeerScoreTracker interface {
 
 type peerScoreTracker struct {
 	sk *dtypes.ScoreKeeper
-	lt LotusTracer
+	lt tracer.LotusTracer
 }
 
-func newPeerScoreTracker(lt LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
+func newPeerScoreTracker(lt tracer.LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
 	return &peerScoreTracker{
 		sk: sk,
 		lt: lt,
@@ -67,7 +69,7 @@ func newPeerScoreTracker(lt LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracke
 
 func (pst *peerScoreTracker) UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
 	if pst.lt != nil {
-		pst.lt.TracePeerScore(scores)
+		pst.lt.PeerScores(scores)
 	}
 
 	pst.sk.Update(scores)
@@ -364,6 +366,17 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
 				100)))
 
+	var lt tracer.LotusTracer
+	if in.Cfg.JsonTracerFile != "" {
+		out, err := os.OpenFile(in.Cfg.JsonTracerFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+		if err != nil {
+			return nil, err
+		}
+
+		jsonTransport := tracer.NewJsonTracerTransport(out)
+		lt = tracer.NewLotusTracer(jsonTransport, in.Host.ID())
+	}
+
 	// tracer
 	if in.Cfg.RemoteTracer != "" {
 		a, err := ma.NewMultiaddr(in.Cfg.RemoteTracer)
@@ -381,7 +394,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 			return nil, err
 		}
 
-		lt := newLotusTracer(tr)
 		pst := newPeerScoreTracker(lt, in.Sk)
 		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
 

From ab86c5419d6ac0fcfdd789e4f421a8cc2db61cef Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 13:41:03 +0200
Subject: [PATCH 07/51] Implement json tracer transport

---
 node/modules/tracer/json_transport.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 node/modules/tracer/json_transport.go

diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
new file mode 100644
index 000000000..2ab004d9c
--- /dev/null
+++ b/node/modules/tracer/json_transport.go
@@ -0,0 +1,20 @@
+package tracer
+
+import (
+	"os"
+)
+
+type jsonTracerTransport struct {
+	out *os.File
+}
+
+func NewJsonTracerTransport(out *os.File) TracerTransport {
+	return &jsonTracerTransport{
+		out: out,
+	}
+}
+
+func (jtt *jsonTracerTransport) Transport(jsonEvent []byte) error {
+	_, err := jtt.out.Write(jsonEvent)
+	return err
+}

From 8a657b90a7d5eba00db902afff5ddd7673ea0e22 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 13:41:56 +0200
Subject: [PATCH 08/51] Refactor lotus tracer to behave like lp2p tracer

---
 node/modules/tracer/tracer.go | 50 ++++++++++++++++++++++++++++-------
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 6d60352e7..af3a987b0 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -2,34 +2,66 @@ package tracer
 
 import (
 	"encoding/json"
+	"time"
 
 	logging "github.com/ipfs/go-log/v2"
-	peer "github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
 var log = logging.Logger("lotus-tracer")
 
-func newLotusTracer(et pubsub.EventTracer, tt TracerTransport) LotusTracer {
+func NewLotusTracer(tt TracerTransport, pid peer.ID) LotusTracer {
 	return &lotusTracer{
-		et: et,
-		tt: tt,
+		tt:  tt,
+		pid: pid,
 	}
 }
 
 type lotusTracer struct {
-	et pubsub.EventTracer
-	tt TracerTransport
+	tt  TracerTransport
+	pid peer.ID
+}
+
+const (
+	TraceEvent_PEER_SCORES pubsub_pb.TraceEvent_Type = 100
+)
+
+type LotusTraceEvent struct {
+	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
+	PeerID     []byte                    `json:"peerID,omitempty"`
+	Timestamp  *int64                    `json:"timestamp,omitempty"`
+	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
+}
+
+type TraceEvent_PeerScores struct {
+	Scores map[peer.ID]*pubsub.PeerScoreSnapshot `json:"scores,omitempty"`
 }
 
 type LotusTracer interface {
-	TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 	Trace(evt *pubsub_pb.TraceEvent)
+	TraceLotusEvent(evt *LotusTraceEvent)
+
+	PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 }
 
-func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
-	jsonEvent, err := json.Marshal(scores)
+func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	now := time.Now().UnixNano()
+	evt := &LotusTraceEvent{
+		Type:      *TraceEvent_PEER_SCORES.Enum(),
+		PeerID:    []byte(lt.pid),
+		Timestamp: &now,
+		PeerScores: &TraceEvent_PeerScores{
+			Scores: scores,
+		},
+	}
+
+	lt.TraceLotusEvent(evt)
+}
+
+func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
+	jsonEvent, err := json.Marshal(evt)
 	if err != nil {
 		log.Errorf("error while marshaling peer score: %s", err)
 		return

From c064ce485283cb96a751c1924996c9cc28fd6a44 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 14:22:48 +0200
Subject: [PATCH 09/51] Add generated documentation

---
 node/config/doc_gen.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index 9efe4e03c..05e7afff4 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -525,6 +525,12 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "RemoteTracer",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "JsonTracerFile",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},

From bb85ca031d9466cc77fec59b04361680822a7f2f Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 14:50:27 +0200
Subject: [PATCH 10/51] Add elasticsearch transport

---
 go.mod                                        |  1 +
 go.sum                                        |  2 +
 node/config/doc_gen.go                        |  6 ++
 node/config/types.go                          |  1 +
 node/modules/lp2p/pubsub.go                   |  3 +
 .../modules/tracer/elasticsearch_transport.go | 59 +++++++++++++++++++
 node/modules/tracer/tracer.go                 | 21 ++++---
 7 files changed, 85 insertions(+), 8 deletions(-)
 create mode 100644 node/modules/tracer/elasticsearch_transport.go

diff --git a/go.mod b/go.mod
index f73511574..a5adcca6f 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/drand/drand v1.2.1
 	github.com/drand/kyber v1.1.4
 	github.com/dustin/go-humanize v1.0.0
+	github.com/elastic/go-elasticsearch/v7 v7.14.0
 	github.com/elastic/go-sysinfo v1.3.0
 	github.com/elastic/gosigar v0.12.0
 	github.com/etclabscore/go-openrpc-reflect v0.0.36
diff --git a/go.sum b/go.sum
index f16a81a9f..6347ac4f7 100644
--- a/go.sum
+++ b/go.sum
@@ -235,6 +235,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/go-elasticsearch/v7 v7.14.0 h1:extp3jos/rwJn3J+lgbaGlwAgs0TVsIHme00GyNAyX4=
+github.com/elastic/go-elasticsearch/v7 v7.14.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE=
 github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
 github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY=
diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index 05e7afff4..fd4ec0366 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -531,6 +531,12 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "JsonTracerFile",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "ElasticSearchTracer",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},
diff --git a/node/config/types.go b/node/config/types.go
index 05ea472f3..025a9c1af 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -310,6 +310,7 @@ type Pubsub struct {
 	IPColocationWhitelist []string
 	RemoteTracer          string
 	JsonTracerFile        string
+	ElasticSearchTracer   string
 }
 
 type Chainstore struct {
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 6cf52fdf0..04f21c84f 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -375,6 +375,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		lt = tracer.NewLotusTracer(jsonTransport, in.Host.ID())
+	} else if in.Cfg.ElasticSearchTracer != "" {
+		elasticSearchTransport := tracer.NewElasticSearchTransport()
+		lt = tracer.NewLotusTracer(elasticSearchTransport, in.Host.ID())
 	}
 
 	// tracer
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
new file mode 100644
index 000000000..2cba23478
--- /dev/null
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -0,0 +1,59 @@
+package tracer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var rpclog = logging.Logger("elasticsearch")
+
+func NewElasticSearchTransport() TracerTransport {
+	es, err := elasticsearch.NewDefaultClient()
+
+	if err != nil {
+		rpclog.Fatalf("Error on creating elastic search client: %+v", err)
+	}
+
+	return &elasticSearchTransport{
+		cl: es,
+	}
+}
+
+type elasticSearchTransport struct {
+	cl *elasticsearch.Client
+}
+
+func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
+	req := esapi.IndexRequest{
+		Index:      "PeerScore",
+		DocumentID: "1", // todo
+		Body:       strings.NewReader(string(jsonEvent)),
+		Refresh:    "true",
+	}
+
+	// Perform the request with the client.
+	res, err := req.Do(context.Background(), est.cl)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	if res.IsError() {
+		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
+	} else {
+		// Deserialize the response into a map.
+		var r map[string]interface{}
+		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
+			return err
+		} else {
+			rpclog.Infof("[%s] %s; version=%d", res.Status(), r["result"], int(r["_version"].(float64)))
+		}
+	}
+	return nil
+}
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index af3a987b0..8c6250151 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -12,7 +12,7 @@ import (
 
 var log = logging.Logger("lotus-tracer")
 
-func NewLotusTracer(tt TracerTransport, pid peer.ID) LotusTracer {
+func NewLotusTracer(tt []TracerTransport, pid peer.ID) LotusTracer {
 	return &lotusTracer{
 		tt:  tt,
 		pid: pid,
@@ -20,7 +20,7 @@ func NewLotusTracer(tt TracerTransport, pid peer.ID) LotusTracer {
 }
 
 type lotusTracer struct {
-	tt  TracerTransport
+	tt  []TracerTransport
 	pid peer.ID
 }
 
@@ -67,10 +67,13 @@ func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
 		return
 	}
 
-	err = lt.tt.Transport(jsonEvent)
-	if err != nil {
-		log.Errorf("error while transporting peer scores: %s", err)
+	for _, t := range lt.tt {
+		err = t.Transport(jsonEvent)
+		if err != nil {
+			log.Errorf("error while transporting peer scores: %s", err)
+		}
 	}
+
 }
 
 func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
@@ -80,8 +83,10 @@ func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
 		return
 	}
 
-	err = lt.tt.Transport(jsonEvent)
-	if err != nil {
-		log.Errorf("error while transporting trace event: %s", err)
+	for _, t := range lt.tt {
+		err = t.Transport(jsonEvent)
+		if err != nil {
+			log.Errorf("error while transporting trace event: %s", err)
+		}
 	}
 }

From cf060965658c933290ddeea205effb97bb54c0ca Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 14:54:20 +0200
Subject: [PATCH 11/51] Fix tracer initialization

---
 node/modules/lp2p/pubsub.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 04f21c84f..c9755a680 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -366,19 +366,19 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
 				100)))
 
-	var lt tracer.LotusTracer
+	var transports []tracer.TracerTransport
 	if in.Cfg.JsonTracerFile != "" {
 		out, err := os.OpenFile(in.Cfg.JsonTracerFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
 		if err != nil {
 			return nil, err
 		}
-
 		jsonTransport := tracer.NewJsonTracerTransport(out)
-		lt = tracer.NewLotusTracer(jsonTransport, in.Host.ID())
+		transports = append(transports, jsonTransport)
 	} else if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport := tracer.NewElasticSearchTransport()
-		lt = tracer.NewLotusTracer(elasticSearchTransport, in.Host.ID())
+		transports = append(transports, elasticSearchTransport)
 	}
+	lt := tracer.NewLotusTracer(transports, in.Host.ID())
 
 	// tracer
 	if in.Cfg.RemoteTracer != "" {

From e20cd0ef27c6abb0f8135c801796d128d81c657c Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 14:58:36 +0200
Subject: [PATCH 12/51] Propagate error on creating transport

---
 node/modules/lp2p/pubsub.go                    | 5 ++++-
 node/modules/tracer/elasticsearch_transport.go | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index c9755a680..7f2a4d678 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -375,7 +375,10 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		transports = append(transports, jsonTransport)
 	} else if in.Cfg.ElasticSearchTracer != "" {
-		elasticSearchTransport := tracer.NewElasticSearchTransport()
+		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
+		if err != nil {
+			return nil, err
+		}
 		transports = append(transports, elasticSearchTransport)
 	}
 	lt := tracer.NewLotusTracer(transports, in.Host.ID())
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 2cba23478..f45dd9ef5 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -13,16 +13,16 @@ import (
 
 var rpclog = logging.Logger("elasticsearch")
 
-func NewElasticSearchTransport() TracerTransport {
+func NewElasticSearchTransport() (TracerTransport, error) {
 	es, err := elasticsearch.NewDefaultClient()
 
 	if err != nil {
-		rpclog.Fatalf("Error on creating elastic search client: %+v", err)
+		return nil, err
 	}
 
 	return &elasticSearchTransport{
 		cl: es,
-	}
+	}, nil
 }
 
 type elasticSearchTransport struct {

From 707faf57b54967a384bf46196b79ff8ec80c79b1 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 15:11:34 +0200
Subject: [PATCH 13/51] Code cleanup

---
 node/modules/lp2p/pubsub.go                    | 4 +++-
 node/modules/tracer/elasticsearch_transport.go | 9 ++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 7f2a4d678..6656abf01 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -374,7 +374,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		}
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		transports = append(transports, jsonTransport)
-	} else if in.Cfg.ElasticSearchTracer != "" {
+	}
+
+	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
 		if err != nil {
 			return nil, err
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index f45dd9ef5..8e214a4e9 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -8,10 +8,11 @@ import (
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
-	logging "github.com/ipfs/go-log/v2"
 )
 
-var rpclog = logging.Logger("elasticsearch")
+const (
+	ElasticSearch_INDEX = "pubsub"
+)
 
 func NewElasticSearchTransport() (TracerTransport, error) {
 	es, err := elasticsearch.NewDefaultClient()
@@ -31,7 +32,7 @@ type elasticSearchTransport struct {
 
 func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
 	req := esapi.IndexRequest{
-		Index:      "PeerScore",
+		Index:      ElasticSearch_INDEX,
 		DocumentID: "1", // todo
 		Body:       strings.NewReader(string(jsonEvent)),
 		Refresh:    "true",
@@ -51,8 +52,6 @@ func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
 		var r map[string]interface{}
 		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
 			return err
-		} else {
-			rpclog.Infof("[%s] %s; version=%d", res.Status(), r["result"], int(r["_version"].(float64)))
 		}
 	}
 	return nil

From 71a2ca9e7cd96b277a6c29049283645d235fe2e4 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:03:51 +0200
Subject: [PATCH 14/51] Add new wrapper structure in tracer transport

---
 .../modules/tracer/elasticsearch_transport.go | 24 +++++++++++++++++--
 node/modules/tracer/json_transport.go         | 20 ++++++++++++++--
 node/modules/tracer/tracer.go                 | 23 +++++++-----------
 node/modules/tracer/transport.go              |  9 ++++++-
 4 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 8e214a4e9..f9cf621eb 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,6 +12,9 @@ import (
 
 const (
 	ElasticSearch_INDEX = "pubsub"
+
+	ElasticSearch_DOC_LOTUS  = "doc_lotus"
+	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
 )
 
 func NewElasticSearchTransport() (TracerTransport, error) {
@@ -30,10 +33,27 @@ type elasticSearchTransport struct {
 	cl *elasticsearch.Client
 }
 
-func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
+func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
+	var e interface{}
+	var docId string
+	if event.lotusTraceEvent != nil {
+		e = *event.lotusTraceEvent
+		docId = ElasticSearch_DOC_LOTUS
+	} else if event.pubsubTraceEvent != nil {
+		e = *event.pubsubTraceEvent
+		docId = ElasticSearch_DOC_PUBSUB
+	} else {
+		return nil
+	}
+
+	jsonEvent, err := json.Marshal(e)
+	if err != nil {
+		return fmt.Errorf("error while marshaling peer score: %s", err)
+	}
+
 	req := esapi.IndexRequest{
 		Index:      ElasticSearch_INDEX,
-		DocumentID: "1", // todo
+		DocumentID: docId,
 		Body:       strings.NewReader(string(jsonEvent)),
 		Refresh:    "true",
 	}
diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 2ab004d9c..128c37473 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -1,6 +1,8 @@
 package tracer
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 )
 
@@ -14,7 +16,21 @@ func NewJsonTracerTransport(out *os.File) TracerTransport {
 	}
 }
 
-func (jtt *jsonTracerTransport) Transport(jsonEvent []byte) error {
-	_, err := jtt.out.Write(jsonEvent)
+func (jtt *jsonTracerTransport) Transport(event TracerTransportEvent) error {
+	var e interface{}
+	if event.lotusTraceEvent != nil {
+		e = *event.lotusTraceEvent
+	} else if event.pubsubTraceEvent != nil {
+		e = *event.pubsubTraceEvent
+	} else {
+		return nil
+	}
+
+	jsonEvent, err := json.Marshal(e)
+	if err != nil {
+		return fmt.Errorf("error while marshaling event: %s", err)
+	}
+
+	_, err = jtt.out.Write(jsonEvent)
 	return err
 }
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 8c6250151..ad5f5c6b7 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -1,7 +1,6 @@
 package tracer
 
 import (
-	"encoding/json"
 	"time"
 
 	logging "github.com/ipfs/go-log/v2"
@@ -61,14 +60,11 @@ func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 }
 
 func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
-	jsonEvent, err := json.Marshal(evt)
-	if err != nil {
-		log.Errorf("error while marshaling peer score: %s", err)
-		return
-	}
-
 	for _, t := range lt.tt {
-		err = t.Transport(jsonEvent)
+		err := t.Transport(TracerTransportEvent{
+			lotusTraceEvent:  evt,
+			pubsubTraceEvent: nil,
+		})
 		if err != nil {
 			log.Errorf("error while transporting peer scores: %s", err)
 		}
@@ -77,14 +73,11 @@ func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
 }
 
 func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
-	jsonEvent, err := json.Marshal(evt)
-	if err != nil {
-		log.Errorf("error while marshaling tracer event: %s", err)
-		return
-	}
-
 	for _, t := range lt.tt {
-		err = t.Transport(jsonEvent)
+		err := t.Transport(TracerTransportEvent{
+			lotusTraceEvent:  nil,
+			pubsubTraceEvent: evt,
+		})
 		if err != nil {
 			log.Errorf("error while transporting trace event: %s", err)
 		}
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
index c8495821e..e67867fd7 100644
--- a/node/modules/tracer/transport.go
+++ b/node/modules/tracer/transport.go
@@ -1,5 +1,12 @@
 package tracer
 
+import pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+
 type TracerTransport interface {
-	Transport(jsonEvent []byte) error
+	Transport(jsonEvent TracerTransportEvent) error
+}
+
+type TracerTransportEvent struct {
+	lotusTraceEvent  *LotusTraceEvent
+	pubsubTraceEvent *pubsub_pb.TraceEvent
 }

From add13c2646cbbe4157fe7a86d0aae26e3d056c3a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:05:44 +0200
Subject: [PATCH 15/51] Code cleanup

---
 node/modules/lp2p/pubsub.go                    | 1 -
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 6656abf01..905643ed0 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -375,7 +375,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		transports = append(transports, jsonTransport)
 	}
-
 	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
 		if err != nil {
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index f9cf621eb..4fe8149be 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -48,7 +48,7 @@ func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
 
 	jsonEvent, err := json.Marshal(e)
 	if err != nil {
-		return fmt.Errorf("error while marshaling peer score: %s", err)
+		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
 	req := esapi.IndexRequest{

From 67c1d633c68a77baff415af5560444be93914122 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:16:12 +0200
Subject: [PATCH 16/51] Rename jsonEvent to evt

---
 node/modules/tracer/elasticsearch_transport.go | 14 +++++++-------
 node/modules/tracer/json_transport.go          | 14 +++++++-------
 node/modules/tracer/transport.go               |  2 +-
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 4fe8149be..c2cfdb87a 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -33,20 +33,20 @@ type elasticSearchTransport struct {
 	cl *elasticsearch.Client
 }
 
-func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
+func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	var e interface{}
 	var docId string
-	if event.lotusTraceEvent != nil {
-		e = *event.lotusTraceEvent
+	if evt.lotusTraceEvent != nil {
+		e = *evt.lotusTraceEvent
 		docId = ElasticSearch_DOC_LOTUS
-	} else if event.pubsubTraceEvent != nil {
-		e = *event.pubsubTraceEvent
+	} else if evt.pubsubTraceEvent != nil {
+		e = *evt.pubsubTraceEvent
 		docId = ElasticSearch_DOC_PUBSUB
 	} else {
 		return nil
 	}
 
-	jsonEvent, err := json.Marshal(e)
+	jsonEvt, err := json.Marshal(e)
 	if err != nil {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
@@ -54,7 +54,7 @@ func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
 	req := esapi.IndexRequest{
 		Index:      ElasticSearch_INDEX,
 		DocumentID: docId,
-		Body:       strings.NewReader(string(jsonEvent)),
+		Body:       strings.NewReader(string(jsonEvt)),
 		Refresh:    "true",
 	}
 
diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 128c37473..26537a600 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -16,21 +16,21 @@ func NewJsonTracerTransport(out *os.File) TracerTransport {
 	}
 }
 
-func (jtt *jsonTracerTransport) Transport(event TracerTransportEvent) error {
+func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {
 	var e interface{}
-	if event.lotusTraceEvent != nil {
-		e = *event.lotusTraceEvent
-	} else if event.pubsubTraceEvent != nil {
-		e = *event.pubsubTraceEvent
+	if evt.lotusTraceEvent != nil {
+		e = *evt.lotusTraceEvent
+	} else if evt.pubsubTraceEvent != nil {
+		e = *evt.pubsubTraceEvent
 	} else {
 		return nil
 	}
 
-	jsonEvent, err := json.Marshal(e)
+	jsonEvt, err := json.Marshal(e)
 	if err != nil {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
-	_, err = jtt.out.Write(jsonEvent)
+	_, err = jtt.out.Write(jsonEvt)
 	return err
 }
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
index e67867fd7..56d926afc 100644
--- a/node/modules/tracer/transport.go
+++ b/node/modules/tracer/transport.go
@@ -3,7 +3,7 @@ package tracer
 import pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 
 type TracerTransport interface {
-	Transport(jsonEvent TracerTransportEvent) error
+	Transport(evt TracerTransportEvent) error
 }
 
 type TracerTransportEvent struct {

From 5cbd2519daf3fc0560317c249d7c031b6d52085c Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:20:36 +0200
Subject: [PATCH 17/51] Remove processing elasticsearch result

---
 node/modules/tracer/elasticsearch_transport.go | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index c2cfdb87a..3bf86ee29 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -67,12 +67,6 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 
 	if res.IsError() {
 		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
-	} else {
-		// Deserialize the response into a map.
-		var r map[string]interface{}
-		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
-			return err
-		}
 	}
 	return nil
 }

From c25a5e06cf5f3b84699bf187bd9129f9ea65e29a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Thu, 16 Sep 2021 12:15:42 +0200
Subject: [PATCH 18/51] Use config string for elasticsearch client

---
 node/modules/lp2p/pubsub.go                    |  4 +++-
 node/modules/tracer/elasticsearch_transport.go | 17 +++++++++++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 905643ed0..d12a3fa1c 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -376,7 +376,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		transports = append(transports, jsonTransport)
 	}
 	if in.Cfg.ElasticSearchTracer != "" {
-		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
+		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
+			in.Cfg.ElasticSearchTracer,
+		)
 		if err != nil {
 			return nil, err
 		}
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 3bf86ee29..45ddd36aa 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v7"
@@ -17,8 +18,20 @@ const (
 	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
 )
 
-func NewElasticSearchTransport() (TracerTransport, error) {
-	es, err := elasticsearch.NewDefaultClient()
+func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
+	conUrl, err := url.Parse(connectionString)
+
+	username := conUrl.User.Username()
+	password, _ := conUrl.User.Password()
+	cfg := elasticsearch.Config{
+		Addresses: []string{
+			"https://" + conUrl.Host,
+		},
+		Username: username,
+		Password: password,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
 
 	if err != nil {
 		return nil, err

From e2206147cc6dc6346dd11aadbee7226f04792c65 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 11:51:59 +0200
Subject: [PATCH 19/51] Add source auth token to lotus traces

---
 node/config/types.go          |  1 +
 node/modules/lp2p/pubsub.go   |  2 +-
 node/modules/tracer/tracer.go | 12 ++++++++----
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/node/config/types.go b/node/config/types.go
index 025a9c1af..aa061baef 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -311,6 +311,7 @@ type Pubsub struct {
 	RemoteTracer          string
 	JsonTracerFile        string
 	ElasticSearchTracer   string
+	TracerSourceAuth      string
 }
 
 type Chainstore struct {
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index d12a3fa1c..aa391a42b 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -384,7 +384,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		}
 		transports = append(transports, elasticSearchTransport)
 	}
-	lt := tracer.NewLotusTracer(transports, in.Host.ID())
+	lt := tracer.NewLotusTracer(transports, in.Host.ID(), in.Cfg.TracerSourceAuth)
 
 	// tracer
 	if in.Cfg.RemoteTracer != "" {
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index ad5f5c6b7..69967565a 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -11,16 +11,18 @@ import (
 
 var log = logging.Logger("lotus-tracer")
 
-func NewLotusTracer(tt []TracerTransport, pid peer.ID) LotusTracer {
+func NewLotusTracer(tt []TracerTransport, pid peer.ID, sourceAuth string) LotusTracer {
 	return &lotusTracer{
 		tt:  tt,
 		pid: pid,
+		sa:  sourceAuth,
 	}
 }
 
 type lotusTracer struct {
 	tt  []TracerTransport
 	pid peer.ID
+	sa  string
 }
 
 const (
@@ -32,6 +34,7 @@ type LotusTraceEvent struct {
 	PeerID     []byte                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
 	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
+	SourceAuth string                    `json:"sourceAuth,omitempty"`
 }
 
 type TraceEvent_PeerScores struct {
@@ -48,9 +51,10 @@ type LotusTracer interface {
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
 	now := time.Now().UnixNano()
 	evt := &LotusTraceEvent{
-		Type:      *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:    []byte(lt.pid),
-		Timestamp: &now,
+		Type:       *TraceEvent_PEER_SCORES.Enum(),
+		PeerID:     []byte(lt.pid),
+		Timestamp:  &now,
+		SourceAuth: lt.sa,
 		PeerScores: &TraceEvent_PeerScores{
 			Scores: scores,
 		},

From d067bc9f0109ad17a9ea368d89e6045f8969f50b Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 15:22:18 +0200
Subject: [PATCH 20/51] Move creating and opening file to json transport
 constructor

---
 node/modules/lp2p/pubsub.go           | 5 ++---
 node/modules/tracer/json_transport.go | 9 +++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index d12a3fa1c..0a3f5e042 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"net"
-	"os"
 	"time"
 
 	host "github.com/libp2p/go-libp2p-core/host"
@@ -368,11 +367,11 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 
 	var transports []tracer.TracerTransport
 	if in.Cfg.JsonTracerFile != "" {
-		out, err := os.OpenFile(in.Cfg.JsonTracerFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracerFile)
 		if err != nil {
 			return nil, err
 		}
-		jsonTransport := tracer.NewJsonTracerTransport(out)
+
 		transports = append(transports, jsonTransport)
 	}
 	if in.Cfg.ElasticSearchTracer != "" {
diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 26537a600..1d5e81f30 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -10,10 +10,15 @@ type jsonTracerTransport struct {
 	out *os.File
 }
 
-func NewJsonTracerTransport(out *os.File) TracerTransport {
+func NewJsonTracerTransport(file string) (TracerTransport, error) {
+	out, err := os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+	if err != nil {
+		return nil, err
+	}
+
 	return &jsonTracerTransport{
 		out: out,
-	}
+	}, nil
 }
 
 func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {

From 9cdc5261c44050acd7aab47418a74b48a70bccd3 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 15:53:39 +0200
Subject: [PATCH 21/51] Fix lotus tracer being nil when remote tracer is not
 configured

---
 node/modules/lp2p/pubsub.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 0a3f5e042..a071b36a5 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -409,7 +409,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	} else {
 		// still instantiate a tracer for collecting metrics
-		trw := newTracerWrapper(nil, nil)
+		trw := newTracerWrapper(nil, lt)
 		options = append(options, pubsub.WithEventTracer(trw))
 
 		pst := newPeerScoreTracker(nil, in.Sk)

From 941a0f715327a341accc371cd727a3eeee42a2d2 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 16:31:36 +0200
Subject: [PATCH 22/51] Update peer score tracker creation to include lotus
 tracer

---
 node/modules/lp2p/pubsub.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index a071b36a5..25ee4963f 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -412,7 +412,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		trw := newTracerWrapper(nil, lt)
 		options = append(options, pubsub.WithEventTracer(trw))
 
-		pst := newPeerScoreTracker(nil, in.Sk)
+		pst := newPeerScoreTracker(lt, in.Sk)
 		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	}
 

From 9efa495b0ebbdbb82f0e6047a352b7c5b3c5cc9b Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 16:42:18 +0200
Subject: [PATCH 23/51] Write every trace to new line when using json transport

---
 node/modules/tracer/json_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 1d5e81f30..ca8535f4b 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -36,6 +36,6 @@ func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
-	_, err = jtt.out.Write(jsonEvt)
+	_, err = jtt.out.WriteString(string(jsonEvt) + "\n")
 	return err
 }

From 8e2b474fc774a92f815c93df513d9770ebdec5b3 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Thu, 16 Sep 2021 16:44:35 +0200
Subject: [PATCH 24/51] Use only one document on elastic

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 45ddd36aa..f57ce85de 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -54,7 +54,7 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 		docId = ElasticSearch_DOC_LOTUS
 	} else if evt.pubsubTraceEvent != nil {
 		e = *evt.pubsubTraceEvent
-		docId = ElasticSearch_DOC_PUBSUB
+		docId = ElasticSearch_DOC_LOTUS
 	} else {
 		return nil
 	}

From f09f5c2d1144bbceef2f013ad23325bdc080dd8a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Thu, 16 Sep 2021 16:49:47 +0200
Subject: [PATCH 25/51] Change elastic document name

---
 node/modules/tracer/elasticsearch_transport.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index f57ce85de..612080ef8 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -51,10 +51,10 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	var docId string
 	if evt.lotusTraceEvent != nil {
 		e = *evt.lotusTraceEvent
-		docId = ElasticSearch_DOC_LOTUS
+		docId = ElasticSearch_DOC_PUBSUB
 	} else if evt.pubsubTraceEvent != nil {
 		e = *evt.pubsubTraceEvent
-		docId = ElasticSearch_DOC_LOTUS
+		docId = ElasticSearch_DOC_PUBSUB
 	} else {
 		return nil
 	}

From dc79ea363d0b8803a405847bc9ef611bf4a4c01a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 17 Sep 2021 11:35:36 +0200
Subject: [PATCH 26/51] Change index name

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 612080ef8..3f9a47e67 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "pubsub"
+	ElasticSearch_INDEX = "lotus"
 
 	ElasticSearch_DOC_LOTUS  = "doc_lotus"
 	ElasticSearch_DOC_PUBSUB = "doc_pubsub"

From a91ffa60d67f8b3fe1f90460e5a3f70ba1e2bb4f Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 17 Sep 2021 12:11:14 +0200
Subject: [PATCH 27/51] Set peer id event as string in lotus trace event

---
 node/modules/tracer/tracer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index ad5f5c6b7..7d8d9e2c4 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -29,7 +29,7 @@ const (
 
 type LotusTraceEvent struct {
 	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
-	PeerID     []byte                    `json:"peerID,omitempty"`
+	PeerID     string                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
 	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
 }
@@ -49,7 +49,7 @@ func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 	now := time.Now().UnixNano()
 	evt := &LotusTraceEvent{
 		Type:      *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:    []byte(lt.pid),
+		PeerID:    string(lt.pid),
 		Timestamp: &now,
 		PeerScores: &TraceEvent_PeerScores{
 			Scores: scores,

From 7d0aefacbb9216221fb9d3d7ee1b863a452579b7 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 17 Sep 2021 13:46:26 +0200
Subject: [PATCH 28/51] Change peer score event type to avoid exceeding
 elastisearch field limit

---
 node/modules/tracer/tracer.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 7d8d9e2c4..4367e8fda 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -31,11 +31,12 @@ type LotusTraceEvent struct {
 	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
 	PeerID     string                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
-	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
+	PeerScores []TraceEvent_PeerScore    `json:"peerScores,omitempty"`
 }
 
-type TraceEvent_PeerScores struct {
-	Scores map[peer.ID]*pubsub.PeerScoreSnapshot `json:"scores,omitempty"`
+type TraceEvent_PeerScore struct {
+	PeerID string  `json:"peerID"`
+	Score  float32 `json:"score"`
 }
 
 type LotusTracer interface {
@@ -46,14 +47,17 @@ type LotusTracer interface {
 }
 
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	var peerScores []TraceEvent_PeerScore
+	for pid, score := range scores {
+		peerScores = append(peerScores, TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)})
+	}
+
 	now := time.Now().UnixNano()
 	evt := &LotusTraceEvent{
-		Type:      *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:    string(lt.pid),
-		Timestamp: &now,
-		PeerScores: &TraceEvent_PeerScores{
-			Scores: scores,
-		},
+		Type:       *TraceEvent_PEER_SCORES.Enum(),
+		PeerID:     string(lt.pid),
+		Timestamp:  &now,
+		PeerScores: peerScores,
 	}
 
 	lt.TraceLotusEvent(evt)

From 0962e73e3e8a0ddb506ef044030014229c147555 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 17 Sep 2021 14:22:57 +0200
Subject: [PATCH 29/51] Add loger to elastic search client

---
 node/modules/tracer/elasticsearch_transport.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 3f9a47e67..a1c271afc 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -5,14 +5,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"os"
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/elastic/go-elasticsearch/v7/estransport"
 )
 
 const (
-	ElasticSearch_INDEX = "lotus"
+	ElasticSearch_INDEX = "lotus2"
 
 	ElasticSearch_DOC_LOTUS  = "doc_lotus"
 	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
@@ -29,6 +31,11 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		},
 		Username: username,
 		Password: password,
+		Logger: &estransport.CurlLogger{
+			Output:             os.Stdout,
+			EnableRequestBody:  true,
+			EnableResponseBody: true,
+		},
 	}
 
 	es, err := elasticsearch.NewClient(cfg)

From a685ccc9555b52c62a0ffec8f448dfbc46426adb Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 17 Sep 2021 15:14:39 +0200
Subject: [PATCH 30/51] Remove document id

---
 node/modules/tracer/elasticsearch_transport.go | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index a1c271afc..4709ef756 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -14,10 +14,7 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "lotus2"
-
-	ElasticSearch_DOC_LOTUS  = "doc_lotus"
-	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
+	ElasticSearch_INDEX = "lotus-pubsub"
 )
 
 func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
@@ -55,13 +52,11 @@ type elasticSearchTransport struct {
 
 func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	var e interface{}
-	var docId string
+
 	if evt.lotusTraceEvent != nil {
 		e = *evt.lotusTraceEvent
-		docId = ElasticSearch_DOC_PUBSUB
 	} else if evt.pubsubTraceEvent != nil {
 		e = *evt.pubsubTraceEvent
-		docId = ElasticSearch_DOC_PUBSUB
 	} else {
 		return nil
 	}
@@ -72,10 +67,9 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	}
 
 	req := esapi.IndexRequest{
-		Index:      ElasticSearch_INDEX,
-		DocumentID: docId,
-		Body:       strings.NewReader(string(jsonEvt)),
-		Refresh:    "true",
+		Index:   ElasticSearch_INDEX,
+		Body:    strings.NewReader(string(jsonEvt)),
+		Refresh: "true",
 	}
 
 	// Perform the request with the client.

From 082f70757c485827552c9731a85c796673de7735 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Mon, 20 Sep 2021 12:29:12 +0200
Subject: [PATCH 31/51] Revert additional logging in elastic client

---
 node/modules/tracer/elasticsearch_transport.go | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 4709ef756..8823b8199 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -5,12 +5,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
-	"os"
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
-	"github.com/elastic/go-elasticsearch/v7/estransport"
 )
 
 const (
@@ -28,11 +26,6 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		},
 		Username: username,
 		Password: password,
-		Logger: &estransport.CurlLogger{
-			Output:             os.Stdout,
-			EnableRequestBody:  true,
-			EnableResponseBody: true,
-		},
 	}
 
 	es, err := elasticsearch.NewClient(cfg)

From d65d9aa6e0a25fe5df8d955e2cd79a07fbd44932 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Tue, 21 Sep 2021 12:30:50 +0200
Subject: [PATCH 32/51] Send peerID peer score as separate events to enable
 usage of kibana runtime fields

---
 node/modules/tracer/tracer.go | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 4367e8fda..01ed1196a 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -28,10 +28,10 @@ const (
 )
 
 type LotusTraceEvent struct {
-	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
-	PeerID     string                    `json:"peerID,omitempty"`
-	Timestamp  *int64                    `json:"timestamp,omitempty"`
-	PeerScores []TraceEvent_PeerScore    `json:"peerScores,omitempty"`
+	Type      pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
+	PeerID    string                    `json:"peerID,omitempty"`
+	Timestamp *int64                    `json:"timestamp,omitempty"`
+	PeerScore TraceEvent_PeerScore      `json:"peerScore,omitempty"`
 }
 
 type TraceEvent_PeerScore struct {
@@ -47,20 +47,17 @@ type LotusTracer interface {
 }
 
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
-	var peerScores []TraceEvent_PeerScore
-	for pid, score := range scores {
-		peerScores = append(peerScores, TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)})
-	}
-
 	now := time.Now().UnixNano()
-	evt := &LotusTraceEvent{
-		Type:       *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:     string(lt.pid),
-		Timestamp:  &now,
-		PeerScores: peerScores,
-	}
+	for pid, score := range scores {
+		evt := &LotusTraceEvent{
+			Type:      *TraceEvent_PEER_SCORES.Enum(),
+			PeerID:    lt.pid.Pretty(),
+			Timestamp: &now,
+			PeerScore: TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)},
+		}
 
-	lt.TraceLotusEvent(evt)
+		lt.TraceLotusEvent(evt)
+	}
 }
 
 func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {

From e46da5a1812efd54723eae3949bd40cb2dba9731 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 12:37:16 +0200
Subject: [PATCH 33/51] Add cli flags

---
 cmd/lotus/daemon.go           | 28 ++++++++++++++++++++++++++++
 node/config/doc_gen.go        | 12 ++++++++++++
 node/config/types.go          |  1 +
 node/modules/dtypes/tracer.go |  6 ++++++
 4 files changed, 47 insertions(+)
 create mode 100644 node/modules/dtypes/tracer.go

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 486ac8ed7..89901a6f5 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -153,6 +153,22 @@ var DaemonCmd = &cli.Command{
 			Name:  "restore-config",
 			Usage: "config file to use when restoring from backup",
 		},
+		&cli.StringFlag{
+			Name:  "trace-to-json",
+			Usage: "starts tracer and outputs to json file defined with this flag",
+		},
+		&cli.StringFlag{
+			Name:  "trace-to-elasticsearch",
+			Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch",
+		},
+		&cli.StringFlag{
+			Name:  "elasticsearch-index",
+			Usage: "configure elasticearch index name if elasticsearch tracer is configured",
+		},
+		&cli.StringFlag{
+			Name:  "trace-source-auth",
+			Usage: "auth token for trusted source of traces",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		isLite := cctx.Bool("lite")
@@ -193,6 +209,14 @@ var DaemonCmd = &cli.Command{
 			return fmt.Errorf("unrecognized profile type: %q", profile)
 		}
 
+		traceToJsonFile := cctx.String("trace-to-json")
+
+		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
+
+		elasticsearchIndex := cctx.String("elasticsearch-index")
+
+		traceSourceAuth := cctx.String("trace-source-auth")
+
 		ctx, _ := tag.New(context.Background(),
 			tag.Insert(metrics.Version, build.BuildVersion),
 			tag.Insert(metrics.Commit, build.CurrentCommit),
@@ -319,6 +343,10 @@ var DaemonCmd = &cli.Command{
 
 			node.Override(new(dtypes.Bootstrapper), isBootstrapper),
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
+			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
+			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
+			node.Override(new(dtypes.ElasticSearchTracer), elasticsearchIndex),
+			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
 			genesis,
 			liteModeDeps,
diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index fd4ec0366..f22156170 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -537,6 +537,18 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "ElasticSearchTracer",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "ElasticSearchIndex",
+			Type: "string",
+
+			Comment: ``,
+		},
+		{
+			Name: "TracerSourceAuth",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},
diff --git a/node/config/types.go b/node/config/types.go
index aa061baef..630ae61fd 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -311,6 +311,7 @@ type Pubsub struct {
 	RemoteTracer          string
 	JsonTracerFile        string
 	ElasticSearchTracer   string
+	ElasticSearchIndex    string
 	TracerSourceAuth      string
 }
 
diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
new file mode 100644
index 000000000..9317924f7
--- /dev/null
+++ b/node/modules/dtypes/tracer.go
@@ -0,0 +1,6 @@
+package dtypes
+
+type JsonTracerFile string
+type ElasticSearchTracer string
+type ElasticSearchIndex string
+type TracerSourceAuth string

From 94a1601ab268314734370d13412f8ee5ebf972ad Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 12:40:33 +0200
Subject: [PATCH 34/51] Remove hardcoded url schema

---
 node/modules/tracer/elasticsearch_transport.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 8823b8199..dbacf5823 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -18,11 +18,15 @@ const (
 func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
 	conUrl, err := url.Parse(connectionString)
 
+	if err != nil {
+		return nil, err
+	}
+
 	username := conUrl.User.Username()
 	password, _ := conUrl.User.Password()
 	cfg := elasticsearch.Config{
 		Addresses: []string{
-			"https://" + conUrl.Host,
+			conUrl.Scheme + conUrl.Host,
 		},
 		Username: username,
 		Password: password,

From 22bbb113e7afc20f73d91583b60915fcf8d0546b Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 12:58:32 +0200
Subject: [PATCH 35/51] Fix schema creation

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index dbacf5823..621b1d437 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -26,7 +26,7 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 	password, _ := conUrl.User.Password()
 	cfg := elasticsearch.Config{
 		Addresses: []string{
-			conUrl.Scheme + conUrl.Host,
+			conUrl.Scheme + "://" + conUrl.Host,
 		},
 		Username: username,
 		Password: password,

From 4438c4bd87b637a9231caadca0785b6c102b4ae7 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 13:43:25 +0200
Subject: [PATCH 36/51] Move index name to config only and add default value

---
 cmd/lotus/daemon.go                            |  7 -------
 node/modules/dtypes/tracer.go                  |  1 -
 node/modules/lp2p/pubsub.go                    |  1 +
 node/modules/tracer/elasticsearch_transport.go | 17 +++++++++++++----
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 89901a6f5..9579d9534 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -161,10 +161,6 @@ var DaemonCmd = &cli.Command{
 			Name:  "trace-to-elasticsearch",
 			Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch",
 		},
-		&cli.StringFlag{
-			Name:  "elasticsearch-index",
-			Usage: "configure elasticearch index name if elasticsearch tracer is configured",
-		},
 		&cli.StringFlag{
 			Name:  "trace-source-auth",
 			Usage: "auth token for trusted source of traces",
@@ -213,8 +209,6 @@ var DaemonCmd = &cli.Command{
 
 		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
 
-		elasticsearchIndex := cctx.String("elasticsearch-index")
-
 		traceSourceAuth := cctx.String("trace-source-auth")
 
 		ctx, _ := tag.New(context.Background(),
@@ -345,7 +339,6 @@ var DaemonCmd = &cli.Command{
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
 			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
 			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
-			node.Override(new(dtypes.ElasticSearchTracer), elasticsearchIndex),
 			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
 			genesis,
diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
index 9317924f7..622f4baf4 100644
--- a/node/modules/dtypes/tracer.go
+++ b/node/modules/dtypes/tracer.go
@@ -2,5 +2,4 @@ package dtypes
 
 type JsonTracerFile string
 type ElasticSearchTracer string
-type ElasticSearchIndex string
 type TracerSourceAuth string
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index a8ba126e0..818832562 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -377,6 +377,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
 			in.Cfg.ElasticSearchTracer,
+			in.Cfg.ElasticSearchIndex,
 		)
 		if err != nil {
 			return nil, err
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 621b1d437..d0d7b84a1 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,10 +12,10 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "lotus-pubsub"
+	ElasticSearch_INDEX_DEFAULT = "lotus-pubsub"
 )
 
-func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
+func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
 	conUrl, err := url.Parse(connectionString)
 
 	if err != nil {
@@ -38,13 +38,22 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		return nil, err
 	}
 
+	var esIndex string
+	if elasticsearchIndex != "" {
+		esIndex = elasticsearchIndex
+	} else {
+		esIndex = ElasticSearch_INDEX_DEFAULT
+	}
+
 	return &elasticSearchTransport{
-		cl: es,
+		cl:      es,
+		esIndex: esIndex,
 	}, nil
 }
 
 type elasticSearchTransport struct {
-	cl *elasticsearch.Client
+	cl      *elasticsearch.Client
+	esIndex string
 }
 
 func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {

From 9332b9f39976af76221dc225c898a98e4590cee0 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 13:53:31 +0200
Subject: [PATCH 37/51] Change json config naming

---
 node/config/doc_gen.go      | 8 +++++++-
 node/config/types.go        | 2 +-
 node/modules/lp2p/pubsub.go | 4 ++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index fd4ec0366..21e9ccc6b 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -528,7 +528,7 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Comment: ``,
 		},
 		{
-			Name: "JsonTracerFile",
+			Name: "JsonTracer",
 			Type: "string",
 
 			Comment: ``,
@@ -537,6 +537,12 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "ElasticSearchTracer",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "TracerSourceAuth",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},
diff --git a/node/config/types.go b/node/config/types.go
index aa061baef..b421a7e8e 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -309,7 +309,7 @@ type Pubsub struct {
 	DirectPeers           []string
 	IPColocationWhitelist []string
 	RemoteTracer          string
-	JsonTracerFile        string
+	JsonTracer            string
 	ElasticSearchTracer   string
 	TracerSourceAuth      string
 }
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index a8ba126e0..b91779ac5 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -366,8 +366,8 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				100)))
 
 	var transports []tracer.TracerTransport
-	if in.Cfg.JsonTracerFile != "" {
-		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracerFile)
+	if in.Cfg.JsonTracer != "" {
+		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracer)
 		if err != nil {
 			return nil, err
 		}

From 72d8e620752174d8f561a6c833351ae4f6ebd298 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 24 Sep 2021 14:07:47 +0200
Subject: [PATCH 38/51] Add kibana dashboard configurations

---
 tools/kibana/block-propagation-dashboard.ndjson | 2 ++
 tools/kibana/peer-scores-dashboard.ndjson       | 3 +++
 2 files changed, 5 insertions(+)
 create mode 100644 tools/kibana/block-propagation-dashboard.ndjson
 create mode 100644 tools/kibana/peer-scores-dashboard.ndjson

diff --git a/tools/kibana/block-propagation-dashboard.ndjson b/tools/kibana/block-propagation-dashboard.ndjson
new file mode 100644
index 000000000..dc5121183
--- /dev/null
+++ b/tools/kibana/block-propagation-dashboard.ndjson
@@ -0,0 +1,2 @@
+{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":42,\"i\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\"},\"panelIndex\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"Block propagation\",\"description\":\"\",\"type\":\"vega\",\"params\":{\"spec\":\"{\\n  $schema: https://vega.github.io/schema/vega/v5.json\\n  title: Node block propagation\\n\\n\\n  \\\"signals\\\": [\\n    { \\\"name\\\": \\\"cx\\\", \\\"update\\\": \\\"width / 2\\\" },\\n    { \\\"name\\\": \\\"cy\\\", \\\"update\\\": \\\"height / 2\\\" },\\n    { \\\"name\\\": \\\"nodeRadius\\\", \\\"value\\\": 10,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 20, \\\"step\\\": 1} },\\n    { \\\"name\\\": \\\"propagationMultiplier\\\", \\\"value\\\": 50,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 100, \\\"step\\\": 1} },\\n  ],\\n\\n  data: [\\n    {\\n    \\n      name: \\\"node-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 0,\\n          aggs: {\\n            unique_peerID: {\\n              terms: { \\n                field: \\\"peerID\\\",\\n                size: 10000\\n              },\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"aggregations.unique_peerID.buckets\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"key\\\"],\\n          \\\"as\\\": [\\\"peerID\\\"]\\n        },\\n        {\\n          type: \\\"identifier\\\",\\n          as: \\\"id\\\"\\n        },\\n        {\\n          type: \\\"formula\\\",\\n          expr: \\\"datum.id - 1\\\",\\n          as: \\\"index\\\"\\n        }\\n      ]\\n    },\\n      {\\n      name: \\\"published-message-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 0\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n    },\\n    {\\n      name: \\\"link-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 1\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"published-message-data\\\",\\n          \\\"key\\\": \\\"_source.publishMessage.messageID\\\",\\n          \\\"fields\\\": [\\\"_source.deliverMessage.messageID\\\"],\\n          \\\"as\\\": [\\\"publishedMessage\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"_source.peerID\\\"],\\n          \\\"as\\\": [\\\"source\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"publishedMessage._source.peerID\\\"],\\n          \\\"as\\\": [\\\"target\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"formula\\\",\\n          \\\"expr\\\": \\\"(datum._source.timestamp - datum.publishedMessage._source.timestamp) * propagationMultiplier\\\",\\n          \\\"as\\\": \\\"distance\\\"\\n        },\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"source.index\\\", \\\"target.index\\\", \\\"distance\\\"]\\n          \\\"as\\\": [\\\"source\\\", \\\"target\\\", \\\"distance\\\"]\\n        },\\n        {\\n          \\\"type\\\": \\\"aggregate\\\",\\n          \\\"ops\\\": [\\\"average\\\"],\\n          \\\"fields\\\": [\\\"distance\\\"],\\n          \\\"groupby\\\": [\\\"source\\\", \\\"target\\\"],\\n          \\\"as\\\": [\\\"distance\\\"]\\n        }\\n      ]\\n    }\\n  ]\\n\\n  scales: [\\n    {\\n      \\\"name\\\": \\\"color\\\",\\n      \\\"type\\\": \\\"ordinal\\\",\\n      \\\"domain\\\": {\\\"data\\\": \\\"node-data\\\", \\\"field\\\": \\\"id\\\"},\\n      \\\"range\\\": {\\\"scheme\\\": \\\"category20c\\\"}\\n    }\\n  ]\\n\\n  marks: [\\n    {\\n      \\\"name\\\": \\\"nodes\\\",\\n      \\\"type\\\": \\\"symbol\\\",\\n      \\\"zindex\\\": 1,\\n      \\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n          \\\"stroke\\\": {\\\"value\\\": \\\"black\\\"},\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'PeerID': datum.peerID}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"size\\\": {\\\"signal\\\": \\\"2 * nodeRadius * nodeRadius\\\"},\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n        },\\n        \\\"hover\\\": { \\\"fill\\\": {\\\"value\\\": \\\"red\\\"} },\\n      },\\n\\n      \\\"from\\\": {\\\"data\\\": \\\"node-data\\\"},\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"force\\\",\\n          \\\"restart\\\": false,\\n          \\\"static\\\": true,\\n          \\\"signal\\\": \\\"force\\\",\\n          \\\"forces\\\": [\\n            {\\\"force\\\": \\\"link\\\", \\\"links\\\": \\\"link-data\\\", distance: {field: \\\"distance\\\"}},\\n            {\\\"force\\\": \\\"center\\\", \\\"x\\\": {\\\"signal\\\": \\\"cx\\\"}, \\\"y\\\": {\\\"signal\\\": \\\"cy\\\"}},\\n          ]\\n        }\\n      ]\\n    },\\n    {\\n      \\\"type\\\": \\\"path\\\",\\n      \\\"from\\\": {\\\"data\\\": \\\"link-data\\\"},\\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'Block propagation': datum.distance}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"stroke\\\": {\\\"value\\\": \\\"#ccc\\\"},\\n          \\\"strokeWidth\\\": {\\\"value\\\": 1.5}\\n        },\\n        \\\"hover\\\": { \\\"stroke\\\": { \\\"value\\\": \\\"#0B33A0\\\" }}\\n      },\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"linkpath\\\",\\n          \\\"require\\\": {\\\"signal\\\": \\\"force\\\"},\\n          \\\"shape\\\": \\\"line\\\",\\n          \\\"sourceX\\\": \\\"datum.source.x\\\", \\\"sourceY\\\": \\\"datum.source.y\\\",\\n          \\\"targetX\\\": \\\"datum.target.x\\\", \\\"targetY\\\": \\\"datum.target.y\\\"\\n        }\\n      ]\\n    }\\n  ]\\n}\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}}},\"enhancements\":{}}}]","timeRestore":false,"title":"Block propagation","version":1},"coreMigrationVersion":"7.14.1","id":"eff2ade0-19fa-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[],"type":"dashboard","updated_at":"2021-09-24T09:46:01.693Z","version":"WzYzMjAsMV0="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]}
diff --git a/tools/kibana/peer-scores-dashboard.ndjson b/tools/kibana/peer-scores-dashboard.ndjson
new file mode 100644
index 000000000..c4d491706
--- /dev/null
+++ b/tools/kibana/peer-scores-dashboard.ndjson
@@ -0,0 +1,3 @@
+{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{\"peerScore.weightedScore\":{\"type\":\"double\",\"script\":{\"source\":\"if (doc['type'].value == 100) {\\n    def score = doc['peerScore.score'].value;\\n    if (doc['sourceAuth'] == \\\"<password>\\\") {\\n\\n        emit(score * 1.2)\\n    } else {\\n        emit(score)\\n    }\\n}\\n\\n\"}},\"sourceAuth\":{\"type\":\"keyword\"}}","title":"lotus-pubsub*","typeMeta":"{}"},"coreMigrationVersion":"7.14.1","id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-09-24T12:03:02.575Z","version":"WzcwMzksMV0="}
+{"attributes":{"description":"Average peer score table per node peerID","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":43,\"i\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\"},\"panelIndex\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsDatatable\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"666e2f39-8868-45ad-b747-fe124830b0ae\":{\"columns\":{\"504c50bd-14c1-4119-820e-c961866fc3b4\":{\"label\":\"peerID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"peerScore.peerID.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\"},\"orderDirection\":\"desc\",\"otherBucket\":false,\"missingBucket\":false},\"customLabel\":true},\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\":{\"label\":\"Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.score\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\":{\"label\":\"Weighted Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.weightedScore\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true}},\"columnOrder\":[\"504c50bd-14c1-4119-820e-c961866fc3b4\",\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"504c50bd-14c1-4119-820e-c961866fc3b4\"},{\"isTransposed\":false,\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}},{\"columnId\":\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\",\"isTransposed\":false,\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}}],\"layerId\":\"666e2f39-8868-45ad-b747-fe124830b0ae\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"9890c040-17b7-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"2c407db0-1acb-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Peer Scores\"}]","timeRestore":false,"title":"Peer Scores","version":1},"coreMigrationVersion":"7.14.1","id":"e7e4fd70-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-09-24T11:59:46.187Z","version":"WzY5NjcsMV0="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file

From 6c680e306dd6b6cb6f20c3f0ecc20bf8d9b4186c Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 24 Sep 2021 16:25:41 +0200
Subject: [PATCH 39/51] Rename json tracer flag

---
 cmd/lotus/daemon.go           | 13 ++++++-------
 node/modules/dtypes/tracer.go |  2 +-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 9579d9534..e58523999 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -1,3 +1,4 @@
+//go:build !nodaemon
 // +build !nodaemon
 
 package main
@@ -205,12 +206,6 @@ var DaemonCmd = &cli.Command{
 			return fmt.Errorf("unrecognized profile type: %q", profile)
 		}
 
-		traceToJsonFile := cctx.String("trace-to-json")
-
-		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
-
-		traceSourceAuth := cctx.String("trace-source-auth")
-
 		ctx, _ := tag.New(context.Background(),
 			tag.Insert(metrics.Version, build.BuildVersion),
 			tag.Insert(metrics.Commit, build.CurrentCommit),
@@ -328,6 +323,10 @@ var DaemonCmd = &cli.Command{
 			log.Warnf("unable to inject prometheus ipfs/go-metrics exporter; some metrics will be unavailable; err: %s", err)
 		}
 
+		traceToJsonFile := cctx.String("trace-to-json")
+		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
+		traceSourceAuth := cctx.String("trace-source-auth")
+
 		var api api.FullNode
 		stop, err := node.New(ctx,
 			node.FullAPI(&api, node.Lite(isLite)),
@@ -337,7 +336,7 @@ var DaemonCmd = &cli.Command{
 
 			node.Override(new(dtypes.Bootstrapper), isBootstrapper),
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
-			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
+			node.Override(new(dtypes.JsonTracer), traceToJsonFile),
 			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
 			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
index 622f4baf4..735ff05c3 100644
--- a/node/modules/dtypes/tracer.go
+++ b/node/modules/dtypes/tracer.go
@@ -1,5 +1,5 @@
 package dtypes
 
-type JsonTracerFile string
+type JsonTracer string
 type ElasticSearchTracer string
 type TracerSourceAuth string

From 36913f4efce511d5e1fbcb6c3b892ffb699e1d8b Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Mon, 27 Sep 2021 13:28:28 +0200
Subject: [PATCH 40/51] Fix elasticsearch index setup

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index d0d7b84a1..49060a13d 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -73,7 +73,7 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	}
 
 	req := esapi.IndexRequest{
-		Index:   ElasticSearch_INDEX,
+		Index:   est.esIndex,
 		Body:    strings.NewReader(string(jsonEvt)),
 		Refresh: "true",
 	}

From 581a54663c7d0dcfdb46e4107de7c9ecededfc34 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 27 Sep 2021 15:09:03 +0200
Subject: [PATCH 41/51] Add README for creating kibana dashboards

---
 tools/kibana/README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 tools/kibana/README.md

diff --git a/tools/kibana/README.md b/tools/kibana/README.md
new file mode 100644
index 000000000..e4e451bbb
--- /dev/null
+++ b/tools/kibana/README.md
@@ -0,0 +1,14 @@
+## Lotus Kibana Dashboard
+
+This folder contains configuration files to create Kibana dashboards to track peer scores and block propagation
+throughout Filecoin network.
+
+### Importing dashboard
+
+The peer score and block propagation dashboard configuration is imported via Kibana import saved object [functionality](https://www.elastic.co/guide/en/kibana/current/managing-saved-objects.html#managing-saved-objects-export-objects).
+
+The index patterns will be created automatically when importing dashboards.
+
+#### Custom index
+
+By default, the dashboards target `lotus-pubsub` index which is the default one when running node. The index can be customised via edit on dashboard visualizations.

From 45bb8c3ebf6fe44f4783c35893909f978b96a720 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 29 Sep 2021 12:50:30 +0200
Subject: [PATCH 42/51] Remove cli flags

---
 cmd/lotus/daemon.go | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index e58523999..51995136b 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -154,18 +154,6 @@ var DaemonCmd = &cli.Command{
 			Name:  "restore-config",
 			Usage: "config file to use when restoring from backup",
 		},
-		&cli.StringFlag{
-			Name:  "trace-to-json",
-			Usage: "starts tracer and outputs to json file defined with this flag",
-		},
-		&cli.StringFlag{
-			Name:  "trace-to-elasticsearch",
-			Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch",
-		},
-		&cli.StringFlag{
-			Name:  "trace-source-auth",
-			Usage: "auth token for trusted source of traces",
-		},
 	},
 	Action: func(cctx *cli.Context) error {
 		isLite := cctx.Bool("lite")
@@ -323,10 +311,6 @@ var DaemonCmd = &cli.Command{
 			log.Warnf("unable to inject prometheus ipfs/go-metrics exporter; some metrics will be unavailable; err: %s", err)
 		}
 
-		traceToJsonFile := cctx.String("trace-to-json")
-		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
-		traceSourceAuth := cctx.String("trace-source-auth")
-
 		var api api.FullNode
 		stop, err := node.New(ctx,
 			node.FullAPI(&api, node.Lite(isLite)),
@@ -336,9 +320,6 @@ var DaemonCmd = &cli.Command{
 
 			node.Override(new(dtypes.Bootstrapper), isBootstrapper),
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
-			node.Override(new(dtypes.JsonTracer), traceToJsonFile),
-			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
-			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
 			genesis,
 			liteModeDeps,

From 3433fa4a6e6f1fc07619ab8aadf9c1a4c8572242 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 29 Sep 2021 12:51:43 +0200
Subject: [PATCH 43/51] Add stringdocs to config props and generate docs

---
 node/config/doc_gen.go | 17 ++++++++++++++---
 node/config/types.go   | 17 +++++++++++++----
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index 21e9ccc6b..c9079dac4 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -531,19 +531,30 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "JsonTracer",
 			Type: "string",
 
-			Comment: ``,
+			Comment: `Path to file that will be used to output tracer content in JSON format.
+If present tracer will save data to defined file.
+Format: file path`,
 		},
 		{
 			Name: "ElasticSearchTracer",
 			Type: "string",
 
-			Comment: ``,
+			Comment: `Connection string for elasticsearch instance.
+If present tracer will save data to elasticsearch.
+Format: https://<username>:<password>@<elasticsearch_url>:<port>/`,
+		},
+		{
+			Name: "ElasticSearchIndex",
+			Type: "string",
+
+			Comment: `Name of elasticsearch index that will be used to save tracer data.
+This property is used only if ElasticSearchTracer propery is set.`,
 		},
 		{
 			Name: "TracerSourceAuth",
 			Type: "string",
 
-			Comment: ``,
+			Comment: `Auth token that will be passed with logs to elasticsearch - used for weighted peers score.`,
 		},
 	},
 	"RetrievalPricing": []DocField{
diff --git a/node/config/types.go b/node/config/types.go
index a7de81328..b8320be20 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -309,10 +309,19 @@ type Pubsub struct {
 	DirectPeers           []string
 	IPColocationWhitelist []string
 	RemoteTracer          string
-	JsonTracer            string
-	ElasticSearchTracer   string
-	ElasticSearchIndex    string
-	TracerSourceAuth      string
+	// Path to file that will be used to output tracer content in JSON format.
+	// If present tracer will save data to defined file.
+	// Format: file path
+	JsonTracer string
+	// Connection string for elasticsearch instance.
+	// If present tracer will save data to elasticsearch.
+	// Format: https://<username>:<password>@<elasticsearch_url>:<port>/
+	ElasticSearchTracer string
+	// Name of elasticsearch index that will be used to save tracer data.
+	// This property is used only if ElasticSearchTracer propery is set.
+	ElasticSearchIndex string
+	// Auth token that will be passed with logs to elasticsearch - used for weighted peers score.
+	TracerSourceAuth string
 }
 
 type Chainstore struct {

From 7b19beaf5d122b98ce6bc36976b6b460e6eed9e2 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 29 Sep 2021 13:12:42 +0200
Subject: [PATCH 44/51] Fix linter errors

---
 node/modules/tracer/elasticsearch_transport.go | 11 ++++++++---
 node/modules/tracer/tracer.go                  | 10 +++++-----
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 49060a13d..1f6f9a157 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX_DEFAULT = "lotus-pubsub"
+	ElasticSearchDefaultIndex = "lotus-pubsub"
 )
 
 func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
@@ -42,7 +42,7 @@ func NewElasticSearchTransport(connectionString string, elasticsearchIndex strin
 	if elasticsearchIndex != "" {
 		esIndex = elasticsearchIndex
 	} else {
-		esIndex = ElasticSearch_INDEX_DEFAULT
+		esIndex = ElasticSearchDefaultIndex
 	}
 
 	return &elasticSearchTransport{
@@ -83,10 +83,15 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	if err != nil {
 		return err
 	}
-	defer res.Body.Close()
+
+	err = res.Body.Close()
+	if err != nil {
+		return err
+	}
 
 	if res.IsError() {
 		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
 	}
+
 	return nil
 }
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 9badfc9cb..417c5c28c 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -26,18 +26,18 @@ type lotusTracer struct {
 }
 
 const (
-	TraceEvent_PEER_SCORES pubsub_pb.TraceEvent_Type = 100
+	TraceEventPeerScores pubsub_pb.TraceEvent_Type = 100
 )
 
 type LotusTraceEvent struct {
 	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
 	PeerID     string                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
-	PeerScore  TraceEvent_PeerScore      `json:"peerScore,omitempty"`
+	PeerScore  TraceEventPeerScore       `json:"peerScore,omitempty"`
 	SourceAuth string                    `json:"sourceAuth,omitempty"`
 }
 
-type TraceEvent_PeerScore struct {
+type TraceEventPeerScore struct {
 	PeerID string  `json:"peerID"`
 	Score  float32 `json:"score"`
 }
@@ -53,11 +53,11 @@ func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 	now := time.Now().UnixNano()
 	for pid, score := range scores {
 		evt := &LotusTraceEvent{
-			Type:       *TraceEvent_PEER_SCORES.Enum(),
+			Type:       *TraceEventPeerScores.Enum(),
 			PeerID:     lt.pid.Pretty(),
 			Timestamp:  &now,
 			SourceAuth: lt.sa,
-			PeerScore:  TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)},
+			PeerScore:  TraceEventPeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)},
 		}
 
 		lt.TraceLotusEvent(evt)

From 181b35dc5e7b20aebd2a5f1e24436a58677c5509 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 29 Sep 2021 16:20:43 +0200
Subject: [PATCH 45/51] Extend peer score event with score components

---
 node/modules/tracer/tracer.go | 36 ++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 417c5c28c..e31b4ea52 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -37,9 +37,21 @@ type LotusTraceEvent struct {
 	SourceAuth string                    `json:"sourceAuth,omitempty"`
 }
 
+type TopicScore struct {
+	Topic                    string        `json:"topic"`
+	TimeInMesh               time.Duration `json:"timeInMesh"`
+	FirstMessageDeliveries   float64       `json:"firstMessageDeliveries"`
+	MeshMessageDeliveries    float64       `json:"meshMessageDeliveries"`
+	InvalidMessageDeliveries float64       `json:"invalidMessageDeliveries"`
+}
+
 type TraceEventPeerScore struct {
-	PeerID string  `json:"peerID"`
-	Score  float32 `json:"score"`
+	PeerID             string       `json:"peerID"`
+	Score              float64      `json:"score"`
+	AppSpecificScore   float64      `json:"appSpecificScore"`
+	IPColocationFactor float64      `json:"ipColocationFactor"`
+	BehaviourPenalty   float64      `json:"behaviourPenalty"`
+	Topics             []TopicScore `json:"topics"`
 }
 
 type LotusTracer interface {
@@ -52,12 +64,30 @@ type LotusTracer interface {
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
 	now := time.Now().UnixNano()
 	for pid, score := range scores {
+		var topics []TopicScore
+		for topic, snapshot := range score.Topics {
+			topics = append(topics, TopicScore{
+				Topic:                    topic,
+				TimeInMesh:               snapshot.TimeInMesh,
+				FirstMessageDeliveries:   snapshot.FirstMessageDeliveries,
+				MeshMessageDeliveries:    snapshot.MeshMessageDeliveries,
+				InvalidMessageDeliveries: snapshot.InvalidMessageDeliveries,
+			})
+		}
+
 		evt := &LotusTraceEvent{
 			Type:       *TraceEventPeerScores.Enum(),
 			PeerID:     lt.pid.Pretty(),
 			Timestamp:  &now,
 			SourceAuth: lt.sa,
-			PeerScore:  TraceEventPeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)},
+			PeerScore: TraceEventPeerScore{
+				PeerID:             pid.Pretty(),
+				Score:              score.Score,
+				AppSpecificScore:   score.AppSpecificScore,
+				IPColocationFactor: score.IPColocationFactor,
+				BehaviourPenalty:   score.BehaviourPenalty,
+				Topics:             topics,
+			},
 		}
 
 		lt.TraceLotusEvent(evt)

From c34e15bbb78b4e555720c921b173209d98504bc5 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 29 Sep 2021 16:21:16 +0200
Subject: [PATCH 46/51] Add elasticsearch index template and update dashboards

---
 tools/kibana/README.md                        | 12 +++-
 .../kibana/block-propagation-dashboard.ndjson |  2 +-
 tools/kibana/index-template.json              | 66 +++++++++++++++++++
 tools/kibana/peer-scores-dashboard.ndjson     |  5 +-
 4 files changed, 78 insertions(+), 7 deletions(-)
 create mode 100644 tools/kibana/index-template.json

diff --git a/tools/kibana/README.md b/tools/kibana/README.md
index e4e451bbb..c556ae10c 100644
--- a/tools/kibana/README.md
+++ b/tools/kibana/README.md
@@ -3,12 +3,18 @@
 This folder contains configuration files to create Kibana dashboards to track peer scores and block propagation
 throughout Filecoin network.
 
+### Importing index template
+
+Index template needs to be imported into Elasticsearch for score weights and to
+prevent Elasticsearch from infering wrong field type.
+
+The [template](./index-template.json) is loaded via [Kibana Index Management](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-mgmt.html) and pasted
+into newly created Index Template.
+
 ### Importing dashboard
 
 The peer score and block propagation dashboard configuration is imported via Kibana import saved object [functionality](https://www.elastic.co/guide/en/kibana/current/managing-saved-objects.html#managing-saved-objects-export-objects).
 
-The index patterns will be created automatically when importing dashboards.
-
 #### Custom index
 
-By default, the dashboards target `lotus-pubsub` index which is the default one when running node. The index can be customised via edit on dashboard visualizations.
+By default, the dashboards and index template target `lotus-pubsub` index which is the default one when running node. The index can be customised via edit on dashboard visualizations and also index template needs to be updated to target new index.
diff --git a/tools/kibana/block-propagation-dashboard.ndjson b/tools/kibana/block-propagation-dashboard.ndjson
index dc5121183..c7b14c0c3 100644
--- a/tools/kibana/block-propagation-dashboard.ndjson
+++ b/tools/kibana/block-propagation-dashboard.ndjson
@@ -1,2 +1,2 @@
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":42,\"i\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\"},\"panelIndex\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"Block propagation\",\"description\":\"\",\"type\":\"vega\",\"params\":{\"spec\":\"{\\n  $schema: https://vega.github.io/schema/vega/v5.json\\n  title: Node block propagation\\n\\n\\n  \\\"signals\\\": [\\n    { \\\"name\\\": \\\"cx\\\", \\\"update\\\": \\\"width / 2\\\" },\\n    { \\\"name\\\": \\\"cy\\\", \\\"update\\\": \\\"height / 2\\\" },\\n    { \\\"name\\\": \\\"nodeRadius\\\", \\\"value\\\": 10,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 20, \\\"step\\\": 1} },\\n    { \\\"name\\\": \\\"propagationMultiplier\\\", \\\"value\\\": 50,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 100, \\\"step\\\": 1} },\\n  ],\\n\\n  data: [\\n    {\\n    \\n      name: \\\"node-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 0,\\n          aggs: {\\n            unique_peerID: {\\n              terms: { \\n                field: \\\"peerID\\\",\\n                size: 10000\\n              },\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"aggregations.unique_peerID.buckets\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"key\\\"],\\n          \\\"as\\\": [\\\"peerID\\\"]\\n        },\\n        {\\n          type: \\\"identifier\\\",\\n          as: \\\"id\\\"\\n        },\\n        {\\n          type: \\\"formula\\\",\\n          expr: \\\"datum.id - 1\\\",\\n          as: \\\"index\\\"\\n        }\\n      ]\\n    },\\n      {\\n      name: \\\"published-message-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 0\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n    },\\n    {\\n      name: \\\"link-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 1\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"published-message-data\\\",\\n          \\\"key\\\": \\\"_source.publishMessage.messageID\\\",\\n          \\\"fields\\\": [\\\"_source.deliverMessage.messageID\\\"],\\n          \\\"as\\\": [\\\"publishedMessage\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"_source.peerID\\\"],\\n          \\\"as\\\": [\\\"source\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"publishedMessage._source.peerID\\\"],\\n          \\\"as\\\": [\\\"target\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"formula\\\",\\n          \\\"expr\\\": \\\"(datum._source.timestamp - datum.publishedMessage._source.timestamp) * propagationMultiplier\\\",\\n          \\\"as\\\": \\\"distance\\\"\\n        },\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"source.index\\\", \\\"target.index\\\", \\\"distance\\\"]\\n          \\\"as\\\": [\\\"source\\\", \\\"target\\\", \\\"distance\\\"]\\n        },\\n        {\\n          \\\"type\\\": \\\"aggregate\\\",\\n          \\\"ops\\\": [\\\"average\\\"],\\n          \\\"fields\\\": [\\\"distance\\\"],\\n          \\\"groupby\\\": [\\\"source\\\", \\\"target\\\"],\\n          \\\"as\\\": [\\\"distance\\\"]\\n        }\\n      ]\\n    }\\n  ]\\n\\n  scales: [\\n    {\\n      \\\"name\\\": \\\"color\\\",\\n      \\\"type\\\": \\\"ordinal\\\",\\n      \\\"domain\\\": {\\\"data\\\": \\\"node-data\\\", \\\"field\\\": \\\"id\\\"},\\n      \\\"range\\\": {\\\"scheme\\\": \\\"category20c\\\"}\\n    }\\n  ]\\n\\n  marks: [\\n    {\\n      \\\"name\\\": \\\"nodes\\\",\\n      \\\"type\\\": \\\"symbol\\\",\\n      \\\"zindex\\\": 1,\\n      \\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n          \\\"stroke\\\": {\\\"value\\\": \\\"black\\\"},\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'PeerID': datum.peerID}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"size\\\": {\\\"signal\\\": \\\"2 * nodeRadius * nodeRadius\\\"},\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n        },\\n        \\\"hover\\\": { \\\"fill\\\": {\\\"value\\\": \\\"red\\\"} },\\n      },\\n\\n      \\\"from\\\": {\\\"data\\\": \\\"node-data\\\"},\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"force\\\",\\n          \\\"restart\\\": false,\\n          \\\"static\\\": true,\\n          \\\"signal\\\": \\\"force\\\",\\n          \\\"forces\\\": [\\n            {\\\"force\\\": \\\"link\\\", \\\"links\\\": \\\"link-data\\\", distance: {field: \\\"distance\\\"}},\\n            {\\\"force\\\": \\\"center\\\", \\\"x\\\": {\\\"signal\\\": \\\"cx\\\"}, \\\"y\\\": {\\\"signal\\\": \\\"cy\\\"}},\\n          ]\\n        }\\n      ]\\n    },\\n    {\\n      \\\"type\\\": \\\"path\\\",\\n      \\\"from\\\": {\\\"data\\\": \\\"link-data\\\"},\\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'Block propagation': datum.distance}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"stroke\\\": {\\\"value\\\": \\\"#ccc\\\"},\\n          \\\"strokeWidth\\\": {\\\"value\\\": 1.5}\\n        },\\n        \\\"hover\\\": { \\\"stroke\\\": { \\\"value\\\": \\\"#0B33A0\\\" }}\\n      },\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"linkpath\\\",\\n          \\\"require\\\": {\\\"signal\\\": \\\"force\\\"},\\n          \\\"shape\\\": \\\"line\\\",\\n          \\\"sourceX\\\": \\\"datum.source.x\\\", \\\"sourceY\\\": \\\"datum.source.y\\\",\\n          \\\"targetX\\\": \\\"datum.target.x\\\", \\\"targetY\\\": \\\"datum.target.y\\\"\\n        }\\n      ]\\n    }\\n  ]\\n}\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}}},\"enhancements\":{}}}]","timeRestore":false,"title":"Block propagation","version":1},"coreMigrationVersion":"7.14.1","id":"eff2ade0-19fa-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[],"type":"dashboard","updated_at":"2021-09-24T09:46:01.693Z","version":"WzYzMjAsMV0="}
+{"attributes":{"description":"Force layout vega graph where link force distance between peers is block propagation","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":42,\"i\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\"},\"panelIndex\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"Block propagation\",\"description\":\"\",\"type\":\"vega\",\"params\":{\"spec\":\"{\\n  $schema: https://vega.github.io/schema/vega/v5.json\\n  title: Node block propagation\\n\\n\\n  \\\"signals\\\": [\\n    { \\\"name\\\": \\\"cx\\\", \\\"update\\\": \\\"width / 2\\\" },\\n    { \\\"name\\\": \\\"cy\\\", \\\"update\\\": \\\"height / 2\\\" },\\n    { \\\"name\\\": \\\"nodeRadius\\\", \\\"value\\\": 10,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 20, \\\"step\\\": 1} },\\n    { \\\"name\\\": \\\"propagationMultiplier\\\", \\\"value\\\": 1,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 100, \\\"step\\\": 1} },\\n  ],\\n\\n  data: [\\n    {\\n    \\n      name: \\\"node-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 0,\\n          aggs: {\\n            unique_peerID: {\\n              terms: { \\n                field: \\\"peerID\\\",\\n                size: 10000\\n              },\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"aggregations.unique_peerID.buckets\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"key\\\"],\\n          \\\"as\\\": [\\\"peerID\\\"]\\n        },\\n        {\\n          type: \\\"identifier\\\",\\n          as: \\\"id\\\"\\n        },\\n        {\\n          type: \\\"formula\\\",\\n          expr: \\\"datum.id - 1\\\",\\n          as: \\\"index\\\"\\n        }\\n      ]\\n    },\\n      {\\n      name: \\\"published-message-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 0\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n    },\\n    {\\n      name: \\\"link-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 1\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"published-message-data\\\",\\n          \\\"key\\\": \\\"_source.publishMessage.messageID\\\",\\n          \\\"fields\\\": [\\\"_source.deliverMessage.messageID\\\"],\\n          \\\"as\\\": [\\\"publishedMessage\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"filter\\\",\\n          \\\"expr\\\": \\\"datum.publishedMessage != null\\\"\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"_source.peerID\\\"],\\n          \\\"as\\\": [\\\"source\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"publishedMessage._source.peerID\\\"],\\n          \\\"as\\\": [\\\"target\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"formula\\\",\\n          \\\"expr\\\": \\\"(datum._source.timestamp - datum.publishedMessage._source.timestamp) * propagationMultiplier\\\",\\n          \\\"as\\\": \\\"distance\\\"\\n        },\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"source.index\\\", \\\"target.index\\\", \\\"distance\\\"]\\n          \\\"as\\\": [\\\"source\\\", \\\"target\\\", \\\"distance\\\"]\\n        },\\n        {\\n          \\\"type\\\": \\\"aggregate\\\",\\n          \\\"ops\\\": [\\\"average\\\"],\\n          \\\"fields\\\": [\\\"distance\\\"],\\n          \\\"groupby\\\": [\\\"source\\\", \\\"target\\\"],\\n          \\\"as\\\": [\\\"distance\\\"]\\n        }\\n      ]\\n    }\\n  ]\\n\\n  scales: [\\n    {\\n      \\\"name\\\": \\\"color\\\",\\n      \\\"type\\\": \\\"ordinal\\\",\\n      \\\"domain\\\": {\\\"data\\\": \\\"node-data\\\", \\\"field\\\": \\\"id\\\"},\\n      \\\"range\\\": {\\\"scheme\\\": \\\"category20c\\\"}\\n    }\\n  ]\\n\\n  marks: [\\n    {\\n      \\\"name\\\": \\\"nodes\\\",\\n      \\\"type\\\": \\\"symbol\\\",\\n      \\\"zindex\\\": 1,\\n      \\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n          \\\"stroke\\\": {\\\"value\\\": \\\"black\\\"},\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'PeerID': datum.peerID}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"size\\\": {\\\"signal\\\": \\\"2 * nodeRadius * nodeRadius\\\"},\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n        },\\n        \\\"hover\\\": { \\\"fill\\\": {\\\"value\\\": \\\"red\\\"} },\\n      },\\n\\n      \\\"from\\\": {\\\"data\\\": \\\"node-data\\\"},\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"force\\\",\\n          \\\"restart\\\": false,\\n          \\\"static\\\": true,\\n          \\\"signal\\\": \\\"force\\\",\\n          \\\"forces\\\": [\\n            {\\\"force\\\": \\\"link\\\", \\\"links\\\": \\\"link-data\\\", distance: {field: \\\"distance\\\"}},\\n            {\\\"force\\\": \\\"center\\\", \\\"x\\\": {\\\"signal\\\": \\\"cx\\\"}, \\\"y\\\": {\\\"signal\\\": \\\"cy\\\"}},\\n          ]\\n        }\\n      ]\\n    },\\n    {\\n      \\\"type\\\": \\\"path\\\",\\n      \\\"from\\\": {\\\"data\\\": \\\"link-data\\\"},\\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'Block propagation': datum.distance}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"stroke\\\": {\\\"value\\\": \\\"#ccc\\\"},\\n          \\\"strokeWidth\\\": {\\\"value\\\": 1.5}\\n        },\\n        \\\"hover\\\": { \\\"stroke\\\": { \\\"value\\\": \\\"#0B33A0\\\" }}\\n      },\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"linkpath\\\",\\n          \\\"require\\\": {\\\"signal\\\": \\\"force\\\"},\\n          \\\"shape\\\": \\\"line\\\",\\n          \\\"sourceX\\\": \\\"datum.source.x\\\", \\\"sourceY\\\": \\\"datum.source.y\\\",\\n          \\\"targetX\\\": \\\"datum.target.x\\\", \\\"targetY\\\": \\\"datum.target.y\\\"\\n        }\\n      ]\\n    }\\n  ]\\n}\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}}},\"enhancements\":{}}}]","timeRestore":false,"title":"Peer block propagation","version":1},"coreMigrationVersion":"7.14.1","id":"937b1470-212b-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[],"type":"dashboard","updated_at":"2021-09-29T13:45:58.342Z","version":"Wzg4NzMsMV0="}
 {"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]}
diff --git a/tools/kibana/index-template.json b/tools/kibana/index-template.json
new file mode 100644
index 000000000..f08298ad1
--- /dev/null
+++ b/tools/kibana/index-template.json
@@ -0,0 +1,66 @@
+{
+  "template": {
+    "settings": {},
+    "mappings": {
+      "runtime": {
+        "peerScore.weightedScore": {
+          "type": "double",
+          "script": {
+            "source": "if (doc['type'].value == 100) {\n    def score = doc['peerScore.score'].value;\n    if (doc['sourceAuth'] == \"<password>\") {\n\n        emit(score * 1.2)\n    } else {\n        emit(score)\n    }\n}\n",
+            "lang": "painless"
+          }
+        }
+      },
+      "properties": {
+        "peerScore": {
+          "properties": {
+            "appSpecificScore": {
+              "type": "double"
+            },
+            "behaviourPenalty": {
+              "type": "double"
+            },
+            "ipColocationFactor": {
+              "type": "double"
+            },
+            "score": {
+              "type": "double"
+            },
+            "topics": {
+              "type": "nested",
+              "properties": {
+                "firstMessageDeliveries": {
+                  "type": "double",
+                  "ignore_malformed": false,
+                  "coerce": true
+                },
+                "invalidMessageDeliveries": {
+                  "type": "double",
+                  "ignore_malformed": false,
+                  "coerce": true
+                },
+                "meshMessageDeliveries": {
+                  "type": "double",
+                  "ignore_malformed": false,
+                  "coerce": true
+                },
+                "timeInMesh": {
+                  "type": "double",
+                  "ignore_malformed": false,
+                  "coerce": true
+                },
+                "topic": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
+        "sourceAuth": {
+          "type": "keyword"
+        }
+      }
+    },
+    "aliases": {}
+  }
+}
diff --git a/tools/kibana/peer-scores-dashboard.ndjson b/tools/kibana/peer-scores-dashboard.ndjson
index c4d491706..4df376ab3 100644
--- a/tools/kibana/peer-scores-dashboard.ndjson
+++ b/tools/kibana/peer-scores-dashboard.ndjson
@@ -1,3 +1,2 @@
-{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{\"peerScore.weightedScore\":{\"type\":\"double\",\"script\":{\"source\":\"if (doc['type'].value == 100) {\\n    def score = doc['peerScore.score'].value;\\n    if (doc['sourceAuth'] == \\\"<password>\\\") {\\n\\n        emit(score * 1.2)\\n    } else {\\n        emit(score)\\n    }\\n}\\n\\n\"}},\"sourceAuth\":{\"type\":\"keyword\"}}","title":"lotus-pubsub*","typeMeta":"{}"},"coreMigrationVersion":"7.14.1","id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-09-24T12:03:02.575Z","version":"WzcwMzksMV0="}
-{"attributes":{"description":"Average peer score table per node peerID","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":43,\"i\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\"},\"panelIndex\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsDatatable\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"666e2f39-8868-45ad-b747-fe124830b0ae\":{\"columns\":{\"504c50bd-14c1-4119-820e-c961866fc3b4\":{\"label\":\"peerID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"peerScore.peerID.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\"},\"orderDirection\":\"desc\",\"otherBucket\":false,\"missingBucket\":false},\"customLabel\":true},\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\":{\"label\":\"Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.score\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\":{\"label\":\"Weighted Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.weightedScore\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true}},\"columnOrder\":[\"504c50bd-14c1-4119-820e-c961866fc3b4\",\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"504c50bd-14c1-4119-820e-c961866fc3b4\"},{\"isTransposed\":false,\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}},{\"columnId\":\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\",\"isTransposed\":false,\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}}],\"layerId\":\"666e2f39-8868-45ad-b747-fe124830b0ae\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"9890c040-17b7-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"2c407db0-1acb-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Peer Scores\"}]","timeRestore":false,"title":"Peer Scores","version":1},"coreMigrationVersion":"7.14.1","id":"e7e4fd70-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-09-24T11:59:46.187Z","version":"WzY5NjcsMV0="}
-{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file
+{"attributes":{"description":"Average peer score table per node peerID","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":43,\"i\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\"},\"panelIndex\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsDatatable\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"666e2f39-8868-45ad-b747-fe124830b0ae\":{\"columns\":{\"504c50bd-14c1-4119-820e-c961866fc3b4\":{\"label\":\"peerID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"peerScore.peerID.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\"},\"orderDirection\":\"desc\",\"otherBucket\":false,\"missingBucket\":false},\"customLabel\":true},\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\":{\"label\":\"Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.score\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\":{\"label\":\"Weighted Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.weightedScore\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true}},\"columnOrder\":[\"504c50bd-14c1-4119-820e-c961866fc3b4\",\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"504c50bd-14c1-4119-820e-c961866fc3b4\"},{\"isTransposed\":false,\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}},{\"columnId\":\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\",\"isTransposed\":false,\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}}],\"layerId\":\"666e2f39-8868-45ad-b747-fe124830b0ae\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"9890c040-17b7-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"2c407db0-1acb-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Peer Scores\"}]","timeRestore":false,"title":"Peer Scores","version":1},"coreMigrationVersion":"7.14.1","id":"e7e4fd70-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-09-24T12:06:00.625Z","version":"WzczNDgsMV0="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]}

From 53c8e68fba566d8f0f07492294832deff5866b53 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 29 Sep 2021 16:23:08 +0200
Subject: [PATCH 47/51] Remove extra tracer dtypes

---
 node/modules/dtypes/tracer.go | 5 -----
 1 file changed, 5 deletions(-)
 delete mode 100644 node/modules/dtypes/tracer.go

diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
deleted file mode 100644
index 735ff05c3..000000000
--- a/node/modules/dtypes/tracer.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package dtypes
-
-type JsonTracer string
-type ElasticSearchTracer string
-type TracerSourceAuth string

From ea7673618d125277ae0d124ad5cabb255dd1c296 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 29 Sep 2021 16:39:44 +0200
Subject: [PATCH 48/51] Add unit tests for lotus tracer

---
 node/modules/tracer/tracer_test.go | 110 +++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 node/modules/tracer/tracer_test.go

diff --git a/node/modules/tracer/tracer_test.go b/node/modules/tracer/tracer_test.go
new file mode 100644
index 000000000..8ce809d3e
--- /dev/null
+++ b/node/modules/tracer/tracer_test.go
@@ -0,0 +1,110 @@
+package tracer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+	"github.com/stretchr/testify/require"
+)
+
+type testTracerTransport struct {
+	t           *testing.T
+	executeTest func(t *testing.T, evt TracerTransportEvent)
+}
+
+const peerIdA peer.ID = "12D3KooWAbSVMgRejb6ECg6fRTkCPGCfu8396msZVryu8ivcz44G"
+
+func NewTestTraceTransport(t *testing.T, executeTest func(t *testing.T, evt TracerTransportEvent)) TracerTransport {
+	return &testTracerTransport{
+		t:           t,
+		executeTest: executeTest,
+	}
+}
+
+func (ttt *testTracerTransport) Transport(evt TracerTransportEvent) error {
+	ttt.executeTest(ttt.t, evt)
+	return nil
+}
+
+func TestTracer_PeerScores(t *testing.T) {
+
+	testTransport := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, peerIdA.Pretty(), evt.lotusTraceEvent.PeerID)
+		require.Equal(t, "source-auth-token-test", evt.lotusTraceEvent.SourceAuth)
+		require.Equal(t, float64(32), evt.lotusTraceEvent.PeerScore.Score)
+
+		n := time.Now().UnixNano()
+		require.LessOrEqual(t, *evt.lotusTraceEvent.Timestamp, n)
+
+		require.Equal(t, peerIdA.Pretty(), evt.lotusTraceEvent.PeerScore.PeerID)
+		require.Equal(t, 1, len(evt.lotusTraceEvent.PeerScore.Topics))
+
+		topic := evt.lotusTraceEvent.PeerScore.Topics[0]
+		require.Equal(t, "topicA", topic.Topic)
+		require.Equal(t, float64(100), topic.FirstMessageDeliveries)
+	})
+
+	lt := NewLotusTracer(
+		[]TracerTransport{testTransport},
+		peerIdA,
+		"source-auth-token-test",
+	)
+
+	topics := make(map[string]*pubsub.TopicScoreSnapshot)
+	topics["topicA"] = &pubsub.TopicScoreSnapshot{
+		FirstMessageDeliveries: float64(100),
+	}
+
+	m := make(map[peer.ID]*pubsub.PeerScoreSnapshot)
+	m[peerIdA] = &pubsub.PeerScoreSnapshot{
+		Score:  float64(32),
+		Topics: topics,
+	}
+
+	lt.PeerScores(m)
+}
+
+func TestTracer_PubSubTrace(t *testing.T) {
+	now := time.Now()
+	n := int64(now.Unix())
+
+	testTransport := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, []byte(peerIdA), evt.pubsubTraceEvent.PeerID)
+		require.Equal(t, &n, evt.pubsubTraceEvent.Timestamp)
+	})
+
+	lt := NewLotusTracer(
+		[]TracerTransport{testTransport},
+		"pid",
+		"source-auth",
+	)
+
+	lt.Trace(&pubsub_pb.TraceEvent{
+		PeerID:    []byte(peerIdA),
+		Timestamp: &n,
+	})
+
+}
+
+func TestTracer_MultipleTransports(t *testing.T) {
+	testTransportA := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, []byte(peerIdA), evt.pubsubTraceEvent.PeerID)
+	})
+
+	testTransportB := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
+		require.Equal(t, []byte(peerIdA), evt.pubsubTraceEvent.PeerID)
+	})
+
+	executeTest := NewLotusTracer(
+		[]TracerTransport{testTransportA, testTransportB},
+		"pid",
+		"source-auth",
+	)
+
+	executeTest.Trace(&pubsub_pb.TraceEvent{
+		PeerID: []byte(peerIdA),
+	})
+}

From 530df4298e2703ffef7ed388d859e80d809daa23 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 29 Sep 2021 16:49:03 +0200
Subject: [PATCH 49/51] Fix linter errors

---
 node/modules/tracer/tracer_test.go | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/node/modules/tracer/tracer_test.go b/node/modules/tracer/tracer_test.go
index 8ce809d3e..d5faf8a62 100644
--- a/node/modules/tracer/tracer_test.go
+++ b/node/modules/tracer/tracer_test.go
@@ -15,7 +15,7 @@ type testTracerTransport struct {
 	executeTest func(t *testing.T, evt TracerTransportEvent)
 }
 
-const peerIdA peer.ID = "12D3KooWAbSVMgRejb6ECg6fRTkCPGCfu8396msZVryu8ivcz44G"
+const peerIDA peer.ID = "12D3KooWAbSVMgRejb6ECg6fRTkCPGCfu8396msZVryu8ivcz44G"
 
 func NewTestTraceTransport(t *testing.T, executeTest func(t *testing.T, evt TracerTransportEvent)) TracerTransport {
 	return &testTracerTransport{
@@ -32,14 +32,14 @@ func (ttt *testTracerTransport) Transport(evt TracerTransportEvent) error {
 func TestTracer_PeerScores(t *testing.T) {
 
 	testTransport := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
-		require.Equal(t, peerIdA.Pretty(), evt.lotusTraceEvent.PeerID)
+		require.Equal(t, peerIDA.Pretty(), evt.lotusTraceEvent.PeerID)
 		require.Equal(t, "source-auth-token-test", evt.lotusTraceEvent.SourceAuth)
 		require.Equal(t, float64(32), evt.lotusTraceEvent.PeerScore.Score)
 
 		n := time.Now().UnixNano()
 		require.LessOrEqual(t, *evt.lotusTraceEvent.Timestamp, n)
 
-		require.Equal(t, peerIdA.Pretty(), evt.lotusTraceEvent.PeerScore.PeerID)
+		require.Equal(t, peerIDA.Pretty(), evt.lotusTraceEvent.PeerScore.PeerID)
 		require.Equal(t, 1, len(evt.lotusTraceEvent.PeerScore.Topics))
 
 		topic := evt.lotusTraceEvent.PeerScore.Topics[0]
@@ -49,7 +49,7 @@ func TestTracer_PeerScores(t *testing.T) {
 
 	lt := NewLotusTracer(
 		[]TracerTransport{testTransport},
-		peerIdA,
+		peerIDA,
 		"source-auth-token-test",
 	)
 
@@ -59,7 +59,7 @@ func TestTracer_PeerScores(t *testing.T) {
 	}
 
 	m := make(map[peer.ID]*pubsub.PeerScoreSnapshot)
-	m[peerIdA] = &pubsub.PeerScoreSnapshot{
+	m[peerIDA] = &pubsub.PeerScoreSnapshot{
 		Score:  float64(32),
 		Topics: topics,
 	}
@@ -68,11 +68,10 @@ func TestTracer_PeerScores(t *testing.T) {
 }
 
 func TestTracer_PubSubTrace(t *testing.T) {
-	now := time.Now()
-	n := int64(now.Unix())
+	n := time.Now().Unix()
 
 	testTransport := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
-		require.Equal(t, []byte(peerIdA), evt.pubsubTraceEvent.PeerID)
+		require.Equal(t, []byte(peerIDA), evt.pubsubTraceEvent.PeerID)
 		require.Equal(t, &n, evt.pubsubTraceEvent.Timestamp)
 	})
 
@@ -83,7 +82,7 @@ func TestTracer_PubSubTrace(t *testing.T) {
 	)
 
 	lt.Trace(&pubsub_pb.TraceEvent{
-		PeerID:    []byte(peerIdA),
+		PeerID:    []byte(peerIDA),
 		Timestamp: &n,
 	})
 
@@ -91,11 +90,11 @@ func TestTracer_PubSubTrace(t *testing.T) {
 
 func TestTracer_MultipleTransports(t *testing.T) {
 	testTransportA := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
-		require.Equal(t, []byte(peerIdA), evt.pubsubTraceEvent.PeerID)
+		require.Equal(t, []byte(peerIDA), evt.pubsubTraceEvent.PeerID)
 	})
 
 	testTransportB := NewTestTraceTransport(t, func(t *testing.T, evt TracerTransportEvent) {
-		require.Equal(t, []byte(peerIdA), evt.pubsubTraceEvent.PeerID)
+		require.Equal(t, []byte(peerIDA), evt.pubsubTraceEvent.PeerID)
 	})
 
 	executeTest := NewLotusTracer(
@@ -105,6 +104,6 @@ func TestTracer_MultipleTransports(t *testing.T) {
 	)
 
 	executeTest.Trace(&pubsub_pb.TraceEvent{
-		PeerID: []byte(peerIdA),
+		PeerID: []byte(peerIDA),
 	})
 }

From 59040d44196978d310dc6b4abc0e8c3c44401228 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 10 Nov 2021 13:45:40 +0100
Subject: [PATCH 50/51] Trace reject messages in pubsub

---
 node/modules/lp2p/pubsub.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 9d9350074..017375940 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -484,6 +484,15 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 		}
 	case pubsub_pb.TraceEvent_REJECT_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1))
+		if trw.traceMessage(evt.GetRejectMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
 	case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1))
 	case pubsub_pb.TraceEvent_JOIN:

From 62e98c1964961ac3c60e66f38f53ec398df439a1 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 10 Nov 2021 14:11:15 +0100
Subject: [PATCH 51/51] Regenerate lotus config

---
 documentation/en/default-lotus-config.toml    | 29 +++++++++++++++++++
 .../en/default-lotus-miner-config.toml        | 29 +++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml
index 3cb8977b2..b84d124af 100644
--- a/documentation/en/default-lotus-config.toml
+++ b/documentation/en/default-lotus-config.toml
@@ -89,6 +89,35 @@
   # env var: LOTUS_PUBSUB_REMOTETRACER
   #RemoteTracer = ""
 
+  # Path to file that will be used to output tracer content in JSON format.
+  # If present tracer will save data to defined file.
+  # Format: file path
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_JSONTRACER
+  #JsonTracer = ""
+
+  # Connection string for elasticsearch instance.
+  # If present tracer will save data to elasticsearch.
+  # Format: https://<username>:<password>@<elasticsearch_url>:<port>/
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_ELASTICSEARCHTRACER
+  #ElasticSearchTracer = ""
+
+  # Name of elasticsearch index that will be used to save tracer data.
+  # This property is used only if ElasticSearchTracer propery is set.
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_ELASTICSEARCHINDEX
+  #ElasticSearchIndex = ""
+
+  # Auth token that will be passed with logs to elasticsearch - used for weighted peers score.
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_TRACERSOURCEAUTH
+  #TracerSourceAuth = ""
+
 
 [Client]
   # type: bool
diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml
index d402f65ed..08617715b 100644
--- a/documentation/en/default-lotus-miner-config.toml
+++ b/documentation/en/default-lotus-miner-config.toml
@@ -89,6 +89,35 @@
   # env var: LOTUS_PUBSUB_REMOTETRACER
   #RemoteTracer = ""
 
+  # Path to file that will be used to output tracer content in JSON format.
+  # If present tracer will save data to defined file.
+  # Format: file path
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_JSONTRACER
+  #JsonTracer = ""
+
+  # Connection string for elasticsearch instance.
+  # If present tracer will save data to elasticsearch.
+  # Format: https://<username>:<password>@<elasticsearch_url>:<port>/
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_ELASTICSEARCHTRACER
+  #ElasticSearchTracer = ""
+
+  # Name of elasticsearch index that will be used to save tracer data.
+  # This property is used only if ElasticSearchTracer propery is set.
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_ELASTICSEARCHINDEX
+  #ElasticSearchIndex = ""
+
+  # Auth token that will be passed with logs to elasticsearch - used for weighted peers score.
+  #
+  # type: string
+  # env var: LOTUS_PUBSUB_TRACERSOURCEAUTH
+  #TracerSourceAuth = ""
+
 
 [Subsystems]
   # type: bool