From 522858c8711e5f56f3b5d6d32ec6f75b5013527f Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 30 Aug 2021 14:35:15 +0200
Subject: [PATCH 01/41] Add lotus tracer interface

---
 node/modules/lp2p/tracer.go | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 node/modules/lp2p/tracer.go

diff --git a/node/modules/lp2p/tracer.go b/node/modules/lp2p/tracer.go
new file mode 100644
index 000000000..36dd6d456
--- /dev/null
+++ b/node/modules/lp2p/tracer.go
@@ -0,0 +1,26 @@
+package lp2p
+
+import (
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+)
+
+func newLotusTracer(tr pubsub.EventTracer) LotusTracer {
+	return &lotusTracer{
+		tr: tr,
+	}
+}
+
+type lotusTracer struct {
+	tr pubsub.EventTracer
+}
+
+type LotusTracer interface {
+	TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
+	Trace(evt *pubsub_pb.TraceEvent)
+}
+
+func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {}
+
+func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {}

From 99fbd7039e524daca65106ff1846d20178a726a1 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 30 Aug 2021 14:38:15 +0200
Subject: [PATCH 02/41] Update pubsub tracking to trace from lotus and lp2p
 tracers

---
 node/modules/lp2p/pubsub.go | 70 +++++++++++++++++++++++++++----------
 1 file changed, 52 insertions(+), 18 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 32b85daf3..5bee17298 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -358,11 +358,12 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 			return nil, err
 		}
 
-		trw := newTracerWrapper(tr, build.BlocksTopic(in.Nn))
+		lt := newLotusTracer(tr)
+		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
 		options = append(options, pubsub.WithEventTracer(trw))
 	} else {
 		// still instantiate a tracer for collecting metrics
-		trw := newTracerWrapper(nil)
+		trw := newTracerWrapper(nil, nil)
 		options = append(options, pubsub.WithEventTracer(trw))
 	}
 
@@ -374,7 +375,11 @@ func HashMsgId(m *pubsub_pb.Message) string {
 	return string(hash[:])
 }
 
-func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTracer {
+func newTracerWrapper(
+	lp2pTracer pubsub.EventTracer,
+	lotusTracer pubsub.EventTracer,
+	topics ...string,
+) pubsub.EventTracer {
 	var topicsMap map[string]struct{}
 	if len(topics) > 0 {
 		topicsMap = make(map[string]struct{})
@@ -383,12 +388,13 @@ func newTracerWrapper(tr pubsub.EventTracer, topics ...string) pubsub.EventTrace
 		}
 	}
 
-	return &tracerWrapper{tr: tr, topics: topicsMap}
+	return &tracerWrapper{lp2pTracer: lp2pTracer, lotusTracer: lotusTracer, topics: topicsMap}
 }
 
 type tracerWrapper struct {
-	tr     pubsub.EventTracer
-	topics map[string]struct{}
+	lp2pTracer  pubsub.EventTracer
+	lotusTracer pubsub.EventTracer
+	topics      map[string]struct{}
 }
 
 func (trw *tracerWrapper) traceMessage(topic string) bool {
@@ -406,33 +412,61 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 	switch evt.GetType() {
 	case pubsub_pb.TraceEvent_PUBLISH_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1))
-		if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) {
-			trw.tr.Trace(evt)
+		if trw.traceMessage(evt.GetPublishMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
 		}
 	case pubsub_pb.TraceEvent_DELIVER_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1))
-		if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
-			trw.tr.Trace(evt)
+		if trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
 		}
 	case pubsub_pb.TraceEvent_REJECT_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1))
 	case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1))
 	case pubsub_pb.TraceEvent_JOIN:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_LEAVE:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_GRAFT:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_PRUNE:
-		if trw.tr != nil {
-			trw.tr.Trace(evt)
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
 		}
 	case pubsub_pb.TraceEvent_RECV_RPC:
 		stats.Record(context.TODO(), metrics.PubsubRecvRPC.M(1))

From b1dafd81b84cbe298fc514e0d2840d0c4aeaa4d1 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 30 Aug 2021 14:49:10 +0200
Subject: [PATCH 03/41] Add peer score tracing on pubsub peer score inspect

---
 node/modules/lp2p/pubsub.go | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 5bee17298..321108e4a 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -49,6 +49,30 @@ func ScoreKeeper() *dtypes.ScoreKeeper {
 	return new(dtypes.ScoreKeeper)
 }
 
+type PeerScoreTracker interface {
+	UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
+}
+
+type peerScoreTracker struct {
+	sk *dtypes.ScoreKeeper
+	lt LotusTracer
+}
+
+func newPeerScoreTracker(lt LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
+	return &peerScoreTracker{
+		sk: sk,
+		lt: lt,
+	}
+}
+
+func (pst *peerScoreTracker) UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	if pst.lt != nil {
+		pst.lt.TracePeerScore(scores)
+	}
+
+	pst.sk.Update(scores)
+}
+
 type GossipIn struct {
 	fx.In
 	Mctx helpers.MetricsCtx
@@ -272,7 +296,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				OpportunisticGraftThreshold: OpportunisticGraftScoreThreshold,
 			},
 		),
-		pubsub.WithPeerScoreInspect(in.Sk.Update, 10*time.Second),
 	}
 
 	// enable Peer eXchange on bootstrappers
@@ -359,12 +382,18 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		}
 
 		lt := newLotusTracer(tr)
+		pst := newPeerScoreTracker(lt, in.Sk)
 		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
+
 		options = append(options, pubsub.WithEventTracer(trw))
+		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	} else {
 		// still instantiate a tracer for collecting metrics
 		trw := newTracerWrapper(nil, nil)
 		options = append(options, pubsub.WithEventTracer(trw))
+
+		pst := newPeerScoreTracker(nil, in.Sk)
+		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	}
 
 	return pubsub.NewGossipSub(helpers.LifecycleCtx(in.Mctx, in.Lc), in.Host, options...)

From c367533a3cd5f901ea474d17d7ea510331573439 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 10 Sep 2021 12:36:38 +0200
Subject: [PATCH 04/41] Add transport interface

---
 node/modules/{lp2p => tracer}/tracer.go | 10 ++++++----
 node/modules/tracer/transport.go        |  5 +++++
 2 files changed, 11 insertions(+), 4 deletions(-)
 rename node/modules/{lp2p => tracer}/tracer.go (75%)
 create mode 100644 node/modules/tracer/transport.go

diff --git a/node/modules/lp2p/tracer.go b/node/modules/tracer/tracer.go
similarity index 75%
rename from node/modules/lp2p/tracer.go
rename to node/modules/tracer/tracer.go
index 36dd6d456..a19ae065d 100644
--- a/node/modules/lp2p/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -1,4 +1,4 @@
-package lp2p
+package tracer
 
 import (
 	peer "github.com/libp2p/go-libp2p-core/peer"
@@ -6,14 +6,16 @@ import (
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
-func newLotusTracer(tr pubsub.EventTracer) LotusTracer {
+func newLotusTracer(et pubsub.EventTracer, tt TracerTransport) LotusTracer {
 	return &lotusTracer{
-		tr: tr,
+		et: et,
+		tt: tt,
 	}
 }
 
 type lotusTracer struct {
-	tr pubsub.EventTracer
+	et pubsub.EventTracer
+	tt TracerTransport
 }
 
 type LotusTracer interface {
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
new file mode 100644
index 000000000..73ef80690
--- /dev/null
+++ b/node/modules/tracer/transport.go
@@ -0,0 +1,5 @@
+package tracer
+
+type TracerTransport interface {
+	Transport(jsonEvent []byte)
+}

From d5a4b0637abf50b418756d02c796c15230febc0c Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 10 Sep 2021 15:37:38 +0200
Subject: [PATCH 05/41] Add lotus tracer transport calls

---
 node/modules/tracer/tracer.go    | 31 +++++++++++++++++++++++++++++--
 node/modules/tracer/transport.go |  2 +-
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index a19ae065d..6d60352e7 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -1,11 +1,16 @@
 package tracer
 
 import (
+	"encoding/json"
+
+	logging "github.com/ipfs/go-log/v2"
 	peer "github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
+var log = logging.Logger("lotus-tracer")
+
 func newLotusTracer(et pubsub.EventTracer, tt TracerTransport) LotusTracer {
 	return &lotusTracer{
 		et: et,
@@ -23,6 +28,28 @@ type LotusTracer interface {
 	Trace(evt *pubsub_pb.TraceEvent)
 }
 
-func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {}
+func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	jsonEvent, err := json.Marshal(scores)
+	if err != nil {
+		log.Errorf("error while marshaling peer score: %s", err)
+		return
+	}
 
-func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {}
+	err = lt.tt.Transport(jsonEvent)
+	if err != nil {
+		log.Errorf("error while transporting peer scores: %s", err)
+	}
+}
+
+func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
+	jsonEvent, err := json.Marshal(evt)
+	if err != nil {
+		log.Errorf("error while marshaling tracer event: %s", err)
+		return
+	}
+
+	err = lt.tt.Transport(jsonEvent)
+	if err != nil {
+		log.Errorf("error while transporting trace event: %s", err)
+	}
+}
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
index 73ef80690..c8495821e 100644
--- a/node/modules/tracer/transport.go
+++ b/node/modules/tracer/transport.go
@@ -1,5 +1,5 @@
 package tracer
 
 type TracerTransport interface {
-	Transport(jsonEvent []byte)
+	Transport(jsonEvent []byte) error
 }

From 8e6a01de6ddd156f721c9039db0407a8277874eb Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 13:40:54 +0200
Subject: [PATCH 06/41] Add configuration in pubsub to trace locally to json
 file

---
 node/config/types.go        |  1 +
 node/modules/lp2p/pubsub.go | 20 ++++++++++++++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/node/config/types.go b/node/config/types.go
index 1576169e6..05ea472f3 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -309,6 +309,7 @@ type Pubsub struct {
 	DirectPeers           []string
 	IPColocationWhitelist []string
 	RemoteTracer          string
+	JsonTracerFile        string
 }
 
 type Chainstore struct {
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 321108e4a..6cf52fdf0 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"net"
+	"os"
 	"time"
 
 	host "github.com/libp2p/go-libp2p-core/host"
@@ -21,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
+	"github.com/filecoin-project/lotus/node/modules/tracer"
 )
 
 func init() {
@@ -55,10 +57,10 @@ type PeerScoreTracker interface {
 
 type peerScoreTracker struct {
 	sk *dtypes.ScoreKeeper
-	lt LotusTracer
+	lt tracer.LotusTracer
 }
 
-func newPeerScoreTracker(lt LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
+func newPeerScoreTracker(lt tracer.LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracker {
 	return &peerScoreTracker{
 		sk: sk,
 		lt: lt,
@@ -67,7 +69,7 @@ func newPeerScoreTracker(lt LotusTracer, sk *dtypes.ScoreKeeper) PeerScoreTracke
 
 func (pst *peerScoreTracker) UpdatePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
 	if pst.lt != nil {
-		pst.lt.TracePeerScore(scores)
+		pst.lt.PeerScores(scores)
 	}
 
 	pst.sk.Update(scores)
@@ -364,6 +366,17 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
 				100)))
 
+	var lt tracer.LotusTracer
+	if in.Cfg.JsonTracerFile != "" {
+		out, err := os.OpenFile(in.Cfg.JsonTracerFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+		if err != nil {
+			return nil, err
+		}
+
+		jsonTransport := tracer.NewJsonTracerTransport(out)
+		lt = tracer.NewLotusTracer(jsonTransport, in.Host.ID())
+	}
+
 	// tracer
 	if in.Cfg.RemoteTracer != "" {
 		a, err := ma.NewMultiaddr(in.Cfg.RemoteTracer)
@@ -381,7 +394,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 			return nil, err
 		}
 
-		lt := newLotusTracer(tr)
 		pst := newPeerScoreTracker(lt, in.Sk)
 		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
 

From ab86c5419d6ac0fcfdd789e4f421a8cc2db61cef Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 13:41:03 +0200
Subject: [PATCH 07/41] Implement json tracer transport

---
 node/modules/tracer/json_transport.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 node/modules/tracer/json_transport.go

diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
new file mode 100644
index 000000000..2ab004d9c
--- /dev/null
+++ b/node/modules/tracer/json_transport.go
@@ -0,0 +1,20 @@
+package tracer
+
+import (
+	"os"
+)
+
+type jsonTracerTransport struct {
+	out *os.File
+}
+
+func NewJsonTracerTransport(out *os.File) TracerTransport {
+	return &jsonTracerTransport{
+		out: out,
+	}
+}
+
+func (jtt *jsonTracerTransport) Transport(jsonEvent []byte) error {
+	_, err := jtt.out.Write(jsonEvent)
+	return err
+}

From 8a657b90a7d5eba00db902afff5ddd7673ea0e22 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 13:41:56 +0200
Subject: [PATCH 08/41] Refactor lotus tracer to behave like lp2p tracer

---
 node/modules/tracer/tracer.go | 50 ++++++++++++++++++++++++++++-------
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 6d60352e7..af3a987b0 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -2,34 +2,66 @@ package tracer
 
 import (
 	"encoding/json"
+	"time"
 
 	logging "github.com/ipfs/go-log/v2"
-	peer "github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
 var log = logging.Logger("lotus-tracer")
 
-func newLotusTracer(et pubsub.EventTracer, tt TracerTransport) LotusTracer {
+func NewLotusTracer(tt TracerTransport, pid peer.ID) LotusTracer {
 	return &lotusTracer{
-		et: et,
-		tt: tt,
+		tt:  tt,
+		pid: pid,
 	}
 }
 
 type lotusTracer struct {
-	et pubsub.EventTracer
-	tt TracerTransport
+	tt  TracerTransport
+	pid peer.ID
+}
+
+const (
+	TraceEvent_PEER_SCORES pubsub_pb.TraceEvent_Type = 100
+)
+
+type LotusTraceEvent struct {
+	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
+	PeerID     []byte                    `json:"peerID,omitempty"`
+	Timestamp  *int64                    `json:"timestamp,omitempty"`
+	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
+}
+
+type TraceEvent_PeerScores struct {
+	Scores map[peer.ID]*pubsub.PeerScoreSnapshot `json:"scores,omitempty"`
 }
 
 type LotusTracer interface {
-	TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 	Trace(evt *pubsub_pb.TraceEvent)
+	TraceLotusEvent(evt *LotusTraceEvent)
+
+	PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 }
 
-func (lt *lotusTracer) TracePeerScore(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
-	jsonEvent, err := json.Marshal(scores)
+func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	now := time.Now().UnixNano()
+	evt := &LotusTraceEvent{
+		Type:      *TraceEvent_PEER_SCORES.Enum(),
+		PeerID:    []byte(lt.pid),
+		Timestamp: &now,
+		PeerScores: &TraceEvent_PeerScores{
+			Scores: scores,
+		},
+	}
+
+	lt.TraceLotusEvent(evt)
+}
+
+func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
+	jsonEvent, err := json.Marshal(evt)
 	if err != nil {
 		log.Errorf("error while marshaling peer score: %s", err)
 		return

From c064ce485283cb96a751c1924996c9cc28fd6a44 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Wed, 15 Sep 2021 14:22:48 +0200
Subject: [PATCH 09/41] Add generated documentation

---
 node/config/doc_gen.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index 9efe4e03c..05e7afff4 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -525,6 +525,12 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "RemoteTracer",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "JsonTracerFile",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},

From bb85ca031d9466cc77fec59b04361680822a7f2f Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 14:50:27 +0200
Subject: [PATCH 10/41] Add elasticsearch transport

---
 go.mod                                        |  1 +
 go.sum                                        |  2 +
 node/config/doc_gen.go                        |  6 ++
 node/config/types.go                          |  1 +
 node/modules/lp2p/pubsub.go                   |  3 +
 .../modules/tracer/elasticsearch_transport.go | 59 +++++++++++++++++++
 node/modules/tracer/tracer.go                 | 21 ++++---
 7 files changed, 85 insertions(+), 8 deletions(-)
 create mode 100644 node/modules/tracer/elasticsearch_transport.go

diff --git a/go.mod b/go.mod
index f73511574..a5adcca6f 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/drand/drand v1.2.1
 	github.com/drand/kyber v1.1.4
 	github.com/dustin/go-humanize v1.0.0
+	github.com/elastic/go-elasticsearch/v7 v7.14.0
 	github.com/elastic/go-sysinfo v1.3.0
 	github.com/elastic/gosigar v0.12.0
 	github.com/etclabscore/go-openrpc-reflect v0.0.36
diff --git a/go.sum b/go.sum
index f16a81a9f..6347ac4f7 100644
--- a/go.sum
+++ b/go.sum
@@ -235,6 +235,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/go-elasticsearch/v7 v7.14.0 h1:extp3jos/rwJn3J+lgbaGlwAgs0TVsIHme00GyNAyX4=
+github.com/elastic/go-elasticsearch/v7 v7.14.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE=
 github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
 github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY=
diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index 05e7afff4..fd4ec0366 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -531,6 +531,12 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "JsonTracerFile",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "ElasticSearchTracer",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},
diff --git a/node/config/types.go b/node/config/types.go
index 05ea472f3..025a9c1af 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -310,6 +310,7 @@ type Pubsub struct {
 	IPColocationWhitelist []string
 	RemoteTracer          string
 	JsonTracerFile        string
+	ElasticSearchTracer   string
 }
 
 type Chainstore struct {
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 6cf52fdf0..04f21c84f 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -375,6 +375,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		lt = tracer.NewLotusTracer(jsonTransport, in.Host.ID())
+	} else if in.Cfg.ElasticSearchTracer != "" {
+		elasticSearchTransport := tracer.NewElasticSearchTransport()
+		lt = tracer.NewLotusTracer(elasticSearchTransport, in.Host.ID())
 	}
 
 	// tracer
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
new file mode 100644
index 000000000..2cba23478
--- /dev/null
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -0,0 +1,59 @@
+package tracer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esapi"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var rpclog = logging.Logger("elasticsearch")
+
+func NewElasticSearchTransport() TracerTransport {
+	es, err := elasticsearch.NewDefaultClient()
+
+	if err != nil {
+		rpclog.Fatalf("Error on creating elastic search client: %+v", err)
+	}
+
+	return &elasticSearchTransport{
+		cl: es,
+	}
+}
+
+type elasticSearchTransport struct {
+	cl *elasticsearch.Client
+}
+
+func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
+	req := esapi.IndexRequest{
+		Index:      "PeerScore",
+		DocumentID: "1", // todo
+		Body:       strings.NewReader(string(jsonEvent)),
+		Refresh:    "true",
+	}
+
+	// Perform the request with the client.
+	res, err := req.Do(context.Background(), est.cl)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	if res.IsError() {
+		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
+	} else {
+		// Deserialize the response into a map.
+		var r map[string]interface{}
+		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
+			return err
+		} else {
+			rpclog.Infof("[%s] %s; version=%d", res.Status(), r["result"], int(r["_version"].(float64)))
+		}
+	}
+	return nil
+}
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index af3a987b0..8c6250151 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -12,7 +12,7 @@ import (
 
 var log = logging.Logger("lotus-tracer")
 
-func NewLotusTracer(tt TracerTransport, pid peer.ID) LotusTracer {
+func NewLotusTracer(tt []TracerTransport, pid peer.ID) LotusTracer {
 	return &lotusTracer{
 		tt:  tt,
 		pid: pid,
@@ -20,7 +20,7 @@ func NewLotusTracer(tt TracerTransport, pid peer.ID) LotusTracer {
 }
 
 type lotusTracer struct {
-	tt  TracerTransport
+	tt  []TracerTransport
 	pid peer.ID
 }
 
@@ -67,10 +67,13 @@ func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
 		return
 	}
 
-	err = lt.tt.Transport(jsonEvent)
-	if err != nil {
-		log.Errorf("error while transporting peer scores: %s", err)
+	for _, t := range lt.tt {
+		err = t.Transport(jsonEvent)
+		if err != nil {
+			log.Errorf("error while transporting peer scores: %s", err)
+		}
 	}
+
 }
 
 func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
@@ -80,8 +83,10 @@ func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
 		return
 	}
 
-	err = lt.tt.Transport(jsonEvent)
-	if err != nil {
-		log.Errorf("error while transporting trace event: %s", err)
+	for _, t := range lt.tt {
+		err = t.Transport(jsonEvent)
+		if err != nil {
+			log.Errorf("error while transporting trace event: %s", err)
+		}
 	}
 }

From cf060965658c933290ddeea205effb97bb54c0ca Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 14:54:20 +0200
Subject: [PATCH 11/41] Fix tracer initialization

---
 node/modules/lp2p/pubsub.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 04f21c84f..c9755a680 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -366,19 +366,19 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
 				100)))
 
-	var lt tracer.LotusTracer
+	var transports []tracer.TracerTransport
 	if in.Cfg.JsonTracerFile != "" {
 		out, err := os.OpenFile(in.Cfg.JsonTracerFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
 		if err != nil {
 			return nil, err
 		}
-
 		jsonTransport := tracer.NewJsonTracerTransport(out)
-		lt = tracer.NewLotusTracer(jsonTransport, in.Host.ID())
+		transports = append(transports, jsonTransport)
 	} else if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport := tracer.NewElasticSearchTransport()
-		lt = tracer.NewLotusTracer(elasticSearchTransport, in.Host.ID())
+		transports = append(transports, elasticSearchTransport)
 	}
+	lt := tracer.NewLotusTracer(transports, in.Host.ID())
 
 	// tracer
 	if in.Cfg.RemoteTracer != "" {

From e20cd0ef27c6abb0f8135c801796d128d81c657c Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 14:58:36 +0200
Subject: [PATCH 12/41] Propagate error on creating transport

---
 node/modules/lp2p/pubsub.go                    | 5 ++++-
 node/modules/tracer/elasticsearch_transport.go | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index c9755a680..7f2a4d678 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -375,7 +375,10 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		transports = append(transports, jsonTransport)
 	} else if in.Cfg.ElasticSearchTracer != "" {
-		elasticSearchTransport := tracer.NewElasticSearchTransport()
+		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
+		if err != nil {
+			return nil, err
+		}
 		transports = append(transports, elasticSearchTransport)
 	}
 	lt := tracer.NewLotusTracer(transports, in.Host.ID())
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 2cba23478..f45dd9ef5 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -13,16 +13,16 @@ import (
 
 var rpclog = logging.Logger("elasticsearch")
 
-func NewElasticSearchTransport() TracerTransport {
+func NewElasticSearchTransport() (TracerTransport, error) {
 	es, err := elasticsearch.NewDefaultClient()
 
 	if err != nil {
-		rpclog.Fatalf("Error on creating elastic search client: %+v", err)
+		return nil, err
 	}
 
 	return &elasticSearchTransport{
 		cl: es,
-	}
+	}, nil
 }
 
 type elasticSearchTransport struct {

From 707faf57b54967a384bf46196b79ff8ec80c79b1 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 15:11:34 +0200
Subject: [PATCH 13/41] Code cleanup

---
 node/modules/lp2p/pubsub.go                    | 4 +++-
 node/modules/tracer/elasticsearch_transport.go | 9 ++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 7f2a4d678..6656abf01 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -374,7 +374,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		}
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		transports = append(transports, jsonTransport)
-	} else if in.Cfg.ElasticSearchTracer != "" {
+	}
+
+	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
 		if err != nil {
 			return nil, err
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index f45dd9ef5..8e214a4e9 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -8,10 +8,11 @@ import (
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
-	logging "github.com/ipfs/go-log/v2"
 )
 
-var rpclog = logging.Logger("elasticsearch")
+const (
+	ElasticSearch_INDEX = "pubsub"
+)
 
 func NewElasticSearchTransport() (TracerTransport, error) {
 	es, err := elasticsearch.NewDefaultClient()
@@ -31,7 +32,7 @@ type elasticSearchTransport struct {
 
 func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
 	req := esapi.IndexRequest{
-		Index:      "PeerScore",
+		Index:      ElasticSearch_INDEX,
 		DocumentID: "1", // todo
 		Body:       strings.NewReader(string(jsonEvent)),
 		Refresh:    "true",
@@ -51,8 +52,6 @@ func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
 		var r map[string]interface{}
 		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
 			return err
-		} else {
-			rpclog.Infof("[%s] %s; version=%d", res.Status(), r["result"], int(r["_version"].(float64)))
 		}
 	}
 	return nil

From 71a2ca9e7cd96b277a6c29049283645d235fe2e4 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:03:51 +0200
Subject: [PATCH 14/41] Add new wrapper structure in tracer transport

---
 .../modules/tracer/elasticsearch_transport.go | 24 +++++++++++++++++--
 node/modules/tracer/json_transport.go         | 20 ++++++++++++++--
 node/modules/tracer/tracer.go                 | 23 +++++++-----------
 node/modules/tracer/transport.go              |  9 ++++++-
 4 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 8e214a4e9..f9cf621eb 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,6 +12,9 @@ import (
 
 const (
 	ElasticSearch_INDEX = "pubsub"
+
+	ElasticSearch_DOC_LOTUS  = "doc_lotus"
+	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
 )
 
 func NewElasticSearchTransport() (TracerTransport, error) {
@@ -30,10 +33,27 @@ type elasticSearchTransport struct {
 	cl *elasticsearch.Client
 }
 
-func (est *elasticSearchTransport) Transport(jsonEvent []byte) error {
+func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
+	var e interface{}
+	var docId string
+	if event.lotusTraceEvent != nil {
+		e = *event.lotusTraceEvent
+		docId = ElasticSearch_DOC_LOTUS
+	} else if event.pubsubTraceEvent != nil {
+		e = *event.pubsubTraceEvent
+		docId = ElasticSearch_DOC_PUBSUB
+	} else {
+		return nil
+	}
+
+	jsonEvent, err := json.Marshal(e)
+	if err != nil {
+		return fmt.Errorf("error while marshaling peer score: %s", err)
+	}
+
 	req := esapi.IndexRequest{
 		Index:      ElasticSearch_INDEX,
-		DocumentID: "1", // todo
+		DocumentID: docId,
 		Body:       strings.NewReader(string(jsonEvent)),
 		Refresh:    "true",
 	}
diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 2ab004d9c..128c37473 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -1,6 +1,8 @@
 package tracer
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 )
 
@@ -14,7 +16,21 @@ func NewJsonTracerTransport(out *os.File) TracerTransport {
 	}
 }
 
-func (jtt *jsonTracerTransport) Transport(jsonEvent []byte) error {
-	_, err := jtt.out.Write(jsonEvent)
+func (jtt *jsonTracerTransport) Transport(event TracerTransportEvent) error {
+	var e interface{}
+	if event.lotusTraceEvent != nil {
+		e = *event.lotusTraceEvent
+	} else if event.pubsubTraceEvent != nil {
+		e = *event.pubsubTraceEvent
+	} else {
+		return nil
+	}
+
+	jsonEvent, err := json.Marshal(e)
+	if err != nil {
+		return fmt.Errorf("error while marshaling event: %s", err)
+	}
+
+	_, err = jtt.out.Write(jsonEvent)
 	return err
 }
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 8c6250151..ad5f5c6b7 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -1,7 +1,6 @@
 package tracer
 
 import (
-	"encoding/json"
 	"time"
 
 	logging "github.com/ipfs/go-log/v2"
@@ -61,14 +60,11 @@ func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 }
 
 func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
-	jsonEvent, err := json.Marshal(evt)
-	if err != nil {
-		log.Errorf("error while marshaling peer score: %s", err)
-		return
-	}
-
 	for _, t := range lt.tt {
-		err = t.Transport(jsonEvent)
+		err := t.Transport(TracerTransportEvent{
+			lotusTraceEvent:  evt,
+			pubsubTraceEvent: nil,
+		})
 		if err != nil {
 			log.Errorf("error while transporting peer scores: %s", err)
 		}
@@ -77,14 +73,11 @@ func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {
 }
 
 func (lt *lotusTracer) Trace(evt *pubsub_pb.TraceEvent) {
-	jsonEvent, err := json.Marshal(evt)
-	if err != nil {
-		log.Errorf("error while marshaling tracer event: %s", err)
-		return
-	}
-
 	for _, t := range lt.tt {
-		err = t.Transport(jsonEvent)
+		err := t.Transport(TracerTransportEvent{
+			lotusTraceEvent:  nil,
+			pubsubTraceEvent: evt,
+		})
 		if err != nil {
 			log.Errorf("error while transporting trace event: %s", err)
 		}
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
index c8495821e..e67867fd7 100644
--- a/node/modules/tracer/transport.go
+++ b/node/modules/tracer/transport.go
@@ -1,5 +1,12 @@
 package tracer
 
+import pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
+
 type TracerTransport interface {
-	Transport(jsonEvent []byte) error
+	Transport(jsonEvent TracerTransportEvent) error
+}
+
+type TracerTransportEvent struct {
+	lotusTraceEvent  *LotusTraceEvent
+	pubsubTraceEvent *pubsub_pb.TraceEvent
 }

From add13c2646cbbe4157fe7a86d0aae26e3d056c3a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:05:44 +0200
Subject: [PATCH 15/41] Code cleanup

---
 node/modules/lp2p/pubsub.go                    | 1 -
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 6656abf01..905643ed0 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -375,7 +375,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		jsonTransport := tracer.NewJsonTracerTransport(out)
 		transports = append(transports, jsonTransport)
 	}
-
 	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
 		if err != nil {
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index f9cf621eb..4fe8149be 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -48,7 +48,7 @@ func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
 
 	jsonEvent, err := json.Marshal(e)
 	if err != nil {
-		return fmt.Errorf("error while marshaling peer score: %s", err)
+		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
 	req := esapi.IndexRequest{

From 67c1d633c68a77baff415af5560444be93914122 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:16:12 +0200
Subject: [PATCH 16/41] Rename jsonEvent to evt

---
 node/modules/tracer/elasticsearch_transport.go | 14 +++++++-------
 node/modules/tracer/json_transport.go          | 14 +++++++-------
 node/modules/tracer/transport.go               |  2 +-
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 4fe8149be..c2cfdb87a 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -33,20 +33,20 @@ type elasticSearchTransport struct {
 	cl *elasticsearch.Client
 }
 
-func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
+func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	var e interface{}
 	var docId string
-	if event.lotusTraceEvent != nil {
-		e = *event.lotusTraceEvent
+	if evt.lotusTraceEvent != nil {
+		e = *evt.lotusTraceEvent
 		docId = ElasticSearch_DOC_LOTUS
-	} else if event.pubsubTraceEvent != nil {
-		e = *event.pubsubTraceEvent
+	} else if evt.pubsubTraceEvent != nil {
+		e = *evt.pubsubTraceEvent
 		docId = ElasticSearch_DOC_PUBSUB
 	} else {
 		return nil
 	}
 
-	jsonEvent, err := json.Marshal(e)
+	jsonEvt, err := json.Marshal(e)
 	if err != nil {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
@@ -54,7 +54,7 @@ func (est *elasticSearchTransport) Transport(event TracerTransportEvent) error {
 	req := esapi.IndexRequest{
 		Index:      ElasticSearch_INDEX,
 		DocumentID: docId,
-		Body:       strings.NewReader(string(jsonEvent)),
+		Body:       strings.NewReader(string(jsonEvt)),
 		Refresh:    "true",
 	}
 
diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 128c37473..26537a600 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -16,21 +16,21 @@ func NewJsonTracerTransport(out *os.File) TracerTransport {
 	}
 }
 
-func (jtt *jsonTracerTransport) Transport(event TracerTransportEvent) error {
+func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {
 	var e interface{}
-	if event.lotusTraceEvent != nil {
-		e = *event.lotusTraceEvent
-	} else if event.pubsubTraceEvent != nil {
-		e = *event.pubsubTraceEvent
+	if evt.lotusTraceEvent != nil {
+		e = *evt.lotusTraceEvent
+	} else if evt.pubsubTraceEvent != nil {
+		e = *evt.pubsubTraceEvent
 	} else {
 		return nil
 	}
 
-	jsonEvent, err := json.Marshal(e)
+	jsonEvt, err := json.Marshal(e)
 	if err != nil {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
-	_, err = jtt.out.Write(jsonEvent)
+	_, err = jtt.out.Write(jsonEvt)
 	return err
 }
diff --git a/node/modules/tracer/transport.go b/node/modules/tracer/transport.go
index e67867fd7..56d926afc 100644
--- a/node/modules/tracer/transport.go
+++ b/node/modules/tracer/transport.go
@@ -3,7 +3,7 @@ package tracer
 import pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 
 type TracerTransport interface {
-	Transport(jsonEvent TracerTransportEvent) error
+	Transport(evt TracerTransportEvent) error
 }
 
 type TracerTransportEvent struct {

From 5cbd2519daf3fc0560317c249d7c031b6d52085c Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Wed, 15 Sep 2021 16:20:36 +0200
Subject: [PATCH 17/41] Remove processing elasticsearch result

---
 node/modules/tracer/elasticsearch_transport.go | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index c2cfdb87a..3bf86ee29 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -67,12 +67,6 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 
 	if res.IsError() {
 		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
-	} else {
-		// Deserialize the response into a map.
-		var r map[string]interface{}
-		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
-			return err
-		}
 	}
 	return nil
 }

From c25a5e06cf5f3b84699bf187bd9129f9ea65e29a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Thu, 16 Sep 2021 12:15:42 +0200
Subject: [PATCH 18/41] Use config string for elasticsearch client

---
 node/modules/lp2p/pubsub.go                    |  4 +++-
 node/modules/tracer/elasticsearch_transport.go | 17 +++++++++++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 905643ed0..d12a3fa1c 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -376,7 +376,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		transports = append(transports, jsonTransport)
 	}
 	if in.Cfg.ElasticSearchTracer != "" {
-		elasticSearchTransport, err := tracer.NewElasticSearchTransport()
+		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
+			in.Cfg.ElasticSearchTracer,
+		)
 		if err != nil {
 			return nil, err
 		}
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 3bf86ee29..45ddd36aa 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v7"
@@ -17,8 +18,20 @@ const (
 	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
 )
 
-func NewElasticSearchTransport() (TracerTransport, error) {
-	es, err := elasticsearch.NewDefaultClient()
+func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
+	conUrl, err := url.Parse(connectionString)
+
+	username := conUrl.User.Username()
+	password, _ := conUrl.User.Password()
+	cfg := elasticsearch.Config{
+		Addresses: []string{
+			"https://" + conUrl.Host,
+		},
+		Username: username,
+		Password: password,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
 
 	if err != nil {
 		return nil, err

From e2206147cc6dc6346dd11aadbee7226f04792c65 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 11:51:59 +0200
Subject: [PATCH 19/41] Add source auth token to lotus traces

---
 node/config/types.go          |  1 +
 node/modules/lp2p/pubsub.go   |  2 +-
 node/modules/tracer/tracer.go | 12 ++++++++----
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/node/config/types.go b/node/config/types.go
index 025a9c1af..aa061baef 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -311,6 +311,7 @@ type Pubsub struct {
 	RemoteTracer          string
 	JsonTracerFile        string
 	ElasticSearchTracer   string
+	TracerSourceAuth      string
 }
 
 type Chainstore struct {
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index d12a3fa1c..aa391a42b 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -384,7 +384,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		}
 		transports = append(transports, elasticSearchTransport)
 	}
-	lt := tracer.NewLotusTracer(transports, in.Host.ID())
+	lt := tracer.NewLotusTracer(transports, in.Host.ID(), in.Cfg.TracerSourceAuth)
 
 	// tracer
 	if in.Cfg.RemoteTracer != "" {
diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index ad5f5c6b7..69967565a 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -11,16 +11,18 @@ import (
 
 var log = logging.Logger("lotus-tracer")
 
-func NewLotusTracer(tt []TracerTransport, pid peer.ID) LotusTracer {
+func NewLotusTracer(tt []TracerTransport, pid peer.ID, sourceAuth string) LotusTracer {
 	return &lotusTracer{
 		tt:  tt,
 		pid: pid,
+		sa:  sourceAuth,
 	}
 }
 
 type lotusTracer struct {
 	tt  []TracerTransport
 	pid peer.ID
+	sa  string
 }
 
 const (
@@ -32,6 +34,7 @@ type LotusTraceEvent struct {
 	PeerID     []byte                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
 	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
+	SourceAuth string                    `json:"sourceAuth,omitempty"`
 }
 
 type TraceEvent_PeerScores struct {
@@ -48,9 +51,10 @@ type LotusTracer interface {
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
 	now := time.Now().UnixNano()
 	evt := &LotusTraceEvent{
-		Type:      *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:    []byte(lt.pid),
-		Timestamp: &now,
+		Type:       *TraceEvent_PEER_SCORES.Enum(),
+		PeerID:     []byte(lt.pid),
+		Timestamp:  &now,
+		SourceAuth: lt.sa,
 		PeerScores: &TraceEvent_PeerScores{
 			Scores: scores,
 		},

From d067bc9f0109ad17a9ea368d89e6045f8969f50b Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 15:22:18 +0200
Subject: [PATCH 20/41] Move creating and opening file to json transport
 constructor

---
 node/modules/lp2p/pubsub.go           | 5 ++---
 node/modules/tracer/json_transport.go | 9 +++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index d12a3fa1c..0a3f5e042 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"net"
-	"os"
 	"time"
 
 	host "github.com/libp2p/go-libp2p-core/host"
@@ -368,11 +367,11 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 
 	var transports []tracer.TracerTransport
 	if in.Cfg.JsonTracerFile != "" {
-		out, err := os.OpenFile(in.Cfg.JsonTracerFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracerFile)
 		if err != nil {
 			return nil, err
 		}
-		jsonTransport := tracer.NewJsonTracerTransport(out)
+
 		transports = append(transports, jsonTransport)
 	}
 	if in.Cfg.ElasticSearchTracer != "" {
diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 26537a600..1d5e81f30 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -10,10 +10,15 @@ type jsonTracerTransport struct {
 	out *os.File
 }
 
-func NewJsonTracerTransport(out *os.File) TracerTransport {
+func NewJsonTracerTransport(file string) (TracerTransport, error) {
+	out, err := os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660)
+	if err != nil {
+		return nil, err
+	}
+
 	return &jsonTracerTransport{
 		out: out,
-	}
+	}, nil
 }
 
 func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {

From 9cdc5261c44050acd7aab47418a74b48a70bccd3 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 15:53:39 +0200
Subject: [PATCH 21/41] Fix lotus tracer being nil when remote tracer is not
 configured

---
 node/modules/lp2p/pubsub.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 0a3f5e042..a071b36a5 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -409,7 +409,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	} else {
 		// still instantiate a tracer for collecting metrics
-		trw := newTracerWrapper(nil, nil)
+		trw := newTracerWrapper(nil, lt)
 		options = append(options, pubsub.WithEventTracer(trw))
 
 		pst := newPeerScoreTracker(nil, in.Sk)

From 941a0f715327a341accc371cd727a3eeee42a2d2 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 16:31:36 +0200
Subject: [PATCH 22/41] Update peer score tracker creation to include lotus
 tracer

---
 node/modules/lp2p/pubsub.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index a071b36a5..25ee4963f 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -412,7 +412,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		trw := newTracerWrapper(nil, lt)
 		options = append(options, pubsub.WithEventTracer(trw))
 
-		pst := newPeerScoreTracker(nil, in.Sk)
+		pst := newPeerScoreTracker(lt, in.Sk)
 		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	}
 

From 9efa495b0ebbdbb82f0e6047a352b7c5b3c5cc9b Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Thu, 16 Sep 2021 16:42:18 +0200
Subject: [PATCH 23/41] Write every trace to new line when using json transport

---
 node/modules/tracer/json_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/json_transport.go b/node/modules/tracer/json_transport.go
index 1d5e81f30..ca8535f4b 100644
--- a/node/modules/tracer/json_transport.go
+++ b/node/modules/tracer/json_transport.go
@@ -36,6 +36,6 @@ func (jtt *jsonTracerTransport) Transport(evt TracerTransportEvent) error {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
-	_, err = jtt.out.Write(jsonEvt)
+	_, err = jtt.out.WriteString(string(jsonEvt) + "\n")
 	return err
 }

From 8e2b474fc774a92f815c93df513d9770ebdec5b3 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Thu, 16 Sep 2021 16:44:35 +0200
Subject: [PATCH 24/41] Use only one document on elastic

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 45ddd36aa..f57ce85de 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -54,7 +54,7 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 		docId = ElasticSearch_DOC_LOTUS
 	} else if evt.pubsubTraceEvent != nil {
 		e = *evt.pubsubTraceEvent
-		docId = ElasticSearch_DOC_PUBSUB
+		docId = ElasticSearch_DOC_LOTUS
 	} else {
 		return nil
 	}

From f09f5c2d1144bbceef2f013ad23325bdc080dd8a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Thu, 16 Sep 2021 16:49:47 +0200
Subject: [PATCH 25/41] Change elastic document name

---
 node/modules/tracer/elasticsearch_transport.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index f57ce85de..612080ef8 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -51,10 +51,10 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	var docId string
 	if evt.lotusTraceEvent != nil {
 		e = *evt.lotusTraceEvent
-		docId = ElasticSearch_DOC_LOTUS
+		docId = ElasticSearch_DOC_PUBSUB
 	} else if evt.pubsubTraceEvent != nil {
 		e = *evt.pubsubTraceEvent
-		docId = ElasticSearch_DOC_LOTUS
+		docId = ElasticSearch_DOC_PUBSUB
 	} else {
 		return nil
 	}

From dc79ea363d0b8803a405847bc9ef611bf4a4c01a Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 17 Sep 2021 11:35:36 +0200
Subject: [PATCH 26/41] Change index name

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 612080ef8..3f9a47e67 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "pubsub"
+	ElasticSearch_INDEX = "lotus"
 
 	ElasticSearch_DOC_LOTUS  = "doc_lotus"
 	ElasticSearch_DOC_PUBSUB = "doc_pubsub"

From a91ffa60d67f8b3fe1f90460e5a3f70ba1e2bb4f Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 17 Sep 2021 12:11:14 +0200
Subject: [PATCH 27/41] Set peer id event as string in lotus trace event

---
 node/modules/tracer/tracer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index ad5f5c6b7..7d8d9e2c4 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -29,7 +29,7 @@ const (
 
 type LotusTraceEvent struct {
 	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
-	PeerID     []byte                    `json:"peerID,omitempty"`
+	PeerID     string                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
 	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
 }
@@ -49,7 +49,7 @@ func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot)
 	now := time.Now().UnixNano()
 	evt := &LotusTraceEvent{
 		Type:      *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:    []byte(lt.pid),
+		PeerID:    string(lt.pid),
 		Timestamp: &now,
 		PeerScores: &TraceEvent_PeerScores{
 			Scores: scores,

From 7d0aefacbb9216221fb9d3d7ee1b863a452579b7 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 17 Sep 2021 13:46:26 +0200
Subject: [PATCH 28/41] Change peer score event type to avoid exceeding
 elastisearch field limit

---
 node/modules/tracer/tracer.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 7d8d9e2c4..4367e8fda 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -31,11 +31,12 @@ type LotusTraceEvent struct {
 	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
 	PeerID     string                    `json:"peerID,omitempty"`
 	Timestamp  *int64                    `json:"timestamp,omitempty"`
-	PeerScores *TraceEvent_PeerScores    `json:"peerScores,omitempty"`
+	PeerScores []TraceEvent_PeerScore    `json:"peerScores,omitempty"`
 }
 
-type TraceEvent_PeerScores struct {
-	Scores map[peer.ID]*pubsub.PeerScoreSnapshot `json:"scores,omitempty"`
+type TraceEvent_PeerScore struct {
+	PeerID string  `json:"peerID"`
+	Score  float32 `json:"score"`
 }
 
 type LotusTracer interface {
@@ -46,14 +47,17 @@ type LotusTracer interface {
 }
 
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
+	var peerScores []TraceEvent_PeerScore
+	for pid, score := range scores {
+		peerScores = append(peerScores, TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)})
+	}
+
 	now := time.Now().UnixNano()
 	evt := &LotusTraceEvent{
-		Type:      *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:    string(lt.pid),
-		Timestamp: &now,
-		PeerScores: &TraceEvent_PeerScores{
-			Scores: scores,
-		},
+		Type:       *TraceEvent_PEER_SCORES.Enum(),
+		PeerID:     string(lt.pid),
+		Timestamp:  &now,
+		PeerScores: peerScores,
 	}
 
 	lt.TraceLotusEvent(evt)

From 0962e73e3e8a0ddb506ef044030014229c147555 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 17 Sep 2021 14:22:57 +0200
Subject: [PATCH 29/41] Add loger to elastic search client

---
 node/modules/tracer/elasticsearch_transport.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 3f9a47e67..a1c271afc 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -5,14 +5,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"os"
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/elastic/go-elasticsearch/v7/estransport"
 )
 
 const (
-	ElasticSearch_INDEX = "lotus"
+	ElasticSearch_INDEX = "lotus2"
 
 	ElasticSearch_DOC_LOTUS  = "doc_lotus"
 	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
@@ -29,6 +31,11 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		},
 		Username: username,
 		Password: password,
+		Logger: &estransport.CurlLogger{
+			Output:             os.Stdout,
+			EnableRequestBody:  true,
+			EnableResponseBody: true,
+		},
 	}
 
 	es, err := elasticsearch.NewClient(cfg)

From a685ccc9555b52c62a0ffec8f448dfbc46426adb Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 17 Sep 2021 15:14:39 +0200
Subject: [PATCH 30/41] Remove document id

---
 node/modules/tracer/elasticsearch_transport.go | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index a1c271afc..4709ef756 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -14,10 +14,7 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "lotus2"
-
-	ElasticSearch_DOC_LOTUS  = "doc_lotus"
-	ElasticSearch_DOC_PUBSUB = "doc_pubsub"
+	ElasticSearch_INDEX = "lotus-pubsub"
 )
 
 func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
@@ -55,13 +52,11 @@ type elasticSearchTransport struct {
 
 func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	var e interface{}
-	var docId string
+
 	if evt.lotusTraceEvent != nil {
 		e = *evt.lotusTraceEvent
-		docId = ElasticSearch_DOC_PUBSUB
 	} else if evt.pubsubTraceEvent != nil {
 		e = *evt.pubsubTraceEvent
-		docId = ElasticSearch_DOC_PUBSUB
 	} else {
 		return nil
 	}
@@ -72,10 +67,9 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	}
 
 	req := esapi.IndexRequest{
-		Index:      ElasticSearch_INDEX,
-		DocumentID: docId,
-		Body:       strings.NewReader(string(jsonEvt)),
-		Refresh:    "true",
+		Index:   ElasticSearch_INDEX,
+		Body:    strings.NewReader(string(jsonEvt)),
+		Refresh: "true",
 	}
 
 	// Perform the request with the client.

From 082f70757c485827552c9731a85c796673de7735 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Mon, 20 Sep 2021 12:29:12 +0200
Subject: [PATCH 31/41] Revert additional logging in elastic client

---
 node/modules/tracer/elasticsearch_transport.go | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 4709ef756..8823b8199 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -5,12 +5,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
-	"os"
 	"strings"
 
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
-	"github.com/elastic/go-elasticsearch/v7/estransport"
 )
 
 const (
@@ -28,11 +26,6 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		},
 		Username: username,
 		Password: password,
-		Logger: &estransport.CurlLogger{
-			Output:             os.Stdout,
-			EnableRequestBody:  true,
-			EnableResponseBody: true,
-		},
 	}
 
 	es, err := elasticsearch.NewClient(cfg)

From d65d9aa6e0a25fe5df8d955e2cd79a07fbd44932 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Tue, 21 Sep 2021 12:30:50 +0200
Subject: [PATCH 32/41] Send peerID peer score as separate events to enable
 usage of kibana runtime fields

---
 node/modules/tracer/tracer.go | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/node/modules/tracer/tracer.go b/node/modules/tracer/tracer.go
index 4367e8fda..01ed1196a 100644
--- a/node/modules/tracer/tracer.go
+++ b/node/modules/tracer/tracer.go
@@ -28,10 +28,10 @@ const (
 )
 
 type LotusTraceEvent struct {
-	Type       pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
-	PeerID     string                    `json:"peerID,omitempty"`
-	Timestamp  *int64                    `json:"timestamp,omitempty"`
-	PeerScores []TraceEvent_PeerScore    `json:"peerScores,omitempty"`
+	Type      pubsub_pb.TraceEvent_Type `json:"type,omitempty"`
+	PeerID    string                    `json:"peerID,omitempty"`
+	Timestamp *int64                    `json:"timestamp,omitempty"`
+	PeerScore TraceEvent_PeerScore      `json:"peerScore,omitempty"`
 }
 
 type TraceEvent_PeerScore struct {
@@ -47,20 +47,17 @@ type LotusTracer interface {
 }
 
 func (lt *lotusTracer) PeerScores(scores map[peer.ID]*pubsub.PeerScoreSnapshot) {
-	var peerScores []TraceEvent_PeerScore
-	for pid, score := range scores {
-		peerScores = append(peerScores, TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)})
-	}
-
 	now := time.Now().UnixNano()
-	evt := &LotusTraceEvent{
-		Type:       *TraceEvent_PEER_SCORES.Enum(),
-		PeerID:     string(lt.pid),
-		Timestamp:  &now,
-		PeerScores: peerScores,
-	}
+	for pid, score := range scores {
+		evt := &LotusTraceEvent{
+			Type:      *TraceEvent_PEER_SCORES.Enum(),
+			PeerID:    lt.pid.Pretty(),
+			Timestamp: &now,
+			PeerScore: TraceEvent_PeerScore{PeerID: pid.Pretty(), Score: float32(score.Score)},
+		}
 
-	lt.TraceLotusEvent(evt)
+		lt.TraceLotusEvent(evt)
+	}
 }
 
 func (lt *lotusTracer) TraceLotusEvent(evt *LotusTraceEvent) {

From e46da5a1812efd54723eae3949bd40cb2dba9731 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 12:37:16 +0200
Subject: [PATCH 33/41] Add cli flags

---
 cmd/lotus/daemon.go           | 28 ++++++++++++++++++++++++++++
 node/config/doc_gen.go        | 12 ++++++++++++
 node/config/types.go          |  1 +
 node/modules/dtypes/tracer.go |  6 ++++++
 4 files changed, 47 insertions(+)
 create mode 100644 node/modules/dtypes/tracer.go

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 486ac8ed7..89901a6f5 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -153,6 +153,22 @@ var DaemonCmd = &cli.Command{
 			Name:  "restore-config",
 			Usage: "config file to use when restoring from backup",
 		},
+		&cli.StringFlag{
+			Name:  "trace-to-json",
+			Usage: "starts tracer and outputs to json file defined with this flag",
+		},
+		&cli.StringFlag{
+			Name:  "trace-to-elasticsearch",
+			Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch",
+		},
+		&cli.StringFlag{
+			Name:  "elasticsearch-index",
+			Usage: "configure elasticearch index name if elasticsearch tracer is configured",
+		},
+		&cli.StringFlag{
+			Name:  "trace-source-auth",
+			Usage: "auth token for trusted source of traces",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		isLite := cctx.Bool("lite")
@@ -193,6 +209,14 @@ var DaemonCmd = &cli.Command{
 			return fmt.Errorf("unrecognized profile type: %q", profile)
 		}
 
+		traceToJsonFile := cctx.String("trace-to-json")
+
+		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
+
+		elasticsearchIndex := cctx.String("elasticsearch-index")
+
+		traceSourceAuth := cctx.String("trace-source-auth")
+
 		ctx, _ := tag.New(context.Background(),
 			tag.Insert(metrics.Version, build.BuildVersion),
 			tag.Insert(metrics.Commit, build.CurrentCommit),
@@ -319,6 +343,10 @@ var DaemonCmd = &cli.Command{
 
 			node.Override(new(dtypes.Bootstrapper), isBootstrapper),
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
+			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
+			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
+			node.Override(new(dtypes.ElasticSearchTracer), elasticsearchIndex),
+			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
 			genesis,
 			liteModeDeps,
diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index fd4ec0366..f22156170 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -537,6 +537,18 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "ElasticSearchTracer",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "ElasticSearchIndex",
+			Type: "string",
+
+			Comment: ``,
+		},
+		{
+			Name: "TracerSourceAuth",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},
diff --git a/node/config/types.go b/node/config/types.go
index aa061baef..630ae61fd 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -311,6 +311,7 @@ type Pubsub struct {
 	RemoteTracer          string
 	JsonTracerFile        string
 	ElasticSearchTracer   string
+	ElasticSearchIndex    string
 	TracerSourceAuth      string
 }
 
diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
new file mode 100644
index 000000000..9317924f7
--- /dev/null
+++ b/node/modules/dtypes/tracer.go
@@ -0,0 +1,6 @@
+package dtypes
+
+type JsonTracerFile string
+type ElasticSearchTracer string
+type ElasticSearchIndex string
+type TracerSourceAuth string

From 94a1601ab268314734370d13412f8ee5ebf972ad Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 12:40:33 +0200
Subject: [PATCH 34/41] Remove hardcoded url schema

---
 node/modules/tracer/elasticsearch_transport.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 8823b8199..dbacf5823 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -18,11 +18,15 @@ const (
 func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
 	conUrl, err := url.Parse(connectionString)
 
+	if err != nil {
+		return nil, err
+	}
+
 	username := conUrl.User.Username()
 	password, _ := conUrl.User.Password()
 	cfg := elasticsearch.Config{
 		Addresses: []string{
-			"https://" + conUrl.Host,
+			conUrl.Scheme + conUrl.Host,
 		},
 		Username: username,
 		Password: password,

From 22bbb113e7afc20f73d91583b60915fcf8d0546b Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 12:58:32 +0200
Subject: [PATCH 35/41] Fix schema creation

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index dbacf5823..621b1d437 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -26,7 +26,7 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 	password, _ := conUrl.User.Password()
 	cfg := elasticsearch.Config{
 		Addresses: []string{
-			conUrl.Scheme + conUrl.Host,
+			conUrl.Scheme + "://" + conUrl.Host,
 		},
 		Username: username,
 		Password: password,

From 4438c4bd87b637a9231caadca0785b6c102b4ae7 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 13:43:25 +0200
Subject: [PATCH 36/41] Move index name to config only and add default value

---
 cmd/lotus/daemon.go                            |  7 -------
 node/modules/dtypes/tracer.go                  |  1 -
 node/modules/lp2p/pubsub.go                    |  1 +
 node/modules/tracer/elasticsearch_transport.go | 17 +++++++++++++----
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 89901a6f5..9579d9534 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -161,10 +161,6 @@ var DaemonCmd = &cli.Command{
 			Name:  "trace-to-elasticsearch",
 			Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch",
 		},
-		&cli.StringFlag{
-			Name:  "elasticsearch-index",
-			Usage: "configure elasticearch index name if elasticsearch tracer is configured",
-		},
 		&cli.StringFlag{
 			Name:  "trace-source-auth",
 			Usage: "auth token for trusted source of traces",
@@ -213,8 +209,6 @@ var DaemonCmd = &cli.Command{
 
 		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
 
-		elasticsearchIndex := cctx.String("elasticsearch-index")
-
 		traceSourceAuth := cctx.String("trace-source-auth")
 
 		ctx, _ := tag.New(context.Background(),
@@ -345,7 +339,6 @@ var DaemonCmd = &cli.Command{
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
 			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
 			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
-			node.Override(new(dtypes.ElasticSearchTracer), elasticsearchIndex),
 			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
 			genesis,
diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
index 9317924f7..622f4baf4 100644
--- a/node/modules/dtypes/tracer.go
+++ b/node/modules/dtypes/tracer.go
@@ -2,5 +2,4 @@ package dtypes
 
 type JsonTracerFile string
 type ElasticSearchTracer string
-type ElasticSearchIndex string
 type TracerSourceAuth string
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index a8ba126e0..818832562 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -377,6 +377,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
 			in.Cfg.ElasticSearchTracer,
+			in.Cfg.ElasticSearchIndex,
 		)
 		if err != nil {
 			return nil, err
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 621b1d437..d0d7b84a1 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -12,10 +12,10 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "lotus-pubsub"
+	ElasticSearch_INDEX_DEFAULT = "lotus-pubsub"
 )
 
-func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
+func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
 	conUrl, err := url.Parse(connectionString)
 
 	if err != nil {
@@ -38,13 +38,22 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		return nil, err
 	}
 
+	var esIndex string
+	if elasticsearchIndex != "" {
+		esIndex = elasticsearchIndex
+	} else {
+		esIndex = ElasticSearch_INDEX_DEFAULT
+	}
+
 	return &elasticSearchTransport{
-		cl: es,
+		cl:      es,
+		esIndex: esIndex,
 	}, nil
 }
 
 type elasticSearchTransport struct {
-	cl *elasticsearch.Client
+	cl      *elasticsearch.Client
+	esIndex string
 }
 
 func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {

From 9332b9f39976af76221dc225c898a98e4590cee0 Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Fri, 24 Sep 2021 13:53:31 +0200
Subject: [PATCH 37/41] Change json config naming

---
 node/config/doc_gen.go      | 8 +++++++-
 node/config/types.go        | 2 +-
 node/modules/lp2p/pubsub.go | 4 ++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go
index fd4ec0366..21e9ccc6b 100644
--- a/node/config/doc_gen.go
+++ b/node/config/doc_gen.go
@@ -528,7 +528,7 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Comment: ``,
 		},
 		{
-			Name: "JsonTracerFile",
+			Name: "JsonTracer",
 			Type: "string",
 
 			Comment: ``,
@@ -537,6 +537,12 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K...
 			Name: "ElasticSearchTracer",
 			Type: "string",
 
+			Comment: ``,
+		},
+		{
+			Name: "TracerSourceAuth",
+			Type: "string",
+
 			Comment: ``,
 		},
 	},
diff --git a/node/config/types.go b/node/config/types.go
index aa061baef..b421a7e8e 100644
--- a/node/config/types.go
+++ b/node/config/types.go
@@ -309,7 +309,7 @@ type Pubsub struct {
 	DirectPeers           []string
 	IPColocationWhitelist []string
 	RemoteTracer          string
-	JsonTracerFile        string
+	JsonTracer            string
 	ElasticSearchTracer   string
 	TracerSourceAuth      string
 }
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index a8ba126e0..b91779ac5 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -366,8 +366,8 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 				100)))
 
 	var transports []tracer.TracerTransport
-	if in.Cfg.JsonTracerFile != "" {
-		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracerFile)
+	if in.Cfg.JsonTracer != "" {
+		jsonTransport, err := tracer.NewJsonTracerTransport(in.Cfg.JsonTracer)
 		if err != nil {
 			return nil, err
 		}

From 72d8e620752174d8f561a6c833351ae4f6ebd298 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 24 Sep 2021 14:07:47 +0200
Subject: [PATCH 38/41] Add kibana dashboard configurations

---
 tools/kibana/block-propagation-dashboard.ndjson | 2 ++
 tools/kibana/peer-scores-dashboard.ndjson       | 3 +++
 2 files changed, 5 insertions(+)
 create mode 100644 tools/kibana/block-propagation-dashboard.ndjson
 create mode 100644 tools/kibana/peer-scores-dashboard.ndjson

diff --git a/tools/kibana/block-propagation-dashboard.ndjson b/tools/kibana/block-propagation-dashboard.ndjson
new file mode 100644
index 000000000..dc5121183
--- /dev/null
+++ b/tools/kibana/block-propagation-dashboard.ndjson
@@ -0,0 +1,2 @@
+{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":42,\"i\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\"},\"panelIndex\":\"c7e4001d-38c9-4fa0-a488-e069dd50d274\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"Block propagation\",\"description\":\"\",\"type\":\"vega\",\"params\":{\"spec\":\"{\\n  $schema: https://vega.github.io/schema/vega/v5.json\\n  title: Node block propagation\\n\\n\\n  \\\"signals\\\": [\\n    { \\\"name\\\": \\\"cx\\\", \\\"update\\\": \\\"width / 2\\\" },\\n    { \\\"name\\\": \\\"cy\\\", \\\"update\\\": \\\"height / 2\\\" },\\n    { \\\"name\\\": \\\"nodeRadius\\\", \\\"value\\\": 10,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 20, \\\"step\\\": 1} },\\n    { \\\"name\\\": \\\"propagationMultiplier\\\", \\\"value\\\": 50,\\n      \\\"bind\\\": {\\\"input\\\": \\\"range\\\", \\\"min\\\": 1, \\\"max\\\": 100, \\\"step\\\": 1} },\\n  ],\\n\\n  data: [\\n    {\\n    \\n      name: \\\"node-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 0,\\n          aggs: {\\n            unique_peerID: {\\n              terms: { \\n                field: \\\"peerID\\\",\\n                size: 10000\\n              },\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"aggregations.unique_peerID.buckets\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"key\\\"],\\n          \\\"as\\\": [\\\"peerID\\\"]\\n        },\\n        {\\n          type: \\\"identifier\\\",\\n          as: \\\"id\\\"\\n        },\\n        {\\n          type: \\\"formula\\\",\\n          expr: \\\"datum.id - 1\\\",\\n          as: \\\"index\\\"\\n        }\\n      ]\\n    },\\n      {\\n      name: \\\"published-message-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 0\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n    },\\n    {\\n      name: \\\"link-data\\\",\\n      url: {\\n        index: lotus-pubsub\\n        body: {\\n          size: 10000,\\n          query: {\\n            bool: {\\n              must: [\\n                {\\n                  match: {\\n                    type: 1\\n                  }\\n                }\\n              ]\\n            }\\n          }\\n        }\\n      },\\n      format: {\\\"property\\\": \\\"hits.hits\\\"},\\n      transform: [\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"published-message-data\\\",\\n          \\\"key\\\": \\\"_source.publishMessage.messageID\\\",\\n          \\\"fields\\\": [\\\"_source.deliverMessage.messageID\\\"],\\n          \\\"as\\\": [\\\"publishedMessage\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"_source.peerID\\\"],\\n          \\\"as\\\": [\\\"source\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"lookup\\\",\\n          \\\"from\\\": \\\"node-data\\\",\\n          \\\"key\\\": \\\"peerID\\\",\\n          \\\"fields\\\": [\\\"publishedMessage._source.peerID\\\"],\\n          \\\"as\\\": [\\\"target\\\"],\\n        },\\n        {\\n          \\\"type\\\": \\\"formula\\\",\\n          \\\"expr\\\": \\\"(datum._source.timestamp - datum.publishedMessage._source.timestamp) * propagationMultiplier\\\",\\n          \\\"as\\\": \\\"distance\\\"\\n        },\\n        {\\n          \\\"type\\\": \\\"project\\\",\\n          \\\"fields\\\": [\\\"source.index\\\", \\\"target.index\\\", \\\"distance\\\"]\\n          \\\"as\\\": [\\\"source\\\", \\\"target\\\", \\\"distance\\\"]\\n        },\\n        {\\n          \\\"type\\\": \\\"aggregate\\\",\\n          \\\"ops\\\": [\\\"average\\\"],\\n          \\\"fields\\\": [\\\"distance\\\"],\\n          \\\"groupby\\\": [\\\"source\\\", \\\"target\\\"],\\n          \\\"as\\\": [\\\"distance\\\"]\\n        }\\n      ]\\n    }\\n  ]\\n\\n  scales: [\\n    {\\n      \\\"name\\\": \\\"color\\\",\\n      \\\"type\\\": \\\"ordinal\\\",\\n      \\\"domain\\\": {\\\"data\\\": \\\"node-data\\\", \\\"field\\\": \\\"id\\\"},\\n      \\\"range\\\": {\\\"scheme\\\": \\\"category20c\\\"}\\n    }\\n  ]\\n\\n  marks: [\\n    {\\n      \\\"name\\\": \\\"nodes\\\",\\n      \\\"type\\\": \\\"symbol\\\",\\n      \\\"zindex\\\": 1,\\n      \\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n          \\\"stroke\\\": {\\\"value\\\": \\\"black\\\"},\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'PeerID': datum.peerID}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"size\\\": {\\\"signal\\\": \\\"2 * nodeRadius * nodeRadius\\\"},\\n          \\\"fill\\\": {\\\"scale\\\": \\\"color\\\", \\\"field\\\": \\\"id\\\"},\\n        },\\n        \\\"hover\\\": { \\\"fill\\\": {\\\"value\\\": \\\"red\\\"} },\\n      },\\n\\n      \\\"from\\\": {\\\"data\\\": \\\"node-data\\\"},\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"force\\\",\\n          \\\"restart\\\": false,\\n          \\\"static\\\": true,\\n          \\\"signal\\\": \\\"force\\\",\\n          \\\"forces\\\": [\\n            {\\\"force\\\": \\\"link\\\", \\\"links\\\": \\\"link-data\\\", distance: {field: \\\"distance\\\"}},\\n            {\\\"force\\\": \\\"center\\\", \\\"x\\\": {\\\"signal\\\": \\\"cx\\\"}, \\\"y\\\": {\\\"signal\\\": \\\"cy\\\"}},\\n          ]\\n        }\\n      ]\\n    },\\n    {\\n      \\\"type\\\": \\\"path\\\",\\n      \\\"from\\\": {\\\"data\\\": \\\"link-data\\\"},\\n      \\\"encode\\\": {\\n        \\\"enter\\\": {\\n          \\\"tooltip\\\": {\\\"signal\\\": \\\"{'Block propagation': datum.distance}\\\"}\\n        },\\n        \\\"update\\\": {\\n          \\\"stroke\\\": {\\\"value\\\": \\\"#ccc\\\"},\\n          \\\"strokeWidth\\\": {\\\"value\\\": 1.5}\\n        },\\n        \\\"hover\\\": { \\\"stroke\\\": { \\\"value\\\": \\\"#0B33A0\\\" }}\\n      },\\n      \\\"transform\\\": [\\n        {\\n          \\\"type\\\": \\\"linkpath\\\",\\n          \\\"require\\\": {\\\"signal\\\": \\\"force\\\"},\\n          \\\"shape\\\": \\\"line\\\",\\n          \\\"sourceX\\\": \\\"datum.source.x\\\", \\\"sourceY\\\": \\\"datum.source.y\\\",\\n          \\\"targetX\\\": \\\"datum.target.x\\\", \\\"targetY\\\": \\\"datum.target.y\\\"\\n        }\\n      ]\\n    }\\n  ]\\n}\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}}},\"enhancements\":{}}}]","timeRestore":false,"title":"Block propagation","version":1},"coreMigrationVersion":"7.14.1","id":"eff2ade0-19fa-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[],"type":"dashboard","updated_at":"2021-09-24T09:46:01.693Z","version":"WzYzMjAsMV0="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]}
diff --git a/tools/kibana/peer-scores-dashboard.ndjson b/tools/kibana/peer-scores-dashboard.ndjson
new file mode 100644
index 000000000..c4d491706
--- /dev/null
+++ b/tools/kibana/peer-scores-dashboard.ndjson
@@ -0,0 +1,3 @@
+{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{\"peerScore.weightedScore\":{\"type\":\"double\",\"script\":{\"source\":\"if (doc['type'].value == 100) {\\n    def score = doc['peerScore.score'].value;\\n    if (doc['sourceAuth'] == \\\"<password>\\\") {\\n\\n        emit(score * 1.2)\\n    } else {\\n        emit(score)\\n    }\\n}\\n\\n\"}},\"sourceAuth\":{\"type\":\"keyword\"}}","title":"lotus-pubsub*","typeMeta":"{}"},"coreMigrationVersion":"7.14.1","id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-09-24T12:03:02.575Z","version":"WzcwMzksMV0="}
+{"attributes":{"description":"Average peer score table per node peerID","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":43,\"i\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\"},\"panelIndex\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsDatatable\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"666e2f39-8868-45ad-b747-fe124830b0ae\":{\"columns\":{\"504c50bd-14c1-4119-820e-c961866fc3b4\":{\"label\":\"peerID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"peerScore.peerID.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\"},\"orderDirection\":\"desc\",\"otherBucket\":false,\"missingBucket\":false},\"customLabel\":true},\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\":{\"label\":\"Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.score\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\":{\"label\":\"Weighted Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.weightedScore\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true}},\"columnOrder\":[\"504c50bd-14c1-4119-820e-c961866fc3b4\",\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"504c50bd-14c1-4119-820e-c961866fc3b4\"},{\"isTransposed\":false,\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}},{\"columnId\":\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\",\"isTransposed\":false,\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}}],\"layerId\":\"666e2f39-8868-45ad-b747-fe124830b0ae\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"9890c040-17b7-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"2c407db0-1acb-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Peer Scores\"}]","timeRestore":false,"title":"Peer Scores","version":1},"coreMigrationVersion":"7.14.1","id":"e7e4fd70-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-09-24T11:59:46.187Z","version":"WzY5NjcsMV0="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file

From 6c680e306dd6b6cb6f20c3f0ecc20bf8d9b4186c Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Fri, 24 Sep 2021 16:25:41 +0200
Subject: [PATCH 39/41] Rename json tracer flag

---
 cmd/lotus/daemon.go           | 13 ++++++-------
 node/modules/dtypes/tracer.go |  2 +-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 9579d9534..e58523999 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -1,3 +1,4 @@
+//go:build !nodaemon
 // +build !nodaemon
 
 package main
@@ -205,12 +206,6 @@ var DaemonCmd = &cli.Command{
 			return fmt.Errorf("unrecognized profile type: %q", profile)
 		}
 
-		traceToJsonFile := cctx.String("trace-to-json")
-
-		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
-
-		traceSourceAuth := cctx.String("trace-source-auth")
-
 		ctx, _ := tag.New(context.Background(),
 			tag.Insert(metrics.Version, build.BuildVersion),
 			tag.Insert(metrics.Commit, build.CurrentCommit),
@@ -328,6 +323,10 @@ var DaemonCmd = &cli.Command{
 			log.Warnf("unable to inject prometheus ipfs/go-metrics exporter; some metrics will be unavailable; err: %s", err)
 		}
 
+		traceToJsonFile := cctx.String("trace-to-json")
+		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
+		traceSourceAuth := cctx.String("trace-source-auth")
+
 		var api api.FullNode
 		stop, err := node.New(ctx,
 			node.FullAPI(&api, node.Lite(isLite)),
@@ -337,7 +336,7 @@ var DaemonCmd = &cli.Command{
 
 			node.Override(new(dtypes.Bootstrapper), isBootstrapper),
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
-			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
+			node.Override(new(dtypes.JsonTracer), traceToJsonFile),
 			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
 			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go
index 622f4baf4..735ff05c3 100644
--- a/node/modules/dtypes/tracer.go
+++ b/node/modules/dtypes/tracer.go
@@ -1,5 +1,5 @@
 package dtypes
 
-type JsonTracerFile string
+type JsonTracer string
 type ElasticSearchTracer string
 type TracerSourceAuth string

From 36913f4efce511d5e1fbcb6c3b892ffb699e1d8b Mon Sep 17 00:00:00 2001
From: Mak Muftic <mak@nodefactory.io>
Date: Mon, 27 Sep 2021 13:28:28 +0200
Subject: [PATCH 40/41] Fix elasticsearch index setup

---
 node/modules/tracer/elasticsearch_transport.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index d0d7b84a1..49060a13d 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -73,7 +73,7 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 	}
 
 	req := esapi.IndexRequest{
-		Index:   ElasticSearch_INDEX,
+		Index:   est.esIndex,
 		Body:    strings.NewReader(string(jsonEvt)),
 		Refresh: "true",
 	}

From 581a54663c7d0dcfdb46e4107de7c9ecededfc34 Mon Sep 17 00:00:00 2001
From: Matija Petrunic <matija.petrunic@fer.hr>
Date: Mon, 27 Sep 2021 15:09:03 +0200
Subject: [PATCH 41/41] Add README for creating kibana dashboards

---
 tools/kibana/README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 tools/kibana/README.md

diff --git a/tools/kibana/README.md b/tools/kibana/README.md
new file mode 100644
index 000000000..e4e451bbb
--- /dev/null
+++ b/tools/kibana/README.md
@@ -0,0 +1,14 @@
+## Lotus Kibana Dashboard
+
+This folder contains configuration files to create Kibana dashboards to track peer scores and block propagation
+throughout Filecoin network.
+
+### Importing dashboard
+
+The peer score and block propagation dashboard configuration is imported via Kibana import saved object [functionality](https://www.elastic.co/guide/en/kibana/current/managing-saved-objects.html#managing-saved-objects-export-objects).
+
+The index patterns will be created automatically when importing dashboards.
+
+#### Custom index
+
+By default, the dashboards target `lotus-pubsub` index which is the default one when running node. The index can be customised via edit on dashboard visualizations.