Merge branch 'mpetrun5/lotus-extended-tracer' of github.com:ChainSafe/lotus into mpetrun5/lotus-extended-tracer

cmd/lotus/daemon.go

@@ -153,6 +153,18 @@ var DaemonCmd = &cli.Command{
 			Name:  "restore-config",
 			Usage: "config file to use when restoring from backup",
 		},
+		&cli.StringFlag{
+			Name:  "trace-to-json",
+			Usage: "starts tracer and outputs to json file defined with this flag",
+		},
+		&cli.StringFlag{
+			Name:  "trace-to-elasticsearch",
+			Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch",
+		},
+		&cli.StringFlag{
+			Name:  "trace-source-auth",
+			Usage: "auth token for trusted source of traces",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		isLite := cctx.Bool("lite")
@@ -193,6 +205,12 @@ var DaemonCmd = &cli.Command{
 			return fmt.Errorf("unrecognized profile type: %q", profile)
 		}
 
+		traceToJsonFile := cctx.String("trace-to-json")
+
+		traceToElasticsearch := cctx.String("trace-to-elasticsearch")
+
+		traceSourceAuth := cctx.String("trace-source-auth")
+
 		ctx, _ := tag.New(context.Background(),
 			tag.Insert(metrics.Version, build.BuildVersion),
 			tag.Insert(metrics.Commit, build.CurrentCommit),
@@ -319,6 +337,9 @@ var DaemonCmd = &cli.Command{
 
 			node.Override(new(dtypes.Bootstrapper), isBootstrapper),
 			node.Override(new(dtypes.ShutdownChan), shutdownChan),
+			node.Override(new(dtypes.JsonTracerFile), traceToJsonFile),
+			node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch),
+			node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth),
 
 			genesis,
 			liteModeDeps,

node/config/types.go

@@ -311,6 +311,7 @@ type Pubsub struct {
 	RemoteTracer          string
 	JsonTracer            string
 	ElasticSearchTracer   string
+	ElasticSearchIndex    string
 	TracerSourceAuth      string
 }
 

node/modules/dtypes/tracer.go

@@ -0,0 +1,5 @@
+package dtypes
+
+type JsonTracerFile string
+type ElasticSearchTracer string
+type TracerSourceAuth string

node/modules/lp2p/pubsub.go

@@ -377,6 +377,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
 			in.Cfg.ElasticSearchTracer,
+			in.Cfg.ElasticSearchIndex,
 		)
 		if err != nil {
 			return nil, err

node/modules/tracer/elasticsearch_transport.go

@@ -12,17 +12,21 @@ import (
 )
 
 const (
-	ElasticSearch_INDEX = "lotus-pubsub"
+	ElasticSearch_INDEX_DEFAULT = "lotus-pubsub"
 )
 
-func NewElasticSearchTransport(connectionString string) (TracerTransport, error) {
+func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
 	conUrl, err := url.Parse(connectionString)
 
+	if err != nil {
+		return nil, err
+	}
+
 	username := conUrl.User.Username()
 	password, _ := conUrl.User.Password()
 	cfg := elasticsearch.Config{
 		Addresses: []string{
-			"https://" + conUrl.Host,
+			conUrl.Scheme + "://" + conUrl.Host,
 		},
 		Username: username,
 		Password: password,
@@ -34,13 +38,22 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error)
 		return nil, err
 	}
 
+	var esIndex string
+	if elasticsearchIndex != "" {
+		esIndex = elasticsearchIndex
+	} else {
+		esIndex = ElasticSearch_INDEX_DEFAULT
+	}
+
 	return &elasticSearchTransport{
-		cl: es,
+		cl:      es,
+		esIndex: esIndex,
 	}, nil
 }
 
 type elasticSearchTransport struct {
-	cl *elasticsearch.Client
+	cl      *elasticsearch.Client
+	esIndex string
 }
 
 func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {

tools/kibana/peer-scores-dashboard.ndjson

@@ -0,0 +1,3 @@
+{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{\"peerScore.weightedScore\":{\"type\":\"double\",\"script\":{\"source\":\"if (doc['type'].value == 100) {\\n    def score = doc['peerScore.score'].value;\\n    if (doc['sourceAuth'] == \\\"<password>\\\") {\\n\\n        emit(score * 1.2)\\n    } else {\\n        emit(score)\\n    }\\n}\\n\\n\"}},\"sourceAuth\":{\"type\":\"keyword\"}}","title":"lotus-pubsub*","typeMeta":"{}"},"coreMigrationVersion":"7.14.1","id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-09-24T12:03:02.575Z","version":"WzcwMzksMV0="}
+{"attributes":{"description":"Average peer score table per node peerID","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.14.1\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":43,\"i\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\"},\"panelIndex\":\"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsDatatable\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"666e2f39-8868-45ad-b747-fe124830b0ae\":{\"columns\":{\"504c50bd-14c1-4119-820e-c961866fc3b4\":{\"label\":\"peerID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"peerScore.peerID.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\"},\"orderDirection\":\"desc\",\"otherBucket\":false,\"missingBucket\":false},\"customLabel\":true},\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\":{\"label\":\"Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.score\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\":{\"label\":\"Weighted Score\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"peerScore.weightedScore\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true}},\"columnOrder\":[\"504c50bd-14c1-4119-820e-c961866fc3b4\",\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"504c50bd-14c1-4119-820e-c961866fc3b4\"},{\"isTransposed\":false,\"columnId\":\"ec82c5f7-b3c4-4715-8646-a8fe584400fc\",\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}},{\"columnId\":\"e22a19e8-1d71-43d6-9ca0-b30ddd423447\",\"isTransposed\":false,\"colorMode\":\"cell\",\"palette\":{\"type\":\"palette\",\"name\":\"positive\",\"params\":{\"stops\":[{\"color\":\"#d6e9e4\",\"stop\":20},{\"color\":\"#aed3ca\",\"stop\":40},{\"color\":\"#85bdb1\",\"stop\":60},{\"color\":\"#5aa898\",\"stop\":80},{\"color\":\"#209280\",\"stop\":100}]}}}],\"layerId\":\"666e2f39-8868-45ad-b747-fe124830b0ae\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"9890c040-17b7-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"2c407db0-1acb-11ec-99f4-75d57f0cd0d8\",\"name\":\"indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Peer Scores\"}]","timeRestore":false,"title":"Peer Scores","version":1},"coreMigrationVersion":"7.14.1","id":"e7e4fd70-1acb-11ec-99f4-75d57f0cd0d8","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"2c407db0-1acb-11ec-99f4-75d57f0cd0d8","name":"cddc98a5-45f3-4ba7-a7c6-6b9d216b19da:indexpattern-datasource-layer-666e2f39-8868-45ad-b747-fe124830b0ae","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-09-24T11:59:46.187Z","version":"WzY5NjcsMV0="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]}