From e46da5a1812efd54723eae3949bd40cb2dba9731 Mon Sep 17 00:00:00 2001 From: Mak Muftic Date: Fri, 24 Sep 2021 12:37:16 +0200 Subject: [PATCH 1/4] Add cli flags --- cmd/lotus/daemon.go | 28 ++++++++++++++++++++++++++++ node/config/doc_gen.go | 12 ++++++++++++ node/config/types.go | 1 + node/modules/dtypes/tracer.go | 6 ++++++ 4 files changed, 47 insertions(+) create mode 100644 node/modules/dtypes/tracer.go diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 486ac8ed7..89901a6f5 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -153,6 +153,22 @@ var DaemonCmd = &cli.Command{ Name: "restore-config", Usage: "config file to use when restoring from backup", }, + &cli.StringFlag{ + Name: "trace-to-json", + Usage: "starts tracer and outputs to json file defined with this flag", + }, + &cli.StringFlag{ + Name: "trace-to-elasticsearch", + Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch", + }, + &cli.StringFlag{ + Name: "elasticsearch-index", + Usage: "configure elasticearch index name if elasticsearch tracer is configured", + }, + &cli.StringFlag{ + Name: "trace-source-auth", + Usage: "auth token for trusted source of traces", + }, }, Action: func(cctx *cli.Context) error { isLite := cctx.Bool("lite") @@ -193,6 +209,14 @@ var DaemonCmd = &cli.Command{ return fmt.Errorf("unrecognized profile type: %q", profile) } + traceToJsonFile := cctx.String("trace-to-json") + + traceToElasticsearch := cctx.String("trace-to-elasticsearch") + + elasticsearchIndex := cctx.String("elasticsearch-index") + + traceSourceAuth := cctx.String("trace-source-auth") + ctx, _ := tag.New(context.Background(), tag.Insert(metrics.Version, build.BuildVersion), tag.Insert(metrics.Commit, build.CurrentCommit), @@ -319,6 +343,10 @@ var DaemonCmd = &cli.Command{ node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), + node.Override(new(dtypes.JsonTracerFile), traceToJsonFile), + node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch), + node.Override(new(dtypes.ElasticSearchTracer), elasticsearchIndex), + node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth), genesis, liteModeDeps, diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index fd4ec0366..f22156170 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -537,6 +537,18 @@ Type: Array of multiaddress peerinfo strings, must include peerid (/p2p/12D3K... Name: "ElasticSearchTracer", Type: "string", + Comment: ``, + }, + { + Name: "ElasticSearchIndex", + Type: "string", + + Comment: ``, + }, + { + Name: "TracerSourceAuth", + Type: "string", + Comment: ``, }, }, diff --git a/node/config/types.go b/node/config/types.go index aa061baef..630ae61fd 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -311,6 +311,7 @@ type Pubsub struct { RemoteTracer string JsonTracerFile string ElasticSearchTracer string + ElasticSearchIndex string TracerSourceAuth string } diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go new file mode 100644 index 000000000..9317924f7 --- /dev/null +++ b/node/modules/dtypes/tracer.go @@ -0,0 +1,6 @@ +package dtypes + +type JsonTracerFile string +type ElasticSearchTracer string +type ElasticSearchIndex string +type TracerSourceAuth string From 94a1601ab268314734370d13412f8ee5ebf972ad Mon Sep 17 00:00:00 2001 From: Mak Muftic Date: Fri, 24 Sep 2021 12:40:33 +0200 Subject: [PATCH 2/4] Remove hardcoded url schema --- node/modules/tracer/elasticsearch_transport.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go index 8823b8199..dbacf5823 100644 --- a/node/modules/tracer/elasticsearch_transport.go +++ b/node/modules/tracer/elasticsearch_transport.go @@ -18,11 +18,15 @@ const ( func NewElasticSearchTransport(connectionString string) (TracerTransport, error) { conUrl, err := url.Parse(connectionString) + if err != nil { + return nil, err + } + username := conUrl.User.Username() password, _ := conUrl.User.Password() cfg := elasticsearch.Config{ Addresses: []string{ - "https://" + conUrl.Host, + conUrl.Scheme + conUrl.Host, }, Username: username, Password: password, From 22bbb113e7afc20f73d91583b60915fcf8d0546b Mon Sep 17 00:00:00 2001 From: Mak Muftic Date: Fri, 24 Sep 2021 12:58:32 +0200 Subject: [PATCH 3/4] Fix schema creation --- node/modules/tracer/elasticsearch_transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go index dbacf5823..621b1d437 100644 --- a/node/modules/tracer/elasticsearch_transport.go +++ b/node/modules/tracer/elasticsearch_transport.go @@ -26,7 +26,7 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error) password, _ := conUrl.User.Password() cfg := elasticsearch.Config{ Addresses: []string{ - conUrl.Scheme + conUrl.Host, + conUrl.Scheme + "://" + conUrl.Host, }, Username: username, Password: password, From 4438c4bd87b637a9231caadca0785b6c102b4ae7 Mon Sep 17 00:00:00 2001 From: Mak Muftic Date: Fri, 24 Sep 2021 13:43:25 +0200 Subject: [PATCH 4/4] Move index name to config only and add default value --- cmd/lotus/daemon.go | 7 ------- node/modules/dtypes/tracer.go | 1 - node/modules/lp2p/pubsub.go | 1 + node/modules/tracer/elasticsearch_transport.go | 17 +++++++++++++---- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 89901a6f5..9579d9534 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -161,10 +161,6 @@ var DaemonCmd = &cli.Command{ Name: "trace-to-elasticsearch", Usage: "starts tracer and outputs to elasticsearch, flag must contain connection string for elasticsearch", }, - &cli.StringFlag{ - Name: "elasticsearch-index", - Usage: "configure elasticearch index name if elasticsearch tracer is configured", - }, &cli.StringFlag{ Name: "trace-source-auth", Usage: "auth token for trusted source of traces", @@ -213,8 +209,6 @@ var DaemonCmd = &cli.Command{ traceToElasticsearch := cctx.String("trace-to-elasticsearch") - elasticsearchIndex := cctx.String("elasticsearch-index") - traceSourceAuth := cctx.String("trace-source-auth") ctx, _ := tag.New(context.Background(), @@ -345,7 +339,6 @@ var DaemonCmd = &cli.Command{ node.Override(new(dtypes.ShutdownChan), shutdownChan), node.Override(new(dtypes.JsonTracerFile), traceToJsonFile), node.Override(new(dtypes.ElasticSearchTracer), traceToElasticsearch), - node.Override(new(dtypes.ElasticSearchTracer), elasticsearchIndex), node.Override(new(dtypes.TracerSourceAuth), traceSourceAuth), genesis, diff --git a/node/modules/dtypes/tracer.go b/node/modules/dtypes/tracer.go index 9317924f7..622f4baf4 100644 --- a/node/modules/dtypes/tracer.go +++ b/node/modules/dtypes/tracer.go @@ -2,5 +2,4 @@ package dtypes type JsonTracerFile string type ElasticSearchTracer string -type ElasticSearchIndex string type TracerSourceAuth string diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index a8ba126e0..818832562 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -377,6 +377,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { if in.Cfg.ElasticSearchTracer != "" { elasticSearchTransport, err := tracer.NewElasticSearchTransport( in.Cfg.ElasticSearchTracer, + in.Cfg.ElasticSearchIndex, ) if err != nil { return nil, err diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go index 621b1d437..d0d7b84a1 100644 --- a/node/modules/tracer/elasticsearch_transport.go +++ b/node/modules/tracer/elasticsearch_transport.go @@ -12,10 +12,10 @@ import ( ) const ( - ElasticSearch_INDEX = "lotus-pubsub" + ElasticSearch_INDEX_DEFAULT = "lotus-pubsub" ) -func NewElasticSearchTransport(connectionString string) (TracerTransport, error) { +func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) { conUrl, err := url.Parse(connectionString) if err != nil { @@ -38,13 +38,22 @@ func NewElasticSearchTransport(connectionString string) (TracerTransport, error) return nil, err } + var esIndex string + if elasticsearchIndex != "" { + esIndex = elasticsearchIndex + } else { + esIndex = ElasticSearch_INDEX_DEFAULT + } + return &elasticSearchTransport{ - cl: es, + cl: es, + esIndex: esIndex, }, nil } type elasticSearchTransport struct { - cl *elasticsearch.Client + cl *elasticsearch.Client + esIndex string } func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {