fix(server): telemetry setup (#1497)

* fix(server): telemetry setup

* more fixes

* fix

* changelog

* update standalone process

CHANGELOG.md

@@ -80,6 +80,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### Bug Fixes
 
+* (server) [#1497](https://github.com/evmos/ethermint/pull/1497) Fix telemetry server setup for observability
 * (rpc) [#1442](https://github.com/evmos/ethermint/pull/1442) Fix decoding of `finalized` block number.
 * (rpc) [#1179](https://github.com/evmos/ethermint/pull/1179) Fix gas used in traceTransaction response.
 * (rpc) [#1284](https://github.com/evmos/ethermint/pull/1284) Fix internal trace response upon incomplete `eth_sendTransaction` call.

server/start.go

@@ -9,11 +9,11 @@ import (
 	"os"
 	"path/filepath"
 	"runtime/pprof"
-	"strings"
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
+	"github.com/cosmos/cosmos-sdk/telemetry"
 
 	"github.com/spf13/cobra"
 
@@ -44,6 +44,7 @@ import (
 	serverconfig "github.com/cosmos/cosmos-sdk/server/config"
 	servergrpc "github.com/cosmos/cosmos-sdk/server/grpc"
 	"github.com/cosmos/cosmos-sdk/server/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
 
 	"github.com/evmos/ethermint/indexer"
 	ethdebug "github.com/evmos/ethermint/rpc/namespaces/ethereum/debug"
@@ -199,9 +200,10 @@ func startStandAlone(ctx *server.Context, appCreator types.AppCreator) error {
 	if err != nil {
 		return err
 	}
+
 	defer func() {
 		if err := db.Close(); err != nil {
-			ctx.Logger.With("error", err).Error("error closing db")
+			ctx.Logger.Error("error closing db", "error", err.Error())
 		}
 	}()
 
@@ -213,6 +215,22 @@ func startStandAlone(ctx *server.Context, appCreator types.AppCreator) error {
 
 	app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper)
 
+	config, err := config.GetConfig(ctx.Viper)
+	if err != nil {
+		ctx.Logger.Error("failed to get server config", "error", err.Error())
+		return err
+	}
+
+	if err := config.ValidateBasic(); err != nil {
+		ctx.Logger.Error("invalid server config", "error", err.Error())
+		return err
+	}
+
+	_, err = startTelemetry(config)
+	if err != nil {
+		return err
+	}
+
 	svr, err := abciserver.NewServer(addr, transport, app)
 	if err != nil {
 		return fmt.Errorf("error creating listener: %v", err)
@@ -240,12 +258,14 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 	cfg := ctx.Config
 	home := cfg.RootDir
 	logger := ctx.Logger
+
 	if cpuProfile := ctx.Viper.GetString(srvflags.CPUProfile); cpuProfile != "" {
 		fp, err := ethdebug.ExpandHome(cpuProfile)
 		if err != nil {
 			ctx.Logger.Debug("failed to get filepath for the CPU profile file", "error", err.Error())
 			return err
 		}
+
 		f, err := os.Create(fp)
 		if err != nil {
 			return err
@@ -265,18 +285,19 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 		}()
 	}
 
-	traceWriterFile := ctx.Viper.GetString(srvflags.TraceStore)
 	db, err := openDB(home, server.GetAppDBBackend(ctx.Viper))
 	if err != nil {
 		logger.Error("failed to open DB", "error", err.Error())
 		return err
 	}
+
 	defer func() {
 		if err := db.Close(); err != nil {
 			ctx.Logger.With("error", err).Error("error closing db")
 		}
 	}()
 
+	traceWriterFile := ctx.Viper.GetString(srvflags.TraceStore)
 	traceWriter, err := openTraceWriter(traceWriterFile)
 	if err != nil {
 		logger.Error("failed to open trace writer", "error", err.Error())
@@ -290,15 +311,8 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 	}
 
 	if err := config.ValidateBasic(); err != nil {
-		if strings.Contains(err.Error(), "set min gas price in app.toml or flag or env variable") {
+		logger.Error("invalid server config", "error", err.Error())
-			ctx.Logger.Error(
+		return err
-				"WARNING: The minimum-gas-prices config in app.toml is set to the empty string. " +
-					"This defaults to 0 in the current version, but will error in the next version " +
-					"(SDK v0.44). Please explicitly put the desired minimum-gas-prices in your app.toml.",
-			)
-		} else {
-			return err
-		}
 	}
 
 	app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper)
@@ -310,15 +324,19 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 	}
 
 	genDocProvider := node.DefaultGenesisDocProviderFunc(cfg)
+
 	var (
 		tmNode   *node.Node
 		gRPCOnly = ctx.Viper.GetBool(srvflags.GRPCOnly)
 	)
+
 	if gRPCOnly {
-		ctx.Logger.Info("starting node in query only mode; Tendermint is disabled")
+		logger.Info("starting node in query only mode; Tendermint is disabled")
 		config.GRPC.Enable = true
 		config.JSONRPC.EnableIndexer = false
 	} else {
+		logger.Info("starting node with ABCI Tendermint in-process")
+
 		tmNode, err = node.NewNode(
 			cfg,
 			pvm.LoadOrGenFilePV(cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()),
@@ -343,7 +361,6 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 			if tmNode.IsRunning() {
 				_ = tmNode.Stop()
 			}
-			logger.Info("Bye!")
 		}()
 	}
 
@@ -355,6 +372,15 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 
 		app.RegisterTxService(clientCtx)
 		app.RegisterTendermintService(clientCtx)
+
+		if a, ok := app.(types.ApplicationQueryService); ok {
+			a.RegisterNodeService(clientCtx)
+		}
+	}
+
+	metrics, err := startTelemetry(config)
+	if err != nil {
+		return err
 	}
 
 	// Enable metrics if JSONRPC is enabled and --metrics is passed
@@ -370,7 +396,8 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 			logger.Error("failed to open evm indexer DB", "error", err.Error())
 			return err
 		}
-		idxLogger := ctx.Logger.With("module", "evmindex")
+
+		idxLogger := ctx.Logger.With("indexer", "evm")
 		idxer = indexer.NewKVIndexer(idxDB, idxLogger, clientCtx)
 		indexerService := NewEVMIndexerService(idxer, clientCtx.Client)
 		indexerService.SetLogger(idxLogger)
@@ -442,6 +469,11 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 	if config.API.Enable {
 		apiSrv = api.New(clientCtx, ctx.Logger.With("server", "api"))
 		app.RegisterAPIRoutes(apiSrv, config.API)
+
+		if config.Telemetry.Enabled {
+			apiSrv.SetTelemetry(metrics)
+		}
+
 		errCh := make(chan error)
 		go func() {
 			if err := apiSrv.Start(config.Config); err != nil {
@@ -454,6 +486,7 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 			return err
 		case <-time.After(types.ServerStartTime): // assume server started successfully
 		}
+
 		defer apiSrv.Close()
 	}
 
@@ -461,6 +494,7 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 		grpcSrv    *grpc.Server
 		grpcWebSrv *http.Server
 	)
+
 	if config.GRPC.Enable {
 		grpcSrv, err = servergrpc.StartGRPCServer(clientCtx, app, config.GRPC)
 		if err != nil {
@@ -470,12 +504,13 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 		if config.GRPCWeb.Enable {
 			grpcWebSrv, err = servergrpc.StartGRPCWeb(grpcSrv, config.Config)
 			if err != nil {
-				ctx.Logger.Error("failed to start grpc-web http server", "error", err)
+				ctx.Logger.Error("failed to start grpc-web http server", "error", err.Error())
 				return err
 			}
+
 			defer func() {
 				if err := grpcWebSrv.Close(); err != nil {
-					logger.Error("failed to close the grpcWebSrc", "error", err.Error())
+					logger.Error("failed to close the grpc-web http server", "error", err.Error())
 				}
 			}()
 		}
@@ -525,25 +560,39 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 	var rosettaSrv crgserver.Server
 	if config.Rosetta.Enable {
 		offlineMode := config.Rosetta.Offline
-		if !config.GRPC.Enable { // If GRPC is not enabled rosetta cannot work in online mode, so it works in offline mode.
+
+		// If GRPC is not enabled rosetta cannot work in online mode, so it works in
+		// offline mode.
+		if !config.GRPC.Enable {
 			offlineMode = true
 		}
 
-		conf := &rosetta.Config{
+		minGasPrices, err := sdk.ParseDecCoins(config.MinGasPrices)
-			Blockchain:    config.Rosetta.Blockchain,
+		if err != nil {
-			Network:       config.Rosetta.Network,
+			ctx.Logger.Error("failed to parse minimum-gas-prices", "error", err.Error())
-			TendermintRPC: ctx.Config.RPC.ListenAddress,
+			return err
-			GRPCEndpoint:  config.GRPC.Address,
+		}
-			Addr:          config.Rosetta.Address,
+
-			Retries:       config.Rosetta.Retries,
+		conf := &rosetta.Config{
-			Offline:       offlineMode,
+			Blockchain:          config.Rosetta.Blockchain,
+			Network:             config.Rosetta.Network,
+			TendermintRPC:       ctx.Config.RPC.ListenAddress,
+			GRPCEndpoint:        config.GRPC.Address,
+			Addr:                config.Rosetta.Address,
+			Retries:             config.Rosetta.Retries,
+			Offline:             offlineMode,
+			GasToSuggest:        config.Rosetta.GasToSuggest,
+			EnableFeeSuggestion: config.Rosetta.EnableFeeSuggestion,
+			GasPrices:           minGasPrices.Sort(),
+			Codec:               clientCtx.Codec.(*codec.ProtoCodec),
+			InterfaceRegistry:   clientCtx.InterfaceRegistry,
 		}
-		conf.WithCodec(clientCtx.InterfaceRegistry, clientCtx.Codec.(*codec.ProtoCodec))
 
 		rosettaSrv, err = rosetta.ServerFromConfig(conf)
 		if err != nil {
 			return err
 		}
+
 		errCh := make(chan error)
 		go func() {
 			if err := rosettaSrv.Start(); err != nil {
@@ -584,3 +633,10 @@ func openTraceWriter(traceWriterFile string) (w io.Writer, err error) {
 		0o600,
 	)
 }
+
+func startTelemetry(cfg config.Config) (*telemetry.Metrics, error) {
+	if !cfg.Telemetry.Enabled {
+		return nil, nil
+	}
+	return telemetry.New(cfg.Telemetry)
+}