From 5040ff87c4bfa5cc9600194cfd9c9189a63da11b Mon Sep 17 00:00:00 2001 From: Alexander Bezobchuk Date: Thu, 18 Jun 2020 14:12:44 -0400 Subject: [PATCH] Merge PR #6449: Initial Metrics --- baseapp/abci.go | 24 ++++++++ docs/core/README.md | 7 ++- docs/core/ocap.md | 4 +- docs/core/runtx_panics.md | 2 +- docs/core/telemetry.md | 118 +++++++++++++++++++++++++++++++++++++ server/config/config.go | 15 ++++- server/config/toml.go | 9 +++ simapp/cmd/simd/testnet.go | 1 + store/cachekv/store.go | 6 +- store/gaskv/store.go | 7 +++ store/iavl/store.go | 9 +++ telemetry/metrics.go | 20 +++++++ telemetry/wrapper.go | 70 ++++++++++++++++++++++ x/auth/ante/fee.go | 1 + x/bank/handler.go | 13 ++++ x/bank/keeper/send.go | 3 + x/crisis/abci.go | 4 ++ x/distribution/abci.go | 4 ++ x/distribution/handler.go | 27 ++++++++- x/evidence/abci.go | 3 + x/gov/abci.go | 3 + x/gov/handler.go | 22 +++++++ x/mint/abci.go | 5 ++ x/slashing/abci.go | 4 ++ x/staking/abci.go | 6 ++ x/staking/handler.go | 29 +++++++++ x/upgrade/abci.go | 4 ++ 27 files changed, 410 insertions(+), 10 deletions(-) create mode 100644 docs/core/telemetry.md create mode 100644 telemetry/wrapper.go diff --git a/baseapp/abci.go b/baseapp/abci.go index f2ae94f8f9..a357fba055 100644 --- a/baseapp/abci.go +++ b/baseapp/abci.go @@ -10,6 +10,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) @@ -102,6 +103,8 @@ func (app *BaseApp) FilterPeerByID(info string) abci.ResponseQuery { // BeginBlock implements the ABCI application interface. func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeginBlock) { + defer telemetry.MeasureSince("abci", "begin_block") + if app.cms.TracingEnabled() { app.cms.SetTracingContext(sdk.TraceContext( map[string]interface{}{"blockHeight": req.Header.Height}, @@ -145,6 +148,8 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg // EndBlock implements the ABCI interface. func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBlock) { + defer telemetry.MeasureSince("abci", "end_block") + if app.deliverState.ms.TracingEnabled() { app.deliverState.ms = app.deliverState.ms.SetTracingContext(nil).(sdk.CacheMultiStore) } @@ -163,6 +168,8 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc // will contain releveant error information. Regardless of tx execution outcome, // the ResponseCheckTx will contain relevant gas execution context. func (app *BaseApp) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { + defer telemetry.MeasureSince("abci", "check_tx") + tx, err := app.txDecoder(req.Tx) if err != nil { return sdkerrors.ResponseCheckTx(err, 0, 0) @@ -201,13 +208,26 @@ func (app *BaseApp) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { // Regardless of tx execution outcome, the ResponseDeliverTx will contain relevant // gas execution context. func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx { + defer telemetry.MeasureSince("abci", "deliver_tx") + tx, err := app.txDecoder(req.Tx) if err != nil { return sdkerrors.ResponseDeliverTx(err, 0, 0) } + gInfo := sdk.GasInfo{} + resultStr := "successful" + + defer func() { + telemetry.IncrCounter(1, "tx", "count") + telemetry.IncrCounter(1, "tx", resultStr) + telemetry.SetGauge(float32(gInfo.GasUsed), "tx", "gas", "used") + telemetry.SetGauge(float32(gInfo.GasWanted), "tx", "gas", "wanted") + }() + gInfo, result, err := app.runTx(runTxModeDeliver, req.Tx, tx) if err != nil { + resultStr = "failed" return sdkerrors.ResponseDeliverTx(err, gInfo.GasWanted, gInfo.GasUsed) } @@ -228,6 +248,8 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx // against that height and gracefully halt if it matches the latest committed // height. func (app *BaseApp) Commit() (res abci.ResponseCommit) { + defer telemetry.MeasureSince("abci", "commit") + header := app.deliverState.ctx.BlockHeader() // Write the DeliverTx state which is cache-wrapped and commit the MultiStore. @@ -294,6 +316,8 @@ func (app *BaseApp) halt() { // Query implements the ABCI interface. It delegates to CommitMultiStore if it // implements Queryable. func (app *BaseApp) Query(req abci.RequestQuery) abci.ResponseQuery { + defer telemetry.MeasureSince("abci", "query") + // handle gRPC routes first rather than calling splitPath because '/' characters // are used as part of gRPC paths if grpcHandler := app.grpcQueryRouter.Route(req.Path); grpcHandler != nil { diff --git a/docs/core/README.md b/docs/core/README.md index f8a024d59e..3ca4023bf9 100644 --- a/docs/core/README.md +++ b/docs/core/README.md @@ -15,7 +15,8 @@ This repository contains reference documentation on the core concepts of the Cos 5. [Store](./store.md) 6. [Encoding](./encoding.md) 7. [Events](./events.md) -8. [Object-Capabilities](./ocap.md) -9. [RunTx recovery middleware](./runtx_panics.md) +8. [Telemetry](./telemetry.md) +9. [Object-Capabilities](./ocap.md) +10. [RunTx recovery middleware](./runtx_panics.md) -After reading about the core concepts, head on to the [Building Modules documentation](../building-modules/README.md) to learn more about the process of building modules. \ No newline at end of file +After reading about the core concepts, head on to the [Building Modules documentation](../building-modules/README.md) to learn more about the process of building modules. diff --git a/docs/core/ocap.md b/docs/core/ocap.md index c3a6cf08ee..adf77d0b02 100644 --- a/docs/core/ocap.md +++ b/docs/core/ocap.md @@ -1,5 +1,5 @@ # Object-Capability Model @@ -72,4 +72,4 @@ gaia app. ## Next -Learn about [building modules](../building-modules/intro.md) {hide} \ No newline at end of file +Learn about [building modules](../building-modules/intro.md) {hide} diff --git a/docs/core/runtx_panics.md b/docs/core/runtx_panics.md index 66a7576ca6..678125206e 100644 --- a/docs/core/runtx_panics.md +++ b/docs/core/runtx_panics.md @@ -1,5 +1,5 @@ # RunTx recovery middleware diff --git a/docs/core/telemetry.md b/docs/core/telemetry.md new file mode 100644 index 0000000000..f2986e30f8 --- /dev/null +++ b/docs/core/telemetry.md @@ -0,0 +1,118 @@ + + +# Telemetry + +The Cosmos SDK enables operators and developers to gain insight into the performance and behavior of +their application through the use of the `telemetry` package. The Cosmos SDK currently supports +enabling in-memory and prometheus as telemetry sinks. This allows the ability to query for and scrape +metrics from a single exposed API endpoint -- `/metrics?format={text|prometheus}`, the default being +`text`. + +If telemetry is enabled via configuration, a single global metrics collector is registered via the +[go-metrics](https://github.com/armon/go-metrics) library. This allows emitting and collecting +metrics through simple API calls. + +Example: + +```go +func EndBlocker(ctx sdk.Context, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyEndBlocker) + + // ... +} +``` + +Developers may use the `telemetry` package directly, which provides wrappers around metric APIs +that include adding useful labels, or they must use the `go-metrics` library directly. It is preferable +to add as much context and adequate dimensionality to metrics as possible, so the `telemetry` package +is advised. Regardless of the package or method used, the Cosmos SDK supports the following metrics +types: + +* gauges +* summaries +* counters + +## Labels + +Certain components of modules will have their name automatically added as a label (e.g. `BeginBlock`). +Operators may also supply the application with a global set of labels that will be applied to all +metrics emitted using the `telemetry` package (e.g. chain-id). Global labels are supplied as a list +of [name, value] tuples. + +Example: + +```toml +global-labels = [ + ["chain_id", "chain-OfXo4V"], +] +``` + +## Cardinality + +Cardinality is key, specifically label and key cardinality. Cardinality is how many unique values of +something there are. So there is naturally a tradeoff between granularity and how much stress is put +on the telemetry sink in terms of indexing, scrape, and query performance. + +Developers should take care to support metrics with enough dimensionality and granularity to be +useful, but not increase the cardinality beyond the sink's limits. A general rule of thumb is to not +exceed a cardinality of 10. + +Consider the following examples with enough granularity and adequate cardinality: + +* begin/end blocker time +* tx gas used +* block gas used +* amount of tokens minted +* amount of accounts created + +The following examples expose too much cardinality and may not even prove to be useful: + +* transfers between accounts with amount +* voting/deposit amount from unique addresses + +## Supported Metrics + +| Metric | Description | Unit | Type | +| :------------------------------ | :------------------------------------------------------------------------------------- | :----------- | :------ | +| `tx_count` | Total number of txs processed via `DeliverTx` | tx | counter | +| `tx_successful` | Total number of successful txs processed via `DeliverTx` | tx | counter | +| `tx_failed` | Total number of failed txs processed via `DeliverTx` | tx | counter | +| `tx_gas_used` | The total amount of gas used by a tx | gas | gauge | +| `tx_gas_wanted` | The total amount of gas requested by a tx | gas | gauge | +| `tx_msg_send` | The total amount of tokens sent in a `MsgSend` (per denom) | token | gauge | +| `tx_msg_withdraw_reward` | The total amount of tokens withdrawn in a `MsgWithdrawDelegatorReward` (per denom) | token | gauge | +| `tx_msg_withdraw_commission` | The total amount of tokens withdrawn in a `MsgWithdrawValidatorCommission` (per denom) | token | gauge | +| `tx_msg_delegate` | The total amount of tokens delegated in a `MsgDelegate` | token | gauge | +| `tx_msg_begin_unbonding` | The total amount of tokens undelegated in a `MsgUndelegate` | token | gauge | +| `tx_msg_begin_begin_redelegate` | The total amount of tokens redelegated in a `MsgBeginRedelegate` | token | gauge | +| `new_account` | Total number of new accounts created | account | counter | +| `gov_proposal` | Total number of governance proposals | proposal | counter | +| `gov_vote` | Total number of governance votes for a proposal | vote | counter | +| `gov_deposit` | Total number of governance deposits for a proposal | deposit | counter | +| `staking_delegate` | Total number of delegations | delegation | counter | +| `staking_undelegate` | Total number of undelegations | undelegation | counter | +| `staking_redelegate` | Total number of redelegations | redelegation | counter | +| `abci_check_tx` | Duration of ABCI `CheckTx` | ms | summary | +| `abci_deliver_tx` | Duration of ABCI `DeliverTx` | ms | summary | +| `abci_commit` | Duration of ABCI `Commit` | ms | summary | +| `abci_query` | Duration of ABCI `Query` | ms | summary | +| `abci_begin_block` | Duration of ABCI `BeginBlock` | ms | summary | +| `abci_end_block` | Duration of ABCI `EndBlock` | ms | summary | +| `begin_blocker` | Duration of `BeginBlock` for a given module | ms | summary | +| `end_blocker` | Duration of `EndBlock` for a given module | ms | summary | +| `store_iavl_get` | Duration of an IAVL `Store#Get` call | ms | summary | +| `store_iavl_set` | Duration of an IAVL `Store#Set` call | ms | summary | +| `store_iavl_has` | Duration of an IAVL `Store#Has` call | ms | summary | +| `store_iavl_delete` | Duration of an IAVL `Store#Delete` call | ms | summary | +| `store_iavl_commit` | Duration of an IAVL `Store#Commit` call | ms | summary | +| `store_iavl_query` | Duration of an IAVL `Store#Query` call | ms | summary | +| `store_gaskv_get` | Duration of a GasKV `Store#Get` call | ms | summary | +| `store_gaskv_set` | Duration of a GasKV `Store#Set` call | ms | summary | +| `store_gaskv_has` | Duration of a GasKV `Store#Has` call | ms | summary | +| `store_gaskv_delete` | Duration of a GasKV `Store#Delete` call | ms | summary | +| `store_cachekv_get` | Duration of a CacheKV `Store#Get` call | ms | summary | +| `store_cachekv_set` | Duration of a CacheKV `Store#Set` call | ms | summary | +| `store_cachekv_write` | Duration of a CacheKV `Store#Write` call | ms | summary | +| `store_cachekv_delete` | Duration of a CacheKV `Store#Delete` call | ms | summary | diff --git a/server/config/config.go b/server/config/config.go index 6ae3737dee..edc621dbd0 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -120,7 +120,10 @@ func DefaultConfig() *Config { PruningKeepEvery: "0", PruningSnapshotEvery: "0", }, - Telemetry: telemetry.Config{}, + Telemetry: telemetry.Config{ + Enabled: false, + GlobalLabels: [][]string{}, + }, API: APIConfig{ Enable: false, Swagger: false, @@ -134,6 +137,15 @@ func DefaultConfig() *Config { // GetConfig returns a fully parsed Config object. func GetConfig() Config { + globalLabelsRaw := viper.Get("telemetry.global-labels").([]interface{}) + globalLabels := make([][]string, 0, len(globalLabelsRaw)) + for _, glr := range globalLabelsRaw { + labelsRaw := glr.([]interface{}) + if len(labelsRaw) == 2 { + globalLabels = append(globalLabels, []string{labelsRaw[0].(string), labelsRaw[1].(string)}) + } + } + return Config{ BaseConfig: BaseConfig{ MinGasPrices: viper.GetString("minimum-gas-prices"), @@ -151,6 +163,7 @@ func GetConfig() Config { EnableHostnameLabel: viper.GetBool("telemetry.enable-hostname-label"), EnableServiceLabel: viper.GetBool("telemetry.enable-service-label"), PrometheusRetentionTime: viper.GetInt64("telemetry.prometheus-retention-time"), + GlobalLabels: globalLabels, }, API: APIConfig{ Enable: viper.GetBool("api.enable"), diff --git a/server/config/toml.go b/server/config/toml.go index af958f44cf..61ad1bb9df 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -73,6 +73,15 @@ enable-service-label = {{ .Telemetry.EnableServiceLabel }} # PrometheusRetentionTime, when positive, enables a Prometheus metrics sink. prometheus-retention-time = {{ .Telemetry.PrometheusRetentionTime }} +# GlobalLabels defines a global set of name/value label tuples applied to all +# metrics emitted using the wrapper functions defined in telemetry package. +# +# Example: +# [["chain_id", "cosmoshub-1"]] +global-labels = [{{ range $k, $v := .Telemetry.GlobalLabels }} + ["{{index $v 0 }}", "{{ index $v 1}}"],{{ end }} +] + ############################################################################### ### API Configuration ### ############################################################################### diff --git a/simapp/cmd/simd/testnet.go b/simapp/cmd/simd/testnet.go index 6ad3c1c4fe..09b1feca29 100644 --- a/simapp/cmd/simd/testnet.go +++ b/simapp/cmd/simd/testnet.go @@ -124,6 +124,7 @@ func InitTestnet( simappConfig.Telemetry.Enabled = true simappConfig.Telemetry.PrometheusRetentionTime = 60 simappConfig.Telemetry.EnableHostnameLabel = false + simappConfig.Telemetry.GlobalLabels = [][]string{{"chain_id", chainID}} var ( genAccounts []authtypes.GenesisAccount diff --git a/store/cachekv/store.go b/store/cachekv/store.go index f39d1af8c6..3c5ec57723 100644 --- a/store/cachekv/store.go +++ b/store/cachekv/store.go @@ -12,6 +12,7 @@ import ( "github.com/cosmos/cosmos-sdk/store/tracekv" "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" ) // If value is nil but deleted is false, it means the parent doesn't have the @@ -51,6 +52,7 @@ func (store *Store) GetStoreType() types.StoreType { func (store *Store) Get(key []byte) (value []byte) { store.mtx.Lock() defer store.mtx.Unlock() + defer telemetry.MeasureSince("store", "cachekv", "get") types.AssertValidKey(key) @@ -69,6 +71,7 @@ func (store *Store) Get(key []byte) (value []byte) { func (store *Store) Set(key []byte, value []byte) { store.mtx.Lock() defer store.mtx.Unlock() + defer telemetry.MeasureSince("store", "cachekv", "set") types.AssertValidKey(key) types.AssertValidValue(value) @@ -86,9 +89,9 @@ func (store *Store) Has(key []byte) bool { func (store *Store) Delete(key []byte) { store.mtx.Lock() defer store.mtx.Unlock() + defer telemetry.MeasureSince("store", "cachekv", "delete") types.AssertValidKey(key) - store.setCacheValue(key, nil, true, true) } @@ -96,6 +99,7 @@ func (store *Store) Delete(key []byte) { func (store *Store) Write() { store.mtx.Lock() defer store.mtx.Unlock() + defer telemetry.MeasureSince("store", "cachekv", "write") // We need a copy of all of the keys. // Not the best, but probably not a bottleneck depending. diff --git a/store/gaskv/store.go b/store/gaskv/store.go index 13f7602a26..22bd182f55 100644 --- a/store/gaskv/store.go +++ b/store/gaskv/store.go @@ -4,6 +4,7 @@ import ( "io" "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" ) var _ types.KVStore = &Store{} @@ -34,6 +35,8 @@ func (gs *Store) GetStoreType() types.StoreType { // Implements KVStore. func (gs *Store) Get(key []byte) (value []byte) { + defer telemetry.MeasureSince("store", "gaskv", "get") + gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostFlat, types.GasReadCostFlatDesc) value = gs.parent.Get(key) @@ -45,6 +48,8 @@ func (gs *Store) Get(key []byte) (value []byte) { // Implements KVStore. func (gs *Store) Set(key []byte, value []byte) { + defer telemetry.MeasureSince("store", "gaskv", "set") + types.AssertValidValue(value) gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostFlat, types.GasWriteCostFlatDesc) // TODO overflow-safe math? @@ -54,12 +59,14 @@ func (gs *Store) Set(key []byte, value []byte) { // Implements KVStore. func (gs *Store) Has(key []byte) bool { + defer telemetry.MeasureSince("store", "gaskv", "has") gs.gasMeter.ConsumeGas(gs.gasConfig.HasCost, types.GasHasDesc) return gs.parent.Has(key) } // Implements KVStore. func (gs *Store) Delete(key []byte) { + defer telemetry.MeasureSince("store", "gaskv", "delete") // charge gas to prevent certain attack vectors even though space is being freed gs.gasMeter.ConsumeGas(gs.gasConfig.DeleteCost, types.GasDeleteDesc) gs.parent.Delete(key) diff --git a/store/iavl/store.go b/store/iavl/store.go index 0548b3d016..38d974e10e 100644 --- a/store/iavl/store.go +++ b/store/iavl/store.go @@ -17,6 +17,7 @@ import ( "github.com/cosmos/cosmos-sdk/store/cachekv" "github.com/cosmos/cosmos-sdk/store/tracekv" "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) @@ -124,6 +125,8 @@ func (st *Store) GetImmutable(version int64) (*Store, error) { // Commit commits the current store state and returns a CommitID with the new // version and hash. func (st *Store) Commit() types.CommitID { + defer telemetry.MeasureSince("store", "iavl", "commit") + hash, version, err := st.tree.SaveVersion() if err != nil { // TODO: Do we want to extend Commit to allow returning errors? @@ -187,23 +190,27 @@ func (st *Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.Ca // Implements types.KVStore. func (st *Store) Set(key, value []byte) { + defer telemetry.MeasureSince("store", "iavl", "set") types.AssertValidValue(value) st.tree.Set(key, value) } // Implements types.KVStore. func (st *Store) Get(key []byte) []byte { + defer telemetry.MeasureSince("store", "iavl", "get") _, value := st.tree.Get(key) return value } // Implements types.KVStore. func (st *Store) Has(key []byte) (exists bool) { + defer telemetry.MeasureSince("store", "iavl", "has") return st.tree.Has(key) } // Implements types.KVStore. func (st *Store) Delete(key []byte) { + defer telemetry.MeasureSince("store", "iavl", "delete") st.tree.Remove(key) } @@ -257,6 +264,8 @@ func getHeight(tree Tree, req abci.RequestQuery) int64 { // if you care to have the latest data to see a tx results, you must // explicitly set the height you want to see func (st *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { + defer telemetry.MeasureSince("store", "iavl", "query") + if len(req.Data) == 0 { return sdkerrors.QueryResult(sdkerrors.Wrap(sdkerrors.ErrTxDecode, "query cannot be zero length")) } diff --git a/telemetry/metrics.go b/telemetry/metrics.go index deeceff5be..f402e92909 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -12,6 +12,10 @@ import ( "github.com/prometheus/common/expfmt" ) +// globalLabels defines the set of global labels that will be applied to all +// metrics emitted using the telemetry package function wrappers. +var globalLabels = []metrics.Label{} + // Metrics supported format types. const ( FormatDefault = "" @@ -41,6 +45,13 @@ type Config struct { // PrometheusRetentionTime, when positive, enables a Prometheus metrics sink. // It defines the retention duration in seconds. PrometheusRetentionTime int64 `mapstructure:"prometheus-retention-time"` + + // GlobalLabels defines a global set of name/value label tuples applied to all + // metrics emitted using the wrapper functions defined in telemetry package. + // + // Example: + // [["chain_id", "cosmoshub-1"]] + GlobalLabels [][]string `mapstructure:"global-labels"` } // Metrics defines a wrapper around application telemetry functionality. It allows @@ -63,6 +74,15 @@ func New(cfg Config) (*Metrics, error) { return nil, nil } + if numGlobalLables := len(cfg.GlobalLabels); numGlobalLables > 0 { + parsedGlobalLabels := make([]metrics.Label, numGlobalLables) + for i, gl := range cfg.GlobalLabels { + parsedGlobalLabels[i] = NewLabel(gl[0], gl[1]) + } + + globalLabels = parsedGlobalLabels + } + metricsConf := metrics.DefaultConfig(cfg.ServiceName) metricsConf.EnableHostname = cfg.EnableHostname metricsConf.EnableHostnameLabel = cfg.EnableHostnameLabel diff --git a/telemetry/wrapper.go b/telemetry/wrapper.go new file mode 100644 index 0000000000..db66cf5854 --- /dev/null +++ b/telemetry/wrapper.go @@ -0,0 +1,70 @@ +package telemetry + +import ( + "time" + + "github.com/armon/go-metrics" +) + +// Common metric key constants +const ( + MetricKeyBeginBlocker = "begin_blocker" + MetricKeyEndBlocker = "end_blocker" + MetricLabelNameModule = "module" +) + +func NewLabel(name, value string) metrics.Label { + return metrics.Label{Name: name, Value: value} +} + +// ModuleMeasureSince provides a short hand method for emitting a time measure +// metric for a module with a given set of keys. If any global labels are defined, +// they will be added to the module label. +func ModuleMeasureSince(module string, keys ...string) { + metrics.MeasureSinceWithLabels( + keys, + time.Now().UTC(), + append([]metrics.Label{NewLabel(MetricLabelNameModule, module)}, globalLabels...), + ) +} + +// ModuleSetGauge provides a short hand method for emitting a gauge metric for a +// module with a given set of keys. If any global labels are defined, they will +// be added to the module label. +func ModuleSetGauge(module string, val float32, keys ...string) { + metrics.SetGaugeWithLabels( + keys, + val, + append([]metrics.Label{NewLabel(MetricLabelNameModule, module)}, globalLabels...), + ) +} + +// IncrCounter provides a wrapper functionality for emitting a counter metric with +// global labels (if any). +func IncrCounter(val float32, keys ...string) { + metrics.IncrCounterWithLabels(keys, val, globalLabels) +} + +// IncrCounterWithLabels provides a wrapper functionality for emitting a counter +// metric with global labels (if any) along with the provided labels. +func IncrCounterWithLabels(keys []string, val float32, labels []metrics.Label) { + metrics.IncrCounterWithLabels(keys, val, append(labels, globalLabels...)) +} + +// SetGauge provides a wrapper functionality for emitting a gauge metric with +// global labels (if any). +func SetGauge(val float32, keys ...string) { + metrics.SetGaugeWithLabels(keys, val, globalLabels) +} + +// SetGaugeWithLabels provides a wrapper functionality for emitting a gauge +// metric with global labels (if any) along with the provided labels. +func SetGaugeWithLabels(keys []string, val float32, labels []metrics.Label) { + metrics.SetGaugeWithLabels(keys, val, append(labels, globalLabels...)) +} + +// MeasureSince provides a wrapper functionality for emitting a a time measure +// metric with global labels (if any). +func MeasureSince(keys ...string) { + metrics.MeasureSinceWithLabels(keys, time.Now().UTC(), globalLabels) +} diff --git a/x/auth/ante/fee.go b/x/auth/ante/fee.go index 5f323c5ca3..47ef35b736 100644 --- a/x/auth/ante/fee.go +++ b/x/auth/ante/fee.go @@ -29,6 +29,7 @@ func (mfd MempoolFeeDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate b if !ok { return ctx, sdkerrors.Wrap(sdkerrors.ErrTxDecode, "Tx must be a FeeTx") } + feeCoins := feeTx.GetFee() gas := feeTx.GetGas() diff --git a/x/bank/handler.go b/x/bank/handler.go index 643d118f61..a11ac6d4e9 100644 --- a/x/bank/handler.go +++ b/x/bank/handler.go @@ -1,6 +1,9 @@ package bank import ( + "github.com/armon/go-metrics" + + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/x/bank/keeper" @@ -40,6 +43,16 @@ func handleMsgSend(ctx sdk.Context, k keeper.Keeper, msg *types.MsgSend) (*sdk.R return nil, err } + defer func() { + for _, a := range msg.Amount { + telemetry.SetGaugeWithLabels( + []string{"tx", "msg", "send"}, + float32(a.Amount.Int64()), + []metrics.Label{telemetry.NewLabel("denom", a.Denom)}, + ) + } + }() + ctx.EventManager().EmitEvent( sdk.NewEvent( sdk.EventTypeMessage, diff --git a/x/bank/keeper/send.go b/x/bank/keeper/send.go index 293b524245..1ab637bc57 100644 --- a/x/bank/keeper/send.go +++ b/x/bank/keeper/send.go @@ -3,6 +3,7 @@ package keeper import ( "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/store/prefix" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/x/bank/types" @@ -103,6 +104,7 @@ func (k BaseSendKeeper) InputOutputCoins(ctx sdk.Context, inputs []types.Input, // such as delegated fee messages. acc := k.ak.GetAccount(ctx, out.Address) if acc == nil { + defer telemetry.IncrCounter(1, "new", "account") k.ak.SetAccount(ctx, k.ak.NewAccountWithAddress(ctx, out.Address)) } } @@ -141,6 +143,7 @@ func (k BaseSendKeeper) SendCoins(ctx sdk.Context, fromAddr sdk.AccAddress, toAd // such as delegated fee messages. acc := k.ak.GetAccount(ctx, toAddr) if acc == nil { + defer telemetry.IncrCounter(1, "new", "account") k.ak.SetAccount(ctx, k.ak.NewAccountWithAddress(ctx, toAddr)) } diff --git a/x/crisis/abci.go b/x/crisis/abci.go index 482a069b50..2844b3c1f5 100644 --- a/x/crisis/abci.go +++ b/x/crisis/abci.go @@ -1,12 +1,16 @@ package crisis import ( + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/crisis/keeper" + "github.com/cosmos/cosmos-sdk/x/crisis/types" ) // check all registered invariants func EndBlocker(ctx sdk.Context, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyEndBlocker) + if k.InvCheckPeriod() == 0 || ctx.BlockHeight()%int64(k.InvCheckPeriod()) != 0 { // skip running the invariant check return diff --git a/x/distribution/abci.go b/x/distribution/abci.go index 44a2286cad..c81dc6b303 100644 --- a/x/distribution/abci.go +++ b/x/distribution/abci.go @@ -3,13 +3,17 @@ package distribution import ( abci "github.com/tendermint/tendermint/abci/types" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/distribution/keeper" + "github.com/cosmos/cosmos-sdk/x/distribution/types" ) // BeginBlocker sets the proposer for determining distribution during endblock // and distribute rewards for the previous block func BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyBeginBlocker) + // determine the total power signing the block var previousTotalPower, sumPreviousPrecommitPower int64 for _, voteInfo := range req.LastCommitInfo.GetVotes() { diff --git a/x/distribution/handler.go b/x/distribution/handler.go index dd96a2aa21..121629b193 100644 --- a/x/distribution/handler.go +++ b/x/distribution/handler.go @@ -1,6 +1,9 @@ package distribution import ( + "github.com/armon/go-metrics" + + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/x/distribution/keeper" @@ -51,11 +54,21 @@ func handleMsgModifyWithdrawAddress(ctx sdk.Context, msg *types.MsgSetWithdrawAd } func handleMsgWithdrawDelegatorReward(ctx sdk.Context, msg *types.MsgWithdrawDelegatorReward, k keeper.Keeper) (*sdk.Result, error) { - _, err := k.WithdrawDelegationRewards(ctx, msg.DelegatorAddress, msg.ValidatorAddress) + amount, err := k.WithdrawDelegationRewards(ctx, msg.DelegatorAddress, msg.ValidatorAddress) if err != nil { return nil, err } + defer func() { + for _, a := range amount { + telemetry.SetGaugeWithLabels( + []string{"tx", "msg", "withdraw_reward"}, + float32(a.Amount.Int64()), + []metrics.Label{telemetry.NewLabel("denom", a.Denom)}, + ) + } + }() + ctx.EventManager().EmitEvent( sdk.NewEvent( sdk.EventTypeMessage, @@ -68,11 +81,21 @@ func handleMsgWithdrawDelegatorReward(ctx sdk.Context, msg *types.MsgWithdrawDel } func handleMsgWithdrawValidatorCommission(ctx sdk.Context, msg *types.MsgWithdrawValidatorCommission, k keeper.Keeper) (*sdk.Result, error) { - _, err := k.WithdrawValidatorCommission(ctx, msg.ValidatorAddress) + amount, err := k.WithdrawValidatorCommission(ctx, msg.ValidatorAddress) if err != nil { return nil, err } + defer func() { + for _, a := range amount { + telemetry.SetGaugeWithLabels( + []string{"tx", "msg", "withdraw_commission"}, + float32(a.Amount.Int64()), + []metrics.Label{telemetry.NewLabel("denom", a.Denom)}, + ) + } + }() + ctx.EventManager().EmitEvent( sdk.NewEvent( sdk.EventTypeMessage, diff --git a/x/evidence/abci.go b/x/evidence/abci.go index bc1422cef3..da5bd22e41 100644 --- a/x/evidence/abci.go +++ b/x/evidence/abci.go @@ -3,6 +3,7 @@ package evidence import ( "fmt" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" abci "github.com/tendermint/tendermint/abci/types" @@ -15,6 +16,8 @@ import ( // BeginBlocker iterates through and handles any newly discovered evidence of // misbehavior submitted by Tendermint. Currently, only equivocation is handled. func BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyBeginBlocker) + for _, tmEvidence := range req.ByzantineValidators { switch tmEvidence.Type { case tmtypes.ABCIEvidenceTypeDuplicateVote: diff --git a/x/gov/abci.go b/x/gov/abci.go index b89a661d65..77dfd663da 100644 --- a/x/gov/abci.go +++ b/x/gov/abci.go @@ -3,6 +3,7 @@ package gov import ( "fmt" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/gov/keeper" "github.com/cosmos/cosmos-sdk/x/gov/types" @@ -10,6 +11,8 @@ import ( // EndBlocker called every block, process inflation, update validator set. func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyEndBlocker) + logger := keeper.Logger(ctx) // delete inactive proposal from store and its deposits diff --git a/x/gov/handler.go b/x/gov/handler.go index 8a0e4a2795..d2b1b234a7 100644 --- a/x/gov/handler.go +++ b/x/gov/handler.go @@ -2,7 +2,11 @@ package gov import ( "fmt" + "strconv" + "github.com/armon/go-metrics" + + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/x/gov/keeper" @@ -36,6 +40,8 @@ func handleMsgSubmitProposal(ctx sdk.Context, keeper keeper.Keeper, msg types.Ms return nil, err } + defer telemetry.IncrCounter(1, types.ModuleName, "proposal") + votingStarted, err := keeper.AddDeposit(ctx, proposal.ProposalID, msg.GetProposer(), msg.GetInitialDeposit()) if err != nil { return nil, err @@ -70,6 +76,14 @@ func handleMsgDeposit(ctx sdk.Context, keeper keeper.Keeper, msg *types.MsgDepos return nil, err } + defer telemetry.IncrCounterWithLabels( + []string{types.ModuleName, "deposit"}, + 1, + []metrics.Label{ + telemetry.NewLabel("proposal_id", strconv.Itoa(int(msg.ProposalID))), + }, + ) + ctx.EventManager().EmitEvent( sdk.NewEvent( sdk.EventTypeMessage, @@ -96,6 +110,14 @@ func handleMsgVote(ctx sdk.Context, keeper keeper.Keeper, msg *types.MsgVote) (* return nil, err } + defer telemetry.IncrCounterWithLabels( + []string{types.ModuleName, "vote"}, + 1, + []metrics.Label{ + telemetry.NewLabel("proposal_id", strconv.Itoa(int(msg.ProposalID))), + }, + ) + ctx.EventManager().EmitEvent( sdk.NewEvent( sdk.EventTypeMessage, diff --git a/x/mint/abci.go b/x/mint/abci.go index 4c1ceee8dc..fa34ec1613 100644 --- a/x/mint/abci.go +++ b/x/mint/abci.go @@ -1,6 +1,7 @@ package mint import ( + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/mint/keeper" "github.com/cosmos/cosmos-sdk/x/mint/types" @@ -8,6 +9,8 @@ import ( // BeginBlocker mints new tokens for the previous block. func BeginBlocker(ctx sdk.Context, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyBeginBlocker) + // fetch stored minter & params minter := k.GetMinter(ctx) params := k.GetParams(ctx) @@ -34,6 +37,8 @@ func BeginBlocker(ctx sdk.Context, k keeper.Keeper) { panic(err) } + defer telemetry.ModuleSetGauge(types.ModuleName, float32(mintedCoin.Amount.Int64()), "minted_tokens") + ctx.EventManager().EmitEvent( sdk.NewEvent( types.EventTypeMint, diff --git a/x/slashing/abci.go b/x/slashing/abci.go index 94b8d1e59a..77a774b491 100644 --- a/x/slashing/abci.go +++ b/x/slashing/abci.go @@ -3,13 +3,17 @@ package slashing import ( abci "github.com/tendermint/tendermint/abci/types" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/slashing/keeper" + "github.com/cosmos/cosmos-sdk/x/slashing/types" ) // BeginBlocker check for infraction evidence or downtime of validators // on every begin block func BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyBeginBlocker) + // Iterate over all the validators which *should* have signed this block // store whether or not they have actually signed it and slash/unbond any // which have missed too many blocks in a row (downtime slashing) diff --git a/x/staking/abci.go b/x/staking/abci.go index 6e4b1067e9..cebaa8cb2b 100644 --- a/x/staking/abci.go +++ b/x/staking/abci.go @@ -3,17 +3,23 @@ package staking import ( abci "github.com/tendermint/tendermint/abci/types" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/staking/keeper" + "github.com/cosmos/cosmos-sdk/x/staking/types" ) // BeginBlocker will persist the current header and validator set as a historical entry // and prune the oldest entry based on the HistoricalEntries parameter func BeginBlocker(ctx sdk.Context, k keeper.Keeper) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyBeginBlocker) + k.TrackHistoricalInfo(ctx) } // Called every block, update validator set func EndBlocker(ctx sdk.Context, k keeper.Keeper) []abci.ValidatorUpdate { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyEndBlocker) + return k.BlockValidatorUpdates(ctx) } diff --git a/x/staking/handler.go b/x/staking/handler.go index 71efc39fd1..cf8de1d5cf 100644 --- a/x/staking/handler.go +++ b/x/staking/handler.go @@ -3,10 +3,12 @@ package staking import ( "time" + "github.com/armon/go-metrics" gogotypes "github.com/gogo/protobuf/types" tmstrings "github.com/tendermint/tendermint/libs/strings" tmtypes "github.com/tendermint/tendermint/types" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" "github.com/cosmos/cosmos-sdk/x/staking/keeper" @@ -194,6 +196,15 @@ func handleMsgDelegate(ctx sdk.Context, msg *types.MsgDelegate, k keeper.Keeper) return nil, err } + defer func() { + telemetry.IncrCounter(1, types.ModuleName, "delegate") + telemetry.SetGaugeWithLabels( + []string{"tx", "msg", msg.Type()}, + float32(msg.Amount.Amount.Int64()), + []metrics.Label{telemetry.NewLabel("denom", msg.Amount.Denom)}, + ) + }() + ctx.EventManager().EmitEvents(sdk.Events{ sdk.NewEvent( types.EventTypeDelegate, @@ -232,6 +243,15 @@ func handleMsgUndelegate(ctx sdk.Context, msg *types.MsgUndelegate, k keeper.Kee return nil, types.ErrBadRedelegationAddr } + defer func() { + telemetry.IncrCounter(1, types.ModuleName, "undelegate") + telemetry.SetGaugeWithLabels( + []string{"tx", "msg", msg.Type()}, + float32(msg.Amount.Amount.Int64()), + []metrics.Label{telemetry.NewLabel("denom", msg.Amount.Denom)}, + ) + }() + completionTimeBz := types.ModuleCdc.MustMarshalBinaryLengthPrefixed(ts) ctx.EventManager().EmitEvents(sdk.Events{ sdk.NewEvent( @@ -274,6 +294,15 @@ func handleMsgBeginRedelegate(ctx sdk.Context, msg *types.MsgBeginRedelegate, k return nil, types.ErrBadRedelegationAddr } + defer func() { + telemetry.IncrCounter(1, types.ModuleName, "redelegate") + telemetry.SetGaugeWithLabels( + []string{"tx", "msg", msg.Type()}, + float32(msg.Amount.Amount.Int64()), + []metrics.Label{telemetry.NewLabel("denom", msg.Amount.Denom)}, + ) + }() + completionTimeBz := types.ModuleCdc.MustMarshalBinaryLengthPrefixed(ts) ctx.EventManager().EmitEvents(sdk.Events{ sdk.NewEvent( diff --git a/x/upgrade/abci.go b/x/upgrade/abci.go index c52220d05a..0da6d5155a 100644 --- a/x/upgrade/abci.go +++ b/x/upgrade/abci.go @@ -5,8 +5,10 @@ import ( abci "github.com/tendermint/tendermint/abci/types" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/upgrade/keeper" + "github.com/cosmos/cosmos-sdk/x/upgrade/types" ) // BeginBlock will check if there is a scheduled plan and if it is ready to be executed. @@ -18,6 +20,8 @@ import ( // a migration to be executed if needed upon this switch (migration defined in the new binary) // skipUpgradeHeightArray is a set of block heights for which the upgrade must be skipped func BeginBlocker(k keeper.Keeper, ctx sdk.Context, _ abci.RequestBeginBlock) { + defer telemetry.ModuleMeasureSince(types.ModuleName, telemetry.MetricKeyBeginBlocker) + plan, found := k.GetUpgradePlan(ctx) if !found { return