test: add systemtests (#23686)

2025-02-14 14:27:03 -08:00 · 2025-02-14 14:27:03 -08:00 · b4bae1a55c
commit b4bae1a55c
parent a5661db02a
22 changed files with 5310 additions and 52 deletions
--- a/9
+++ b/9
@ -467,7 +467,7 @@ localnet-build-dlv:

 localnet-build-nodes:
 	$(DOCKER) run --rm -v $(CURDIR)/.testnets:/data cosmossdk/simd \
-			  testnet init-files --v 4 -o /data --starting-ip-address 192.168.10.2 --keyring-backend=test
+			  testnet init-files --validator-count 4 -o /data --starting-ip-address 192.168.10.2 --keyring-backend=test
 	docker compose up -d

 localnet-stop:
@ -482,3 +482,10 @@ localnet-start: localnet-stop localnet-build-env localnet-build-nodes
 localnet-debug: localnet-stop localnet-build-dlv localnet-build-nodes

 .PHONY: localnet-start localnet-stop localnet-debug localnet-build-env localnet-build-dlv localnet-build-nodes
+
+test-system: build
+	mkdir -p ./tests/systemtests/binaries/
+	cp $(BUILDDIR)/simd ./tests/systemtests/binaries/
+	$(MAKE) -C tests/systemtests test
+.PHONY: test-system
+
--- a/simapp/simd/cmd/testnet.go
+++ b/simapp/simd/cmd/testnet.go
@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"time"

 	cmtconfig "github.com/cometbft/cometbft/config"
 	cmttime "github.com/cometbft/cometbft/types/time"
@ -17,37 +18,42 @@ import (
 	"cosmossdk.io/math/unsafe"
 	"cosmossdk.io/simapp"

+	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/cosmos/cosmos-sdk/client/tx"
 	"github.com/cosmos/cosmos-sdk/crypto/hd"
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
 	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
-	"github.com/cosmos/cosmos-sdk/runtime"
 	"github.com/cosmos/cosmos-sdk/server"
 	srvconfig "github.com/cosmos/cosmos-sdk/server/config"
 	"github.com/cosmos/cosmos-sdk/testutil"
 	"github.com/cosmos/cosmos-sdk/testutil/network"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/types/module"
-	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
-	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+	"github.com/cosmos/cosmos-sdk/version"
 	"github.com/cosmos/cosmos-sdk/x/genutil"
 	genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
-	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 )

 var (
 	flagNodeDirPrefix     = "node-dir-prefix"
-	flagNumValidators     = "v"
+	flagNumValidators     = "validator-count"
 	flagOutputDir         = "output-dir"
 	flagNodeDaemonHome    = "node-daemon-home"
 	flagStartingIPAddress = "starting-ip-address"
+	flagListenIPAddress   = "listen-ip-address"
 	flagEnableLogging     = "enable-logging"
 	flagGRPCAddress       = "grpc.address"
 	flagRPCAddress        = "rpc.address"
 	flagAPIAddress        = "api.address"
 	flagPrintMnemonic     = "print-mnemonic"
+	flagStakingDenom      = "staking-denom"
+	flagCommitTimeout     = "commit-timeout"
+	flagSingleHost        = "single-host"
 )

 type initArgs struct {
@ -60,6 +66,9 @@ type initArgs struct {
 	numValidators     int
 	outputDir         string
 	startingIPAddress string
+	listenIPAddress   string
+	singleMachine     bool
+	bondTokenDenom    string
 }

 type startArgs struct {
@ -73,10 +82,11 @@ type startArgs struct {
 	outputDir     string
 	printMnemonic bool
 	rpcAddress    string
+	timeoutCommit time.Duration
 }

 func addTestnetFlagsToCmd(cmd *cobra.Command) {
-	cmd.Flags().Int(flagNumValidators, 4, "Number of validators to initialize the testnet with")
+	cmd.Flags().IntP(flagNumValidators, "n", 4, "Number of validators to initialize the testnet with")
 	cmd.Flags().StringP(flagOutputDir, "o", "./.testnets", "Directory to store initialization data for the testnet")
 	cmd.Flags().String(flags.FlagChainID, "", "genesis file chain-id, if left blank will be randomly created")
 	cmd.Flags().String(server.FlagMinGasPrices, fmt.Sprintf("0.000006%s", sdk.DefaultBondDenom), "Minimum gas prices to accept for transactions; All fees in a tx must meet this minimum (e.g. 0.01photino,0.001stake)")
@ -94,7 +104,7 @@ func addTestnetFlagsToCmd(cmd *cobra.Command) {

 // NewTestnetCmd creates a root testnet command with subcommands to run an in-process testnet or initialize
 // validator configuration files for running a multi-validator testnet in a separate process
-func NewTestnetCmd(mbm module.BasicManager, genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
+func NewTestnetCmd(mm module.BasicManager, genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
 	testnetCmd := &cobra.Command{
 		Use:                        "testnet",
 		Short:                      "subcommands for starting or configuring local testnets",
@ -104,18 +114,18 @@ func NewTestnetCmd(mbm module.BasicManager, genBalIterator banktypes.GenesisBala
 	}

 	testnetCmd.AddCommand(testnetStartCmd())
-	testnetCmd.AddCommand(testnetInitFilesCmd(mbm, genBalIterator))
+	testnetCmd.AddCommand(testnetInitFilesCmd(mm, genBalIterator))

 	return testnetCmd
 }

 // testnetInitFilesCmd returns a cmd to initialize all files for CometBFT testnet and application
-func testnetInitFilesCmd(mbm module.BasicManager, genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
+func testnetInitFilesCmd(mm module.BasicManager, genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "init-files",
 		Short: "Initialize config directories & files for a multi-validator testnet running locally via separate processes (e.g. Docker Compose or similar)",
-		Long: `init-files will setup "v" number of directories and populate each with
-necessary files (private validator, genesis, config, etc.) for running "v" validator nodes.
+		Long: fmt.Sprintf(`init-files will setup one directory per validator and populate each with
+necessary files (private validator, genesis, config, etc.) for running validator nodes.

 Booting up a network with these validator folders is intended to be used with Docker Compose,
 or a similar setup where each node has a manually configurable IP address.
@ -123,8 +133,8 @@ or a similar setup where each node has a manually configurable IP address.
 Note, strict routability for addresses is turned off in the config file.

 Example:
-	simd testnet init-files --v 4 --output-dir ./.testnets --starting-ip-address 192.168.10.2
-	`,
+	%s testnet init-files --validator-count 4 --output-dir ./.testnets --starting-ip-address 192.168.10.2
+	`, version.AppName),
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			clientCtx, err := client.GetClientQueryContext(cmd)
 			if err != nil {
@ -142,18 +152,29 @@ Example:
 			args.nodeDirPrefix, _ = cmd.Flags().GetString(flagNodeDirPrefix)
 			args.nodeDaemonHome, _ = cmd.Flags().GetString(flagNodeDaemonHome)
 			args.startingIPAddress, _ = cmd.Flags().GetString(flagStartingIPAddress)
+			args.listenIPAddress, _ = cmd.Flags().GetString(flagListenIPAddress)
 			args.numValidators, _ = cmd.Flags().GetInt(flagNumValidators)
 			args.algo, _ = cmd.Flags().GetString(flags.FlagKeyType)
+			args.bondTokenDenom, _ = cmd.Flags().GetString(flagStakingDenom)
+			args.singleMachine, _ = cmd.Flags().GetBool(flagSingleHost)
+			config.Consensus.TimeoutCommit, err = cmd.Flags().GetDuration(flagCommitTimeout)
+			if err != nil {
+				return err
+			}

-			return initTestnetFiles(clientCtx, cmd, config, mbm, genBalIterator, clientCtx.TxConfig.SigningContext().ValidatorAddressCodec(), args)
+			return initTestnetFiles(clientCtx, cmd, config, mm, genBalIterator, args)
 		},
 	}

 	addTestnetFlagsToCmd(cmd)
-	cmd.Flags().String(flagNodeDirPrefix, "node", "Prefix the directory name for each node with (node results in node0, node1, ...)")
+	cmd.Flags().String(flagNodeDirPrefix, "node", "Prefix for the name of per-validator subdirectories (to be number-suffixed like node0, node1, ...)")
 	cmd.Flags().String(flagNodeDaemonHome, "simd", "Home directory of the node's daemon configuration")
 	cmd.Flags().String(flagStartingIPAddress, "192.168.0.1", "Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:46656, ID1@192.168.0.2:46656, ...)")
+	cmd.Flags().String(flagListenIPAddress, "0.0.0.0", "TCP or UNIX socket IP address for the RPC server to listen on")
 	cmd.Flags().String(flags.FlagKeyringBackend, flags.DefaultKeyringBackend, "Select keyring's backend (os|file|test)")
+	cmd.Flags().Duration(flagCommitTimeout, 5*time.Second, "Time to wait after a block commit before starting on the new height")
+	cmd.Flags().Bool(flagSingleHost, false, "Cluster runs on a single host machine with different ports")
+	cmd.Flags().String(flagStakingDenom, sdk.DefaultBondDenom, "Default staking token denominator")

 	return cmd
 }
@ -163,14 +184,14 @@ func testnetStartCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "start",
 		Short: "Launch an in-process multi-validator testnet",
-		Long: `testnet will launch an in-process multi-validator testnet,
-and generate "v" directories, populated with necessary validator configuration files
-(private validator, genesis, config, etc.).
+		Long: fmt.Sprintf(`testnet will launch an in-process multi-validator testnet,
+and generate a directory for each validator populated with necessary
+configuration files (private validator, genesis, config, etc.).

 Example:
-	simd testnet --v 4 --output-dir ./.testnets
-	`,
-		RunE: func(cmd *cobra.Command, _ []string) error {
+	%s testnet --validator-count 4 --output-dir ./.testnets
+	`, version.AppName),
+		RunE: func(cmd *cobra.Command, _ []string) (err error) {
 			args := startArgs{}
 			args.outputDir, _ = cmd.Flags().GetString(flagOutputDir)
 			args.chainID, _ = cmd.Flags().GetString(flags.FlagChainID)
@ -203,9 +224,8 @@ func initTestnetFiles(
 	clientCtx client.Context,
 	cmd *cobra.Command,
 	nodeConfig *cmtconfig.Config,
-	mbm module.BasicManager,
+	mm module.BasicManager,
 	genBalIterator banktypes.GenesisBalancesIterator,
-	valAddrCodec runtime.ValidatorAddressCodec,
 	args initArgs,
 ) error {
 	if args.chainID == "" {
@ -214,40 +234,64 @@ func initTestnetFiles(
 	nodeIDs := make([]string, args.numValidators)
 	valPubKeys := make([]cryptotypes.PubKey, args.numValidators)

-	simappConfig := srvconfig.DefaultConfig()
-	simappConfig.MinGasPrices = args.minGasPrices
-	simappConfig.API.Enable = true
-	simappConfig.Telemetry.Enabled = true
-	simappConfig.Telemetry.PrometheusRetentionTime = 60
-	simappConfig.Telemetry.EnableHostnameLabel = false
-	simappConfig.Telemetry.GlobalLabels = [][]string{{"chain_id", args.chainID}}
+	appConfig := srvconfig.DefaultConfig()
+	appConfig.MinGasPrices = args.minGasPrices
+	appConfig.API.Enable = true
+	appConfig.Telemetry.Enabled = true
+	appConfig.Telemetry.PrometheusRetentionTime = 60
+	appConfig.Telemetry.EnableHostnameLabel = false
+	appConfig.Telemetry.GlobalLabels = [][]string{{"chain_id", args.chainID}}

 	var (
 		genAccounts []authtypes.GenesisAccount
 		genBalances []banktypes.Balance
 		genFiles    []string
 	)
+	const (
+		rpcPort  = 26657
+		apiPort  = 1317
+		grpcPort = 9090
+	)
+	p2pPortStart := 26656

 	inBuf := bufio.NewReader(cmd.InOrStdin())
 	// generate private keys, node IDs, and initial transactions
 	for i := 0; i < args.numValidators; i++ {
+		var portOffset int
+		if args.singleMachine {
+			portOffset = i
+			p2pPortStart = 16656 // use different start point to not conflict with rpc port
+			nodeConfig.P2P.AddrBookStrict = false
+			nodeConfig.P2P.PexReactor = false
+			nodeConfig.P2P.AllowDuplicateIP = true
+			appConfig.API.Address = fmt.Sprintf("tcp://0.0.0.0:%d", apiPort+portOffset)
+			appConfig.GRPC.Address = fmt.Sprintf("0.0.0.0:%d", grpcPort+portOffset)
+		}
+
 		nodeDirName := fmt.Sprintf("%s%d", args.nodeDirPrefix, i)
 		nodeDir := filepath.Join(args.outputDir, nodeDirName, args.nodeDaemonHome)
 		gentxsDir := filepath.Join(args.outputDir, "gentxs")

 		nodeConfig.SetRoot(nodeDir)
 		nodeConfig.Moniker = nodeDirName
-		nodeConfig.RPC.ListenAddress = "tcp://0.0.0.0:26657"
+		nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://%s:%d", args.listenIPAddress, rpcPort+portOffset)

 		if err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm); err != nil {
 			_ = os.RemoveAll(args.outputDir)
 			return err
 		}
-
-		ip, err := getIP(i, args.startingIPAddress)
-		if err != nil {
-			_ = os.RemoveAll(args.outputDir)
-			return err
+		var (
+			err error
+			ip  string
+		)
+		if args.singleMachine {
+			ip = "0.0.0.0"
+		} else {
+			ip, err = getIP(i, args.startingIPAddress)
+			if err != nil {
+				_ = os.RemoveAll(args.outputDir)
+				return err
+			}
 		}

 		nodeIDs[i], valPubKeys[i], err = genutil.InitializeNodeValidatorFiles(nodeConfig)
@ -256,7 +300,7 @@ func initTestnetFiles(
 			return err
 		}

-		memo := fmt.Sprintf("%s@%s:26656", nodeIDs[i], ip)
+		memo := fmt.Sprintf("%s@%s:%d", nodeIDs[i], ip, p2pPortStart+portOffset)
 		genFiles = append(genFiles, nodeConfig.GenesisFile())

 		kb, err := keyring.New(sdk.KeyringServiceName(), args.keyringBackend, nodeDir, inBuf, clientCtx.Codec)
@ -292,21 +336,19 @@ func initTestnetFiles(
 		accStakingTokens := sdk.TokensFromConsensusPower(500, sdk.DefaultPowerReduction)
 		coins := sdk.Coins{
 			sdk.NewCoin("testtoken", accTokens),
-			sdk.NewCoin(sdk.DefaultBondDenom, accStakingTokens),
+			sdk.NewCoin(args.bondTokenDenom, accStakingTokens),
 		}

 		genBalances = append(genBalances, banktypes.Balance{Address: addr.String(), Coins: coins.Sort()})
 		genAccounts = append(genAccounts, authtypes.NewBaseAccount(addr, nil, 0, 0))

-		valStr, err := valAddrCodec.BytesToString(sdk.ValAddress(addr))
-		if err != nil {
-			return err
-		}
+		valAddr := sdk.ValAddress(addr)
+		valStr := valAddr.String()
 		valTokens := sdk.TokensFromConsensusPower(100, sdk.DefaultPowerReduction)
 		createValMsg, err := stakingtypes.NewMsgCreateValidator(
 			valStr,
 			valPubKeys[i],
-			sdk.NewCoin(sdk.DefaultBondDenom, valTokens),
+			sdk.NewCoin(args.bondTokenDenom, valTokens),
 			stakingtypes.NewDescription(nodeDirName, "", "", "", ""),
 			stakingtypes.NewCommissionRates(math.LegacyOneDec(), math.LegacyOneDec(), math.LegacyOneDec()),
 			math.OneInt(),
@ -343,16 +385,18 @@ func initTestnetFiles(
 		}

 		srvconfig.SetConfigTemplate(srvconfig.DefaultConfigTemplate)
-		srvconfig.WriteConfigFile(filepath.Join(nodeDir, "config", "app.toml"), simappConfig)
+
+		srvconfig.WriteConfigFile(filepath.Join(nodeDir, "config", "app.toml"), appConfig)
 	}

-	if err := initGenFiles(clientCtx, mbm, args.chainID, genAccounts, genBalances, genFiles, args.numValidators); err != nil {
+	if err := initGenFiles(clientCtx, mm, args.chainID, genAccounts, genBalances, genFiles, args.numValidators); err != nil {
 		return err
 	}

 	err := collectGenFiles(
 		clientCtx, nodeConfig, args.chainID, nodeIDs, valPubKeys, args.numValidators,
-		args.outputDir, args.nodeDirPrefix, args.nodeDaemonHome, genBalIterator, valAddrCodec,
+		args.outputDir, args.nodeDirPrefix, args.nodeDaemonHome, genBalIterator,
+		rpcPort, p2pPortStart, args.singleMachine,
 	)
 	if err != nil {
 		return err
@ -363,11 +407,11 @@ func initTestnetFiles(
 }

 func initGenFiles(
-	clientCtx client.Context, mbm module.BasicManager, chainID string,
+	clientCtx client.Context, mm module.BasicManager, chainID string,
 	genAccounts []authtypes.GenesisAccount, genBalances []banktypes.Balance,
 	genFiles []string, numValidators int,
 ) error {
-	appGenState := mbm.DefaultGenesis(clientCtx.Codec)
+	appGenState := mm.DefaultGenesis(clientCtx.Codec)

 	// set the accounts in the genesis state
 	var authGenState authtypes.GenesisState
@ -386,6 +430,7 @@ func initGenFiles(
 	clientCtx.Codec.MustUnmarshalJSON(appGenState[banktypes.ModuleName], &bankGenState)

 	bankGenState.Balances = banktypes.SanitizeGenesisBalances(genBalances)
+
 	for _, bal := range bankGenState.Balances {
 		bankGenState.Supply = bankGenState.Supply.Add(bal.Coins...)
 	}
@ -409,12 +454,20 @@ func initGenFiles(
 func collectGenFiles(
 	clientCtx client.Context, nodeConfig *cmtconfig.Config, chainID string,
 	nodeIDs []string, valPubKeys []cryptotypes.PubKey, numValidators int,
-	outputDir, nodeDirPrefix, nodeDaemonHome string, genBalIterator banktypes.GenesisBalancesIterator, valAddrCodec runtime.ValidatorAddressCodec,
+	outputDir, nodeDirPrefix, nodeDaemonHome string, genBalIterator banktypes.GenesisBalancesIterator,
+	rpcPortStart, p2pPortStart int,
+	singleMachine bool,
 ) error {
 	var appState json.RawMessage
 	genTime := cmttime.Now()

 	for i := 0; i < numValidators; i++ {
+		if singleMachine {
+			portOffset := i
+			nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://0.0.0.0:%d", rpcPortStart+portOffset)
+			nodeConfig.P2P.ListenAddress = fmt.Sprintf("tcp://0.0.0.0:%d", p2pPortStart+portOffset)
+		}
+
 		nodeDirName := fmt.Sprintf("%s%d", nodeDirPrefix, i)
 		nodeDir := filepath.Join(outputDir, nodeDirName, nodeDaemonHome)
 		gentxsDir := filepath.Join(outputDir, "gentxs")
@ -430,8 +483,16 @@ func collectGenFiles(
 			return err
 		}

-		nodeAppState, err := genutil.GenAppStateFromConfig(clientCtx.Codec, clientCtx.TxConfig, nodeConfig, initCfg, appGenesis, genBalIterator, genutiltypes.DefaultMessageValidator,
-			valAddrCodec)
+		nodeAppState, err := genutil.GenAppStateFromConfig(
+			clientCtx.Codec,
+			clientCtx.TxConfig,
+			nodeConfig,
+			initCfg,
+			appGenesis,
+			genBalIterator,
+			genutiltypes.DefaultMessageValidator,
+			clientCtx.TxConfig.SigningContext().ValidatorAddressCodec(),
+		)
 		if err != nil {
 			return err
 		}
@ -507,6 +568,7 @@ func startTestnet(cmd *cobra.Command, args startArgs) error {
 	networkConfig.APIAddress = args.apiAddress
 	networkConfig.GRPCAddress = args.grpcAddress
 	networkConfig.PrintMnemonic = args.printMnemonic
+	networkConfig.TimeoutCommit = args.timeoutCommit
 	networkLogger := network.NewCLILogger(cmd)

 	baseDir := fmt.Sprintf("%s/%s", args.outputDir, networkConfig.ChainID)
--- a/systemtests/CHANGELOG.md
+++ b/systemtests/CHANGELOG.md
@ -0,0 +1,61 @@
+<!--
+Guiding Principles:
+
+Changelogs are for humans, not machines.
+There should be an entry for every single version.
+The same types of changes should be grouped.
+Versions and sections should be linkable.
+The latest version comes first.
+The release date of each version is displayed.
+Mention whether you follow Semantic Versioning.
+
+Usage:
+
+Changelog entries are generated by git cliff ref: https://github.com/orhun/git-cliff
+
+Each commit should be conventional, the following message groups are supported.
+
+* feat: A new feature
+* fix: A bug fix
+* docs: Documentation only changes
+* style: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
+* refactor: A code change that neither fixes a bug nor adds a feature
+* perf: A code change that improves performance
+* test: Adding missing tests or correcting existing tests
+* build: Changes that affect the build system or external dependencies (example scopes: go, npm)
+* ci: Changes to our CI configuration files and scripts (example scopes: GH Actions)
+* chore: Other changes that don't modify src or test files
+* revert: Reverts a previous commit
+
+When a change is made that affects the API or state machine, the commit message prefix should be suffixed with `!`.
+
+Ref: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
+-->
+
+# Changelog
+
+## [Unreleased]
+
+## [v1.1.0] - 2025-01-05
+
+* [#23359](https://github.com/cosmos/cosmos-sdk/pull/23359) Add `RunCommandWithInputAndArgs` on CLIWrapper.
+
+## [v1.0.0] - 2024-12-19
+
+* [#22849](https://github.com/cosmos/cosmos-sdk/pull/22849) Avoid telemetry server conflicts on port 7180
+
+## [v1.0.0-rc.4] - 2024-12-10
+
+* [#22810](https://github.com/cosmos/cosmos-sdk/pull/22810) Avoid HTTP server conflicts on port 8080
+
+## [v1.0.0-rc.3] - 2024-12-05
+
+* [#22774](https://github.com/cosmos/cosmos-sdk/pull/22774) Add greater than or equal support in Rest test suite
+
+## [v1.0.0-rc.2] - 2024-11-26
+
+* [#22577](https://github.com/cosmos/cosmos-sdk/pull/22577) Support invalid RPC response for CometBFT v1
+
+## [v1.0.0-rc.1] - 2024-11-26
+
+* [#22578](https://github.com/cosmos/cosmos-sdk/pull/22578) Extract system test framework
--- a/systemtests/README.md
+++ b/systemtests/README.md
@ -0,0 +1,63 @@
+# System Tests
+
+This package contains the testing framework for black-box system tests. It includes a test runner that sets up a 
+multi-node blockchain locally for use in tests. The framework provides utilities and helpers for easy access and 
+setup in tests.
+
+## Components
+
+- **CLI**: Command-line interface wrapper for interacting with the chain or keyring
+- **Servers**: Server instances to run the blockchain environment.
+- **Events**: Event listeners
+- **RPC**: Remote Procedure Call setup for communication.
+
+## Dependencies
+
+- **testify**: Testing toolkit.
+- **gjson**: JSON parser.
+- **sjson**: JSON modifier.
+
+Server and client-side operations are executed on the host machine.
+
+## Developer
+
+### Test strategy
+
+System tests cover the full stack via cli and a running (multi node) network. They are more expensive (in terms of time/ cpu) 
+to run compared to unit or integration tests. 
+Therefore, we focus on the **critical path** and do not cover every condition.
+
+## How to use
+
+Read the [getting_started.md](../systemtests/getting_started.md) guide to get started.
+
+### Execute a single test
+
+```sh
+go test -tags system_test -count=1 -v . --run TestStakeUnstake  -verbose
+```
+
+Test cli parameters
+
+* `-verbose` verbose output
+* `-wait-time` duration - time to wait for chain events (default 30s)
+* `-nodes-count` int - number of nodes in the cluster (default 4)
+
+# Port ranges
+
+With *n* nodes:
+
+* `26657` - `26657+n` - RPC
+* `1317` - `1317+n` - API
+* `9090` - `9090+n` - GRPC
+* `16656` - `16656+n` - P2P
+
+For example Node *3* listens on `26660` for RPC calls
+
+## Resources
+
+* [gjson query syntax](https://github.com/tidwall/gjson#path-syntax)
+
+## Disclaimer
+
+This is based on the system test framework in [wasmd](https://github.com/CosmWasm/wasmd) built by Confio.
--- a/systemtests/cli.go
+++ b/systemtests/cli.go
@ -0,0 +1,522 @@
+package systemtests
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+type (
+	// blocks until next block is minted
+	awaitNextBlock func(t *testing.T, timeout ...time.Duration) int64
+	// RunErrorAssert is custom type that is satisfies by testify matchers as well
+	RunErrorAssert func(t assert.TestingT, err error, msgAndArgs ...interface{}) (ok bool)
+)
+
+// CLIWrapper provides a more convenient way to interact with the CLI binary from the Go tests
+type CLIWrapper struct {
+	t               *testing.T
+	nodeAddress     string
+	chainID         string
+	homeDir         string
+	fees            string
+	Debug           bool
+	assertErrorFn   RunErrorAssert
+	awaitNextBlock  awaitNextBlock
+	expTXCommitted  bool
+	execBinary      string
+	nodesCount      int
+	runSingleOutput bool
+}
+
+// NewCLIWrapper constructor
+func NewCLIWrapper(t *testing.T, sut *SystemUnderTest, verbose bool) *CLIWrapper {
+	t.Helper()
+	return NewCLIWrapperX(
+		t,
+		sut.execBinary,
+		sut.rpcAddr,
+		sut.chainID,
+		sut.AwaitNextBlock,
+		sut.nodesCount,
+		filepath.Join(WorkDir, sut.outputDir),
+		"1"+sdk.DefaultBondDenom,
+		verbose,
+		assert.NoError,
+		false,
+		true,
+	)
+}
+
+// NewCLIWrapperX extended constructor
+func NewCLIWrapperX(
+	t *testing.T,
+	execBinary string,
+	nodeAddress string,
+	chainID string,
+	awaiter awaitNextBlock,
+	nodesCount int,
+	homeDir string,
+	fees string,
+	debug bool,
+	assertErrorFn RunErrorAssert,
+	runSingleOutput bool,
+	expTXCommitted bool,
+) *CLIWrapper {
+	t.Helper()
+	if strings.TrimSpace(execBinary) == "" {
+		t.Fatal("name of executable binary must not be empty")
+	}
+	return &CLIWrapper{
+		t:               t,
+		execBinary:      execBinary,
+		nodeAddress:     nodeAddress,
+		chainID:         chainID,
+		homeDir:         homeDir,
+		Debug:           debug,
+		awaitNextBlock:  awaiter,
+		nodesCount:      nodesCount,
+		fees:            fees,
+		assertErrorFn:   assertErrorFn,
+		runSingleOutput: runSingleOutput,
+		expTXCommitted:  expTXCommitted,
+	}
+}
+
+// WithRunErrorsIgnored does not fail on any error
+func (c CLIWrapper) WithRunErrorsIgnored() CLIWrapper {
+	return c.WithRunErrorMatcher(func(t assert.TestingT, err error, msgAndArgs ...interface{}) bool {
+		return true
+	})
+}
+
+// WithRunErrorMatcher assert function to ensure run command error value
+func (c CLIWrapper) WithRunErrorMatcher(f RunErrorAssert) CLIWrapper {
+	return c.clone(func(r *CLIWrapper) {
+		r.assertErrorFn = f
+	})
+}
+
+func (c CLIWrapper) WithRunSingleOutput() CLIWrapper {
+	return c.clone(func(r *CLIWrapper) {
+		r.runSingleOutput = true
+	})
+}
+
+func (c CLIWrapper) WithNodeAddress(nodeAddr string) CLIWrapper {
+	return c.clone(func(r *CLIWrapper) {
+		r.nodeAddress = nodeAddr
+	})
+}
+
+func (c CLIWrapper) WithAssertTXUncommitted() CLIWrapper {
+	return c.clone(func(r *CLIWrapper) {
+		r.expTXCommitted = false
+	})
+}
+
+func (c CLIWrapper) WithChainID(newChainID string) CLIWrapper {
+	return c.clone(func(r *CLIWrapper) {
+		r.chainID = newChainID
+	})
+}
+
+func (c CLIWrapper) clone(mutator ...func(r *CLIWrapper)) CLIWrapper {
+	r := NewCLIWrapperX(
+		c.t,
+		c.execBinary,
+		c.nodeAddress,
+		c.chainID,
+		c.awaitNextBlock,
+		c.nodesCount,
+		c.homeDir,
+		c.fees,
+		c.Debug,
+		c.assertErrorFn,
+		c.runSingleOutput,
+		c.expTXCommitted,
+	)
+	for _, m := range mutator {
+		m(r)
+	}
+	return *r
+}
+
+// Run main entry for executing cli commands.
+// When configured, method blocks until tx is committed.
+func (c CLIWrapper) Run(args ...string) string {
+	c.t.Helper()
+	if c.fees != "" && !slices.ContainsFunc(args, func(s string) bool {
+		return strings.HasPrefix(s, "--fees")
+	}) {
+		args = append(args, "--fees="+c.fees) // add default fee
+	}
+	args = c.WithTXFlags(args...)
+	execOutput, ok := c.run(args)
+	if !ok {
+		return execOutput
+	}
+	rsp, committed := c.AwaitTxCommitted(execOutput, DefaultWaitTime)
+	c.t.Logf("tx committed: %v", committed)
+	require.Equal(c.t, c.expTXCommitted, committed, "expected tx committed: %v", c.expTXCommitted)
+	return rsp
+}
+
+// RunAndWait runs a cli command and waits for the server result when the TX is executed
+// It returns the result of the transaction.
+func (c CLIWrapper) RunAndWait(args ...string) string {
+	rsp := c.Run(args...)
+	RequireTxSuccess(c.t, rsp)
+	txResult, found := c.AwaitTxCommitted(rsp)
+	require.True(c.t, found)
+	return txResult
+}
+
+// RunCommandWithArgs use for run cli command, not tx
+func (c CLIWrapper) RunCommandWithArgs(args ...string) string {
+	c.t.Helper()
+	execOutput, _ := c.run(args)
+	return execOutput
+}
+
+// RunCommandWithInputAndArgs use for run cli command, not tx
+// Takes input as io.Reader for the command
+func (c CLIWrapper) RunCommandWithInputAndArgs(input io.Reader, args ...string) string {
+	c.t.Helper()
+	execOutput, _ := c.runWithInput(args, input)
+	return execOutput
+}
+
+// AwaitTxCommitted wait for tx committed on chain
+// returns the server execution result and true when found within 3 blocks.
+func (c CLIWrapper) AwaitTxCommitted(submitResp string, timeout ...time.Duration) (string, bool) {
+	c.t.Helper()
+	RequireTxSuccess(c.t, submitResp)
+	txHash := gjson.Get(submitResp, "txhash")
+	require.True(c.t, txHash.Exists())
+	var txResult string
+	for i := 0; i < 3; i++ { // max blocks to wait for a commit
+		txResult = c.WithRunErrorsIgnored().CustomQuery("q", "tx", txHash.String())
+		if code := gjson.Get(txResult, "code"); code.Exists() {
+			if code.Int() != 0 { // 0 = success code
+				c.t.Logf("+++ got error response code: %s\n", txResult)
+			}
+			return txResult, true
+		}
+		c.awaitNextBlock(c.t, timeout...)
+	}
+	return "", false
+}
+
+// Keys wasmd keys CLI command
+func (c CLIWrapper) Keys(args ...string) string {
+	args = c.WithKeyringFlags(args...)
+	out, _ := c.run(args)
+	return out
+}
+
+// CustomQuery main entrypoint for wasmd CLI queries
+func (c CLIWrapper) CustomQuery(args ...string) string {
+	args = c.WithQueryFlags(args...)
+	out, _ := c.run(args)
+	return out
+}
+
+// execute shell command
+func (c CLIWrapper) run(args []string) (output string, ok bool) {
+	c.t.Helper()
+	return c.runWithInput(args, nil)
+}
+
+func (c CLIWrapper) runWithInput(args []string, input io.Reader) (output string, ok bool) {
+	c.t.Helper()
+	if c.Debug {
+		c.t.Logf("+++ running `%s %s`", c.execBinary, strings.Join(args, " "))
+	}
+	gotOut, gotErr := func() (out []byte, err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				err = fmt.Errorf("recovered from panic: %v", r)
+			}
+		}()
+		cmd := exec.Command(locateExecutable(c.execBinary), args...) //nolint:gosec // test code only
+		cmd.Dir = WorkDir
+		cmd.Stdin = input
+
+		if c.runSingleOutput {
+			return cmd.Output()
+		}
+
+		return cmd.CombinedOutput()
+	}()
+
+	if c.Debug {
+		if gotErr != nil {
+			c.t.Logf("+++ ERROR output: %s - %s", gotOut, gotErr)
+		} else {
+			c.t.Logf("+++ output: %s", gotOut)
+		}
+	}
+
+	ok = c.assertErrorFn(c.t, gotErr, string(gotOut))
+	return strings.TrimSpace(string(gotOut)), ok
+}
+
+// WithQueryFlags append the test default query flags to the given args
+func (c CLIWrapper) WithQueryFlags(args ...string) []string {
+	args = append(args, "--output", "json")
+	return c.WithTargetNodeFlags(args...)
+}
+
+// WithTXFlags append the test default TX flags to the given args.
+// This includes
+// - broadcast-mode: sync
+// - output: json
+// - chain-id
+// - keyring flags
+// - target-node
+func (c CLIWrapper) WithTXFlags(args ...string) []string {
+	args = append(args,
+		"--broadcast-mode", "sync",
+		"--output", "json",
+		"--yes",
+		"--chain-id", c.chainID,
+	)
+	args = c.WithKeyringFlags(args...)
+	return c.WithTargetNodeFlags(args...)
+}
+
+// WithKeyringFlags append the test default keyring flags to the given args
+func (c CLIWrapper) WithKeyringFlags(args ...string) []string {
+	r := append(args,
+		"--home", c.homeDir,
+		"--keyring-backend", "test",
+	)
+	for _, v := range args {
+		if v == "-a" || v == "--address" { // show address only
+			return r
+		}
+	}
+	return append(r, "--output", "json")
+}
+
+// WithTargetNodeFlags append the test default target node address flags to the given args
+func (c CLIWrapper) WithTargetNodeFlags(args ...string) []string {
+	return append(args,
+		"--node", c.nodeAddress,
+	)
+}
+
+// WasmExecute send MsgExecute to a contract
+func (c CLIWrapper) WasmExecute(contractAddr, msg, from string, args ...string) string {
+	cmd := append([]string{"tx", "wasm", "execute", contractAddr, msg, "--from", from}, args...)
+	return c.Run(cmd...)
+}
+
+// AddKey add key to default keyring. Returns address
+func (c CLIWrapper) AddKey(name string) string {
+	cmd := c.WithKeyringFlags("keys", "add", name, "--no-backup")
+	out, _ := c.run(cmd)
+	addr := gjson.Get(out, "address").String()
+	require.NotEmpty(c.t, addr, "got %q", out)
+	return addr
+}
+
+// AddKeyFromSeed recovers the key from given seed and add it to default keyring. Returns address
+func (c CLIWrapper) AddKeyFromSeed(name, mnemoic string) string {
+	cmd := c.WithKeyringFlags("keys", "add", name, "--recover")
+	out, _ := c.runWithInput(cmd, strings.NewReader(mnemoic))
+	addr := gjson.Get(out, "address").String()
+	require.NotEmpty(c.t, addr, "got %q", out)
+	return addr
+}
+
+// GetKeyAddr returns Acc address
+func (c CLIWrapper) GetKeyAddr(name string) string {
+	cmd := c.WithKeyringFlags("keys", "show", name, "-a")
+	out, _ := c.run(cmd)
+	addr := strings.Trim(out, "\n")
+	require.NotEmpty(c.t, addr, "got %q", out)
+	return addr
+}
+
+// GetKeyAddrPrefix returns key address with Beach32 prefix encoding for a key (acc|val|cons)
+func (c CLIWrapper) GetKeyAddrPrefix(name, prefix string) string {
+	cmd := c.WithKeyringFlags("keys", "show", name, "-a", "--bech="+prefix)
+	out, _ := c.run(cmd)
+	addr := strings.Trim(out, "\n")
+	require.NotEmpty(c.t, addr, "got %q", out)
+	return addr
+}
+
+// GetPubKeyByCustomField returns pubkey in base64 by custom field
+func (c CLIWrapper) GetPubKeyByCustomField(addr, field string) string {
+	keysListOutput := c.Keys("keys", "list")
+	keysList := gjson.Parse(keysListOutput)
+
+	var pubKeyValue string
+	keysList.ForEach(func(_, value gjson.Result) bool {
+		if value.Get(field).String() == addr {
+			pubKeyJSON := gjson.Parse(value.Get("pubkey").String())
+			pubKeyValue = pubKeyJSON.Get("key").String()
+			return false
+		}
+		return true
+	})
+	return pubKeyValue
+}
+
+const defaultSrcAddr = "node0"
+
+// FundAddress sends the token amount to the destination address
+func (c CLIWrapper) FundAddress(destAddr, amount string) string {
+	require.NotEmpty(c.t, destAddr)
+	require.NotEmpty(c.t, amount)
+	cmd := []string{"tx", "bank", "send", defaultSrcAddr, destAddr, amount}
+	rsp := c.Run(cmd...)
+	RequireTxSuccess(c.t, rsp)
+	return rsp
+}
+
+// QueryBalances queries all balances for an account. Returns json response
+// Example:`{"balances":[{"denom":"node0token","amount":"1000000000"},{"denom":"stake","amount":"400000003"}],"pagination":{}}`
+func (c CLIWrapper) QueryBalances(addr string) string {
+	return c.CustomQuery("q", "bank", "balances", addr)
+}
+
+// QueryBalance returns balance amount for given denom.
+// 0 when not found
+func (c CLIWrapper) QueryBalance(addr, denom string) int64 {
+	raw := c.CustomQuery("q", "bank", "balance", addr, denom)
+	require.Contains(c.t, raw, "amount", raw)
+	return gjson.Get(raw, "balance.amount").Int()
+}
+
+// QueryTotalSupply returns total amount of tokens for a given denom.
+// 0 when not found
+func (c CLIWrapper) QueryTotalSupply(denom string) int64 {
+	raw := c.CustomQuery("q", "bank", "total-supply")
+	require.Contains(c.t, raw, "amount", raw)
+	return gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", denom)).Int()
+}
+
+// SubmitGovProposal submit a gov v1 proposal
+func (c CLIWrapper) SubmitGovProposal(proposalJson string, args ...string) string {
+	if len(args) == 0 {
+		args = []string{"--from=" + defaultSrcAddr}
+	}
+
+	pathToProposal := filepath.Join(c.t.TempDir(), "proposal.json")
+	err := os.WriteFile(pathToProposal, []byte(proposalJson), os.FileMode(0o744))
+	require.NoError(c.t, err)
+	c.t.Log("Submit upgrade proposal")
+	return c.Run(append([]string{"tx", "gov", "submit-proposal", pathToProposal}, args...)...)
+}
+
+// SubmitAndVoteGovProposal submit proposal, let all validators vote yes and return proposal id
+func (c CLIWrapper) SubmitAndVoteGovProposal(proposalJson string, args ...string) string {
+	rsp := c.SubmitGovProposal(proposalJson, args...)
+	RequireTxSuccess(c.t, rsp)
+	raw := c.CustomQuery("q", "gov", "proposals", "--depositor", c.GetKeyAddr(defaultSrcAddr))
+	proposals := gjson.Get(raw, "proposals.#.id").Array()
+	require.NotEmpty(c.t, proposals, raw)
+	ourProposalID := proposals[len(proposals)-1].String() // last is ours
+	for i := 0; i < c.nodesCount; i++ {
+		go func(i int) { // do parallel
+			c.t.Logf("Voting: validator %d\n", i)
+			rsp = c.Run("tx", "gov", "vote", ourProposalID, "yes", "--from", c.GetKeyAddr(fmt.Sprintf("node%d", i)))
+			RequireTxSuccess(c.t, rsp)
+		}(i)
+	}
+	return ourProposalID
+}
+
+func (c CLIWrapper) ChainID() string {
+	return c.chainID
+}
+
+// Version returns the current version of the client binary
+func (c CLIWrapper) Version() string {
+	v, ok := c.run([]string{"version"})
+	require.True(c.t, ok)
+	return v
+}
+
+// RequireTxSuccess require the received response to contain the success code
+func RequireTxSuccess(t *testing.T, got string) {
+	t.Helper()
+	code, details := parseResultCode(t, got)
+	require.Equal(t, int64(0), code, "non success tx code : %s", details)
+}
+
+// RequireTxFailure require the received response to contain any failure code and the passed msgs
+// From CometBFT v1, an RPC error won't return ABCI response, and error must be parsed
+func RequireTxFailure(t *testing.T, got string, containsMsgs ...string) {
+	t.Helper()
+	if strings.Contains(got, "broadcast error on transaction validation") {
+		return // tx is invalid, no need to parse
+	}
+
+	code, details := parseResultCode(t, got)
+	require.NotEqual(t, int64(0), code, details)
+	for _, msg := range containsMsgs {
+		require.Contains(t, details, msg)
+	}
+}
+
+func parseResultCode(t *testing.T, got string) (int64, string) {
+	t.Helper()
+	code := gjson.Get(got, "code")
+	require.True(t, code.Exists(), "got response: %q", got)
+
+	details := got
+	if log := gjson.Get(got, "raw_log"); log.Exists() && len(log.String()) != 0 {
+		details = log.String()
+	}
+	return code.Int(), details
+}
+
+var (
+	// ErrOutOfGasMatcher requires error with "out of gas" message
+	ErrOutOfGasMatcher RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const oogMsg = "out of gas"
+		return expErrWithMsg(t, err, args, oogMsg)
+	}
+	// ErrTimeoutMatcher requires time out message
+	ErrTimeoutMatcher RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const expMsg = "timed out waiting for tx to be included in a block"
+		return expErrWithMsg(t, err, args, expMsg)
+	}
+	// ErrPostFailedMatcher requires post failed
+	ErrPostFailedMatcher RunErrorAssert = func(t assert.TestingT, err error, args ...interface{}) bool {
+		const expMsg = "post failed"
+		return expErrWithMsg(t, err, args, expMsg)
+	}
+)
+
+func expErrWithMsg(t assert.TestingT, err error, args []interface{}, expMsg string) bool {
+	if ok := assert.Error(t, err, args); !ok {
+		return false
+	}
+	var found bool
+	for _, v := range args {
+		if strings.Contains(fmt.Sprintf("%s", v), expMsg) {
+			found = true
+			break
+		}
+	}
+	assert.True(t, found, "expected %q but got: %s", expMsg, args)
+	return false // always abort
+}
--- a/systemtests/genesis_io.go
+++ b/systemtests/genesis_io.go
@ -0,0 +1,55 @@
+package systemtests
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+
+	sdkmath "cosmossdk.io/math"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+// SetConsensusMaxGas max gas that can be consumed in a block
+func SetConsensusMaxGas(t *testing.T, max int) GenesisMutator {
+	t.Helper()
+	return func(genesis []byte) []byte {
+		state, err := sjson.SetRawBytes(genesis, "consensus.params.block.max_gas", []byte(fmt.Sprintf(`"%d"`, max)))
+		require.NoError(t, err)
+		return state
+	}
+}
+
+func SetGovVotingPeriod(t *testing.T, period time.Duration) GenesisMutator {
+	t.Helper()
+	return func(genesis []byte) []byte {
+		state, err := sjson.SetRawBytes(genesis, "app_state.gov.params.voting_period", []byte(fmt.Sprintf("%q", period.String())))
+		require.NoError(t, err)
+		return state
+	}
+}
+
+func SetGovExpeditedVotingPeriod(t *testing.T, period time.Duration) GenesisMutator {
+	t.Helper()
+	return func(genesis []byte) []byte {
+		state, err := sjson.SetRawBytes(genesis, "app_state.gov.params.expedited_voting_period", []byte(fmt.Sprintf("%q", period.String())))
+		require.NoError(t, err)
+		return state
+	}
+}
+
+// GetGenesisBalance return the balance amount for an address from the given genesis json
+func GetGenesisBalance(rawGenesis []byte, addr string) sdk.Coins {
+	var r []sdk.Coin
+	balances := gjson.GetBytes(rawGenesis, fmt.Sprintf(`app_state.bank.balances.#[address==%q]#.coins`, addr)).Array()
+	for _, coins := range balances {
+		for _, coin := range coins.Array() {
+			r = append(r, sdk.NewCoin(coin.Get("denom").String(), sdkmath.NewInt(coin.Get("amount").Int())))
+		}
+	}
+	return r
+}
--- a/systemtests/getting_started.md
+++ b/systemtests/getting_started.md
@ -0,0 +1,215 @@
+# Getting started with a new system test
+
+## Preparation
+
+Build a new binary from current branch and copy it to the `tests/systemtests/binaries` folder by running system tests.
+In project root:
+
+```shell
+make test-system
+```
+
+Or via manual steps
+
+```shell
+make build
+mkdir -p ./tests/systemtests/binaries
+cp ./build/simd ./tests/systemtests/binaries/
+```
+
+## Part 1: Writing the first system test
+
+Switch to the `tests/systemtests` folder to work from here.
+
+If there is no test file matching your use case, start a new test file here.
+for example `bank_test.go` to begin with:
+
+```go
+//go:build system_test
+
+package systemtests
+
+import (
+	"testing"
+)
+
+func TestQueryTotalSupply(t *testing.T) {
+	sut.ResetChain(t)
+	sut.StartChain(t)
+
+	cli := NewCLIWrapper(t, sut, verbose)
+	raw := cli.CustomQuery("q", "bank", "total-supply")
+	t.Log("### got: " + raw)
+}
+```
+
+The file begins with a Go build tag to exclude it from regular go test runs.
+All tests in the `systemtests` folder build upon the *test runner* initialized in `main_test.go`.
+This gives you a multi node chain started on your box.
+It is a good practice to reset state in the beginning so that you have a stable base.
+
+The system tests framework comes with a CLI wrapper that makes it easier to interact or parse results.
+In this example we want to execute `simd q bank total-supply --output json --node tcp://localhost:26657` which queries
+the bank module.
+Then print the result to for the next steps
+
+### Run the test
+
+```shell
+go test -mod=readonly -tags='system_test' -v ./...  --run TestQueryTotalSupply --verbose 
+```
+
+This give very verbose output. You would see all simd CLI commands used for starting the server or by the client to interact.
+In the example code, we just log the output. Watch out for 
+
+```shell
+    bank_test.go:15: ### got: {
+          "supply": [
+            {
+              "denom": "stake",
+              "amount": "2000000190"
+            },
+            {
+              "denom": "testtoken",
+              "amount": "4000000000"
+            }
+          ],
+          "pagination": {
+            "total": "2"
+          }
+        }
+```
+
+At the end is a tail from the server log printed. This can sometimes be handy when debugging issues.
+
+
+### Tips
+
+* Passing `--nodes-count=1` overwrites the default node count and can speed up your test for local runs
+
+## Part 2: Working with json
+
+When we have a json response, the [gjson](https://github.com/tidwall/gjson) lib can shine. It comes with jquery like
+syntax that makes it easy to navigation within the document.
+
+For example `gjson.Get(raw, "supply").Array()` gives us all the children to `supply` as an array.
+Or `gjson.Get("supply.#(denom==stake).amount").Int()` for the amount of the stake token as int64 type.
+
+In order to test our assumptions in the system test, we modify the code to use `gjson` to fetch the data:
+
+```go
+	raw := cli.CustomQuery("q", "bank", "total-supply")
+
+	exp := map[string]int64{
+        "stake":     int64(500000000 * sut.nodesCount),
+        "testtoken": int64(1000000000 * sut.nodesCount),
+	}
+	require.Len(t, gjson.Get(raw, "supply").Array(), len(exp), raw)
+
+	for k, v := range exp {
+		got := gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", k)).Int()
+		assert.Equal(t, v, got, raw)
+	}
+```
+
+The assumption on the staking token usually fails due to inflation minted on the staking token. Let's fix this in the next step 
+
+### Run the test
+
+```shell
+go test -mod=readonly -tags='system_test' -v ./...  --run TestQueryTotalSupply --verbose 
+```
+
+### Tips
+
+* Putting the `raw` json response to the assert/require statements helps with debugging on failures. You are usually lacking
+  context when you look at the values only.
+
+
+## Part 3: Setting state via genesis
+
+First step is to disable inflation. This can be done via the `ModifyGenesisJSON` helper. But to add some complexity, 
+we also introduce a new token and update the balance of the account for key `node0`.
+The setup code looks quite big and unreadable now. Usually a good time to think about extracting helper functions for
+common operations. The `genesis_io.go` file contains some examples already. I would skip this and take this to showcase the mix
+of `gjson`, `sjson` and stdlib json operations.
+
+```go
+	sut.ResetChain(t)
+    cli := NewCLIWrapper(t, sut, verbose)
+
+	sut.ModifyGenesisJSON(t, func(genesis []byte) []byte {
+		// disable inflation
+		genesis, err := sjson.SetRawBytes(genesis, "app_state.mint.minter.inflation", []byte(`"0.000000000000000000"`))
+		require.NoError(t, err)
+
+		// add new token to supply
+		var supply []json.RawMessage
+		rawSupply := gjson.Get(string(genesis), "app_state.bank.supply").String()
+		require.NoError(t, json.Unmarshal([]byte(rawSupply), &supply))
+		supply = append(supply, json.RawMessage(`{"denom": "mytoken","amount": "1000000"}`))
+		newSupply, err := json.Marshal(supply)
+		require.NoError(t, err)
+		genesis, err = sjson.SetRawBytes(genesis, "app_state.bank.supply", newSupply)
+		require.NoError(t, err)
+
+		// add amount to any balance
+		anyAddr := cli.GetKeyAddr("node0")
+		newBalances := GetGenesisBalance(genesis, anyAddr).Add(sdk.NewInt64Coin("mytoken", 1000000))
+		newBalancesBz, err := newBalances.MarshalJSON()
+		require.NoError(t, err)
+		newState, err := sjson.SetRawBytes(genesis, fmt.Sprintf("app_state.bank.balances.#[address==%q]#.coins", anyAddr), newBalancesBz)
+		require.NoError(t, err)
+		return newState
+	})
+    sut.StartChain(t)
+```
+
+Next step is to add the new token to the assert map. But we can also make it more resilient to different node counts.
+
+```go
+	exp := map[string]int64{
+		"stake":     int64(500000000 * sut.nodesCount),
+		"testtoken": int64(1000000000 * sut.nodesCount),
+		"mytoken":   1000000,
+	}
+```
+
+```shell
+go test -mod=readonly -tags='system_test' -v ./...  --run TestQueryTotalSupply --verbose --nodes-count=1 
+```
+
+## Part 4: Set state via TX
+
+Complexer workflows and tests require modifying state on a running chain. This works only with builtin logic and operations.
+If we want to burn some of our new tokens, we need to submit a bank burn message to do this.
+The CLI wrapper works similar to the query. Just pass the parameters. It uses the `node0` key as *default*:
+
+```go
+	// and when
+	txHash := cli.Run("tx", "bank", "burn", "node0", "400000mytoken")
+	RequireTxSuccess(t, txHash)
+```
+
+`RequireTxSuccess` or `RequireTxFailure` can be used to ensure the expected result of the operation.
+Next, check that the changes are applied.
+
+```go
+	exp["mytoken"] = 600_000 // update expected state
+	raw = cli.CustomQuery("q", "bank", "total-supply")
+	for k, v := range exp {
+		got := gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", k)).Int()
+		assert.Equal(t, v, got, raw)
+	}
+	assert.Equal(t, int64(600_000), cli.QueryBalance(cli.GetKeyAddr("node0"), "mytoken"))
+```
+
+While tests are still more or less readable, it can gets harder the longer they are. I found it helpful to add
+some comments at the beginning to describe what the intention is. For example:
+
+```go
+	// scenario:
+	// given a chain with a custom token on genesis
+	// when an amount is burned
+	// then this is reflected in the total supply
+```
--- a/systemtests/go.mod
+++ b/systemtests/go.mod
@ -0,0 +1,170 @@
+module cosmossdk.io/systemtests
+
+go 1.23
+
+require (
+	cosmossdk.io/math v1.5.0
+	github.com/cometbft/cometbft v0.38.17
+	github.com/cometbft/cometbft/api v1.0.0-rc.1
+	github.com/cosmos/cosmos-sdk v0.50.11
+	github.com/creachadair/tomledit v0.0.27
+	github.com/stretchr/testify v1.10.0
+	github.com/tidwall/gjson v1.14.2
+	github.com/tidwall/sjson v1.2.5
+	google.golang.org/grpc v1.70.0
+)
+
+require (
+	buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.36.4-20241120201313-68e42a58b301.1 // indirect
+	buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.36.4-20240130113600-88ef6483f90f.1 // indirect
+	cosmossdk.io/api v0.8.2 // indirect
+	cosmossdk.io/collections v0.4.0 // indirect
+	cosmossdk.io/core v0.11.0 // indirect
+	cosmossdk.io/depinject v1.1.0 // indirect
+	cosmossdk.io/errors v1.0.1 // indirect
+	cosmossdk.io/log v1.5.0 // indirect
+	cosmossdk.io/store v1.1.1 // indirect
+	cosmossdk.io/x/tx v1.1.0 // indirect
+	filippo.io/edwards25519 v1.0.0 // indirect
+	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
+	github.com/99designs/keyring v1.2.1 // indirect
+	github.com/DataDog/datadog-go v3.2.0+incompatible // indirect
+	github.com/DataDog/zstd v1.5.6 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bgentry/speakeasy v0.2.0 // indirect
+	github.com/bytedance/sonic v1.12.8 // indirect
+	github.com/bytedance/sonic/loader v0.2.3 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudwego/base64x v0.1.5 // indirect
+	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
+	github.com/cockroachdb/pebble v1.1.2 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/cometbft/cometbft-db v0.14.1 // indirect
+	github.com/cosmos/btcutil v1.0.5 // indirect
+	github.com/cosmos/cosmos-db v1.1.1 // indirect
+	github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
+	github.com/cosmos/go-bip39 v1.0.0 // indirect
+	github.com/cosmos/gogogateway v1.2.0 // indirect
+	github.com/cosmos/gogoproto v1.7.0 // indirect
+	github.com/cosmos/iavl v1.2.2 // indirect
+	github.com/cosmos/ics23/go v0.11.0 // indirect
+	github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect
+	github.com/danieljoos/wincred v1.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
+	github.com/dgraph-io/badger/v4 v4.2.0 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
+	github.com/emicklei/dot v1.6.2 // indirect
+	github.com/fatih/color v1.18.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/getsentry/sentry-go v0.30.0 // indirect
+	github.com/go-kit/kit v0.13.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
+	github.com/gogo/googleapis v1.4.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.2.4 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/orderedcode v0.0.1 // indirect
+	github.com/gorilla/handlers v1.5.1 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
+	github.com/hashicorp/go-hclog v1.6.3 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-metrics v0.5.4 // indirect
+	github.com/hashicorp/go-plugin v1.6.2 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/yamux v0.1.2 // indirect
+	github.com/hdevalence/ed25519consensus v0.1.0 // indirect
+	github.com/huandu/skiplist v1.2.1 // indirect
+	github.com/iancoleman/strcase v0.3.0 // indirect
+	github.com/improbable-eng/grpc-web v0.15.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/linxGnu/grocksdb v1.9.7 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/highwayhash v1.0.3 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mtibben/percent v0.2.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect
+	github.com/oklog/run v1.1.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/rs/cors v1.11.1 // indirect
+	github.com/rs/zerolog v1.33.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sasha-s/go-deadlock v0.3.5 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
+	github.com/tendermint/go-amino v0.16.0 // indirect
+	github.com/tidwall/btree v1.7.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/zondax/hid v0.9.2 // indirect
+	github.com/zondax/ledger-go v0.14.3 // indirect
+	go.etcd.io/bbolt v1.4.0-alpha.0.0.20240404170359-43604f3112c5 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/arch v0.13.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4 // indirect
+	google.golang.org/protobuf v1.36.4 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.5.1 // indirect
+	nhooyr.io/websocket v1.8.6 // indirect
+	pgregory.net/rapid v1.1.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)
--- a/systemtests/go.sum
+++ b/systemtests/go.sum
--- a/systemtests/io_utils.go
+++ b/systemtests/io_utils.go
@ -0,0 +1,65 @@
+package systemtests
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// MustCopyFile copies the file from the source path `src` to the destination path `dest` and returns an open file handle to `dest`.
+func MustCopyFile(src, dest string) *os.File {
+	in, err := os.Open(src)
+	if err != nil {
+		panic(fmt.Sprintf("failed to open %q: %v", src, err))
+	}
+	defer in.Close()
+
+	out, err := os.Create(dest)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create %q: %v", dest, err))
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		panic(fmt.Sprintf("failed to copy from %q to %q: %v", src, dest, err))
+	}
+	return out
+}
+
+// MustCopyFilesInDir copies all files (excluding directories) from the source directory `src` to the destination directory `dest`.
+func MustCopyFilesInDir(src, dest string) {
+	err := os.MkdirAll(dest, 0o750)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create %q: %v", dest, err))
+	}
+
+	fs, err := os.ReadDir(src)
+	if err != nil {
+		panic(fmt.Sprintf("failed to read dir %q: %v", src, err))
+	}
+
+	for _, f := range fs {
+		if f.IsDir() {
+			continue
+		}
+		_ = MustCopyFile(filepath.Join(src, f.Name()), filepath.Join(dest, f.Name()))
+	}
+}
+
+// StoreTempFile creates a temporary file in the test's temporary directory with the provided content.
+// It returns a pointer to the created file. Errors during the process are handled with test assertions.
+func StoreTempFile(t *testing.T, content []byte) *os.File {
+	t.Helper()
+	out, err := os.CreateTemp(t.TempDir(), "")
+	require.NoError(t, err)
+	_, err = io.Copy(out, bytes.NewReader(content))
+	require.NoError(t, err)
+	require.NoError(t, out.Close())
+	return out
+}
--- a/systemtests/node_utils.go
+++ b/systemtests/node_utils.go
@ -0,0 +1,46 @@
+package systemtests
+
+import (
+	"bytes"
+	"path/filepath"
+	"testing"
+
+	"github.com/cometbft/cometbft/privval"
+	"github.com/stretchr/testify/require"
+
+	cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec"
+	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
+)
+
+// LoadValidatorPubKeyForNode load validator nodes consensus pub key for given node number
+func LoadValidatorPubKeyForNode(t *testing.T, sut *SystemUnderTest, nodeNumber int) cryptotypes.PubKey {
+	t.Helper()
+	return LoadValidatorPubKey(t, filepath.Join(WorkDir, sut.nodePath(nodeNumber), "config", "priv_validator_key.json"))
+}
+
+// LoadValidatorPubKey load validator nodes consensus pub key from disk
+func LoadValidatorPubKey(t *testing.T, keyFile string) cryptotypes.PubKey {
+	t.Helper()
+	filePV := privval.LoadFilePVEmptyState(keyFile, "")
+	pubKey, err := filePV.GetPubKey()
+	require.NoError(t, err)
+	valPubKey, err := cryptocodec.FromCmtPubKeyInterface(pubKey)
+	require.NoError(t, err)
+	return valPubKey
+}
+
+// QueryCometValidatorPowerForNode returns the validator's power from tendermint RPC endpoint. 0 when not found
+func QueryCometValidatorPowerForNode(t *testing.T, sut *SystemUnderTest, nodeNumber int) int64 {
+	t.Helper()
+	pubKebBz := LoadValidatorPubKeyForNode(t, sut, nodeNumber).Bytes()
+	return QueryCometValidatorPower(sut.RPCClient(t), pubKebBz)
+}
+
+func QueryCometValidatorPower(c RPCClient, pubKebBz []byte) int64 {
+	for _, v := range c.Validators() {
+		if bytes.Equal(v.PubKey.Bytes(), pubKebBz) {
+			return v.VotingPower
+		}
+	}
+	return 0
+}
--- a/systemtests/rest_support.go
+++ b/systemtests/rest_support.go
@ -0,0 +1,146 @@
+package systemtests
+
+import (
+	"io"
+	"net/http"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type RestTestCase struct {
+	Name       string
+	Url        string
+	ExpCode    int
+	ExpCodeGTE int
+	ExpOut     string
+}
+
+// RunRestQueries runs given Rest testcases by making requests and
+// checking response with expected output
+func RunRestQueries(t *testing.T, testCases ...RestTestCase) {
+	t.Helper()
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			if tc.ExpCodeGTE > 0 && tc.ExpCode > 0 {
+				require.Fail(t, "only one of ExpCode or ExpCodeGTE should be set")
+			}
+
+			var resp []byte
+			if tc.ExpCodeGTE > 0 {
+				resp = GetRequestWithHeadersGreaterThanOrEqual(t, tc.Url, nil, tc.ExpCodeGTE)
+			} else {
+				resp = GetRequestWithHeaders(t, tc.Url, nil, tc.ExpCode)
+			}
+			require.JSONEq(t, tc.ExpOut, string(resp))
+		})
+	}
+}
+
+// RunRestQueriesIgnoreNumbers runs given rest testcases by making requests and
+// checking response with expected output ignoring number values
+// This method is used when number values in response are non-deterministic
+func RunRestQueriesIgnoreNumbers(t *testing.T, testCases ...RestTestCase) {
+	t.Helper()
+
+	// regex for standalone quoted numbers (e.g., "-3" or "0.02")
+	standaloneQuotedNumberRegex := regexp.MustCompile(`"(-?\d+(\.\d+)?)"`)
+	// regex for numbers in escaped strings (e.g., \"-3\")
+	escapedNumberRegex := regexp.MustCompile(`\\\"(-?\d+(\.\d+)?)\\\"`)
+	// regex for unquoted numbers (e.g., 2, -1, 0.02,)
+	unquotedNumberRegex := regexp.MustCompile(`\b-?\d+(\.\d+)?\b,`)
+
+	replaceNumber := func(input string) string {
+		// handle numbers in escaped strings
+		result := escapedNumberRegex.ReplaceAllStringFunc(input, func(match string) string {
+			// replace with escaped "NUMBER"
+			return `\"NUMBER\"`
+		})
+
+		// handle standalone quoted numbers
+		result = standaloneQuotedNumberRegex.ReplaceAllStringFunc(result, func(match string) string {
+			// replace with "NUMBER" (quotes preserved)
+			return `"NUMBER"`
+		})
+
+		// handle unquoted numbers
+		result = unquotedNumberRegex.ReplaceAllStringFunc(result, func(match string) string {
+			// replace with "NUMBER" (add quotes to ensure json validity)
+			return `"NUMBER",`
+		})
+
+		return result
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			if tc.ExpCodeGTE > 0 && tc.ExpCode > 0 {
+				require.Fail(t, "only one of ExpCode or ExpCodeGTE should be set")
+			}
+
+			var resp []byte
+			if tc.ExpCodeGTE > 0 {
+				resp = GetRequestWithHeadersGreaterThanOrEqual(t, tc.Url, nil, tc.ExpCodeGTE)
+			} else {
+				resp = GetRequestWithHeaders(t, tc.Url, nil, tc.ExpCode)
+			}
+
+			expectedJSON := replaceNumber(tc.ExpOut)
+			actualJSON := replaceNumber(string(resp))
+
+			// compare two jsons
+			require.JSONEq(t, expectedJSON, actualJSON)
+		})
+	}
+}
+
+func GetRequest(t *testing.T, url string) []byte {
+	t.Helper()
+	return GetRequestWithHeaders(t, url, nil, http.StatusOK)
+}
+
+func GetRequestWithHeaders(t *testing.T, url string, headers map[string]string, expCode int) []byte {
+	t.Helper()
+	req, err := http.NewRequest("GET", url, nil)
+	require.NoError(t, err)
+
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
+	httpClient := &http.Client{}
+	res, err := httpClient.Do(req)
+	require.NoError(t, err)
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	body, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+	require.Equal(t, expCode, res.StatusCode, "status code should be %d, got: %d, %s", expCode, res.StatusCode, body)
+
+	return body
+}
+
+func GetRequestWithHeadersGreaterThanOrEqual(t *testing.T, url string, headers map[string]string, expCode int) []byte {
+	t.Helper()
+	req, err := http.NewRequest("GET", url, nil)
+	require.NoError(t, err)
+
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
+	httpClient := &http.Client{}
+	res, err := httpClient.Do(req)
+	require.NoError(t, err)
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	body, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+	require.GreaterOrEqual(t, res.StatusCode, expCode, "status code should be greater or equal to %d, got: %d, %s", expCode, res.StatusCode, body)
+
+	return body
+}
--- a/systemtests/rpc_client.go
+++ b/systemtests/rpc_client.go
@ -0,0 +1,113 @@
+package systemtests
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"strconv"
+	"testing"
+
+	abci "github.com/cometbft/cometbft/api/cometbft/abci/v1"
+	rpcclient "github.com/cometbft/cometbft/rpc/client"
+	client "github.com/cometbft/cometbft/rpc/client/http"
+	cmtypes "github.com/cometbft/cometbft/types"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/cosmos/cosmos-sdk/codec"
+	"github.com/cosmos/cosmos-sdk/codec/types"
+	cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec"
+	grpctypes "github.com/cosmos/cosmos-sdk/types/grpc"
+)
+
+// RPCClient is a test helper to interact with a node via the RPC endpoint.
+type RPCClient struct {
+	client *client.HTTP
+	t      *testing.T
+}
+
+// NewRPCClient constructor
+func NewRPCClient(t *testing.T, addr string) RPCClient {
+	t.Helper()
+	httpClient, err := client.New(addr, "/websocket")
+	require.NoError(t, err)
+	require.NoError(t, httpClient.Start())
+	t.Cleanup(func() { _ = httpClient.Stop() })
+	return RPCClient{client: httpClient, t: t}
+}
+
+// Validators returns list of validators
+func (r RPCClient) Validators() []*cmtypes.Validator {
+	v, err := r.client.Validators(context.Background(), nil, nil, nil)
+	require.NoError(r.t, err)
+	return v.Validators
+}
+
+func (r RPCClient) Invoke(ctx context.Context, method string, req, reply interface{}, opts ...grpc.CallOption) error {
+	if reflect.ValueOf(req).IsNil() {
+		return errors.New("request cannot be nil")
+	}
+
+	ir := types.NewInterfaceRegistry()
+	cryptocodec.RegisterInterfaces(ir)
+	cdc := codec.NewProtoCodec(ir).GRPCCodec()
+
+	reqBz, err := cdc.Marshal(req)
+	if err != nil {
+		return err
+	}
+
+	var height int64
+	md, _ := metadata.FromOutgoingContext(ctx)
+	if heights := md.Get(grpctypes.GRPCBlockHeightHeader); len(heights) > 0 {
+		height, err := strconv.ParseInt(heights[0], 10, 64)
+		if err != nil {
+			return err
+		}
+		if height < 0 {
+			return errors.New("height must be greater than or equal to 0")
+		}
+	}
+
+	abciReq := abci.QueryRequest{
+		Path:   method,
+		Data:   reqBz,
+		Height: height,
+	}
+
+	abciOpts := rpcclient.ABCIQueryOptions{
+		Height: height,
+		Prove:  abciReq.Prove,
+	}
+
+	result, err := r.client.ABCIQueryWithOptions(ctx, abciReq.Path, abciReq.Data, abciOpts)
+	if err != nil {
+		return err
+	}
+
+	if !result.Response.IsOK() {
+		return errors.New(result.Response.String())
+	}
+
+	err = cdc.Unmarshal(result.Response.Value, reply)
+	if err != nil {
+		return err
+	}
+
+	md = metadata.Pairs(grpctypes.GRPCBlockHeightHeader, strconv.FormatInt(result.Response.Height, 10))
+	for _, callOpt := range opts {
+		header, ok := callOpt.(grpc.HeaderCallOption)
+		if !ok {
+			continue
+		}
+
+		*header.HeaderAddr = md
+	}
+
+	return types.UnpackInterfaces(reply, ir)
+}
+
+func (r RPCClient) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	return nil, errors.New("not implemented")
+}
--- a/systemtests/system.go
+++ b/systemtests/system.go
@ -0,0 +1,987 @@
+package systemtests
+
+import (
+	"bufio"
+	"bytes"
+	"container/ring"
+	"context"
+	"fmt"
+	"io"
+	"maps"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/cometbft/cometbft/libs/sync"
+	client "github.com/cometbft/cometbft/rpc/client/http"
+	ctypes "github.com/cometbft/cometbft/rpc/core/types"
+	tmtypes "github.com/cometbft/cometbft/types"
+	"github.com/creachadair/tomledit"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/sjson"
+
+	"github.com/cosmos/cosmos-sdk/server"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+var (
+	// WorkDir is the directory where tests are executed. Path should be relative to this dir
+	WorkDir string
+
+	// ExecBinaryUnversionedRegExp regular expression to extract the unversioned binary name
+	ExecBinaryUnversionedRegExp = regexp.MustCompile(`^(\w+)-?.*$`)
+
+	MaxGas = 10_000_000
+	// DefaultApiPort is the port for the node to interact with
+	DefaultApiPort       = 1317
+	DefaultRpcPort       = 26657
+	DefaultTelemetryPort = 7180
+	DefaultRestPort      = 8080
+	DefaultGrpcPort      = 9090
+	DefaultP2PPort       = 16656
+)
+
+type TestnetInitializer interface {
+	Initialize()
+}
+
+// SystemUnderTest blockchain provisioning
+type SystemUnderTest struct {
+	execBinary         string
+	blockListener      *EventListener
+	currentHeight      atomic.Int64
+	outputDir          string
+	testnetInitializer TestnetInitializer
+
+	// blockTime is the expected/desired block time. This is not going to be very precise
+	// since Tendermint consensus does not allow specifying it directly.
+	blockTime         time.Duration
+	rpcAddr           string
+	apiAddr           string
+	initialNodesCount int
+	nodesCount        int
+	minGasPrice       string
+	cleanupFn         []CleanupFn
+	outBuff           *ring.Ring
+	errBuff           *ring.Ring
+	out               io.Writer
+	verbose           bool
+	ChainStarted      bool
+	projectName       string
+	dirty             bool // requires full reset when marked dirty
+
+	pidsLock sync.RWMutex
+	pids     map[int]struct{}
+	chainID  string
+}
+
+func NewSystemUnderTest(execBinary string, verbose bool, nodesCount int, blockTime time.Duration, initer ...TestnetInitializer) *SystemUnderTest {
+	if execBinary == "" {
+		panic("executable binary name must not be empty")
+	}
+	nameTokens := ExecBinaryUnversionedRegExp.FindAllString(execBinary, 1)
+	if len(nameTokens) == 0 || nameTokens[0] == "" {
+		panic("failed to parse project name from binary")
+	}
+
+	execBinary = filepath.Join(WorkDir, "binaries", execBinary)
+	s := &SystemUnderTest{
+		chainID:           "testing",
+		execBinary:        execBinary,
+		outputDir:         "./testnet",
+		blockTime:         blockTime,
+		rpcAddr:           "tcp://localhost:26657",
+		apiAddr:           fmt.Sprintf("http://localhost:%d", DefaultApiPort),
+		initialNodesCount: nodesCount,
+		outBuff:           ring.New(100),
+		errBuff:           ring.New(100),
+		out:               os.Stdout,
+		verbose:           verbose,
+		minGasPrice:       fmt.Sprintf("0.000001%s", sdk.DefaultBondDenom),
+		projectName:       nameTokens[0],
+		pids:              make(map[int]struct{}, nodesCount),
+	}
+	if len(initer) > 0 {
+		s.testnetInitializer = initer[0]
+	} else {
+		s.testnetInitializer = NewSingleHostTestnetCmdInitializer(execBinary, WorkDir, s.chainID, s.outputDir, s.initialNodesCount, s.minGasPrice, s.CommitTimeout(), s.Log)
+	}
+	return s
+}
+
+// SetExecBinary sets the executable binary for the system under test.
+func (s *SystemUnderTest) SetExecBinary(binary string) {
+	s.execBinary = binary
+}
+
+// ExecBinary returns the path of the binary executable associated with the SystemUnderTest instance.
+func (s *SystemUnderTest) ExecBinary() string {
+	return s.execBinary
+}
+
+// SetTestnetInitializer sets the initializer for the testnet configuration.
+func (s *SystemUnderTest) SetTestnetInitializer(testnetInitializer TestnetInitializer) {
+	s.testnetInitializer = testnetInitializer
+}
+
+// TestnetInitializer returns the testnet initializer associated with the SystemUnderTest.
+func (s *SystemUnderTest) TestnetInitializer() TestnetInitializer {
+	return s.testnetInitializer
+}
+
+// CommitTimeout returns the max time to wait for a commit. Default to 90% of block time
+func (s *SystemUnderTest) CommitTimeout() time.Duration {
+	// The commit timeout is a lower bound for the block time. We try to set it to a level that allows us to reach the expected block time.
+	return time.Duration((int64(s.blockTime) * 90) / 100) // leave 10% for all other operations
+}
+
+func (s *SystemUnderTest) SetupChain() {
+	s.Logf("Setup chain: %s\n", s.outputDir)
+	if err := os.RemoveAll(filepath.Join(WorkDir, s.outputDir)); err != nil {
+		panic(err.Error())
+	}
+	s.testnetInitializer.Initialize()
+	s.nodesCount = s.initialNodesCount
+
+	// modify genesis with system test defaults
+	src := filepath.Join(WorkDir, s.nodePath(0), "config", "genesis.json")
+	genesisBz, err := os.ReadFile(src) // #nosec G304
+	if err != nil {
+		panic(fmt.Sprintf("failed to load genesis: %s", err))
+	}
+
+	genesisBz, err = sjson.SetRawBytes(genesisBz, "consensus.params.block.max_gas", []byte(fmt.Sprintf(`"%d"`, MaxGas)))
+	if err != nil {
+		panic(fmt.Sprintf("failed to set block max gas: %s", err))
+	}
+	s.withEachNodeHome(func(i int, home string) {
+		if err := saveGenesis(home, genesisBz); err != nil {
+			panic(err)
+		}
+	})
+
+	// backup genesis
+	dest := filepath.Join(WorkDir, s.nodePath(0), "config", "genesis.json.orig")
+	MustCopyFile(src, dest)
+	// backup keyring
+	src = filepath.Join(WorkDir, s.nodePath(0), "keyring-test")
+	dest = filepath.Join(WorkDir, s.outputDir, "keyring-test")
+	MustCopyFilesInDir(src, dest)
+}
+
+func (s *SystemUnderTest) StartChain(t *testing.T, xargs ...string) {
+	t.Helper()
+	s.Log("Start chain\n")
+	s.ChainStarted = true
+	s.startNodesAsync(t, append([]string{"start", "--log_level=info", "--log_no_color"}, xargs...)...)
+
+	s.AwaitNodeUp(t, s.rpcAddr)
+
+	t.Log("Start new block listener")
+	s.blockListener = NewEventListener(t, s.rpcAddr)
+	s.cleanupFn = append(s.cleanupFn,
+		s.blockListener.Subscribe("tm.event='NewBlock'", func(e ctypes.ResultEvent) (more bool) {
+			newBlock, ok := e.Data.(tmtypes.EventDataNewBlock)
+			require.True(t, ok, "unexpected type %T", e.Data)
+			s.currentHeight.Store(newBlock.Block.Height)
+			return true
+		}),
+	)
+	s.AwaitNextBlock(t, 4e9)
+}
+
+// MarkDirty whole chain will be reset when marked dirty
+func (s *SystemUnderTest) MarkDirty() {
+	s.dirty = true
+}
+
+// IsDirty true when non default genesis or other state modification were applied that might create incompatibility for tests
+func (s *SystemUnderTest) IsDirty() bool {
+	return s.dirty
+}
+
+// watchLogs stores stdout/stderr in a file and in a ring buffer to output the last n lines on test error
+func (s *SystemUnderTest) watchLogs(node int, cmd *exec.Cmd) {
+	logfile, err := os.Create(filepath.Join(WorkDir, s.outputDir, fmt.Sprintf("node%d.out", node)))
+	if err != nil {
+		panic(fmt.Sprintf("open logfile error %#+v", err))
+	}
+
+	errReader, err := cmd.StderrPipe()
+	if err != nil {
+		panic(fmt.Sprintf("stderr reader error %#+v", err))
+	}
+	stopRingBuffer := make(chan struct{})
+	go appendToBuf(io.TeeReader(errReader, logfile), s.errBuff, stopRingBuffer)
+
+	outReader, err := cmd.StdoutPipe()
+	if err != nil {
+		panic(fmt.Sprintf("stdout reader error %#+v", err))
+	}
+	go appendToBuf(io.TeeReader(outReader, logfile), s.outBuff, stopRingBuffer)
+	s.cleanupFn = append(s.cleanupFn, func() {
+		close(stopRingBuffer)
+		_ = logfile.Close()
+	})
+}
+
+func appendToBuf(r io.Reader, b *ring.Ring, stop <-chan struct{}) {
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		select {
+		case <-stop:
+			return
+		default:
+		}
+		text := scanner.Text()
+		// filter out noise
+		if isLogNoise(text) {
+			continue
+		}
+		b.Value = text
+		b = b.Next()
+	}
+}
+
+func isLogNoise(text string) bool {
+	for _, v := range []string{
+		"\x1b[36mmodule=\x1b[0mrpc-server", // "module=rpc-server",
+	} {
+		if strings.Contains(text, v) {
+			return true
+		}
+	}
+	return false
+}
+
+// AwaitUpgradeInfo blocks util an upgrade info file is persisted to disk
+func (s *SystemUnderTest) AwaitUpgradeInfo(t *testing.T) {
+	t.Helper()
+	var found bool
+	for !found {
+		s.withEachNodeHome(func(i int, home string) {
+			_, err := os.Stat(filepath.Join(s.nodePath(0), "data", "upgrade-info.json"))
+			switch {
+			case err == nil:
+				found = true
+			case !os.IsNotExist(err):
+				t.Fatal(err.Error())
+			}
+		})
+		time.Sleep(s.blockTime / 2)
+	}
+}
+
+func (s *SystemUnderTest) AwaitChainStopped() {
+	for s.anyNodeRunning() {
+		time.Sleep(s.blockTime)
+	}
+}
+
+// AwaitNodeUp ensures the node is running
+func (s *SystemUnderTest) AwaitNodeUp(t *testing.T, rpcAddr string) {
+	t.Helper()
+	t.Logf("Await node is up: %s", rpcAddr)
+	timeout := DefaultWaitTime
+	ctx, done := context.WithTimeout(context.Background(), timeout)
+	defer done()
+
+	started := make(chan struct{})
+	go func() { // query for a non empty block on status page
+		t.Logf("Checking node status: %s\n", rpcAddr)
+		for {
+			con, err := client.New(rpcAddr, "/websocket")
+			if err != nil || con.Start() != nil {
+				time.Sleep(time.Second)
+				continue
+			}
+			result, err := con.Status(ctx)
+			if err != nil || result.SyncInfo.LatestBlockHeight < 1 {
+				_ = con.Stop()
+				continue
+			}
+			t.Logf("Node started. Current block: %d\n", result.SyncInfo.LatestBlockHeight)
+			_ = con.Stop()
+			started <- struct{}{}
+		}
+	}()
+	select {
+	case <-started:
+	case <-ctx.Done():
+		require.NoError(t, ctx.Err())
+	case <-time.NewTimer(timeout).C:
+		t.Fatalf("timeout waiting for node start: %s", timeout)
+	}
+}
+
+// StopChain stops the system under test and executes all registered cleanup callbacks
+func (s *SystemUnderTest) StopChain() {
+	s.Log("Stop chain\n")
+	if !s.ChainStarted {
+		return
+	}
+
+	for _, c := range s.cleanupFn {
+		c()
+	}
+	s.cleanupFn = nil
+	// send SIGTERM
+	s.withEachPid(func(p *os.Process) {
+		go func() {
+			if err := p.Signal(syscall.SIGTERM); err != nil {
+				s.Logf("failed to stop node with pid %d: %s\n", p.Pid, err)
+			}
+		}()
+	})
+	// give some final time to shut down
+	s.withEachPid(func(p *os.Process) {
+		time.Sleep(200 * time.Millisecond)
+	})
+	// goodbye
+	for ; s.anyNodeRunning(); time.Sleep(100 * time.Millisecond) {
+		s.withEachPid(func(p *os.Process) {
+			s.Logf("killing node %d\n", p.Pid)
+			if err := p.Kill(); err != nil {
+				s.Logf("failed to kill node with pid %d: %s\n", p.Pid, err)
+			}
+		})
+	}
+	s.ChainStarted = false
+}
+
+func (s *SystemUnderTest) withEachPid(cb func(p *os.Process)) {
+	s.pidsLock.RLock()
+	pids := maps.Keys(s.pids)
+	s.pidsLock.RUnlock()
+
+	for pid := range pids {
+		p, err := os.FindProcess(pid)
+		if err != nil {
+			continue
+		}
+		cb(p)
+	}
+}
+
+// PrintBuffer prints the chain logs to the console
+func (s *SystemUnderTest) PrintBuffer() {
+	s.outBuff.Do(func(v interface{}) {
+		if v != nil {
+			_, _ = fmt.Fprintf(s.out, "out> %s\n", v)
+		}
+	})
+	fmt.Fprint(s.out, "8< chain err -----------------------------------------\n")
+	s.errBuff.Do(func(v interface{}) {
+		if v != nil {
+			_, _ = fmt.Fprintf(s.out, "err> %s\n", v)
+		}
+	})
+}
+
+// AwaitNBlocks blocks until the current height + n block is reached. An optional timeout parameter can be passed to abort early
+func (s *SystemUnderTest) AwaitNBlocks(t *testing.T, n int64, timeout ...time.Duration) {
+	t.Helper()
+	s.AwaitBlockHeight(t, s.CurrentHeight()+n, timeout...)
+}
+
+// AwaitBlockHeight blocks until the target height is reached. An optional timeout parameter can be passed to abort early
+func (s *SystemUnderTest) AwaitBlockHeight(t *testing.T, targetHeight int64, timeout ...time.Duration) {
+	t.Helper()
+	require.Greater(t, targetHeight, s.currentHeight.Load())
+	var maxWaitTime time.Duration
+	if len(timeout) != 0 {
+		maxWaitTime = timeout[0]
+	} else {
+		maxWaitTime = time.Duration(targetHeight-s.currentHeight.Load()+3) * s.blockTime
+	}
+	abort := time.NewTimer(maxWaitTime).C
+	for {
+		select {
+		case <-abort:
+			t.Fatalf("Timeout - block %d not reached within %s", targetHeight, maxWaitTime)
+			return
+		default:
+			if current := s.AwaitNextBlock(t); current >= targetHeight {
+				return
+			}
+		}
+	}
+}
+
+// AwaitNextBlock is a first class function that any caller can use to ensure a new block was minted.
+// Returns the new height
+func (s *SystemUnderTest) AwaitNextBlock(t *testing.T, timeout ...time.Duration) int64 {
+	t.Helper()
+	maxWaitTime := s.blockTime * 3
+	if len(timeout) != 0 { // optional argument to overwrite default timeout
+		maxWaitTime = timeout[0]
+	}
+	done := make(chan int64)
+	go func() {
+		for start, current := s.currentHeight.Load(), s.currentHeight.Load(); current == start; current = s.currentHeight.Load() {
+			time.Sleep(s.blockTime)
+		}
+		done <- s.currentHeight.Load()
+		close(done)
+	}()
+	select {
+	case v := <-done:
+		return v
+	case <-time.NewTimer(maxWaitTime).C:
+		t.Fatalf("Timeout - no block within %s", maxWaitTime)
+		return -1
+	}
+}
+
+// ResetDirtyChain reset chain when non default setup or state (dirty)
+func (s *SystemUnderTest) ResetDirtyChain(t *testing.T) {
+	t.Helper()
+	if s.IsDirty() {
+		s.ResetChain(t)
+	}
+}
+
+// ResetChain stops and clears all nodes state via 'unsafe-reset-all'
+func (s *SystemUnderTest) ResetChain(t *testing.T) {
+	t.Helper()
+	t.Log("Reset chain")
+	s.StopChain()
+	restoreOriginalGenesis(t, s)
+	restoreOriginalKeyring(t, s)
+	s.resetBuffers()
+
+	// remove all additional nodes
+	for i := s.initialNodesCount; i < s.nodesCount; i++ {
+		_ = os.RemoveAll(filepath.Join(WorkDir, s.nodePath(i)))
+		_ = os.Remove(filepath.Join(WorkDir, s.outputDir, fmt.Sprintf("node%d.out", i)))
+	}
+	s.nodesCount = s.initialNodesCount
+
+	// reset all validator nodes
+	s.ForEachNodeExecAndWait(t, []string{"comet", "unsafe-reset-all"})
+	s.currentHeight.Store(0)
+	s.dirty = false
+}
+
+// ModifyGenesisCLI executes the CLI commands to modify the genesis
+func (s *SystemUnderTest) ModifyGenesisCLI(t *testing.T, cmds ...[]string) {
+	t.Helper()
+	s.ForEachNodeExecAndWait(t, cmds...)
+	s.MarkDirty()
+}
+
+type GenesisMutator func([]byte) []byte
+
+// ModifyGenesisJSON resets the chain and executes the callbacks to update the json representation
+// The mutator callbacks after each other receive the genesis as raw bytes and return the updated genesis for the next.
+// example:
+//
+//	return func(genesis []byte) []byte {
+//		val, _ := json.Marshal(sdk.NewDecCoins(fees...))
+//		state, _ := sjson.SetRawBytes(genesis, "app_state.globalfee.params.minimum_gas_prices", val)
+//		return state
+//	}
+func (s *SystemUnderTest) ModifyGenesisJSON(t *testing.T, mutators ...GenesisMutator) {
+	t.Helper()
+	s.ResetChain(t)
+	s.modifyGenesisJSON(t, mutators...)
+}
+
+// modify json without enforcing a reset
+func (s *SystemUnderTest) modifyGenesisJSON(t *testing.T, mutators ...GenesisMutator) {
+	t.Helper()
+	require.Empty(t, s.currentHeight.Load(), "forced chain reset required")
+	current, err := os.ReadFile(filepath.Join(WorkDir, s.nodePath(0), "config", "genesis.json"))
+	require.NoError(t, err)
+	for _, m := range mutators {
+		current = m(current)
+	}
+	out := StoreTempFile(t, current)
+	defer os.Remove(out.Name())
+	s.setGenesis(t, out.Name())
+	s.MarkDirty()
+}
+
+// ReadGenesisJSON returns current genesis.json content as raw string
+func (s *SystemUnderTest) ReadGenesisJSON(t *testing.T) string {
+	t.Helper()
+	content, err := os.ReadFile(filepath.Join(WorkDir, s.nodePath(0), "config", "genesis.json"))
+	require.NoError(t, err)
+	return string(content)
+}
+
+// setGenesis copy genesis file to all nodes
+func (s *SystemUnderTest) setGenesis(t *testing.T, srcPath string) {
+	t.Helper()
+	in, err := os.Open(srcPath)
+	require.NoError(t, err)
+	defer in.Close()
+	var buf bytes.Buffer
+
+	_, err = io.Copy(&buf, in)
+	require.NoError(t, err)
+
+	s.withEachNodeHome(func(i int, home string) {
+		require.NoError(t, saveGenesis(home, buf.Bytes()))
+	})
+}
+
+func saveGenesis(home string, content []byte) error {
+	out, err := os.Create(filepath.Join(WorkDir, home, "config", "genesis.json"))
+	if err != nil {
+		return fmt.Errorf("out file: %w", err)
+	}
+	defer out.Close()
+
+	if _, err = io.Copy(out, bytes.NewReader(content)); err != nil {
+		return fmt.Errorf("write out file: %w", err)
+	}
+
+	if err = out.Close(); err != nil {
+		return fmt.Errorf("close out file: %w", err)
+	}
+	return nil
+}
+
+// ForEachNodeExecAndWait runs the given app executable commands for all cluster nodes synchronously
+// The commands output is returned for each node.
+func (s *SystemUnderTest) ForEachNodeExecAndWait(t *testing.T, cmds ...[]string) [][]string {
+	t.Helper()
+	result := make([][]string, s.nodesCount)
+	s.withEachNodeHome(func(i int, home string) {
+		result[i] = make([]string, len(cmds))
+		for j, xargs := range cmds {
+			xargs = append(xargs, "--home", home)
+			s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(xargs, " "))
+			out := MustRunShellCmd(t, s.execBinary, xargs...)
+			s.Logf("Result: %s\n", out)
+			result[i][j] = out
+		}
+	})
+	return result
+}
+
+func MustRunShellCmd(t *testing.T, cmd string, args ...string) string {
+	t.Helper()
+	out, err := RunShellCmd(cmd, args...)
+	require.NoError(t, err)
+	return out
+}
+
+func RunShellCmd(cmd string, args ...string) (string, error) {
+	c := exec.Command( //nolint:gosec // used by tests only
+		locateExecutable(cmd),
+		args...,
+	)
+	c.Dir = WorkDir
+	out, err := c.Output()
+	if err != nil {
+		return string(out), fmt.Errorf("run `%s %s`: out: %s: %w", cmd, strings.Join(args, " "), string(out), err)
+	}
+	return string(out), nil
+}
+
+// startNodesAsync runs the given app cli command for all cluster nodes and returns without waiting
+func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) {
+	t.Helper()
+	s.withEachNodeHome(func(i int, home string) {
+		args := append(xargs, "--home="+home)
+		s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " "))
+		cmd := exec.Command( //nolint:gosec // used by tests only
+			locateExecutable(s.execBinary),
+			args...,
+		)
+		cmd.Dir = WorkDir
+		s.watchLogs(i, cmd)
+		require.NoError(t, cmd.Start(), "node %d", i)
+		s.Logf("Node started: %d\n", cmd.Process.Pid)
+
+		// cleanup when stopped
+		s.awaitProcessCleanup(cmd)
+	})
+}
+
+// tracks the PID in state with a go routine waiting for the shutdown completion to unregister
+func (s *SystemUnderTest) awaitProcessCleanup(cmd *exec.Cmd) {
+	pid := cmd.Process.Pid
+	s.pidsLock.Lock()
+	s.pids[pid] = struct{}{}
+	s.pidsLock.Unlock()
+	go func() {
+		_ = cmd.Wait() // blocks until shutdown
+		s.Logf("Node stopped: %d\n", pid)
+		s.pidsLock.Lock()
+		delete(s.pids, pid)
+		s.pidsLock.Unlock()
+	}()
+}
+
+func (s *SystemUnderTest) withEachNodeHome(cb func(i int, home string)) {
+	for i := 0; i < s.nodesCount; i++ {
+		cb(i, s.nodePath(i))
+	}
+}
+
+// NodeDir returns the workdir and path to the node home folder.
+func (s *SystemUnderTest) NodeDir(i int) string {
+	return filepath.Join(WorkDir, s.nodePath(i))
+}
+
+// nodePath returns the path of the node within the work dir. not absolute
+func (s *SystemUnderTest) nodePath(i int) string {
+	return NodePath(i, s.outputDir, s.projectName)
+}
+
+func NodePath(n int, outputDir, name string) string {
+	return fmt.Sprintf("%s/node%d/%s", outputDir, n, name)
+}
+
+func (s *SystemUnderTest) Log(msg string) {
+	if s.verbose {
+		_, _ = fmt.Fprint(s.out, msg)
+	}
+}
+
+func (s *SystemUnderTest) Logf(msg string, args ...interface{}) {
+	s.Log(fmt.Sprintf(msg, args...))
+}
+
+func (s *SystemUnderTest) RPCClient(t *testing.T) RPCClient {
+	t.Helper()
+	return NewRPCClient(t, s.rpcAddr)
+}
+
+func (s *SystemUnderTest) APIAddress() string {
+	return s.apiAddr
+}
+
+func (s *SystemUnderTest) AllPeers(t *testing.T) []string {
+	t.Helper()
+	result := make([]string, s.nodesCount)
+	for i, n := range s.AllNodes(t) {
+		result[i] = n.PeerAddr()
+	}
+	return result
+}
+
+func (s *SystemUnderTest) AllNodes(t *testing.T) []Node {
+	t.Helper()
+	return AllNodes(t, s)
+}
+
+func AllNodes(t *testing.T, s *SystemUnderTest) []Node {
+	t.Helper()
+	result := make([]Node, s.nodesCount)
+	outs := s.ForEachNodeExecAndWait(t, []string{"comet", "show-node-id"})
+	ip := "127.0.0.1"
+	if false { // is there still a use case for external ip?
+		var err error
+		ip, err = server.ExternalIP()
+		require.NoError(t, err)
+	}
+	for i, out := range outs {
+		result[i] = Node{
+			ID:      strings.TrimSpace(out[0]),
+			IP:      ip,
+			RPCPort: 26657 + i, // as defined in testnet command
+			P2PPort: 16656 + i, // as defined in testnet command
+		}
+	}
+	return result
+}
+
+func (s *SystemUnderTest) resetBuffers() {
+	s.outBuff = ring.New(100)
+	s.errBuff = ring.New(100)
+}
+
+// AddFullnode starts a new fullnode that connects to the existing chain but is not a validator.
+func (s *SystemUnderTest) AddFullnode(t *testing.T, beforeStart ...func(nodeNumber int, nodePath string)) Node {
+	t.Helper()
+	s.MarkDirty()
+	s.nodesCount++
+	nodeNumber := s.nodesCount - 1
+	nodePath := s.nodePath(nodeNumber)
+	_ = os.RemoveAll(nodePath) // drop any legacy path, just in case
+
+	// prepare new node
+	moniker := fmt.Sprintf("node%d", nodeNumber)
+	args := []string{"init", moniker, "--home=" + nodePath, "--overwrite"}
+	s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " "))
+	cmd := exec.Command( //nolint:gosec // used by tests only
+		locateExecutable(s.execBinary),
+		args...,
+	)
+	cmd.Dir = WorkDir
+	s.watchLogs(nodeNumber, cmd)
+	require.NoError(t, cmd.Run(), "failed to start node with id %d", nodeNumber)
+	require.NoError(t, saveGenesis(nodePath, []byte(s.ReadGenesisJSON(t))))
+
+	configPath := filepath.Join(WorkDir, nodePath, "config")
+
+	// start node
+	allNodes := s.AllNodes(t)
+	node := allNodes[len(allNodes)-1]
+	// quick hack: copy config and overwrite by start params
+	for _, tomlFile := range []string{"config.toml", "app.toml"} {
+		configFile := filepath.Join(configPath, tomlFile)
+		_ = os.Remove(configFile)
+		_ = MustCopyFile(filepath.Join(WorkDir, s.nodePath(0), "config", tomlFile), configFile)
+		if tomlFile == "app.toml" && IsV2() {
+			file := filepath.Join(WorkDir, s.nodePath(nodeNumber), "config", tomlFile)
+			EditToml(file, func(doc *tomledit.Document) {
+				SetValue(doc, fmt.Sprintf("%s:%d", node.IP, DefaultApiPort+nodeNumber), "grpc-gateway", "address")
+				SetValue(doc, fmt.Sprintf("%s:%d", node.IP, DefaultRestPort+nodeNumber), "rest", "address")
+				SetValue(doc, fmt.Sprintf("%s:%d", node.IP, DefaultTelemetryPort+nodeNumber), "telemetry", "address")
+			})
+		}
+	}
+	peers := make([]string, len(allNodes)-1)
+	for i, n := range allNodes[0 : len(allNodes)-1] {
+		peers[i] = n.PeerAddr()
+	}
+	for _, c := range beforeStart {
+		c(nodeNumber, nodePath)
+	}
+	args = []string{
+		"start",
+		"--p2p.persistent_peers=" + strings.Join(peers, ","),
+		fmt.Sprintf("--p2p.laddr=tcp://localhost:%d", node.P2PPort),
+		fmt.Sprintf("--rpc.laddr=tcp://localhost:%d", node.RPCPort),
+		fmt.Sprintf("--grpc.address=localhost:%d", DefaultGrpcPort+nodeNumber),
+		"--p2p.pex=false",
+		"--moniker=" + moniker,
+		"--log_level=info",
+		"--log_no_color",
+		"--home", nodePath,
+	}
+	s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " "))
+	cmd = exec.Command( //nolint:gosec // used by tests only
+		locateExecutable(s.execBinary),
+		args...,
+	)
+	cmd.Dir = WorkDir
+	s.watchLogs(nodeNumber, cmd)
+	require.NoError(t, cmd.Start(), "node %d", nodeNumber)
+	s.awaitProcessCleanup(cmd)
+	return node
+}
+
+// NewEventListener constructor for Eventlistener with system rpc address
+func (s *SystemUnderTest) NewEventListener(t *testing.T) *EventListener {
+	t.Helper()
+	return NewEventListener(t, s.rpcAddr)
+}
+
+// is any process let running?
+func (s *SystemUnderTest) anyNodeRunning() bool {
+	s.pidsLock.RLock()
+	defer s.pidsLock.RUnlock()
+	return len(s.pids) != 0
+}
+
+func (s *SystemUnderTest) CurrentHeight() int64 {
+	return s.currentHeight.Load()
+}
+
+// NodesCount returns the number of node instances used
+func (s *SystemUnderTest) NodesCount() int {
+	return s.nodesCount
+}
+
+func (s *SystemUnderTest) BlockTime() time.Duration {
+	return s.blockTime
+}
+
+type Node struct {
+	ID      string
+	IP      string
+	RPCPort int
+	P2PPort int
+}
+
+func (n Node) PeerAddr() string {
+	return fmt.Sprintf("%s@%s:%d", n.ID, n.IP, n.P2PPort)
+}
+
+func (n Node) RPCAddr() string {
+	return fmt.Sprintf("tcp://%s:%d", n.IP, n.RPCPort)
+}
+
+// locateExecutable looks up the binary on the OS path.
+func locateExecutable(file string) string {
+	if strings.TrimSpace(file) == "" {
+		panic("executable binary name must not be empty")
+	}
+	path, err := exec.LookPath(file)
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error with file %q: %s", file, err.Error()))
+	}
+	if path == "" {
+		panic(fmt.Sprintf("%q not found", file))
+	}
+	return path
+}
+
+// EventListener watches for events on the chain
+type EventListener struct {
+	t      *testing.T
+	client *client.HTTP
+}
+
+// NewEventListener event listener
+func NewEventListener(t *testing.T, rpcAddr string) *EventListener {
+	t.Helper()
+	httpClient, err := client.New(rpcAddr, "/websocket")
+	require.NoError(t, err)
+	require.NoError(t, httpClient.Start())
+	return &EventListener{client: httpClient, t: t}
+}
+
+var DefaultWaitTime = 30 * time.Second
+
+type (
+	CleanupFn     func()
+	EventConsumer func(e ctypes.ResultEvent) (more bool)
+)
+
+// Subscribe to receive events for a topic. Does not block.
+// For query syntax See https://docs.cosmos.network/master/core/events.html#subscribing-to-events
+func (l *EventListener) Subscribe(query string, cb EventConsumer) func() {
+	ctx, done := context.WithCancel(context.Background())
+	l.t.Cleanup(done)
+	eventsChan, err := l.client.WSEvents.Subscribe(ctx, "testing", query)
+	require.NoError(l.t, err)
+	cleanup := func() {
+		ctx, _ := context.WithTimeout(ctx, DefaultWaitTime)     //nolint:govet // used in cleanup only
+		go l.client.WSEvents.Unsubscribe(ctx, "testing", query) //nolint:errcheck // used by tests only
+		done()
+	}
+	go func() {
+		for e := range eventsChan {
+			if !cb(e) {
+				return
+			}
+		}
+	}()
+	return cleanup
+}
+
+// AwaitQuery blocks and waits for a single result or timeout. This can be used with `broadcast-mode=async`.
+// For query syntax See https://docs.cosmos.network/master/core/events.html#subscribing-to-events
+func (l *EventListener) AwaitQuery(query string, optMaxWaitTime ...time.Duration) *ctypes.ResultEvent {
+	c, result := CaptureSingleEventConsumer()
+	maxWaitTime := DefaultWaitTime
+	if len(optMaxWaitTime) != 0 {
+		maxWaitTime = optMaxWaitTime[0]
+	}
+	cleanupFn := l.Subscribe(query, TimeoutConsumer(l.t, maxWaitTime, c))
+	l.t.Cleanup(cleanupFn)
+	return result
+}
+
+// TimeoutConsumer is an event consumer decorator with a max wait time. Panics when wait time exceeded without
+// a result returned
+func TimeoutConsumer(t *testing.T, maxWaitTime time.Duration, next EventConsumer) EventConsumer {
+	t.Helper()
+	ctx, done := context.WithCancel(context.Background())
+	t.Cleanup(done)
+	timeout := time.NewTimer(maxWaitTime)
+	timedOut := make(chan struct{}, 1)
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-timeout.C:
+			timedOut <- struct{}{}
+			close(timedOut)
+		}
+	}()
+	return func(e ctypes.ResultEvent) (more bool) {
+		select {
+		case <-timedOut:
+			t.Fatalf("Timeout waiting for new events %s", maxWaitTime)
+			return false
+		default:
+			timeout.Reset(maxWaitTime)
+			result := next(e)
+			if !result {
+				done()
+			}
+			return result
+		}
+	}
+}
+
+// CaptureSingleEventConsumer consumes one event. No timeout
+func CaptureSingleEventConsumer() (EventConsumer, *ctypes.ResultEvent) {
+	var result ctypes.ResultEvent
+	return func(e ctypes.ResultEvent) (more bool) {
+		return false
+	}, &result
+}
+
+// CaptureAllEventsConsumer is an `EventConsumer` that captures all events until `done()` is called to stop or timeout happens.
+// The consumer works async in the background and returns all the captured events when `done()` is called.
+// This can be used to verify that certain events have happened.
+// Example usage:
+//
+//		c, done := CaptureAllEventsConsumer(t)
+//		query := `tm.event='Tx'`
+//		cleanupFn := l.Subscribe(query, c)
+//		t.Cleanup(cleanupFn)
+//
+//	 // do something in your test that create events
+//
+//		assert.Len(t, done(), 1) // then verify your assumption
+func CaptureAllEventsConsumer(t *testing.T, optMaxWaitTime ...time.Duration) (c EventConsumer, done func() []ctypes.ResultEvent) {
+	t.Helper()
+	maxWaitTime := DefaultWaitTime
+	if len(optMaxWaitTime) != 0 {
+		maxWaitTime = optMaxWaitTime[0]
+	}
+	var (
+		mu             sync.Mutex
+		capturedEvents []ctypes.ResultEvent
+		exit           bool
+	)
+	collectEventsConsumer := func(e ctypes.ResultEvent) (more bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		if exit {
+			return false
+		}
+		capturedEvents = append(capturedEvents, e)
+		return true
+	}
+
+	return TimeoutConsumer(t, maxWaitTime, collectEventsConsumer), func() []ctypes.ResultEvent {
+		mu.Lock()
+		defer mu.Unlock()
+		exit = true
+		return capturedEvents
+	}
+}
+
+// restoreOriginalGenesis replace nodes genesis by the one created on setup
+func restoreOriginalGenesis(t *testing.T, s *SystemUnderTest) {
+	t.Helper()
+	src := filepath.Join(WorkDir, s.nodePath(0), "config", "genesis.json.orig")
+	s.setGenesis(t, src)
+}
+
+// restoreOriginalKeyring replaces test keyring with original
+func restoreOriginalKeyring(t *testing.T, s *SystemUnderTest) {
+	t.Helper()
+	dest := filepath.Join(WorkDir, s.outputDir, "keyring-test")
+	require.NoError(t, os.RemoveAll(dest))
+	for i := 0; i < s.initialNodesCount; i++ {
+		src := filepath.Join(WorkDir, s.nodePath(i), "keyring-test")
+		MustCopyFilesInDir(src, dest)
+	}
+}
--- a/systemtests/test_runner.go
+++ b/systemtests/test_runner.go
@ -0,0 +1,129 @@
+package systemtests
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+var (
+	Sut            *SystemUnderTest
+	Verbose        bool
+	execBinaryName string
+)
+
+func RunTests(m *testing.M) {
+	waitTime := flag.Duration("wait-time", DefaultWaitTime, "time to wait for chain events")
+	nodesCount := flag.Int("nodes-count", 4, "number of nodes in the cluster")
+	blockTime := flag.Duration("block-time", 1000*time.Millisecond, "block creation time")
+	execBinary := flag.String("binary", "simd", "executable binary for server/ client side")
+	bech32Prefix := flag.String("bech32", "cosmos", "bech32 prefix to be used with addresses")
+	flag.BoolVar(&Verbose, "verbose", false, "verbose output")
+	flag.Parse()
+
+	// fail fast on most common setup issue
+	requireEnoughFileHandlers(*nodesCount + 1) // +1 as tests may start another node
+
+	dir, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	WorkDir = dir
+	if Verbose {
+		println("Work dir: ", WorkDir)
+	}
+	initSDKConfig(*bech32Prefix)
+
+	DefaultWaitTime = *waitTime
+	if *execBinary == "" {
+		panic("executable binary name must not be empty")
+	}
+	execBinaryName = *execBinary
+
+	Sut = NewSystemUnderTest(*execBinary, Verbose, *nodesCount, *blockTime)
+	Sut.SetupChain() // setup chain and keyring
+
+	// run tests
+	exitCode := m.Run()
+
+	// postprocess
+	Sut.StopChain()
+	if Verbose || exitCode != 0 {
+		Sut.PrintBuffer()
+		printResultFlag(exitCode == 0)
+	}
+
+	os.Exit(exitCode)
+}
+
+func GetSystemUnderTest() *SystemUnderTest {
+	return Sut
+}
+
+func IsVerbose() bool {
+	return Verbose
+}
+
+func GetExecutableName() string {
+	return execBinaryName
+}
+
+// requireEnoughFileHandlers uses `ulimit`
+func requireEnoughFileHandlers(nodesCount int) {
+	ulimit, err := exec.LookPath("ulimit")
+	if err != nil || ulimit == "" { // skip when not available
+		return
+	}
+
+	cmd := exec.Command(ulimit, "-n")
+	cmd.Dir = WorkDir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error :%#+v, output: %s", err, string(out)))
+	}
+	fileDescrCount, err := strconv.Atoi(strings.Trim(string(out), " \t\n"))
+	if err != nil {
+		panic(fmt.Sprintf("unexpected error :%#+v, output: %s", err, string(out)))
+	}
+	expFH := nodesCount * 260 // random number that worked on my box
+	if fileDescrCount < expFH {
+		panic(fmt.Sprintf("Fail fast. Insufficient setup. Run 'ulimit -n %d'", expFH))
+	}
+}
+
+func initSDKConfig(bech32Prefix string) {
+	config := sdk.GetConfig()
+	config.SetBech32PrefixForAccount(bech32Prefix, bech32Prefix+sdk.PrefixPublic)
+	config.SetBech32PrefixForValidator(bech32Prefix+sdk.PrefixValidator+sdk.PrefixOperator, bech32Prefix+sdk.PrefixValidator+sdk.PrefixOperator+sdk.PrefixPublic)
+	config.SetBech32PrefixForConsensusNode(bech32Prefix+sdk.PrefixValidator+sdk.PrefixConsensus, bech32Prefix+sdk.PrefixValidator+sdk.PrefixConsensus+sdk.PrefixPublic)
+}
+
+const (
+	successFlag = `
+ ___ _   _  ___ ___ ___  ___ ___ 
+/ __| | | |/ __/ __/ _ \/ __/ __|
+\__ \ |_| | (_| (_|  __/\__ \__ \
+|___/\__,_|\___\___\___||___/___/`
+	failureFlag = `
+  __      _ _          _ 
+ / _|    (_) |        | |
+| |_ __ _ _| | ___  __| |
+|  _/ _| | | |/ _ \/ _| |
+| || (_| | | |  __/ (_| |
+|_| \__,_|_|_|\___|\__,_|`
+)
+
+func printResultFlag(ok bool) {
+	if ok {
+		fmt.Println(successFlag)
+	} else {
+		fmt.Println(failureFlag)
+	}
+}
--- a/systemtests/testnet_init.go
+++ b/systemtests/testnet_init.go
@ -0,0 +1,228 @@
+package systemtests
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cometbft/cometbft/p2p"
+	"github.com/creachadair/tomledit"
+	"github.com/creachadair/tomledit/parser"
+)
+
+// IsV2 checks if the tests run with simapp v2
+func IsV2() bool {
+	buildOptions := os.Getenv("COSMOS_BUILD_OPTIONS")
+	return strings.Contains(buildOptions, "v2")
+}
+
+// SingleHostTestnetCmdInitializer default testnet cmd that supports the --single-host param
+type SingleHostTestnetCmdInitializer struct {
+	execBinary        string
+	workDir           string
+	chainID           string
+	outputDir         string
+	initialNodesCount int
+	minGasPrice       string
+	commitTimeout     time.Duration
+	log               func(string)
+}
+
+// NewSingleHostTestnetCmdInitializer constructor
+func NewSingleHostTestnetCmdInitializer(
+	execBinary, workDir, chainID, outputDir string,
+	initialNodesCount int,
+	minGasPrice string,
+	commitTimeout time.Duration,
+	log func(string),
+) *SingleHostTestnetCmdInitializer {
+	return &SingleHostTestnetCmdInitializer{
+		execBinary:        execBinary,
+		workDir:           workDir,
+		chainID:           chainID,
+		outputDir:         outputDir,
+		initialNodesCount: initialNodesCount,
+		minGasPrice:       minGasPrice,
+		commitTimeout:     commitTimeout,
+		log:               log,
+	}
+}
+
+// InitializerWithBinary creates new SingleHostTestnetCmdInitializer from sut with given binary
+func InitializerWithBinary(binary string, sut *SystemUnderTest) TestnetInitializer {
+	return NewSingleHostTestnetCmdInitializer(
+		binary,
+		WorkDir,
+		sut.chainID,
+		sut.outputDir,
+		sut.initialNodesCount,
+		sut.minGasPrice,
+		sut.CommitTimeout(),
+		sut.Log,
+	)
+}
+
+func (s SingleHostTestnetCmdInitializer) Initialize() {
+	args := []string{
+		"testnet",
+		"init-files",
+		"--chain-id=" + s.chainID,
+		"--output-dir=" + s.outputDir,
+		"--validator-count=" + strconv.Itoa(s.initialNodesCount),
+		"--keyring-backend=test",
+		"--commit-timeout=" + s.commitTimeout.String(),
+		"--single-host",
+	}
+
+	if IsV2() {
+		args = append(args, "--server.minimum-gas-prices="+s.minGasPrice)
+	} else {
+		args = append(args, "--minimum-gas-prices="+s.minGasPrice)
+	}
+
+	s.log(fmt.Sprintf("+++ %s %s\n", s.execBinary, strings.Join(args, " ")))
+	out, err := RunShellCmd(s.execBinary, args...)
+	if err != nil {
+		panic(err)
+	}
+	s.log(out)
+}
+
+// ModifyConfigYamlInitializer testnet cmd prior to --single-host param. Modifies the toml files.
+type ModifyConfigYamlInitializer struct {
+	execBinary        string
+	workDir           string
+	chainID           string
+	outputDir         string
+	initialNodesCount int
+	minGasPrice       string
+	commitTimeout     time.Duration
+	log               func(string)
+	projectName       string
+}
+
+func NewModifyConfigYamlInitializer(exec string, s *SystemUnderTest) *ModifyConfigYamlInitializer {
+	return &ModifyConfigYamlInitializer{
+		execBinary:        exec,
+		workDir:           WorkDir,
+		chainID:           s.chainID,
+		outputDir:         s.outputDir,
+		initialNodesCount: s.initialNodesCount,
+		minGasPrice:       s.minGasPrice,
+		commitTimeout:     s.CommitTimeout(),
+		log:               s.Log,
+		projectName:       s.projectName,
+	}
+}
+
+func (s ModifyConfigYamlInitializer) Initialize() {
+	// init with legacy testnet command
+	args := []string{
+		"testnet",
+		"init-files",
+		"--chain-id=" + s.chainID,
+		"--output-dir=" + s.outputDir,
+		"--v=" + strconv.Itoa(s.initialNodesCount),
+		"--keyring-backend=test",
+	}
+
+	if IsV2() {
+		args = append(args, "--server.minimum-gas-prices="+s.minGasPrice)
+	} else {
+		args = append(args, "--minimum-gas-prices="+s.minGasPrice)
+	}
+
+	s.log(fmt.Sprintf("+++ %s %s\n", s.execBinary, strings.Join(args, " ")))
+
+	out, err := RunShellCmd(s.execBinary, args...)
+	if err != nil {
+		panic(err)
+	}
+	s.log(out)
+
+	nodeAddresses := make([]string, s.initialNodesCount)
+	for i := 0; i < s.initialNodesCount; i++ {
+		nodeDir := filepath.Join(WorkDir, NodePath(i, s.outputDir, s.projectName), "config")
+		id := string(mustV(p2p.LoadNodeKey(filepath.Join(nodeDir, "node_key.json"))).ID())
+		nodeAddresses[i] = fmt.Sprintf("%s@127.0.0.1:%d", id, DefaultP2PPort+i)
+	}
+
+	// then update configs
+	for i := 0; i < s.initialNodesCount; i++ {
+		nodeDir := filepath.Join(WorkDir, NodePath(i, s.outputDir, s.projectName), "config")
+		nodeNumber := i
+		EditToml(filepath.Join(nodeDir, "config.toml"), func(doc *tomledit.Document) {
+			UpdatePort(doc, DefaultRpcPort+i, "rpc", "laddr")
+			UpdatePort(doc, DefaultP2PPort+i, "p2p", "laddr")
+			SetBool(doc, false, "p2p", "addr_book_strict")
+			SetBool(doc, false, "p2p", "pex")
+			SetBool(doc, true, "p2p", "allow_duplicate_ip")
+			peers := make([]string, s.initialNodesCount)
+			copy(peers, nodeAddresses[0:nodeNumber])
+			copy(peers[nodeNumber:], nodeAddresses[nodeNumber+1:])
+			SetValue(doc, strings.Join(peers, ","), "p2p", "persistent_peers")
+			SetValue(doc, s.commitTimeout.String(), "consensus", "timeout_commit")
+			SetValue(doc, "goleveldb", "db_backend")
+		})
+		EditToml(filepath.Join(nodeDir, "app.toml"), func(doc *tomledit.Document) {
+			UpdatePort(doc, DefaultApiPort+i, "api", "address")
+			UpdatePort(doc, DefaultGrpcPort+i, "grpc", "address")
+		})
+	}
+}
+
+func EditToml(filename string, f func(doc *tomledit.Document)) {
+	tomlFile := mustV(os.OpenFile(filename, os.O_RDWR, 0o600))
+	defer tomlFile.Close()
+	doc := mustV(tomledit.Parse(tomlFile))
+	f(doc)
+	mustV(tomlFile.Seek(0, 0)) // reset the cursor to the beginning of the file
+	must(tomlFile.Truncate(0))
+	must(tomledit.Format(tomlFile, doc))
+}
+
+func SetBool(doc *tomledit.Document, newVal bool, xpath ...string) {
+	e := doc.First(xpath...)
+	if e == nil {
+		panic(fmt.Sprintf("not found: %v", xpath))
+	}
+	e.Value = parser.MustValue(strconv.FormatBool(newVal))
+}
+
+func SetValue(doc *tomledit.Document, newVal string, xpath ...string) {
+	e := doc.First(xpath...)
+	if e == nil {
+		panic(fmt.Sprintf("not found: %v", xpath))
+	}
+	e.Value = parser.MustValue(fmt.Sprintf("%q", newVal))
+}
+
+func UpdatePort(doc *tomledit.Document, newPort int, xpath ...string) {
+	e := doc.First(xpath...)
+	if e == nil {
+		panic(fmt.Sprintf("not found: %v", xpath))
+	}
+	data := e.Value.X.String()
+	pos := strings.LastIndexAny(data, ":")
+	if pos == -1 {
+		panic("column not found")
+	}
+	data = data[0:pos+1] + strconv.Itoa(newPort)
+	e.Value = parser.MustValue(data + "\"")
+}
+
+// mustV same as must but with value returned
+func mustV[T any](r T, err error) T {
+	must(err)
+	return r
+}
+
+// must simple panic on error for fluent calls
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
--- a/tests/systemtests/.gitignore
+++ b/tests/systemtests/.gitignore
@ -0,0 +1,3 @@
+/testnet
+/binaries
+foo/
--- a/tests/systemtests/Makefile
+++ b/tests/systemtests/Makefile
@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+
+WAIT_TIME ?= 45s
+
+all: test format
+
+test:
+	go test -mod=readonly -failfast -timeout=15m -tags='system_test' ./... --wait-time=$(WAIT_TIME)
+
+format:
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs misspell -w
+	find . -name '*.go' -type f -not -path "./vendor*" -not -path "./tests/system/vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gci write --skip-generated -s standard -s default -s "prefix(cosmossdk.io)" -s "prefix(github.com/cosmos/cosmos-sdk)" --custom-order
+
+.PHONY: all test format
--- a/tests/systemtests/go.mod
+++ b/tests/systemtests/go.mod
@ -0,0 +1,178 @@
+module cosmossdk.io/tests/systemtests
+
+go 1.23
+
+toolchain go1.24.0
+
+require (
+	github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
+	github.com/cosmos/cosmos-sdk v0.50.11
+	github.com/cosmos/gogogateway v1.2.0 // indirect
+	github.com/cosmos/gogoproto v1.7.0 // indirect
+	github.com/cosmos/iavl v1.2.2 // indirect
+	github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/stretchr/testify v1.10.0
+	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
+	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/grpc v1.70.0 // indirect
+)
+
+require (
+	cosmossdk.io/systemtests v0.0.0-00010101000000-000000000000
+	github.com/tidwall/gjson v1.14.2
+)
+
+require (
+	buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.36.4-20241120201313-68e42a58b301.1 // indirect
+	buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.36.4-20240130113600-88ef6483f90f.1 // indirect
+	cosmossdk.io/api v0.8.2 // indirect
+	cosmossdk.io/collections v0.4.0 // indirect
+	cosmossdk.io/core v0.11.0 // indirect
+	cosmossdk.io/depinject v1.1.0 // indirect
+	cosmossdk.io/errors v1.0.1 // indirect
+	cosmossdk.io/log v1.5.0 // indirect
+	cosmossdk.io/math v1.5.0 // indirect
+	cosmossdk.io/store v1.1.1 // indirect
+	cosmossdk.io/x/tx v1.1.0 // indirect
+	filippo.io/edwards25519 v1.0.0 // indirect
+	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
+	github.com/99designs/keyring v1.2.1 // indirect
+	github.com/DataDog/datadog-go v3.2.0+incompatible // indirect
+	github.com/DataDog/zstd v1.5.6 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bgentry/speakeasy v0.2.0 // indirect
+	github.com/bytedance/sonic v1.12.8 // indirect
+	github.com/bytedance/sonic/loader v0.2.3 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudwego/base64x v0.1.5 // indirect
+	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
+	github.com/cockroachdb/pebble v1.1.2 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/cometbft/cometbft v0.38.15 // indirect
+	github.com/cometbft/cometbft-db v0.14.1 // indirect
+	github.com/cometbft/cometbft/api v1.0.0-rc.1 // indirect
+	github.com/cosmos/btcutil v1.0.5 // indirect
+	github.com/cosmos/cosmos-db v1.1.1 // indirect
+	github.com/cosmos/go-bip39 v1.0.0 // indirect
+	github.com/cosmos/ics23/go v0.11.0 // indirect
+	github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect
+	github.com/creachadair/tomledit v0.0.27 // indirect
+	github.com/danieljoos/wincred v1.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
+	github.com/dgraph-io/badger/v4 v4.2.0 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/emicklei/dot v1.6.2 // indirect
+	github.com/fatih/color v1.18.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/getsentry/sentry-go v0.30.0 // indirect
+	github.com/go-kit/kit v0.13.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
+	github.com/gogo/googleapis v1.4.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.2.4 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/orderedcode v0.0.1 // indirect
+	github.com/gorilla/handlers v1.5.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
+	github.com/hashicorp/go-hclog v1.6.3 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-metrics v0.5.4 // indirect
+	github.com/hashicorp/go-plugin v1.6.2 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/yamux v0.1.2 // indirect
+	github.com/hdevalence/ed25519consensus v0.1.0 // indirect
+	github.com/huandu/skiplist v1.2.1 // indirect
+	github.com/iancoleman/strcase v0.3.0 // indirect
+	github.com/improbable-eng/grpc-web v0.15.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/linxGnu/grocksdb v1.9.7 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/highwayhash v1.0.3 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mtibben/percent v0.2.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect
+	github.com/oklog/run v1.1.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/rs/cors v1.11.1 // indirect
+	github.com/rs/zerolog v1.33.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sasha-s/go-deadlock v0.3.5 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tendermint/go-amino v0.16.0 // indirect
+	github.com/tidwall/btree v1.7.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/zondax/hid v0.9.2 // indirect
+	github.com/zondax/ledger-go v0.14.3 // indirect
+	go.etcd.io/bbolt v1.4.0-alpha.0.0.20240404170359-43604f3112c5 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/arch v0.13.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4 // indirect
+	google.golang.org/protobuf v1.36.4 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.5.1 // indirect
+	nhooyr.io/websocket v1.8.6 // indirect
+	pgregory.net/rapid v1.1.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)
+
+replace cosmossdk.io/systemtests => ../../systemtests
--- a/tests/systemtests/go.sum
+++ b/tests/systemtests/go.sum
--- a/tests/systemtests/main_test.go
+++ b/tests/systemtests/main_test.go
@ -0,0 +1,11 @@
+package systemtests
+
+import (
+	"testing"
+
+	"cosmossdk.io/systemtests"
+)
+
+func TestMain(m *testing.M) {
+	systemtests.RunTests(m)
+}
--- a/tests/systemtests/staking_test.go
+++ b/tests/systemtests/staking_test.go
@ -0,0 +1,54 @@
+package systemtests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
+
+	"cosmossdk.io/systemtests"
+)
+
+func TestStakeUnstake(t *testing.T) {
+	// Scenario:
+	// delegate tokens to validator
+	// undelegate some tokens
+	sut := systemtests.Sut
+	sut.ResetChain(t)
+
+	cli := systemtests.NewCLIWrapper(t, sut, systemtests.Verbose)
+
+	// add genesis account with some tokens
+	account1Addr := cli.AddKey("account1")
+	sut.ModifyGenesisCLI(t,
+		[]string{"genesis", "add-genesis-account", account1Addr, "10000000stake"},
+	)
+
+	sut.StartChain(t)
+
+	// query validator address to delegate tokens
+	rsp := cli.CustomQuery("q", "staking", "validators")
+	valAddr := gjson.Get(rsp, "validators.#.operator_address").Array()[0].String()
+
+	// stake tokens
+	rsp = cli.Run("tx", "staking", "delegate", valAddr, "10000stake", "--from="+account1Addr, "--fees=1stake")
+	systemtests.RequireTxSuccess(t, rsp)
+
+	t.Log(cli.QueryBalance(account1Addr, "stake"))
+	assert.Equal(t, int64(9989999), cli.QueryBalance(account1Addr, "stake"))
+
+	rsp = cli.CustomQuery("q", "staking", "delegation", account1Addr, valAddr)
+	assert.Equal(t, "10000", gjson.Get(rsp, "delegation_response.balance.amount").String(), rsp)
+	assert.Equal(t, "stake", gjson.Get(rsp, "delegation_response.balance.denom").String(), rsp)
+
+	// unstake tokens
+	rsp = cli.Run("tx", "staking", "unbond", valAddr, "5000stake", "--from="+account1Addr, "--fees=1stake")
+	systemtests.RequireTxSuccess(t, rsp)
+
+	rsp = cli.CustomQuery("q", "staking", "delegation", account1Addr, valAddr)
+	assert.Equal(t, "5000", gjson.Get(rsp, "delegation_response.balance.amount").String(), rsp)
+	assert.Equal(t, "stake", gjson.Get(rsp, "delegation_response.balance.denom").String(), rsp)
+
+	rsp = cli.CustomQuery("q", "staking", "unbonding-delegation", account1Addr, valAddr)
+	assert.Equal(t, "5000", gjson.Get(rsp, "unbond.entries.#.balance").Array()[0].String(), rsp)
+}