feat(server): in-place testnet creator (#19280)

This commit is contained in:
Adam Tucker 2024-02-12 04:53:07 -06:00 committed by GitHub
parent cfc3739033
commit 89df28ceb9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 405 additions and 51 deletions

View File

@ -52,6 +52,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
* (baseapp) [#18499](https://github.com/cosmos/cosmos-sdk/pull/18499) Add `MsgRouter` response type from message name function.
* (types) [#18768](https://github.com/cosmos/cosmos-sdk/pull/18768) Add MustValAddressFromBech32 function.
* (gRPC) [#19049](https://github.com/cosmos/cosmos-sdk/pull/19049) Add debug log prints for each gRPC request.
* (server) [#19280](https://github.com/cosmos/cosmos-sdk/pull/19280) Adds in-place testnet CLI command.
### Improvements

View File

@ -107,6 +107,11 @@ func SetChainID(chainID string) func(*BaseApp) {
return func(app *BaseApp) { app.chainID = chainID }
}
// SetStoreLoader allows customization of the rootMultiStore initialization.
func SetStoreLoader(loader StoreLoader) func(*BaseApp) {
return func(app *BaseApp) { app.SetStoreLoader(loader) }
}
// SetOptimisticExecution enables optimistic execution.
func SetOptimisticExecution(opts ...func(*oe.OptimisticExecution)) func(*BaseApp) {
return func(app *BaseApp) {

View File

@ -1,23 +1,30 @@
package server
import (
"bufio"
"context"
"fmt"
"io"
"net"
"os"
"runtime/pprof"
"strings"
"time"
"github.com/cometbft/cometbft/abci/server"
cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands"
cmtcfg "github.com/cometbft/cometbft/config"
cmtjson "github.com/cometbft/cometbft/libs/json"
"github.com/cometbft/cometbft/node"
"github.com/cometbft/cometbft/p2p"
pvm "github.com/cometbft/cometbft/privval"
cmtstate "github.com/cometbft/cometbft/proto/tendermint/state"
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
"github.com/cometbft/cometbft/proxy"
rpchttp "github.com/cometbft/cometbft/rpc/client/http"
"github.com/cometbft/cometbft/rpc/client/local"
sm "github.com/cometbft/cometbft/state"
"github.com/cometbft/cometbft/store"
cmttypes "github.com/cometbft/cometbft/types"
dbm "github.com/cosmos/cosmos-db"
"github.com/hashicorp/go-metrics"
@ -90,6 +97,14 @@ const (
// mempool flags
FlagMempoolMaxTxs = "mempool.max-txs"
// testnet keys
KeyIsTestnet = "is-testnet"
KeyNewChainID = "new-chain-ID"
KeyNewOpAddr = "new-operator-addr"
KeyNewValAddr = "new-validator-addr"
KeyUserPubKey = "user-pub-key"
KeyTriggerTestnetUpgrade = "trigger-testnet-upgrade"
)
// StartCmdOptions defines options that can be customized in `StartCmdWithOptions`,
@ -180,56 +195,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled.
},
}
cmd.Flags().Bool(flagWithComet, true, "Run abci app embedded in-process with CometBFT")
cmd.Flags().String(flagAddress, "tcp://127.0.0.1:26658", "Listen address")
cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc")
cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file")
cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)")
cmd.Flags().Uint64(FlagQueryGasLimit, 0, "Maximum gas a Rest/Grpc query can consume. Blank and 0 imply unbounded.")
cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary")
cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node")
cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node")
cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching")
cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file")
cmd.Flags().Bool(FlagTrace, false, "Provide full stack traces for errors in ABCI Log")
cmd.Flags().String(FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)")
cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')")
cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')")
cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks")
cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune CometBFT blocks")
cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled")
cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)")
cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on")
cmd.Flags().Uint(FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections")
cmd.Flags().Uint(FlagRPCReadTimeout, 10, "Define the CometBFT RPC read timeout (in seconds)")
cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the CometBFT RPC write timeout (in seconds)")
cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the CometBFT maximum request body (in bytes)")
cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)")
cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no CometBFT process is started)")
cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled")
cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on")
cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)")
cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval")
cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep")
cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree")
cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool")
cmd.Flags().Duration(FlagShutdownGrace, 0*time.Second, "On Shutdown, duration to wait for resource clean up")
// support old flags name for backwards compatibility
cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName {
if name == "with-tendermint" {
name = flagWithComet
}
return pflag.NormalizedName(name)
})
// add support for all CometBFT-specific command line options
cmtcmd.AddNodeFlags(cmd)
if opts.AddFlags != nil {
opts.AddFlags(cmd)
}
addStartNodeFlags(cmd, opts)
return cmd
}
@ -631,7 +597,15 @@ func startApp(svrCtx *Context, appCreator types.AppCreator, opts StartCmdOptions
return app, traceCleanupFn, err
}
app = appCreator(svrCtx.Logger, db, traceWriter, svrCtx.Viper)
if isTestnet, ok := svrCtx.Viper.Get(KeyIsTestnet).(bool); ok && isTestnet {
app, err = testnetify(svrCtx, home, appCreator, db, traceWriter)
if err != nil {
return app, traceCleanupFn, err
}
} else {
app = appCreator(svrCtx.Logger, db, traceWriter, svrCtx.Viper)
}
cleanupFn = func() {
traceCleanupFn()
if localErr := app.Close(); localErr != nil {
@ -640,3 +614,370 @@ func startApp(svrCtx *Context, appCreator types.AppCreator, opts StartCmdOptions
}
return app, cleanupFn, nil
}
// InPlaceTestnetCreator utilizes the provided chainID and operatorAddress as well as the local private validator key to
// control the network represented in the data folder. This is useful to create testnets nearly identical to your
// mainnet environment.
func InPlaceTestnetCreator(testnetAppCreator types.AppCreator) *cobra.Command {
opts := StartCmdOptions{}
if opts.DBOpener == nil {
opts.DBOpener = OpenDB
}
cmd := &cobra.Command{
Use: "in-place-testnet [newChainID] [newOperatorAddress]",
Short: "Create and start a testnet from current local state",
Long: `Create and start a testnet from current local state.
After utilizing this command the network will start. If the network is stopped,
the normal "start" command should be used. Re-using this command on state that
has already been modified by this command could result in unexpected behavior.
Additionally, the first block may take up to one minute to be committed, depending
on how old the block is. For instance, if a snapshot was taken weeks ago and we want
to turn this into a testnet, it is possible lots of pending state needs to be committed
(expiring locks, etc.). It is recommended that you should wait for this block to be committed
before stopping the daemon.
If the --trigger-testnet-upgrade flag is set, the upgrade handler specified by the flag will be run
on the first block of the testnet.
Regardless of whether the flag is set or not, if any new stores are introduced in the daemon being run,
those stores will be registered in order to prevent panics. Therefore, you only need to set the flag if
you want to test the upgrade handler itself.
`,
Example: "in-place-testnet localosmosis osmo12smx2wdlyttvyzvzg54y2vnqwq2qjateuf7thj",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
serverCtx := GetServerContextFromCmd(cmd)
_, err := GetPruningOptionsFromFlags(serverCtx.Viper)
if err != nil {
return err
}
clientCtx, err := client.GetClientQueryContext(cmd)
if err != nil {
return err
}
withCMT, _ := cmd.Flags().GetBool(flagWithComet)
if !withCMT {
serverCtx.Logger.Info("starting ABCI without CometBFT")
}
newChainID := args[0]
newOperatorAddress := args[1]
skipConfirmation, _ := cmd.Flags().GetBool("skip-confirmation")
if !skipConfirmation {
// Confirmation prompt to prevent accidental modification of state.
reader := bufio.NewReader(os.Stdin)
fmt.Println("This operation will modify state in your data folder and cannot be undone. Do you want to continue? (y/n)")
text, _ := reader.ReadString('\n')
response := strings.TrimSpace(strings.ToLower(text))
if response != "y" && response != "yes" {
fmt.Println("Operation canceled.")
return nil
}
}
// Set testnet keys to be used by the application.
// This is done to prevent changes to existing start API.
serverCtx.Viper.Set(KeyIsTestnet, true)
serverCtx.Viper.Set(KeyNewChainID, newChainID)
serverCtx.Viper.Set(KeyNewOpAddr, newOperatorAddress)
err = wrapCPUProfile(serverCtx, func() error {
return start(serverCtx, clientCtx, testnetAppCreator, withCMT, opts)
})
serverCtx.Logger.Debug("received quit signal")
graceDuration, _ := cmd.Flags().GetDuration(FlagShutdownGrace)
if graceDuration > 0 {
serverCtx.Logger.Info("graceful shutdown start", FlagShutdownGrace, graceDuration)
<-time.After(graceDuration)
serverCtx.Logger.Info("graceful shutdown complete")
}
return err
},
}
addStartNodeFlags(cmd, opts)
cmd.Flags().String(KeyTriggerTestnetUpgrade, "", "If set (example: \"v21\"), triggers the v21 upgrade handler to run on the first block of the testnet")
cmd.Flags().Bool("skip-confirmation", false, "Skip the confirmation prompt")
return cmd
}
// testnetify modifies both state and blockStore, allowing the provided operator address and local validator key to control the network
// that the state in the data folder represents. The chainID of the local genesis file is modified to match the provided chainID.
func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, db dbm.DB, traceWriter io.WriteCloser) (types.Application, error) {
config := ctx.Config
newChainID, ok := ctx.Viper.Get(KeyNewChainID).(string)
if !ok {
return nil, fmt.Errorf("expected string for key %s", KeyNewChainID)
}
// Modify app genesis chain ID and save to genesis file.
genFilePath := config.GenesisFile()
appGen, err := genutiltypes.AppGenesisFromFile(genFilePath)
if err != nil {
return nil, err
}
appGen.ChainID = newChainID
if err := appGen.ValidateAndComplete(); err != nil {
return nil, err
}
if err := appGen.SaveAs(genFilePath); err != nil {
return nil, err
}
// Load the comet genesis doc provider.
genDocProvider := node.DefaultGenesisDocProviderFunc(config)
// Initialize blockStore and stateDB.
blockStoreDB, err := cmtcfg.DefaultDBProvider(&cmtcfg.DBContext{ID: "blockstore", Config: config})
if err != nil {
return nil, err
}
blockStore := store.NewBlockStore(blockStoreDB)
stateDB, err := cmtcfg.DefaultDBProvider(&cmtcfg.DBContext{ID: "state", Config: config})
if err != nil {
return nil, err
}
defer blockStore.Close()
defer stateDB.Close()
privValidator := pvm.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
userPubKey, err := privValidator.GetPubKey()
if err != nil {
return nil, err
}
validatorAddress := userPubKey.Address()
if err != nil {
return nil, err
}
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: config.Storage.DiscardABCIResponses,
})
state, genDoc, err := node.LoadStateFromDBOrGenesisDocProvider(stateDB, genDocProvider)
if err != nil {
return nil, err
}
ctx.Viper.Set(KeyNewValAddr, validatorAddress)
ctx.Viper.Set(KeyUserPubKey, userPubKey)
testnetApp := testnetAppCreator(ctx.Logger, db, traceWriter, ctx.Viper)
// We need to create a temporary proxyApp to get the initial state of the application.
// Depending on how the node was stopped, the application height can differ from the blockStore height.
// This height difference changes how we go about modifying the state.
cmtApp := NewCometABCIWrapper(testnetApp)
_, context := getCtx(ctx, true)
clientCreator := proxy.NewLocalClientCreator(cmtApp)
metrics := node.DefaultMetricsProvider(config.Instrumentation)
_, _, _, _, proxyMetrics, _, _ := metrics(genDoc.ChainID)
proxyApp := proxy.NewAppConns(clientCreator, proxyMetrics)
if err := proxyApp.Start(); err != nil {
return nil, fmt.Errorf("error starting proxy app connections: %v", err)
}
res, err := proxyApp.Query().Info(context, proxy.RequestInfo)
if err != nil {
return nil, fmt.Errorf("error calling Info: %v", err)
}
err = proxyApp.Stop()
if err != nil {
return nil, err
}
appHash := res.LastBlockAppHash
appHeight := res.LastBlockHeight
var block *cmttypes.Block
switch {
case appHeight == blockStore.Height():
// This state occurs when we stop the node with the halt height flag, and need to handle differently
state.LastBlockHeight++
block = blockStore.LoadBlock(blockStore.Height())
block.AppHash = appHash
state.AppHash = appHash
case blockStore.Height() > state.LastBlockHeight:
// This state occurs when we kill the node
err = blockStore.DeleteLatestBlock()
if err != nil {
return nil, err
}
block = blockStore.LoadBlock(blockStore.Height())
default:
// If there is any other state, we just load the block
block = blockStore.LoadBlock(blockStore.Height())
}
block.ChainID = newChainID
state.ChainID = newChainID
block.LastBlockID = state.LastBlockID
block.LastCommit.BlockID = state.LastBlockID
// Create a vote from our validator
vote := cmttypes.Vote{
Type: cmtproto.PrecommitType,
Height: state.LastBlockHeight,
Round: 0,
BlockID: state.LastBlockID,
Timestamp: time.Now(),
ValidatorAddress: validatorAddress,
ValidatorIndex: 0,
Signature: []byte{},
}
// Sign the vote, and copy the proto changes from the act of signing to the vote itself
voteProto := vote.ToProto()
err = privValidator.SignVote(newChainID, voteProto)
if err != nil {
return nil, err
}
vote.Signature = voteProto.Signature
vote.Timestamp = voteProto.Timestamp
// Modify the block's lastCommit to be signed only by our validator
block.LastCommit.Signatures[0].ValidatorAddress = validatorAddress
block.LastCommit.Signatures[0].Signature = vote.Signature
block.LastCommit.Signatures = []cmttypes.CommitSig{block.LastCommit.Signatures[0]}
// Load the seenCommit of the lastBlockHeight and modify it to be signed from our validator
seenCommit := blockStore.LoadSeenCommit(state.LastBlockHeight)
seenCommit.BlockID = state.LastBlockID
seenCommit.Round = vote.Round
seenCommit.Signatures[0].Signature = vote.Signature
seenCommit.Signatures[0].ValidatorAddress = validatorAddress
seenCommit.Signatures[0].Timestamp = vote.Timestamp
seenCommit.Signatures = []cmttypes.CommitSig{seenCommit.Signatures[0]}
err = blockStore.SaveSeenCommit(state.LastBlockHeight, seenCommit)
if err != nil {
return nil, err
}
// Create ValidatorSet struct containing just our valdiator.
newVal := &cmttypes.Validator{
Address: validatorAddress,
PubKey: userPubKey,
VotingPower: 900000000000000,
}
newValSet := &cmttypes.ValidatorSet{
Validators: []*cmttypes.Validator{newVal},
Proposer: newVal,
}
// Replace all valSets in state to be the valSet with just our validator.
state.Validators = newValSet
state.LastValidators = newValSet
state.NextValidators = newValSet
state.LastHeightValidatorsChanged = blockStore.Height()
err = stateStore.Save(state)
if err != nil {
return nil, err
}
// Create a ValidatorsInfo struct to store in stateDB.
valSet, err := state.Validators.ToProto()
if err != nil {
return nil, err
}
valInfo := &cmtstate.ValidatorsInfo{
ValidatorSet: valSet,
LastHeightChanged: state.LastBlockHeight,
}
buf, err := valInfo.Marshal()
if err != nil {
return nil, err
}
// Modfiy Validators stateDB entry.
err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height())), buf)
if err != nil {
return nil, err
}
// Modify LastValidators stateDB entry.
err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height()-1)), buf)
if err != nil {
return nil, err
}
// Modify NextValidators stateDB entry.
err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height()+1)), buf)
if err != nil {
return nil, err
}
// Since we modified the chainID, we set the new genesisDoc in the stateDB.
b, err := cmtjson.Marshal(genDoc)
if err != nil {
return nil, err
}
if err := stateDB.SetSync([]byte("genesisDoc"), b); err != nil {
return nil, err
}
return testnetApp, err
}
// addStartNodeFlags should be added to any CLI commands that start the network.
func addStartNodeFlags(cmd *cobra.Command, opts StartCmdOptions) {
cmd.Flags().Bool(flagWithComet, true, "Run abci app embedded in-process with CometBFT")
cmd.Flags().String(flagAddress, "tcp://127.0.0.1:26658", "Listen address")
cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc")
cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file")
cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)")
cmd.Flags().Uint64(FlagQueryGasLimit, 0, "Maximum gas a Rest/Grpc query can consume. Blank and 0 imply unbounded.")
cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary")
cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node")
cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node")
cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching")
cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file")
cmd.Flags().Bool(FlagTrace, false, "Provide full stack traces for errors in ABCI Log")
cmd.Flags().String(FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)")
cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')")
cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')")
cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks")
cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune CometBFT blocks")
cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled")
cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)")
cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on")
cmd.Flags().Uint(FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections")
cmd.Flags().Uint(FlagRPCReadTimeout, 10, "Define the CometBFT RPC read timeout (in seconds)")
cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the CometBFT RPC write timeout (in seconds)")
cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the CometBFT maximum request body (in bytes)")
cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)")
cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no CometBFT process is started)")
cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled")
cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on")
cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)")
cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval")
cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep")
cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree")
cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool")
cmd.Flags().Duration(FlagShutdownGrace, 0*time.Second, "On Shutdown, duration to wait for resource clean up")
// support old flags name for backwards compatibility
cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName {
if name == "with-tendermint" {
name = flagWithComet
}
return pflag.NormalizedName(name)
})
// add support for all CometBFT-specific command line options
cmtcmd.AddNodeFlags(cmd)
if opts.AddFlags != nil {
opts.AddFlags(cmd)
}
}

View File

@ -354,6 +354,13 @@ func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, addStartFl
)
}
// AddTestnetCreatorCommand allows chains to create a testnet from the state existing in their node's data directory.
func AddTestnetCreatorCommand(rootCmd *cobra.Command, appCreator types.AppCreator, addStartFlags types.ModuleInitFlags) {
testnetCreateCmd := InPlaceTestnetCreator(appCreator)
addStartFlags(testnetCreateCmd)
rootCmd.AddCommand(testnetCreateCmd)
}
// https://stackoverflow.com/questions/23558425/how-do-i-get-the-local-ip-address-in-go
// TODO there must be a better way to get external IP
func ExternalIP() (string, error) {