feat(simapp/v2): Testnet init command (#20866)

This commit is contained in:
Hieu Vu 2024-07-05 15:25:54 +07:00 committed by GitHub
parent d79cd5898f
commit 476edaa90b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 587 additions and 3 deletions

17
scripts/local-testnet.sh Normal file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -x
ROOT=$PWD
SIMD="$ROOT/build/simdv2"
COSMOS_BUILD_OPTIONS=v2 make build
$SIMD testnet init-files --chain-id=testing --output-dir="$HOME/.testnet" --validator-count=3 --keyring-backend=test --minimum-gas-prices=0.000001stake --commit-timeout=900ms --single-host
$SIMD start --log_level=info --home "$HOME/.testnet/node0/simdv2" &
$SIMD start --log_level=info --home "$HOME/.testnet/node1/simdv2" &
$SIMD start --log_level=info --home "$HOME/.testnet/node2/simdv2"

View File

@ -2,7 +2,9 @@ package serverv2
import (
"context"
"errors"
"fmt"
"net"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@ -47,3 +49,59 @@ func GetLoggerFromCmd(cmd *cobra.Command) corelog.Logger {
return logger
}
// ExternalIP https://stackoverflow.com/questions/23558425/how-do-i-get-the-local-ip-address-in-go
// TODO there must be a better way to get external IP
func ExternalIP() (string, error) {
ifaces, err := net.Interfaces()
if err != nil {
return "", err
}
for _, iface := range ifaces {
if skipInterface(iface) {
continue
}
addrs, err := iface.Addrs()
if err != nil {
return "", err
}
for _, addr := range addrs {
ip := addrToIP(addr)
if ip == nil || ip.IsLoopback() {
continue
}
ip = ip.To4()
if ip == nil {
continue // not an ipv4 address
}
return ip.String(), nil
}
}
return "", errors.New("are you connected to the network?")
}
func skipInterface(iface net.Interface) bool {
if iface.Flags&net.FlagUp == 0 {
return true // interface down
}
if iface.Flags&net.FlagLoopback != 0 {
return true // loopback interface
}
return false
}
func addrToIP(addr net.Addr) net.IP {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
return ip
}

View File

@ -8,7 +8,7 @@ require (
cosmossdk.io/core v0.12.1-0.20231114100755-569e3ff6a0d7
cosmossdk.io/depinject v1.0.0-alpha.4
cosmossdk.io/log v1.3.1
cosmossdk.io/math v1.3.0 // indirect
cosmossdk.io/math v1.3.0
cosmossdk.io/runtime/v2 v2.0.0-00010101000000-000000000000
cosmossdk.io/server/v2 v2.0.0-00010101000000-000000000000
cosmossdk.io/server/v2/cometbft v0.0.0-00010101000000-000000000000
@ -31,7 +31,7 @@ require (
cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000
cosmossdk.io/x/tx v0.13.3 // indirect
cosmossdk.io/x/upgrade v0.0.0-20230613133644-0a778132a60f
github.com/cometbft/cometbft v1.0.0-rc1 // indirect
github.com/cometbft/cometbft v1.0.0-rc1
github.com/cosmos/cosmos-db v1.0.2
// this version is not used as it is always replaced by the latest Cosmos SDK version
github.com/cosmos/cosmos-sdk v0.51.0
@ -39,7 +39,7 @@ require (
github.com/golang/mock v1.6.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.19.0
github.com/stretchr/testify v1.9.0
golang.org/x/sync v0.7.0 // indirect

View File

@ -19,6 +19,7 @@ import (
"cosmossdk.io/simapp/v2"
confixcmd "cosmossdk.io/tools/confix/cmd"
authcmd "cosmossdk.io/x/auth/client/cli"
banktypes "cosmossdk.io/x/bank/types"
"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/debug"
@ -90,6 +91,7 @@ func initRootCmd[AppT serverv2.AppI[T], T transaction.Tx](
genutilcli.InitCmd(moduleManager),
debug.Cmd(),
confixcmd.ConfigCommand(),
NewTestnetCmd(moduleManager, banktypes.GenesisBalancesIterator{}),
// pruning.Cmd(newApp), // TODO add to comet server
// snapshot.Cmd(newApp), // TODO add to comet server
)

View File

@ -0,0 +1,507 @@
package cmd
import (
"bufio"
"encoding/json"
"fmt"
"net"
"os"
"path/filepath"
"time"
cmtconfig "github.com/cometbft/cometbft/config"
cmttime "github.com/cometbft/cometbft/types/time"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"cosmossdk.io/core/log"
"cosmossdk.io/core/transaction"
"cosmossdk.io/math"
"cosmossdk.io/math/unsafe"
runtimev2 "cosmossdk.io/runtime/v2"
serverv2 "cosmossdk.io/server/v2"
"cosmossdk.io/server/v2/api/grpc"
"cosmossdk.io/server/v2/cometbft"
authtypes "cosmossdk.io/x/auth/types"
banktypes "cosmossdk.io/x/bank/types"
stakingtypes "cosmossdk.io/x/staking/types"
"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/client/tx"
"github.com/cosmos/cosmos-sdk/crypto/hd"
"github.com/cosmos/cosmos-sdk/crypto/keyring"
cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
// srvconfig "github.com/cosmos/cosmos-sdk/server/config"
"github.com/cosmos/cosmos-sdk/testutil"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/version"
"github.com/cosmos/cosmos-sdk/x/genutil"
genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
)
var (
flagNodeDirPrefix = "node-dir-prefix"
flagNumValidators = "validator-count"
flagOutputDir = "output-dir"
flagNodeDaemonHome = "node-daemon-home"
flagStartingIPAddress = "starting-ip-address"
flagListenIPAddress = "listen-ip-address"
flagStakingDenom = "staking-denom"
flagCommitTimeout = "commit-timeout"
flagSingleHost = "single-host"
)
type initArgs struct {
algo string
chainID string
keyringBackend string
minGasPrices string
nodeDaemonHome string
nodeDirPrefix string
numValidators int
outputDir string
startingIPAddress string
listenIPAddress string
singleMachine bool
bondTokenDenom string
}
func addTestnetFlagsToCmd(cmd *cobra.Command) {
cmd.Flags().IntP(flagNumValidators, "n", 4, "Number of validators to initialize the testnet with")
cmd.Flags().StringP(flagOutputDir, "o", "./.testnets", "Directory to store initialization data for the testnet")
cmd.Flags().String(flags.FlagChainID, "", "genesis file chain-id, if left blank will be randomly created")
cmd.Flags().String(cometbft.FlagMinGasPrices, fmt.Sprintf("0.000006%s", sdk.DefaultBondDenom), "Minimum gas prices to accept for transactions; All fees in a tx must meet this minimum (e.g. 0.01photino,0.001stake)")
cmd.Flags().String(flags.FlagKeyType, string(hd.Secp256k1Type), "Key signing algorithm to generate keys for")
// support old flags name for backwards compatibility
cmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName {
if name == flags.FlagKeyAlgorithm {
name = flags.FlagKeyType
}
return pflag.NormalizedName(name)
})
}
// NewTestnetCmd creates a root testnet command with subcommands to run an in-process testnet or initialize
// validator configuration files for running a multi-validator testnet in a separate process
func NewTestnetCmd[T transaction.Tx](mm *runtimev2.MM[T], genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
testnetCmd := &cobra.Command{
Use: "testnet",
Short: "subcommands for starting or configuring local testnets",
DisableFlagParsing: true,
SuggestionsMinimumDistance: 2,
RunE: client.ValidateCmd,
}
testnetCmd.AddCommand(testnetInitFilesCmd(mm, genBalIterator))
return testnetCmd
}
// testnetInitFilesCmd returns a cmd to initialize all files for CometBFT testnet and application
func testnetInitFilesCmd[T transaction.Tx](mm *runtimev2.MM[T], genBalIterator banktypes.GenesisBalancesIterator) *cobra.Command {
cmd := &cobra.Command{
Use: "init-files",
Short: "Initialize config directories & files for a multi-validator testnet running locally via separate processes (e.g. Docker Compose or similar)",
Long: fmt.Sprintf(`init-files will setup one directory per validator and populate each with
necessary files (private validator, genesis, config, etc.) for running validator nodes.
Booting up a network with these validator folders is intended to be used with Docker Compose,
or a similar setup where each node has a manually configurable IP address.
Note, strict routability for addresses is turned off in the config file.
Example:
%s testnet init-files --validator-count 4 --output-dir ./.testnets --starting-ip-address 192.168.10.2
`, version.AppName),
RunE: func(cmd *cobra.Command, _ []string) error {
clientCtx, err := client.GetClientQueryContext(cmd)
if err != nil {
return err
}
config := client.GetConfigFromCmd(cmd)
args := initArgs{}
args.outputDir, _ = cmd.Flags().GetString(flagOutputDir)
args.keyringBackend, _ = cmd.Flags().GetString(flags.FlagKeyringBackend)
args.chainID, _ = cmd.Flags().GetString(flags.FlagChainID)
args.minGasPrices, _ = cmd.Flags().GetString(cometbft.FlagMinGasPrices)
args.nodeDirPrefix, _ = cmd.Flags().GetString(flagNodeDirPrefix)
args.nodeDaemonHome, _ = cmd.Flags().GetString(flagNodeDaemonHome)
args.startingIPAddress, _ = cmd.Flags().GetString(flagStartingIPAddress)
args.listenIPAddress, _ = cmd.Flags().GetString(flagListenIPAddress)
args.numValidators, _ = cmd.Flags().GetInt(flagNumValidators)
args.algo, _ = cmd.Flags().GetString(flags.FlagKeyType)
args.bondTokenDenom, _ = cmd.Flags().GetString(flagStakingDenom)
args.singleMachine, _ = cmd.Flags().GetBool(flagSingleHost)
config.Consensus.TimeoutCommit, err = cmd.Flags().GetDuration(flagCommitTimeout)
if err != nil {
return err
}
return initTestnetFiles(clientCtx, cmd, config, mm, genBalIterator, args)
},
}
addTestnetFlagsToCmd(cmd)
cmd.Flags().String(flagNodeDirPrefix, "node", "Prefix for the name of per-validator subdirectories (to be number-suffixed like node0, node1, ...)")
cmd.Flags().String(flagNodeDaemonHome, "simdv2", "Home directory of the node's daemon configuration")
cmd.Flags().String(flagStartingIPAddress, "192.168.0.1", "Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:46656, ID1@192.168.0.2:46656, ...)")
cmd.Flags().String(flagListenIPAddress, "127.0.0.1", "TCP or UNIX socket IP address for the RPC server to listen on")
cmd.Flags().String(flags.FlagKeyringBackend, flags.DefaultKeyringBackend, "Select keyring's backend (os|file|test)")
cmd.Flags().Duration(flagCommitTimeout, 5*time.Second, "Time to wait after a block commit before starting on the new height")
cmd.Flags().Bool(flagSingleHost, false, "Cluster runs on a single host machine with different ports")
cmd.Flags().String(flagStakingDenom, sdk.DefaultBondDenom, "Default staking token denominator")
return cmd
}
const nodeDirPerm = 0o755
// initTestnetFiles initializes testnet files for a testnet to be run in a separate process
func initTestnetFiles[T transaction.Tx](
clientCtx client.Context,
cmd *cobra.Command,
nodeConfig *cmtconfig.Config,
mm *runtimev2.MM[T],
genBalIterator banktypes.GenesisBalancesIterator,
args initArgs,
) error {
if args.chainID == "" {
args.chainID = "chain-" + unsafe.Str(6)
}
nodeIDs := make([]string, args.numValidators)
valPubKeys := make([]cryptotypes.PubKey, args.numValidators)
// appConfig := srvconfig.DefaultConfig()
// appConfig.MinGasPrices = args.minGasPrices
// appConfig.API.Enable = true
// appConfig.Telemetry.Enabled = true
// appConfig.Telemetry.PrometheusRetentionTime = 60
// appConfig.Telemetry.EnableHostnameLabel = false
// appConfig.Telemetry.GlobalLabels = [][]string{{"chain_id", args.chainID}}
var (
genAccounts []authtypes.GenesisAccount
genBalances []banktypes.Balance
genFiles []string
)
const (
rpcPort = 26657
apiPort = 1317
grpcPort = 9090
)
p2pPortStart := 26656
inBuf := bufio.NewReader(cmd.InOrStdin())
// generate private keys, node IDs, and initial transactions
for i := 0; i < args.numValidators; i++ {
var portOffset int
var grpcConfig *grpc.Config
if args.singleMachine {
portOffset = i
p2pPortStart = 16656 // use different start point to not conflict with rpc port
nodeConfig.P2P.AddrBookStrict = false
nodeConfig.P2P.PexReactor = false
nodeConfig.P2P.AllowDuplicateIP = true
grpcConfig = &grpc.Config{
Enable: true,
Address: fmt.Sprintf("127.0.0.1:%d", grpcPort+portOffset),
MaxRecvMsgSize: grpc.DefaultConfig().MaxRecvMsgSize,
MaxSendMsgSize: grpc.DefaultConfig().MaxSendMsgSize,
}
}
nodeDirName := fmt.Sprintf("%s%d", args.nodeDirPrefix, i)
nodeDir := filepath.Join(args.outputDir, nodeDirName, args.nodeDaemonHome)
cmd.Println("Node dir", nodeDir)
gentxsDir := filepath.Join(args.outputDir, "gentxs")
nodeConfig.SetRoot(nodeDir)
nodeConfig.Moniker = nodeDirName
nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://%s:%d", args.listenIPAddress, rpcPort+portOffset)
if err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm); err != nil {
_ = os.RemoveAll(args.outputDir)
return err
}
var (
err error
ip string
)
if args.singleMachine {
ip = "127.0.0.1"
} else {
ip, err = getIP(i, args.startingIPAddress)
if err != nil {
_ = os.RemoveAll(args.outputDir)
return err
}
}
nodeIDs[i], valPubKeys[i], err = genutil.InitializeNodeValidatorFiles(nodeConfig, args.algo)
if err != nil {
_ = os.RemoveAll(args.outputDir)
return err
}
memo := fmt.Sprintf("%s@%s:%d", nodeIDs[i], ip, p2pPortStart+portOffset)
genFiles = append(genFiles, nodeConfig.GenesisFile())
kb, err := keyring.New(sdk.KeyringServiceName(), args.keyringBackend, nodeDir, inBuf, clientCtx.Codec)
if err != nil {
return err
}
keyringAlgos, _ := kb.SupportedAlgorithms()
algo, err := keyring.NewSigningAlgoFromString(args.algo, keyringAlgos)
if err != nil {
return err
}
addr, secret, err := testutil.GenerateSaveCoinKey(kb, nodeDirName, "", true, algo, sdk.GetFullBIP44Path())
if err != nil {
_ = os.RemoveAll(args.outputDir)
return err
}
info := map[string]string{"secret": secret}
cliPrint, err := json.Marshal(info)
if err != nil {
return err
}
// save private key seed words
if err := writeFile(fmt.Sprintf("%v.json", "key_seed"), nodeDir, cliPrint); err != nil {
return err
}
accTokens := sdk.TokensFromConsensusPower(1000, sdk.DefaultPowerReduction)
accStakingTokens := sdk.TokensFromConsensusPower(500, sdk.DefaultPowerReduction)
coins := sdk.Coins{
sdk.NewCoin("testtoken", accTokens),
sdk.NewCoin(args.bondTokenDenom, accStakingTokens),
}
genBalances = append(genBalances, banktypes.Balance{Address: addr.String(), Coins: coins.Sort()})
genAccounts = append(genAccounts, authtypes.NewBaseAccount(addr, nil, 0, 0))
valStr, err := clientCtx.ValidatorAddressCodec.BytesToString(addr)
if err != nil {
return err
}
valTokens := sdk.TokensFromConsensusPower(100, sdk.DefaultPowerReduction)
createValMsg, err := stakingtypes.NewMsgCreateValidator(
valStr,
valPubKeys[i],
sdk.NewCoin(args.bondTokenDenom, valTokens),
stakingtypes.NewDescription(nodeDirName, "", "", "", ""),
stakingtypes.NewCommissionRates(math.LegacyOneDec(), math.LegacyOneDec(), math.LegacyOneDec()),
math.OneInt(),
)
if err != nil {
return err
}
txBuilder := clientCtx.TxConfig.NewTxBuilder()
if err := txBuilder.SetMsgs(createValMsg); err != nil {
return err
}
txBuilder.SetMemo(memo)
txBuilder.SetGasLimit(flags.DefaultGasLimit)
txBuilder.SetFeePayer(addr)
txFactory := tx.Factory{}
txFactory = txFactory.
WithChainID(args.chainID).
WithMemo(memo).
WithKeybase(kb).
WithTxConfig(clientCtx.TxConfig)
if err := tx.Sign(cmd.Context(), txFactory, nodeDirName, txBuilder, true); err != nil {
return err
}
txBz, err := clientCtx.TxConfig.TxJSONEncoder()(txBuilder.GetTx())
if err != nil {
return err
}
if err := writeFile(fmt.Sprintf("%v.json", nodeDirName), gentxsDir, txBz); err != nil {
return err
}
// Write server config
cometServer := cometbft.New[serverv2.AppI[T], T](&temporaryTxDecoder[T]{clientCtx.TxConfig}, cometbft.ServerOptions[T]{}, cometbft.OverwriteDefaultCometConfig(nodeConfig))
grpcServer := grpc.New[serverv2.AppI[T], T](grpc.OverwriteDefaultConfig(grpcConfig))
server := serverv2.NewServer(log.NewNopLogger(), cometServer, grpcServer)
err = server.WriteConfig(filepath.Join(nodeDir, "config"))
if err != nil {
return err
}
}
if err := initGenFiles(clientCtx, mm, args.chainID, genAccounts, genBalances, genFiles, args.numValidators); err != nil {
return err
}
err := collectGenFiles(
clientCtx, nodeConfig, args.chainID, nodeIDs, valPubKeys, args.numValidators,
args.outputDir, args.nodeDirPrefix, args.nodeDaemonHome, genBalIterator,
rpcPort, p2pPortStart, args.singleMachine,
)
if err != nil {
return err
}
// Update viper root since root dir become rootdir/node/simd
client.GetViperFromCmd(cmd).Set(flags.FlagHome, nodeConfig.RootDir)
cmd.PrintErrf("Successfully initialized %d node directories\n", args.numValidators)
return nil
}
func initGenFiles[T transaction.Tx](
clientCtx client.Context, mm *runtimev2.MM[T], chainID string,
genAccounts []authtypes.GenesisAccount, genBalances []banktypes.Balance,
genFiles []string, numValidators int,
) error {
appGenState := mm.DefaultGenesis()
// set the accounts in the genesis state
var authGenState authtypes.GenesisState
clientCtx.Codec.MustUnmarshalJSON(appGenState[authtypes.ModuleName], &authGenState)
accounts, err := authtypes.PackAccounts(genAccounts)
if err != nil {
return err
}
authGenState.Accounts = accounts
appGenState[authtypes.ModuleName] = clientCtx.Codec.MustMarshalJSON(&authGenState)
// set the balances in the genesis state
var bankGenState banktypes.GenesisState
clientCtx.Codec.MustUnmarshalJSON(appGenState[banktypes.ModuleName], &bankGenState)
bankGenState.Balances, err = banktypes.SanitizeGenesisBalances(genBalances, clientCtx.AddressCodec)
if err != nil {
return err
}
for _, bal := range bankGenState.Balances {
bankGenState.Supply = bankGenState.Supply.Add(bal.Coins...)
}
appGenState[banktypes.ModuleName] = clientCtx.Codec.MustMarshalJSON(&bankGenState)
appGenStateJSON, err := json.MarshalIndent(appGenState, "", " ")
if err != nil {
return err
}
appGenesis := genutiltypes.NewAppGenesisWithVersion(chainID, appGenStateJSON)
// generate empty genesis files for each validator and save
for i := 0; i < numValidators; i++ {
if err := appGenesis.SaveAs(genFiles[i]); err != nil {
return err
}
}
return nil
}
func collectGenFiles(
clientCtx client.Context, nodeConfig *cmtconfig.Config, chainID string,
nodeIDs []string, valPubKeys []cryptotypes.PubKey, numValidators int,
outputDir, nodeDirPrefix, nodeDaemonHome string, genBalIterator banktypes.GenesisBalancesIterator,
rpcPortStart, p2pPortStart int,
singleMachine bool,
) error {
var appState json.RawMessage
genTime := cmttime.Now()
for i := 0; i < numValidators; i++ {
if singleMachine {
portOffset := i
nodeConfig.RPC.ListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", rpcPortStart+portOffset)
nodeConfig.P2P.ListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", p2pPortStart+portOffset)
}
nodeDirName := fmt.Sprintf("%s%d", nodeDirPrefix, i)
nodeDir := filepath.Join(outputDir, nodeDirName, nodeDaemonHome)
gentxsDir := filepath.Join(outputDir, "gentxs")
nodeConfig.Moniker = nodeDirName
nodeConfig.SetRoot(nodeDir)
nodeID, valPubKey := nodeIDs[i], valPubKeys[i]
initCfg := genutiltypes.NewInitConfig(chainID, gentxsDir, nodeID, valPubKey)
appGenesis, err := genutiltypes.AppGenesisFromFile(nodeConfig.GenesisFile())
if err != nil {
return err
}
nodeAppState, err := genutil.GenAppStateFromConfig(clientCtx.Codec, clientCtx.TxConfig, nodeConfig, initCfg, appGenesis, genBalIterator, genutiltypes.DefaultMessageValidator,
clientCtx.ValidatorAddressCodec, clientCtx.AddressCodec)
if err != nil {
return err
}
if appState == nil {
// set the canonical application state (they should not differ)
appState = nodeAppState
}
genFile := nodeConfig.GenesisFile()
// overwrite each validator's genesis file to have a canonical genesis time
if err := genutil.ExportGenesisFileWithTime(genFile, chainID, nil, appState, genTime); err != nil {
return err
}
}
return nil
}
func getIP(i int, startingIPAddr string) (ip string, err error) {
if len(startingIPAddr) == 0 {
ip, err = serverv2.ExternalIP()
if err != nil {
return "", err
}
return ip, nil
}
return calculateIP(startingIPAddr, i)
}
func calculateIP(ip string, i int) (string, error) {
ipv4 := net.ParseIP(ip).To4()
if ipv4 == nil {
return "", fmt.Errorf("%v: non ipv4 address", ip)
}
for j := 0; j < i; j++ {
ipv4[3]++
}
return ipv4.String(), nil
}
func writeFile(name, dir string, contents []byte) error {
file := filepath.Join(dir, name)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("could not create directory %q: %w", dir, err)
}
if err := os.WriteFile(file, contents, 0o600); err != nil {
return err
}
return nil
}