From 68af247459085e0c46bf9025215b9de3838cdcd5 Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Wed, 5 Apr 2023 15:16:45 -0400 Subject: [PATCH] feat: add testutil/testnet package (#15655) Co-authored-by: Marko --- simapp/internal/testnet/cometstarter_test.go | 144 +++++ simapp/internal/testnet/doc.go | 10 + simapp/internal/testnet/example_basic_test.go | 112 ++++ testutil/testnet/cometstarter.go | 213 +++++++ testutil/testnet/delegator.go | 59 ++ testutil/testnet/doc.go | 4 + testutil/testnet/genesis.go | 528 ++++++++++++++++++ testutil/testnet/genesis_test.go | 256 +++++++++ testutil/testnet/network.go | 116 ++++ testutil/testnet/nodes.go | 70 +++ testutil/testnet/nodes_test.go | 29 + testutil/testnet/validator.go | 187 +++++++ 12 files changed, 1728 insertions(+) create mode 100644 simapp/internal/testnet/cometstarter_test.go create mode 100644 simapp/internal/testnet/doc.go create mode 100644 simapp/internal/testnet/example_basic_test.go create mode 100644 testutil/testnet/cometstarter.go create mode 100644 testutil/testnet/delegator.go create mode 100644 testutil/testnet/doc.go create mode 100644 testutil/testnet/genesis.go create mode 100644 testutil/testnet/genesis_test.go create mode 100644 testutil/testnet/network.go create mode 100644 testutil/testnet/nodes.go create mode 100644 testutil/testnet/nodes_test.go create mode 100644 testutil/testnet/validator.go diff --git a/simapp/internal/testnet/cometstarter_test.go b/simapp/internal/testnet/cometstarter_test.go new file mode 100644 index 0000000000..efd008bed0 --- /dev/null +++ b/simapp/internal/testnet/cometstarter_test.go @@ -0,0 +1,144 @@ +package testnet_test + +import ( + "fmt" + "math/rand" + "net" + "testing" + "time" + + "cosmossdk.io/log" + "cosmossdk.io/simapp" + cmtcfg "github.com/cometbft/cometbft/config" + dbm "github.com/cosmos/cosmos-db" + "github.com/cosmos/cosmos-sdk/baseapp" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + "github.com/cosmos/cosmos-sdk/testutil/testnet" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/stretchr/testify/require" +) + +// Use a limited set of available ports to ensure that +// retries eventually land on a free port. +func TestCometStarter_PortContention(t *testing.T) { + if testing.Short() { + t.Skip("skipping long test in short mode") + } + + const nVals = 4 + + // Find n+1 addresses that should be free. + // Ephemeral port range should start at about 49k+ + // according to `sysctl net.inet.ip.portrange` on macOS, + // and at about 32k+ on Linux + // according to `sysctl net.ipv4.ip_local_port_range`. + // + // Because we attempt to find free addresses outside that range, + // it is unlikely that another process will claim a port + // we discover to be free, during the time this test runs. + const portSeekStart = 19000 + reuseAddrs := make([]string, 0, nVals+1) + for i := portSeekStart; i < portSeekStart+1000; i++ { + addr := fmt.Sprintf("127.0.0.1:%d", i) + ln, err := net.Listen("tcp", addr) + if err != nil { + // No need to log the failure. + continue + } + + // If the port was free, append it to our reusable addresses. + reuseAddrs = append(reuseAddrs, "tcp://"+addr) + _ = ln.Close() + + if len(reuseAddrs) == nVals+1 { + break + } + } + + if len(reuseAddrs) != nVals+1 { + t.Fatalf("needed %d free ports but only found %d", nVals+1, len(reuseAddrs)) + } + + // Now that we have one more port than the number of validators, + // there is a good chance that picking a random port will conflict with a previously chosen one. + // But since CometStarter retries several times, + // it should eventually land on a free port. + + valPKs := testnet.NewValidatorPrivKeys(nVals) + cmtVals := valPKs.CometGenesisValidators() + stakingVals := cmtVals.StakingValidators() + + const chainID = "simapp-cometstarter" + + b := testnet.DefaultGenesisBuilderOnlyValidators( + chainID, + stakingVals, + sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction), + ) + + jGenesis := b.Encode() + + // Use an info-level logger, because the debug logs in comet are noisy + // and there is a data race in comet debug logs, + // due to be fixed in v0.37.1 which is not yet released: + // https://github.com/cometbft/cometbft/pull/532 + logger := log.NewTestLoggerInfo(t) + + const nRuns = 4 + for i := 0; i < nRuns; i++ { + t.Run(fmt.Sprintf("attempt %d", i), func(t *testing.T) { + nodes, err := testnet.NewNetwork(nVals, func(idx int) *testnet.CometStarter { + rootDir := t.TempDir() + + app := simapp.NewSimApp( + logger.With("instance", idx), + dbm.NewMemDB(), + nil, + true, + simtestutil.NewAppOptionsWithFlagHome(rootDir), + baseapp.SetChainID(chainID), + ) + + cfg := cmtcfg.DefaultConfig() + + // memdb is sufficient for this test. + cfg.BaseConfig.DBBackend = "memdb" + + return testnet.NewCometStarter( + app, + cfg, + valPKs[idx].Val, + jGenesis, + rootDir, + ). + Logger(logger.With("rootmodule", fmt.Sprintf("comet_node-%d", idx))). + TCPAddrChooser(func() string { + // This chooser function is the key of this test, + // where there is only one more available address than there are nodes. + // Therefore it is likely that an address will already be in use, + // thereby exercising the address-in-use retry. + return reuseAddrs[rand.Intn(len(reuseAddrs))] + }) + }) + defer nodes.StopAndWait() + require.NoError(t, err) + + heightAdvanced := false + for j := 0; j < 40; j++ { + cs := nodes[0].ConsensusState() + if cs.GetLastHeight() < 2 { + time.Sleep(250 * time.Millisecond) + continue + } + + // Saw height advance. + heightAdvanced = true + break + } + + if !heightAdvanced { + t.Fatalf("consensus height did not advance in approximately 10 seconds") + } + }) + } +} diff --git a/simapp/internal/testnet/doc.go b/simapp/internal/testnet/doc.go new file mode 100644 index 0000000000..19064ddd0f --- /dev/null +++ b/simapp/internal/testnet/doc.go @@ -0,0 +1,10 @@ +// Package testnet contains tests for +// [github.com/cosmos/cosmos-sdk/testutil/testnet]. +// +// Eventually all of these tests will move into that package, +// but that is currently blocked on having a minimal app defined +// in the root cosmos-sdk Go module. +// Once that app is available, the contents of this package +// will be moved to testutil/testnet, +// and references to SimApp will be replaced by the minimal app. +package testnet diff --git a/simapp/internal/testnet/example_basic_test.go b/simapp/internal/testnet/example_basic_test.go new file mode 100644 index 0000000000..99f7ae362c --- /dev/null +++ b/simapp/internal/testnet/example_basic_test.go @@ -0,0 +1,112 @@ +package testnet_test + +import ( + "fmt" + "os" + "path/filepath" + + "cosmossdk.io/log" + "cosmossdk.io/simapp" + cmtcfg "github.com/cometbft/cometbft/config" + dbm "github.com/cosmos/cosmos-db" + "github.com/cosmos/cosmos-sdk/baseapp" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + "github.com/cosmos/cosmos-sdk/testutil/testnet" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +func Example_basicUsage() { + const nVals = 2 + + // Set up new private keys for the set of validators. + valPKs := testnet.NewValidatorPrivKeys(nVals) + + // Comet-style validators. + cmtVals := valPKs.CometGenesisValidators() + + // Cosmos SDK staking validators for genesis. + stakingVals := cmtVals.StakingValidators() + + const chainID = "example-basic" + + // Create a genesis builder that only requires validators, + // without any separate delegator accounts. + // + // If you need further customization, start with testnet.NewGenesisBuilder(). + b := testnet.DefaultGenesisBuilderOnlyValidators( + chainID, + stakingVals, + // The amount to use in each validator's account during gentx. + sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction), + ) + + // JSON-formatted genesis. + jGenesis := b.Encode() + + // In this example, we have an outer root directory for the validators. + // Use t.TempDir() in tests. + rootDir, err := os.MkdirTemp("", "testnet-example-") + if err != nil { + panic(err) + } + defer os.RemoveAll(rootDir) + + // In tests, you probably want to use log.NewTestLoggerInfo(t). + logger := log.NewNopLogger() + + // The NewNetwork function creates a network of validators. + // We have to provide a callback to return CometStarter instances. + // NewNetwork will start all the comet instances concurrently + // and join the nodes together. + nodes, err := testnet.NewNetwork(nVals, func(idx int) *testnet.CometStarter { + // Make a new directory for the validator being created. + // In tests, this would be a simpler call to t.TempDir(). + dir := filepath.Join(rootDir, fmt.Sprintf("val-%d", idx)) + if err := os.Mkdir(dir, 0o755); err != nil { + panic(err) + } + + // TODO: use a different minimal app for this. + app := simapp.NewSimApp( + logger.With("instance", idx), + dbm.NewMemDB(), + nil, + true, + simtestutil.NewAppOptionsWithFlagHome(rootDir), + baseapp.SetChainID(chainID), + ) + + // Each CometStarter instance must be associated with + // a distinct comet Config object, + // as the CometStarter will automatically modify some fields, + // including P2P.ListenAddress. + cfg := cmtcfg.DefaultConfig() + + // No need to persist comet's DB to disk in this example. + cfg.BaseConfig.DBBackend = "memdb" + + return testnet.NewCometStarter( + app, + cfg, + valPKs[idx].Val, // Validator private key for this comet instance. + jGenesis, // Raw bytes of genesis file. + dir, // Where to put files on disk. + ).Logger(logger.With("root_module", fmt.Sprintf("comet_%d", idx))) + }) + // StopAndWait must be deferred before the error check, + // as the nodes value may contain some successfully started instances. + defer nodes.StopAndWait() + if err != nil { + panic(err) + } + + // Now you can begin interacting with the nodes. + // For the sake of this example, we'll just check + // a couple simple properties of one node. + fmt.Println(nodes[0].IsListening()) + fmt.Println(nodes[0].GenesisDoc().ChainID) + + // Output: + // true + // example-basic +} diff --git a/testutil/testnet/cometstarter.go b/testutil/testnet/cometstarter.go new file mode 100644 index 0000000000..428cc0231c --- /dev/null +++ b/testutil/testnet/cometstarter.go @@ -0,0 +1,213 @@ +package testnet + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "syscall" + + "cosmossdk.io/log" + abcitypes "github.com/cometbft/cometbft/abci/types" + cmtcfg "github.com/cometbft/cometbft/config" + cmted25519 "github.com/cometbft/cometbft/crypto/ed25519" + "github.com/cometbft/cometbft/node" + "github.com/cometbft/cometbft/p2p" + "github.com/cometbft/cometbft/privval" + "github.com/cometbft/cometbft/proxy" + cmttypes "github.com/cometbft/cometbft/types" + servercmtlog "github.com/cosmos/cosmos-sdk/server/log" + genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types" +) + +// CometStarter offers a builder-pattern interface to +// starting a Comet instance with an ABCI application running alongside. +// +// As CometStart is more broadly used in the codebase, +// the number of available methods on CometStarter will grow. +type CometStarter struct { + logger log.Logger + + app abcitypes.Application + + cfg *cmtcfg.Config + valPrivKey cmted25519.PrivKey + genesis []byte + + rootDir string + + tcpAddrChooser func() string + + startTries int +} + +// NewCometStarter accepts a minimal set of arguments to start comet with an ABCI app. +// For further configuration, chain other CometStarter methods before calling Start: +// +// NewCometStarter(...).Logger(...).Start() +func NewCometStarter( + app abcitypes.Application, + cfg *cmtcfg.Config, + valPrivKey cmted25519.PrivKey, + genesis []byte, + rootDir string, +) *CometStarter { + cfg.SetRoot(rootDir) + + // CometStarter won't work without these settings, + // so set them unconditionally. + cfg.P2P.AllowDuplicateIP = true + cfg.P2P.AddrBookStrict = false + + // For now, we disallow RPC listening. + // Comet v0.37 uses a global value such that multiple comet nodes in one process + // end up contending over one "rpc environment" and only the last-started validator + // will control the RPC service. + // + // The "rpc environment" was removed as a global in + // https://github.com/cometbft/cometbft/commit/3324f49fb7e7b40189726746493e83b82a61b558 + // which is due to land in v0.38. + // + // At that point, we should keep the default as RPC off, + // but we should add a RPCListen method to opt in to enabling it. + cfg.RPC.ListenAddress = "" + + // defaultStartTries is somewhat arbitrary. + // Occasionally TestCometStarter_PortContention would fail with 10 tries, + // and bumping it up to 12 makes it almost never fail. + const defaultStartTries = 12 + return &CometStarter{ + logger: log.NewNopLogger(), + + app: app, + + cfg: cfg, + genesis: genesis, + valPrivKey: valPrivKey, + + rootDir: rootDir, + + startTries: defaultStartTries, + } +} + +// Logger sets the logger for s and for the eventual started comet instance. +func (s *CometStarter) Logger(logger log.Logger) *CometStarter { + s.logger = logger + return s +} + +// Start returns a started Comet node. +func (s *CometStarter) Start() (*node.Node, error) { + fpv, nodeKey, err := s.initDisk() + if err != nil { + return nil, err + } + + appGenesisProvider := func() (*cmttypes.GenesisDoc, error) { + appGenesis, err := genutiltypes.AppGenesisFromFile(s.cfg.GenesisFile()) + if err != nil { + return nil, err + } + + return appGenesis.ToGenesisDoc() + } + + for i := 0; i < s.startTries; i++ { + s.cfg.P2P.ListenAddress = s.likelyAvailableAddress() + + n, err := node.NewNode( + s.cfg, + fpv, + nodeKey, + proxy.NewLocalClientCreator(s.app), + appGenesisProvider, + node.DefaultDBProvider, + node.DefaultMetricsProvider(s.cfg.Instrumentation), + servercmtlog.CometZeroLogWrapper{Logger: s.logger}, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create comet node: %w", err) + } + + err = n.Start() + if err == nil { + return n, nil + } + + // Error isn't nil -- if it is EADDRINUSE then we can try again. + if errors.Is(err, syscall.EADDRINUSE) { + continue + } + + // Non-nil error that isn't EADDRINUSE, just return the error. + return nil, err + } + + // If we didn't return a node from inside the loop, + // then we must have exhausted our try limit. + return nil, fmt.Errorf("failed to start a comet node within %d tries", s.startTries) +} + +// initDisk creates the config and data directories on disk, +// and other required files, so that comet and the validator work correctly. +// It also generates a node key for validators. +func (s *CometStarter) initDisk() (cmttypes.PrivValidator, *p2p.NodeKey, error) { + if err := os.MkdirAll(filepath.Join(s.rootDir, "config"), 0o750); err != nil { + return nil, nil, fmt.Errorf("failed to make config directory: %w", err) + } + if err := os.MkdirAll(filepath.Join(s.rootDir, "data"), 0o750); err != nil { + return nil, nil, fmt.Errorf("failed to make data directory: %w", err) + } + + fpv := privval.NewFilePV(s.valPrivKey, s.cfg.PrivValidatorKeyFile(), s.cfg.PrivValidatorStateFile()) + fpv.Save() + + if err := os.WriteFile(s.cfg.GenesisFile(), s.genesis, 0600); err != nil { + return nil, nil, fmt.Errorf("failed to write genesis file: %w", err) + } + + nodeKey, err := p2p.LoadOrGenNodeKey(s.cfg.NodeKeyFile()) + if err != nil { + return nil, nil, err + } + + return fpv, nodeKey, nil +} + +// TCPAddrChooser sets the function to use when selecting a (likely to be free) +// TCP address for comet's P2P port. +// +// This should only be used when testing CometStarter. +// +// It must return a string in format "tcp://IP:PORT". +func (s *CometStarter) TCPAddrChooser(fn func() string) *CometStarter { + s.tcpAddrChooser = fn + return s +} + +// likelyAvailableAddress provides a TCP address that is likely to be available +// for comet or other processes to listen on. +// +// Generally, it is better to directly provide a net.Listener that is already bound to an address, +// but unfortunately comet does not offer that as part of its API. +// Instead, we locally bind to :0 and then report that as a "likely available" port. +// If another process steals that port before our comet instance can bind to it, +// the Start method handles retries. +func (s *CometStarter) likelyAvailableAddress() string { + // If s.TCPAddrChooser was called, use that implementation. + if s.tcpAddrChooser != nil { + return s.tcpAddrChooser() + } + + // Fall back to attempting a random port. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + panic(fmt.Errorf("failed to bind to random port: %w", err)) + } + + defer ln.Close() + return "tcp://" + ln.Addr().String() +} diff --git a/testutil/testnet/delegator.go b/testutil/testnet/delegator.go new file mode 100644 index 0000000000..5ed8ef0ed6 --- /dev/null +++ b/testutil/testnet/delegator.go @@ -0,0 +1,59 @@ +package testnet + +import ( + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + sdk "github.com/cosmos/cosmos-sdk/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" +) + +// DelegatorPrivKeys is a slice of secp256k1.PrivKey. +type DelegatorPrivKeys []*secp256k1.PrivKey + +// NewDelegatorPrivKeysreturns a DelegatorPrivKeys of length n, +// where each set of keys is dynamically generated. +func NewDelegatorPrivKeys(n int) DelegatorPrivKeys { + dpk := make(DelegatorPrivKeys, n) + + for i := range dpk { + dpk[i] = secp256k1.GenPrivKey() + } + + return dpk +} + +// BaseAccounts returns the base accounts corresponding to the delegators' public keys. +func (dpk DelegatorPrivKeys) BaseAccounts() BaseAccounts { + ba := make(BaseAccounts, len(dpk)) + + for i, pk := range dpk { + pubKey := pk.PubKey() + + const accountNumber = 0 + const sequenceNumber = 0 + + ba[i] = authtypes.NewBaseAccount( + pubKey.Address().Bytes(), pubKey, accountNumber, sequenceNumber, + ) + } + + return ba +} + +// BaseAccounts is a slice of [*authtypes.BaseAccount]. +type BaseAccounts []*authtypes.BaseAccount + +// Balances creates a slice of [banktypes.Balance] for each account in ba, +// where each balance has an identical Coins value of the singleBalance argument. +func (ba BaseAccounts) Balances(singleBalance sdk.Coins) []banktypes.Balance { + balances := make([]banktypes.Balance, len(ba)) + + for i, b := range ba { + balances[i] = banktypes.Balance{ + Address: b.GetAddress().String(), + Coins: singleBalance, + } + } + + return balances +} diff --git a/testutil/testnet/doc.go b/testutil/testnet/doc.go new file mode 100644 index 0000000000..55d4dbe441 --- /dev/null +++ b/testutil/testnet/doc.go @@ -0,0 +1,4 @@ +// Package testnet provides APIs for easily create and configure +// validators, genesis files, and comet instances, +// to support testing app chain instances in-process. +package testnet diff --git a/testutil/testnet/genesis.go b/testutil/testnet/genesis.go new file mode 100644 index 0000000000..b5b985a98f --- /dev/null +++ b/testutil/testnet/genesis.go @@ -0,0 +1,528 @@ +package testnet + +import ( + "encoding/json" + "fmt" + "strconv" + "time" + + cmttypes "github.com/cometbft/cometbft/types" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/tx/signing" + authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" + "github.com/cosmos/cosmos-sdk/x/auth/tx" + authtx "github.com/cosmos/cosmos-sdk/x/auth/tx" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" + consensusparamtypes "github.com/cosmos/cosmos-sdk/x/consensus/types" + distributiontypes "github.com/cosmos/cosmos-sdk/x/distribution/types" + genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types" + minttypes "github.com/cosmos/cosmos-sdk/x/mint/types" + slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types" + stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" +) + +// GenesisBuilder enables constructing a genesis file, +// following a builder pattern. +// +// None of the methods on GenesisBuilder return an error, +// choosing instead to panic. +// GenesisBuilder is only intended for use in tests, +// where inputs are predetermined and expected to succeed. +type GenesisBuilder struct { + amino *codec.LegacyAmino + codec *codec.ProtoCodec + + // The value used in ChainID. + // Some other require this value, + // so store it as a field instead of re-parsing it from JSON. + chainID string + + // The outer JSON object. + // Most data goes into app_state, but there are some top-level fields. + outer map[string]json.RawMessage + + // Many of GenesisBuilder's methods operate on the app_state JSON object, + // so we track that separately and nest it inside outer upon a call to JSON(). + appState map[string]json.RawMessage + + gentxs []sdk.Tx +} + +// NewGenesisBuilder returns an initialized GenesisBuilder. +// +// The returned GenesisBuilder has an initial height of 1 +// and a genesis_time of the current time when the function was called. +func NewGenesisBuilder() *GenesisBuilder { + ir := codectypes.NewInterfaceRegistry() + cryptocodec.RegisterInterfaces(ir) + stakingtypes.RegisterInterfaces(ir) + banktypes.RegisterInterfaces(ir) + authtypes.RegisterInterfaces(ir) + pCodec := codec.NewProtoCodec(ir) + + return &GenesisBuilder{ + amino: codec.NewLegacyAmino(), + codec: pCodec, + + outer: map[string]json.RawMessage{ + "initial_height": json.RawMessage(`"1"`), + "genesis_time": json.RawMessage( + strconv.AppendQuote(nil, time.Now().UTC().Format(time.RFC3339Nano)), + ), + }, + appState: map[string]json.RawMessage{}, + } +} + +// GenTx emulates the gentx CLI, creating a message to create a validator +// represented by val, with "amount" self delegation, +// and signed by privVal. +func (b *GenesisBuilder) GenTx(privVal secp256k1.PrivKey, val cmttypes.GenesisValidator, amount sdk.Coin) *GenesisBuilder { + if b.chainID == "" { + panic(fmt.Errorf("(*GenesisBuilder).GenTx must not be called before (*GenesisBuilder).ChainID")) + } + + pubKey, err := cryptocodec.FromCmtPubKeyInterface(val.PubKey) + if err != nil { + panic(err) + } + + // Produce the create validator message. + msg, err := stakingtypes.NewMsgCreateValidator( + privVal.PubKey().Address().Bytes(), + pubKey, + amount, + stakingtypes.Description{ + Moniker: "TODO", + }, + stakingtypes.CommissionRates{ + Rate: sdk.MustNewDecFromStr("0.1"), + MaxRate: sdk.MustNewDecFromStr("0.2"), + MaxChangeRate: sdk.MustNewDecFromStr("0.01"), + }, + sdk.OneInt(), + ) + if err != nil { + panic(err) + } + valAddr, err := sdk.ValAddressFromBech32(msg.ValidatorAddress) + if err != nil { + panic(err) + } + + msg.DelegatorAddress = sdk.AccAddress(valAddr).String() + + if err := msg.ValidateBasic(); err != nil { + panic(err) + } + + txConf := authtx.NewTxConfig(b.codec, tx.DefaultSignModes) + + txb := txConf.NewTxBuilder() + if err := txb.SetMsgs(msg); err != nil { + panic(err) + } + + const signMode = signing.SignMode_SIGN_MODE_DIRECT + + // Need to set the signature object on the tx builder first, + // otherwise we end up signing a different total message + // compared to what gets eventually verified. + if err := txb.SetSignatures( + signing.SignatureV2{ + PubKey: privVal.PubKey(), + Data: &signing.SingleSignatureData{ + SignMode: signMode, + }, + }, + ); err != nil { + panic(err) + } + + // Generate bytes to be signed. + bytesToSign, err := txConf.SignModeHandler().GetSignBytes( + signing.SignMode_SIGN_MODE_DIRECT, + authsigning.SignerData{ + ChainID: b.chainID, + PubKey: privVal.PubKey(), + Address: sdk.MustBech32ifyAddressBytes("cosmos", privVal.PubKey().Address()), // TODO: don't hardcode cosmos1! + + // No account or sequence number for gentx. + }, + txb.GetTx(), + ) + if err != nil { + panic(err) + } + + // Produce the signature. + signed, err := privVal.Sign(bytesToSign) + if err != nil { + panic(err) + } + + // Set the signature on the builder. + if err := txb.SetSignatures( + signing.SignatureV2{ + PubKey: privVal.PubKey(), + Data: &signing.SingleSignatureData{ + SignMode: signMode, + Signature: signed, + }, + }, + ); err != nil { + panic(err) + } + + b.gentxs = append(b.gentxs, txb.GetTx()) + + return b +} + +// ChainID sets the genesis's "chain_id" field. +func (b *GenesisBuilder) ChainID(id string) *GenesisBuilder { + b.chainID = id + + var err error + b.outer["chain_id"], err = json.Marshal(id) + if err != nil { + panic(err) + } + + return b +} + +// GenesisTime sets the genesis's "genesis_time" field. +// Note that [NewGenesisBuilder] sets the genesis time to the current time by default. +func (b *GenesisBuilder) GenesisTime(t time.Time) *GenesisBuilder { + var err error + b.outer["genesis_time"], err = json.Marshal(t.Format(time.RFC3339Nano)) + if err != nil { + panic(err) + } + return b +} + +// InitialHeight sets the genesis's "initial_height" field to h. +// Note that [NewGenesisBuilder] sets the initial height to 1 by default. +func (b *GenesisBuilder) InitialHeight(h int64) *GenesisBuilder { + var err error + b.outer["initial_height"], err = json.Marshal(strconv.FormatInt(h, 10)) + if err != nil { + panic(err) + } + return b +} + +// AuthParams sets the auth params on the genesis. +func (b *GenesisBuilder) AuthParams(params authtypes.Params) *GenesisBuilder { + var err error + b.appState[authtypes.ModuleName], err = json.Marshal(map[string]any{ + "params": params, + }) + if err != nil { + panic(err) + } + + return b +} + +// DefaultAuthParams calls b.AuthParams with [authtypes.DefaultParams], +// as a convenience so that callers do not have to import the authtypes package. +func (b *GenesisBuilder) DefaultAuthParams() *GenesisBuilder { + return b.AuthParams(authtypes.DefaultParams()) +} + +// Consensus sets the consensus parameters and initial validators. +// +// If params is nil, [cmttypes.DefaultConsensusParams] is used. +func (b *GenesisBuilder) Consensus(params *cmttypes.ConsensusParams, vals CometGenesisValidators) *GenesisBuilder { + if params == nil { + params = cmttypes.DefaultConsensusParams() + } + + var err error + b.outer[consensusparamtypes.ModuleName], err = (&genutiltypes.ConsensusGenesis{ + Params: params, + Validators: vals.ToComet(), + }).MarshalJSON() + if err != nil { + panic(err) + } + + return b +} + +// Staking sets the staking parameters, validators, and delegations on the genesis. +// +// This also modifies the bank state's balances to include the bonded pool balance. +func (b *GenesisBuilder) Staking( + params stakingtypes.Params, + vals StakingValidators, + delegations []stakingtypes.Delegation, +) *GenesisBuilder { + var err error + b.appState[stakingtypes.ModuleName], err = b.codec.MarshalJSON( + stakingtypes.NewGenesisState(params, vals.ToStakingType(), delegations), + ) + if err != nil { + panic(err) + } + + // Modify bank state for bonded pool. + + var coins sdk.Coins + for _, v := range vals { + coins = coins.Add(sdk.NewCoin(sdk.DefaultBondDenom, v.V.Tokens)) + } + + bondedPoolBalance := banktypes.Balance{ + Address: authtypes.NewModuleAddress(stakingtypes.BondedPoolName).String(), + Coins: coins, + } + + // get bank types genesis, add account + + bankGenesis := banktypes.GetGenesisStateFromAppState(b.codec, b.appState) + bankGenesis.Balances = append(bankGenesis.Balances, bondedPoolBalance) + + b.appState[banktypes.ModuleName], err = b.codec.MarshalJSON(bankGenesis) + if err != nil { + panic(err) + } + + return b +} + +// StakingWithDefaultParams calls b.Staking, providing [stakingtypes.DefaultParams] +// so that callers don't necessarily have to import [stakingtypes]. +func (b *GenesisBuilder) StakingWithDefaultParams(vals StakingValidators, delegations []stakingtypes.Delegation) *GenesisBuilder { + return b.Staking(stakingtypes.DefaultParams(), vals, delegations) +} + +// DefaultStaking is shorthand for b.StakingWithDefaultParams with nil validators and delegations. +func (b *GenesisBuilder) DefaultStaking() *GenesisBuilder { + return b.StakingWithDefaultParams(nil, nil) +} + +// Banking sets the banking genesis state. +func (b *GenesisBuilder) Banking( + params banktypes.Params, + balances []banktypes.Balance, + totalSupply sdk.Coins, + denomMetadata []banktypes.Metadata, + sendEnabled []banktypes.SendEnabled, +) *GenesisBuilder { + var err error + b.appState[banktypes.ModuleName], err = b.codec.MarshalJSON( + banktypes.NewGenesisState( + params, + balances, + totalSupply, + denomMetadata, + sendEnabled, + ), + ) + if err != nil { + panic(err) + } + return b +} + +// BankingWithDefaultParams calls b.Banking with [banktypes.DefaultParams], +// so that callers don't necessarily have to import [banktypes]. +func (b *GenesisBuilder) BankingWithDefaultParams( + balances []banktypes.Balance, + totalSupply sdk.Coins, + denomMetadata []banktypes.Metadata, + sendEnabled []banktypes.SendEnabled, +) *GenesisBuilder { + return b.Banking( + banktypes.DefaultParams(), + balances, + totalSupply, + denomMetadata, + sendEnabled, + ) +} + +// Mint sets the mint genesis state. +func (b *GenesisBuilder) Mint(m minttypes.Minter, p minttypes.Params) *GenesisBuilder { + var err error + b.appState[minttypes.ModuleName], err = b.codec.MarshalJSON( + minttypes.NewGenesisState(m, p), + ) + if err != nil { + panic(err) + } + return b +} + +// DefaultMint calls b.Mint with [minttypes.DefaultInitialMinter] and [minttypes.DefaultParams]. +func (b *GenesisBuilder) DefaultMint() *GenesisBuilder { + return b.Mint(minttypes.DefaultInitialMinter(), minttypes.DefaultParams()) +} + +// Slashing sets the slashing genesis state. +func (b *GenesisBuilder) Slashing( + params slashingtypes.Params, + si []slashingtypes.SigningInfo, + mb []slashingtypes.ValidatorMissedBlocks, +) *GenesisBuilder { + var err error + b.appState[slashingtypes.ModuleName], err = b.codec.MarshalJSON( + slashingtypes.NewGenesisState(params, si, mb), + ) + if err != nil { + panic(err) + } + return b +} + +// SlashingWithDefaultParams calls b.Slashing with [slashingtypes.DefaultParams], +// so that callers don't necessarily have to import [slashingtypes]. +func (b *GenesisBuilder) SlashingWithDefaultParams( + si []slashingtypes.SigningInfo, + mb []slashingtypes.ValidatorMissedBlocks, +) *GenesisBuilder { + return b.Slashing(slashingtypes.DefaultParams(), si, mb) +} + +// DefaultSlashing is shorthand for b.SlashingWithDefaultParams +// with nil signing info and validator missed blocks. +func (b *GenesisBuilder) DefaultSlashing() *GenesisBuilder { + return b.SlashingWithDefaultParams(nil, nil) +} + +// BaseAccounts sets the initial base accounts and balances. +func (b *GenesisBuilder) BaseAccounts(ba BaseAccounts, balances []banktypes.Balance) *GenesisBuilder { + // Logic mostly copied from AddGenesisAccount. + + authGenState := authtypes.GetGenesisStateFromAppState(b.codec, b.appState) + bankGenState := banktypes.GetGenesisStateFromAppState(b.codec, b.appState) + + accs, err := authtypes.UnpackAccounts(authGenState.Accounts) + if err != nil { + panic(err) + } + + for _, a := range ba { + accs = append(accs, a) + } + accs = authtypes.SanitizeGenesisAccounts(accs) + + genAccs, err := authtypes.PackAccounts(accs) + if err != nil { + panic(err) + } + + authGenState.Accounts = genAccs + jAuthGenState, err := b.codec.MarshalJSON(&authGenState) + if err != nil { + panic(err) + } + b.appState[authtypes.ModuleName] = jAuthGenState + + bankGenState.Balances = append(bankGenState.Balances, balances...) + bankGenState.Balances = banktypes.SanitizeGenesisBalances(bankGenState.Balances) + + jBankState, err := b.codec.MarshalJSON(bankGenState) + if err != nil { + panic(err) + } + b.appState[banktypes.ModuleName] = jBankState + return b +} + +func (b *GenesisBuilder) Distribution(g *distributiontypes.GenesisState) *GenesisBuilder { + j, err := b.codec.MarshalJSON(g) + if err != nil { + panic(err) + } + + b.appState[distributiontypes.ModuleName] = j + return b +} + +func (b *GenesisBuilder) DefaultDistribution() *GenesisBuilder { + return b.Distribution(distributiontypes.DefaultGenesisState()) +} + +// JSON returns the map of the genesis after applying some final transformations. +func (b *GenesisBuilder) JSON() map[string]json.RawMessage { + gentxGenesisState := genutiltypes.NewGenesisStateFromTx( + authtx.NewTxConfig(b.codec, tx.DefaultSignModes).TxJSONEncoder(), + b.gentxs, + ) + + if err := genutiltypes.ValidateGenesis( + gentxGenesisState, + authtx.NewTxConfig(b.codec, tx.DefaultSignModes).TxJSONDecoder(), + genutiltypes.DefaultMessageValidator, + ); err != nil { + panic(err) + } + + b.appState = genutiltypes.SetGenesisStateInAppState( + b.codec, b.appState, gentxGenesisState, + ) + + appState, err := b.amino.MarshalJSON(b.appState) + if err != nil { + panic(err) + } + + b.outer["app_state"] = appState + + return b.outer +} + +// Encode returns the JSON-encoded, finalized genesis. +func (b *GenesisBuilder) Encode() []byte { + j, err := b.amino.MarshalJSON(b.JSON()) + if err != nil { + panic(err) + } + + return j +} + +// DefaultGenesisBuilderOnlyValidators returns a GenesisBuilder configured only with the given StakingValidators, +// with default parameters everywhere else. +// validatorAmount is the amount to give each validator during gentx. +// +// This is a convenience for the common case of nothing special in the genesis. +// For anything outside of the defaults, +// the longhand form of NewGenesisBuilder().ChainID(chainID)... should be used. +func DefaultGenesisBuilderOnlyValidators( + chainID string, + sv StakingValidators, + validatorAmount sdk.Coin, +) *GenesisBuilder { + cmtVals := make(CometGenesisValidators, len(sv)) + for i := range sv { + cmtVals[i] = sv[i].C + } + + b := NewGenesisBuilder(). + ChainID(chainID). + DefaultAuthParams(). + Consensus(nil, cmtVals). + BaseAccounts(sv.BaseAccounts(), nil). + StakingWithDefaultParams(nil, nil). + BankingWithDefaultParams(sv.Balances(), nil, nil, nil). + DefaultDistribution(). + DefaultMint(). + SlashingWithDefaultParams(nil, nil) + + for _, v := range sv { + b.GenTx(*v.PK.Del, v.C.V, sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction)) + } + + return b +} diff --git a/testutil/testnet/genesis_test.go b/testutil/testnet/genesis_test.go new file mode 100644 index 0000000000..aed85c54cc --- /dev/null +++ b/testutil/testnet/genesis_test.go @@ -0,0 +1,256 @@ +package testnet_test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "testing" + "time" + + cmted25519 "github.com/cometbft/cometbft/crypto/ed25519" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + "github.com/cosmos/cosmos-sdk/testutil/testnet" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/stretchr/testify/require" +) + +func TestGenesisBuilder_GenesisTime(t *testing.T) { + t.Run("defaults to current time", func(t *testing.T) { + before := time.Now() + time.Sleep(time.Millisecond) // So that the genesis time will be strictly after "before". + gb := testnet.NewGenesisBuilder() + time.Sleep(time.Millisecond) // So that the genesis time will be strictly before "after". + after := time.Now() + + var gts string + require.NoError(t, json.Unmarshal(gb.JSON()["genesis_time"], >s)) + + gt, err := time.Parse(time.RFC3339Nano, gts) + require.NoError(t, err) + + require.True(t, gt.After(before)) + require.True(t, after.After(gt)) + }) + + t.Run("can be set to arbitrary time", func(t *testing.T) { + want := time.Date(2023, 3, 27, 9, 50, 23, 0, time.UTC) + + gb := testnet.NewGenesisBuilder().GenesisTime(want) + + var gts string + require.NoError(t, json.Unmarshal(gb.JSON()["genesis_time"], >s)) + + gt, err := time.Parse(time.RFC3339Nano, gts) + require.NoError(t, err) + + require.True(t, gt.Equal(want)) + }) +} + +func TestGenesisBuilder_InitialHeight(t *testing.T) { + t.Run("defaults to 1", func(t *testing.T) { + var ih string + require.NoError( + t, + json.Unmarshal( + testnet.NewGenesisBuilder().JSON()["initial_height"], + &ih, + ), + ) + + require.Equal(t, ih, "1") + }) + + t.Run("can be set to arbitrary value", func(t *testing.T) { + var ih string + require.NoError( + t, + json.Unmarshal( + testnet.NewGenesisBuilder().InitialHeight(12345).JSON()["initial_height"], + &ih, + ), + ) + + require.Equal(t, ih, "12345") + }) +} + +func TestGenesisBuilder_ChainID(t *testing.T) { + // No default. + gb := testnet.NewGenesisBuilder() + m := gb.JSON() + _, ok := m["chain_id"] + require.False(t, ok) + + m = gb.ChainID("my-chain").JSON() + var id string + require.NoError( + t, + json.Unmarshal( + gb.ChainID("my-chain").JSON()["chain_id"], + &id, + ), + ) + require.Equal(t, id, "my-chain") +} + +// Use known keys and addresses to assert that correct validator and delegator keys +// occur in the expected locations (i.e. we didn't mistakenly swap the keys anywhere). +func TestGenesisBuilder_GentxAddresses(t *testing.T) { + const chainID = "simapp-chain" + + const valSecret0 = "val-secret-0" + const valAddr0 = "3F3B076353767F046477A6E0982F808C24D1870A" + const valPubKey0 = "ZhVhrOUHnUwYw/GlBSBrw/0X6A261gchCRYkAxGF2jk=" + valKey0 := cmted25519.GenPrivKeyFromSecret([]byte(valSecret0)) + if addr := valKey0.PubKey().Address().String(); addr != valAddr0 { + t.Fatalf("unexpected address %q for validator key 0 (expected %q)", addr, valAddr0) + } + if pub := base64.StdEncoding.EncodeToString(valKey0.PubKey().Bytes()); pub != valPubKey0 { + t.Fatalf("unexpected public key %q for validator key 0 (expected %q)", pub, valAddr0) + } + + const delSecret0 = "del-secret-0" + const delAddr0 = "30D7E04DA313C31B59A46408494B4272F0A9A256" + const delPubKey0 = "Aol+ZF9xBuZmYJrT1QFLpZBvSfr/zEKifWyg0Xi1tsFV" + const delAccAddr0 = "cosmos1xrt7qndrz0p3kkdyvsyyjj6zwtc2ngjky8dcpe" + delKey0 := secp256k1.GenPrivKeyFromSecret([]byte(delSecret0)) + if addr := delKey0.PubKey().Address().String(); addr != delAddr0 { + t.Fatalf("unexpected address %q for delegator key 0 (expected %q)", addr, delAddr0) + } + if pub := base64.StdEncoding.EncodeToString(delKey0.PubKey().Bytes()); pub != delPubKey0 { + t.Fatalf("unexpected public key %q for delegator key 0 (expected %q)", pub, delAddr0) + } + da, err := bech32.ConvertAndEncode("cosmos", delKey0.PubKey().Address().Bytes()) + require.NoError(t, err) + if da != delAccAddr0 { + t.Fatalf("unexpected account address %q for delegator key 0 (expected %q)", da, delAccAddr0) + } + + valPKs := testnet.ValidatorPrivKeys{ + &testnet.ValidatorPrivKey{ + Val: valKey0, + Del: delKey0, + }, + } + cmtVals := valPKs.CometGenesisValidators() + stakingVals := cmtVals.StakingValidators() + valBaseAccounts := stakingVals.BaseAccounts() + + b := testnet.NewGenesisBuilder(). + ChainID("my-test-chain"). + DefaultAuthParams(). + Consensus(nil, cmtVals). + BaseAccounts(valBaseAccounts, nil). + StakingWithDefaultParams(stakingVals, nil) + + for i, v := range valPKs { + b.GenTx(*v.Del, cmtVals[i].V, sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction)) + } + + var g struct { + Consensus struct { + Validators []struct { + Address string `json:"address"` + PubKey struct { + Value string `json:"value"` + } `json:"pub_key"` + } `json:"validators"` + } `json:"consensus"` + + AppState struct { + Genutil struct { + GenTxs []struct { + Body struct { + Messages []struct { + Type string `json:"@type"` + DelegatorAddress string `json:"delegator_address"` + ValidatorAddress string `json:"validator_address"` + PubKey struct { + Key string `json:"key"` + } `json:"pubkey"` + } `json:"messages"` + } `json:"body"` + AuthInfo struct { + SignerInfos []struct { + PublicKey struct { + Key string `json:"key"` + } `json:"public_key"` + } `json:"signer_infos"` + } `json:"auth_info"` + } `json:"gen_txs"` + } `json:"genutil"` + + Auth struct { + Accounts []struct { + Address string `json:"address"` + PubKey struct { + Key string `json:"key"` + } `json:"pub_key"` + } `json:"accounts"` + } `json:"auth"` + } `json:"app_state"` + } + if err := json.Unmarshal(b.Encode(), &g); err != nil { + t.Fatal(err) + } + + // Validator encoded as expected. + vals := g.Consensus.Validators + require.Equal(t, vals[0].Address, valAddr0) + require.Equal(t, vals[0].PubKey.Value, valPubKey0) + + // Public keys on gentx message match correct keys (no ed25519/secp256k1 mismatch). + gentxs := g.AppState.Genutil.GenTxs + require.Equal(t, gentxs[0].Body.Messages[0].PubKey.Key, valPubKey0) + require.Equal(t, gentxs[0].AuthInfo.SignerInfos[0].PublicKey.Key, delPubKey0) + + // Delegator is derived from the secp256k1 key, not the ed25519 key. + require.Equal(t, gentxs[0].Body.Messages[0].DelegatorAddress, delAccAddr0) + + // The validator address must match the delegator address. + _, parsedValAddr, err := bech32.DecodeAndConvert(gentxs[0].Body.Messages[0].DelegatorAddress) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("%X", parsedValAddr), delAddr0) + + // The only base account in this genesis, matches the secp256k1 key. + acct := g.AppState.Auth.Accounts[0] + require.Equal(t, acct.Address, delAccAddr0) + require.Equal(t, acct.PubKey.Key, delPubKey0) +} + +func ExampleGenesisBuilder() { + const nVals = 4 + + // Generate random private keys for validators. + valPKs := testnet.NewValidatorPrivKeys(nVals) + + // Produce the comet representation of those validators. + cmtVals := valPKs.CometGenesisValidators() + + stakingVals := cmtVals.StakingValidators() + + // Configure a new genesis builder + // with a fairly thorough set of defaults. + // + // If you only ever need defaults, you can use DefaultGenesisBuilderOnlyValidators(). + b := testnet.NewGenesisBuilder(). + ChainID("my-chain-id"). + DefaultAuthParams(). + Consensus(nil, cmtVals). + BaseAccounts(stakingVals.BaseAccounts(), nil). + DefaultStaking(). + BankingWithDefaultParams(stakingVals.Balances(), nil, nil, nil). + DefaultDistribution(). + DefaultMint(). + DefaultSlashing() + + for i := range stakingVals { + b.GenTx(*valPKs[i].Del, cmtVals[i].V, sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction)) + } + + // Now, you can access b.JSON() if you need to make further modifications + // not (yet) supported by the GenesisBuilder API, + // or you can use b.Encode() for the serialzed JSON of the genesis. +} diff --git a/testutil/testnet/network.go b/testutil/testnet/network.go new file mode 100644 index 0000000000..97e7771b11 --- /dev/null +++ b/testutil/testnet/network.go @@ -0,0 +1,116 @@ +package testnet + +import ( + "errors" + "fmt" + "sync" + + "github.com/cometbft/cometbft/p2p" +) + +// NewNetwork concurrently calls createCometStarter, nVals times; +// then it returns a slice of started comet nodes, +// in order corresponding with the number passed to createCometStarter. +// The returned nodes will all be peered together, +// by dialing each node's [github.com/cometbft/cometbft/p2p/pex.Reactor] to each other. +// +// Every node is attempted to be started, +// and any errors collected are joined together and returned. +// +// In the event of errors, a non-nil Nodes slice may still be returned +// and some elements may be nil. +// Callers should call [Nodes.Stop] and [Nodes.Wait] to perform cleanup, +// regardless of the returned error. +func NewNetwork(nVals int, createCometStarter func(int) *CometStarter) (Nodes, error) { + // The ordered slice of nodes; correct indexing is important. + // The creator goroutines will write directly into this slice. + nodes := make(Nodes, nVals) + + // Every node will be started in its own goroutine. + // We collect the switches so that each node can dial every other node. + switchCh := make(chan *p2p.Switch, nVals) + errCh := make(chan error, nVals) + + var wg sync.WaitGroup + // Start goroutines to populate nodes slice and notify as each node is available. + for i := 0; i < nVals; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + n, err := createCometStarter(i).Start() + if err != nil { + errCh <- fmt.Errorf("failed to start node %d: %w", i, err) + return + } + + // Notify that the new node's switch is available, + // so this node can be peered with the other nodes. + switchCh <- n.PEXReactor().Switch + + // And assign the node into its correct index in the ordered slice. + nodes[i] = n + }(i) + } + + // Once all the creation goroutines are complete, close the channels, + // to signal to the collection goroutines. + go func() { + wg.Wait() + close(errCh) + close(switchCh) + }() + + joinPeersDone := make(chan struct{}) + go joinPeers(switchCh, joinPeersDone) + + finalErrCh := make(chan error, 1) + go collectErrors(errCh, finalErrCh) + + // If there were any errors, return them. + // And return any set nodes, so that they can be cleaned up properly. + if err := <-finalErrCh; err != nil { + return nodes, err + } + + // No errors, so wait for peer joining to complete + // before returning the ordered nodes. + <-joinPeersDone + return nodes, nil +} + +// collectErrors collects all errors that arrive on the in channel, +// joins them, then sends the joined final error on the out channel. +func collectErrors(in <-chan error, out chan<- error) { + var errs []error + for err := range in { + errs = append(errs, err) + } + + var res error + if len(errs) > 0 { + res = errors.Join(errs...) + } + out <- res +} + +// joinPeers collects each switch arriving on newSwitches; +// each time a new switch arrives, it dials every previously seen switch. +// +// This allows each node to be started independently and concurrently +// without predetermined p2p ports. +func joinPeers(newSwitches <-chan *p2p.Switch, done chan<- struct{}) { + defer close(done) + + var readySwitches []*p2p.Switch + for newSwitch := range newSwitches { + newNetAddr := newSwitch.NetAddress() + for _, s := range readySwitches { + // For every new switch, connect with all the previously seen switches. + // It might not be necessary to dial in both directions, but it shouldn't hurt. + _ = s.DialPeerWithAddress(newNetAddr) + _ = newSwitch.DialPeerWithAddress(s.NetAddress()) + } + readySwitches = append(readySwitches, newSwitch) + } +} diff --git a/testutil/testnet/nodes.go b/testutil/testnet/nodes.go new file mode 100644 index 0000000000..d47699fc6f --- /dev/null +++ b/testutil/testnet/nodes.go @@ -0,0 +1,70 @@ +package testnet + +import ( + "errors" + "fmt" + "time" + + "github.com/cometbft/cometbft/node" +) + +// Nodes is a slice of comet nodes, +// with some additional convenience methods. +// +// Nodes may contain nil elements, +// so that a partially failed call to NewNetwork +// can still be properly cleaned up. +type Nodes []*node.Node + +// Stop stops each node sequentially. +// All errors occurring during stop are returned as a joined error. +// +// Nil elements in ns are skipped. +func (ns Nodes) Stop() error { + var errs []error + for i, n := range ns { + if n == nil { + continue + } + if err := n.Stop(); err != nil { + errs = append(errs, fmt.Errorf("failed to stop node %d: %w", i, err)) + } + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} + +// Wait blocks until every node in ns has completely stopped. +// +// Nil elements in ns are skipped. +func (ns Nodes) Wait() { + for _, n := range ns { + if n == nil { + continue + } + n.Wait() + } +} + +// StopAndWait is shorthand for calling both Stop() and Wait(), +// useful as a deferred call in tests. +func (ns Nodes) StopAndWait() error { + err := ns.Stop() + ns.Wait() + + // TODO(mr): remove this sleep call after we are using a version of Comet + // that includes a fix for https://github.com/cometbft/cometbft/issues/646. + // + // On my machine, this sleep appears to completely eliminate the late file write. + // It also almost always works around https://github.com/cometbft/cometbft/issues/652. + // + // Presumably the fix for those two issues will be included in a v0.37.1 release. + // If not, I assume they will be part of the first v0.38 series release. + time.Sleep(250 * time.Millisecond) + + return err +} diff --git a/testutil/testnet/nodes_test.go b/testutil/testnet/nodes_test.go new file mode 100644 index 0000000000..55fd3f4b8e --- /dev/null +++ b/testutil/testnet/nodes_test.go @@ -0,0 +1,29 @@ +package testnet_test + +import ( + "testing" + + "github.com/cometbft/cometbft/node" + "github.com/cosmos/cosmos-sdk/testutil/testnet" + "github.com/stretchr/testify/require" +) + +// Nil entries in a Nodes slice don't fail Stop or Wait. +func TestNodes_StopWaitNil(t *testing.T) { + for _, tc := range []struct { + Name string + Nodes []*node.Node + }{ + {Name: "nil slice", Nodes: nil}, + {Name: "slice with nil elements", Nodes: []*node.Node{nil}}, + } { + ns := testnet.Nodes(tc.Nodes) + t.Run(tc.Name, func(t *testing.T) { + require.NoError(t, ns.Stop()) + + // Nothing special to assert about Wait(). + // It should return immediately, without panicking. + ns.Wait() + }) + } +} diff --git a/testutil/testnet/validator.go b/testutil/testnet/validator.go new file mode 100644 index 0000000000..2e795393a5 --- /dev/null +++ b/testutil/testnet/validator.go @@ -0,0 +1,187 @@ +package testnet + +import ( + "fmt" + + sdkmath "cosmossdk.io/math" + cmted25519 "github.com/cometbft/cometbft/crypto/ed25519" + cmttypes "github.com/cometbft/cometbft/types" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/bech32" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" + stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" +) + +// ValidatorPrivKeys is a slice of [*ValidatorPrivKey]. +type ValidatorPrivKeys []*ValidatorPrivKey + +// ValidatorPrivKey holds a validator key (a comet ed25519 key) +// and the validator's delegator or account key (a Cosmos SDK secp256k1 key). +type ValidatorPrivKey struct { + Val cmted25519.PrivKey + Del *secp256k1.PrivKey +} + +// NewValidatorPrivKeys returns a ValidatorPrivKeys of length n, +// where each set of keys is dynamically generated. +// +// If writing a test where deterministic keys are required, +// the caller should manually construct a slice and assign each key as needed. +func NewValidatorPrivKeys(n int) ValidatorPrivKeys { + vpk := make(ValidatorPrivKeys, n) + + for i := range vpk { + vpk[i] = &ValidatorPrivKey{ + Val: cmted25519.GenPrivKey(), + Del: secp256k1.GenPrivKey(), + } + } + + return vpk +} + +// CometGenesisValidators derives the CometGenesisValidators belonging to vpk. +func (vpk ValidatorPrivKeys) CometGenesisValidators() CometGenesisValidators { + cgv := make(CometGenesisValidators, len(vpk)) + + for i, pk := range vpk { + pubKey := pk.Val.PubKey() + + const votingPower = 1 + cmtVal := cmttypes.NewValidator(pubKey, votingPower) + + cgv[i] = &CometGenesisValidator{ + V: cmttypes.GenesisValidator{ + Address: cmtVal.Address, + PubKey: cmtVal.PubKey, + Power: cmtVal.VotingPower, + Name: fmt.Sprintf("val-%d", i), + }, + PK: pk, + } + } + + return cgv +} + +// CometGenesisValidators is a slice of [*CometGenesisValidator]. +type CometGenesisValidators []*CometGenesisValidator + +// CometGenesisValidator holds a comet GenesisValidator +// and a reference to the ValidatorPrivKey from which the CometGenesisValidator was derived. +type CometGenesisValidator struct { + V cmttypes.GenesisValidator + PK *ValidatorPrivKey +} + +// ToComet returns a new slice of [cmttypes.GenesisValidator], +// useful for some interactions. +func (cgv CometGenesisValidators) ToComet() []cmttypes.GenesisValidator { + vs := make([]cmttypes.GenesisValidator, len(cgv)) + for i, v := range cgv { + vs[i] = v.V + } + return vs +} + +// StakingValidators derives the StakingValidators belonging to cgv. +func (cgv CometGenesisValidators) StakingValidators() StakingValidators { + vals := make(StakingValidators, len(cgv)) + for i, v := range cgv { + pk, err := cryptocodec.FromCmtPubKeyInterface(v.V.PubKey) + if err != nil { + panic(fmt.Errorf("failed to extract comet pub key: %w", err)) + } + + pkAny, err := codectypes.NewAnyWithValue(pk) + if err != nil { + panic(fmt.Errorf("failed to wrap pub key in any type: %w", err)) + } + + vals[i] = &StakingValidator{ + V: stakingtypes.Validator{ + OperatorAddress: sdk.ValAddress(v.V.Address).String(), // TODO: this relies on global bech32 config. + ConsensusPubkey: pkAny, + Status: stakingtypes.Bonded, + Tokens: sdk.DefaultPowerReduction, + DelegatorShares: sdkmath.LegacyOneDec(), + MinSelfDelegation: sdkmath.ZeroInt(), + + // more fields uncopied from testutil/sims/app_helpers.go:220 + }, + C: v, + PK: v.PK, + } + } + + return vals +} + +// StakingValidators is a slice of [*StakingValidator]. +type StakingValidators []*StakingValidator + +// StakingValidator holds a [stakingtypes.Validator], +// and the CometGenesisValidator and ValidatorPrivKey required to derive the StakingValidator. +type StakingValidator struct { + V stakingtypes.Validator + C *CometGenesisValidator + PK *ValidatorPrivKey +} + +// ToStakingType returns a new slice of [stakingtypes.Validator], +// useful for some interactions. +func (sv StakingValidators) ToStakingType() []stakingtypes.Validator { + vs := make([]stakingtypes.Validator, len(sv)) + for i, v := range sv { + vs[i] = v.V + } + return vs +} + +// BaseAccounts returns the BaseAccounts for this set of StakingValidators. +// The base accounts are important for [*GenesisBuilder.BaseAccounts]. +func (sv StakingValidators) BaseAccounts() BaseAccounts { + ba := make(BaseAccounts, len(sv)) + + for i, v := range sv { + const accountNumber = 0 + const sequenceNumber = 0 + + pubKey := v.PK.Del.PubKey() + bech, err := bech32.ConvertAndEncode("cosmos", pubKey.Address().Bytes()) // TODO: this shouldn't be hardcoded to cosmos! + if err != nil { + panic(err) + } + accAddr, err := sdk.AccAddressFromBech32(bech) + if err != nil { + panic(err) + } + ba[i] = authtypes.NewBaseAccount( + accAddr, pubKey, accountNumber, sequenceNumber, + ) + } + + return ba +} + +// Balances returns the balances held by this set of StakingValidators. +func (sv StakingValidators) Balances() []banktypes.Balance { + bals := make([]banktypes.Balance, len(sv)) + + for i, v := range sv { + addr, err := bech32.ConvertAndEncode("cosmos", v.PK.Del.PubKey().Address().Bytes()) // TODO: this shouldn't be hardcoded to cosmos! + if err != nil { + panic(err) + } + bals[i] = banktypes.Balance{ + Address: addr, + Coins: sdk.Coins{sdk.NewCoin(sdk.DefaultBondDenom, v.V.Tokens)}, + } + } + + return bals +}