diff --git a/scripts/build/testing.mk b/scripts/build/testing.mk index 98f1afe2a6..8fcdcb7a61 100644 --- a/scripts/build/testing.mk +++ b/scripts/build/testing.mk @@ -29,7 +29,7 @@ test-all: test-unit test-e2e test-integration test-ledger-mock test-race .PHONY: test-system test-system: build mkdir -p ./tests/systemtests/binaries/ - cp $(BUILDDIR)/simd ./tests/systemtests/binaries/ + cp $(BUILDDIR)/simd$(if $(findstring v2,$(COSMOS_BUILD_OPTIONS)),v2) ./tests/systemtests/binaries/ $(MAKE) -C tests/systemtests test diff --git a/tests/systemtests/Makefile b/tests/systemtests/Makefile index 1a384b8166..948d43bf81 100644 --- a/tests/systemtests/Makefile +++ b/tests/systemtests/Makefile @@ -5,7 +5,7 @@ WAIT_TIME ?= 45s all: test test: - go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose + go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose $(if $(findstring v2,$(COSMOS_BUILD_OPTIONS)),--binary=simdv2) format: find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w diff --git a/tests/systemtests/cli.go b/tests/systemtests/cli.go index 404b7849fb..bae6c55eaf 100644 --- a/tests/systemtests/cli.go +++ b/tests/systemtests/cli.go @@ -223,7 +223,7 @@ func (c CLIWrapper) runWithInput(args []string, input io.Reader) (output string, cmd := exec.Command(locateExecutable(c.execBinary), args...) //nolint:gosec // test code only cmd.Dir = WorkDir cmd.Stdin = input - return cmd.CombinedOutput() + return cmd.Output() }() gotOut = filterProtoNoise(gotOut) ok = c.assertErrorFn(c.t, gotErr, string(gotOut)) diff --git a/tests/systemtests/fraud_test.go b/tests/systemtests/fraud_test.go index 675f3969e4..a383669eef 100644 --- a/tests/systemtests/fraud_test.go +++ b/tests/systemtests/fraud_test.go @@ -29,9 +29,10 @@ func TestValidatorDoubleSign(t *testing.T) { t.Log(rsp) nodePowerBefore := QueryCometValidatorPowerForNode(t, sut, 0) require.NotEmpty(t, nodePowerBefore) + t.Logf("nodePowerBefore: %v", nodePowerBefore) var validatorPubKey cryptotypes.PubKey - newNode := sut.AddFullnode(t, "0.001stake", func(nodeNumber int, nodePath string) { + newNode := sut.AddFullnode(t, func(nodeNumber int, nodePath string) { valKeyFile := filepath.Join(WorkDir, nodePath, "config", "priv_validator_key.json") _ = os.Remove(valKeyFile) _, err := copyFile(filepath.Join(WorkDir, sut.nodePath(0), "config", "priv_validator_key.json"), valKeyFile) @@ -54,9 +55,15 @@ func TestValidatorDoubleSign(t *testing.T) { // then comet status updated nodePowerAfter := QueryCometValidatorPowerForNode(t, sut, 0) require.Empty(t, nodePowerAfter) + t.Logf("nodePowerAfter: %v", nodePowerAfter) // and sdk status updated byzantineOperatorAddr := cli.GetKeyAddrPrefix("node0", "val") rsp = cli.CustomQuery("q", "staking", "validator", byzantineOperatorAddr) assert.True(t, gjson.Get(rsp, "validator.jailed").Bool(), rsp) + + t.Log("let's run for some blocks to confirm all good") + for i := 0; i < 10; i++ { + sut.AwaitNextBlock(t) + } } diff --git a/tests/systemtests/system.go b/tests/systemtests/system.go index 355854cfdd..bc8f4351b3 100644 --- a/tests/systemtests/system.go +++ b/tests/systemtests/system.go @@ -23,6 +23,7 @@ import ( tmtypes "github.com/cometbft/cometbft/types" "github.com/stretchr/testify/require" "github.com/tidwall/sjson" + "golang.org/x/exp/maps" "github.com/cosmos/cosmos-sdk/server" sdk "github.com/cosmos/cosmos-sdk/types" @@ -154,7 +155,7 @@ func (s *SystemUnderTest) StartChain(t *testing.T, xargs ...string) { t.Helper() s.Log("Start chain\n") s.ChainStarted = true - s.startNodesAsync(t, append([]string{"start", "--trace", "--log_level=info", "--log_no_color"}, xargs...)...) + s.startNodesAsync(t, append([]string{"start", "--log_level=info", "--log_no_color"}, xargs...)...) s.AwaitNodeUp(t, s.rpcAddr) @@ -332,10 +333,10 @@ func (s *SystemUnderTest) StopChain() { func (s *SystemUnderTest) withEachPid(cb func(p *os.Process)) { s.pidsLock.RLock() - pids := s.pids + pids := maps.Keys(s.pids) s.pidsLock.RUnlock() - for pid := range pids { + for _, pid := range pids { p, err := os.FindProcess(pid) if err != nil { continue @@ -528,7 +529,7 @@ func (s *SystemUnderTest) ForEachNodeExecAndWait(t *testing.T, cmds ...[]string) for j, xargs := range cmds { xargs = append(xargs, "--home", home) s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(xargs, " ")) - out := runShellCmd(t, s.execBinary, xargs...) + out := MustRunShellCmd(t, s.execBinary, xargs...) s.Logf("Result: %s\n", out) result[i][j] = out } @@ -536,20 +537,20 @@ func (s *SystemUnderTest) ForEachNodeExecAndWait(t *testing.T, cmds ...[]string) return result } -func runShellCmd(t *testing.T, cmd string, args ...string) string { +func MustRunShellCmd(t *testing.T, cmd string, args ...string) string { t.Helper() - out, err := runShellCmdX(cmd, args...) + out, err := RunShellCmd(cmd, args...) require.NoError(t, err) return out } -func runShellCmdX(cmd string, args ...string) (string, error) { +func RunShellCmd(cmd string, args ...string) (string, error) { c := exec.Command( //nolint:gosec // used by tests only locateExecutable(cmd), args..., ) c.Dir = WorkDir - out, err := c.CombinedOutput() + out, err := c.Output() if err != nil { return string(out), fmt.Errorf("run `%s %s`: out: %s: %w", cmd, strings.Join(args, " "), string(out), err) } @@ -560,7 +561,7 @@ func runShellCmdX(cmd string, args ...string) (string, error) { func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) { t.Helper() s.withEachNodeHome(func(i int, home string) { - args := append(xargs, "--home", home) + args := append(xargs, "--home="+home) s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " ")) cmd := exec.Command( //nolint:gosec // used by tests only locateExecutable(s.execBinary), @@ -569,24 +570,27 @@ func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) { cmd.Dir = WorkDir s.watchLogs(i, cmd) require.NoError(t, cmd.Start(), "node %d", i) - - pid := cmd.Process.Pid - s.pidsLock.Lock() - s.pids[pid] = struct{}{} - s.pidsLock.Unlock() - s.Logf("Node started: %d\n", pid) + s.Logf("Node started: %d\n", cmd.Process.Pid) // cleanup when stopped - go func(pid int) { - _ = cmd.Wait() // blocks until shutdown - s.pidsLock.Lock() - delete(s.pids, pid) - s.pidsLock.Unlock() - s.Logf("Node stopped: %d\n", pid) - }(pid) + s.awaitProcessCleanup(cmd) }) } +func (s *SystemUnderTest) awaitProcessCleanup(cmd *exec.Cmd) { + pid := cmd.Process.Pid + s.pidsLock.Lock() + s.pids[pid] = struct{}{} + s.pidsLock.Unlock() + go func() { + _ = cmd.Wait() // blocks until shutdown + s.Logf("Node stopped: %d\n", pid) + s.pidsLock.Lock() + delete(s.pids, pid) + s.pidsLock.Unlock() + }() +} + func (s *SystemUnderTest) withEachNodeHome(cb func(i int, home string)) { for i := 0; i < s.nodesCount; i++ { cb(i, s.nodePath(i)) @@ -658,7 +662,7 @@ func (s *SystemUnderTest) resetBuffers() { } // AddFullnode starts a new fullnode that connects to the existing chain but is not a validator. -func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeStart ...func(nodeNumber int, nodePath string)) Node { +func (s *SystemUnderTest) AddFullnode(t *testing.T, beforeStart ...func(nodeNumber int, nodePath string)) Node { t.Helper() s.MarkDirty() s.nodesCount++ @@ -668,7 +672,7 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS // prepare new node moniker := fmt.Sprintf("node%d", nodeNumber) - args := []string{"init", moniker, "--home", nodePath, "--overwrite"} + args := []string{"init", moniker, "--home=" + nodePath, "--overwrite"} s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " ")) cmd := exec.Command( //nolint:gosec // used by tests only locateExecutable(s.execBinary), @@ -679,12 +683,15 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS require.NoError(t, cmd.Run(), "failed to start node with id %d", nodeNumber) require.NoError(t, saveGenesis(nodePath, []byte(s.ReadGenesisJSON(t)))) - // quick hack: copy config and overwrite by start params - configFile := filepath.Join(WorkDir, nodePath, "config", "config.toml") - _ = os.Remove(configFile) - _, err := copyFile(filepath.Join(WorkDir, s.nodePath(0), "config", "config.toml"), configFile) - require.NoError(t, err) + configPath := filepath.Join(WorkDir, nodePath, "config") + // quick hack: copy config and overwrite by start params + for _, tomlFile := range []string{"config.toml", "app.toml"} { + configFile := filepath.Join(configPath, tomlFile) + _ = os.Remove(configFile) + _, err := copyFile(filepath.Join(WorkDir, s.nodePath(0), "config", tomlFile), configFile) + require.NoError(t, err) + } // start node allNodes := s.AllNodes(t) node := allNodes[len(allNodes)-1] @@ -701,7 +708,6 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS fmt.Sprintf("--p2p.laddr=tcp://localhost:%d", node.P2PPort), fmt.Sprintf("--rpc.laddr=tcp://localhost:%d", node.RPCPort), fmt.Sprintf("--grpc.address=localhost:%d", 9090+nodeNumber), - fmt.Sprintf("--minimum-gas-prices=%s", minGasPrices), "--p2p.pex=false", "--moniker=" + moniker, "--log_level=info", @@ -716,6 +722,7 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS cmd.Dir = WorkDir s.watchLogs(nodeNumber, cmd) require.NoError(t, cmd.Start(), "node %d", nodeNumber) + s.awaitProcessCleanup(cmd) return node } diff --git a/tests/systemtests/testnet_init.go b/tests/systemtests/testnet_init.go index 28a957d631..c3cd9c0f73 100644 --- a/tests/systemtests/testnet_init.go +++ b/tests/systemtests/testnet_init.go @@ -58,7 +58,7 @@ func (s SingleHostTestnetCmdInitializer) Initialize() { "--single-host", } s.log(fmt.Sprintf("+++ %s %s\n", s.execBinary, strings.Join(args, " "))) - out, err := runShellCmdX(s.execBinary, args...) + out, err := RunShellCmd(s.execBinary, args...) if err != nil { panic(err) } @@ -112,7 +112,7 @@ func (s ModifyConfigYamlInitializer) Initialize() { } s.log(fmt.Sprintf("+++ %s %s\n", s.execBinary, strings.Join(args, " "))) - out, err := runShellCmdX(s.execBinary, args...) + out, err := RunShellCmd(s.execBinary, args...) if err != nil { panic(err) } diff --git a/tests/systemtests/upgrade_test.go b/tests/systemtests/upgrade_test.go index 15d759079b..a9a990ef28 100644 --- a/tests/systemtests/upgrade_test.go +++ b/tests/systemtests/upgrade_test.go @@ -106,8 +106,8 @@ func FetchExecutable(t *testing.T, version string) string { } destFile := cacheFile t.Log("+++ version not in cache, downloading from docker image") - runShellCmd(t, "docker", "pull", "ghcr.io/cosmos/simapp:"+version) - runShellCmd(t, "docker", "create", "--name=ci_temp", "ghcr.io/cosmos/simapp:"+version) - runShellCmd(t, "docker", "cp", "ci_temp:/usr/bin/simd", destFile) + MustRunShellCmd(t, "docker", "pull", "ghcr.io/cosmos/simapp:"+version) + MustRunShellCmd(t, "docker", "create", "--name=ci_temp", "ghcr.io/cosmos/simapp:"+version) + MustRunShellCmd(t, "docker", "cp", "ci_temp:/usr/bin/simd", destFile) return destFile }