test: add more sims tests (#23421)

scripts/build/simulations.mk

@@ -53,12 +53,6 @@ test-sim-multi-seed-short:
 	 @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
 		-NumBlocks=50
 
-test-sim-benchmark-invariants:
-	# @echo "Running simulation invariant benchmarks..."
-	# cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
-	# -Enabled=true -NumBlocks=1000 -BlockSize=200 \
-	# -Commit=true -Seed=57 -v -timeout 24h
-
 .PHONY: \
 test-sim-nondeterminism \
 test-sim-nondeterminism-streaming \
@@ -68,29 +62,27 @@ test-sim-after-import \
 test-sim-custom-genesis-multi-seed \
 test-sim-multi-seed-short \
 test-sim-multi-seed-long \
-test-sim-benchmark-invariants
 
 SIM_NUM_BLOCKS ?= 500
 SIM_BLOCK_SIZE ?= 200
-SIM_COMMIT ?= true
 
 #? test-sim-fuzz: Run fuzz test for simapp
 test-sim-fuzz:
-#	@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
+	@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
 #ld flags are a quick fix to make it work on current osx
-#	@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
+	@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -json -tags='sims' -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
 
 #? test-sim-benchmark: Run benchmark test for simapp
 test-sim-benchmark:
-#	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
-#	@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$  \
-		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h
+	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+	@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$  \
+		-NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE)
 
 
 test-sim-profile:
-#	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
-#	@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
+	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+	@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
+		-NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -cpuprofile cpu.out -memprofile mem.out
 
 .PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz
 

simapp/v2/.gitignore

@@ -0,0 +1 @@
+/simapp.test

simapp/v2/sim_bench_test.go

@@ -0,0 +1,17 @@
+//go:build sims
+
+package simapp
+
+import (
+	"github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
+	"testing"
+)
+
+func BenchmarkFullAppSimulation(b *testing.B) {
+	b.ReportAllocs()
+	cfg := cli.NewConfigFromFlags()
+	cfg.ChainID = SimAppChainID
+	for i := 0; i < b.N; i++ {
+		RunWithSeed[Tx](b, NewSimApp[Tx], AppConfig, cfg, 1)
+	}
+}

simapp/v2/sim_fuzz_test.go

@@ -0,0 +1,23 @@
+//go:build sims
+
+package simapp
+
+import (
+	simsxv2 "github.com/cosmos/cosmos-sdk/simsx/v2"
+	simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
+	"testing"
+)
+
+func FuzzFullAppSimulation(f *testing.F) {
+	cfg := simcli.NewConfigFromFlags()
+	cfg.ChainID = SimAppChainID
+
+	f.Fuzz(func(t *testing.T, rawSeed []byte) {
+		if len(rawSeed) < 8 {
+			t.Skip()
+			return
+		}
+		randSource := simsxv2.NewByteSource(cfg.FuzzSeed, cfg.Seed)
+		RunWithRandSource[Tx](t, NewSimApp[Tx], AppConfig, cfg, randSource)
+	})
+}

simapp/v2/sim_runner.go

@@ -106,7 +106,7 @@ type (
 
 	// TestInstance system under test
 	TestInstance[T Tx] struct {
-		Seed          int64
+		RandSource    simsxv2.RandSource
 		App           SimulationApp[T]
 		TxDecoder     transaction.Codec[T]
 		BankKeeper    BankKeeper
@@ -126,7 +126,7 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](
 	tb testing.TB,
 	appFactory AppFactory[T, V],
 	appConfigFactory AppConfigFactory,
-	seed int64,
+	randSource simsxv2.RandSource,
 ) TestInstance[T] {
 	tb.Helper()
 	vp := viper.New()
@@ -153,7 +153,7 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](
 	xapp, err := appFactory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings()))))
 	require.NoError(tb, err)
 	return TestInstance[T]{
-		Seed:          seed,
+		RandSource:    randSource,
 		App:           xapp,
 		BankKeeper:    bankKeeper,
 		AuthKeeper:    authKeeper,
@@ -197,7 +197,7 @@ func (ti TestInstance[T]) InitializeChain(
 	}
 }
 
-// RunWithSeeds runs a series of subtests using the default set of random seeds for deterministic simulation testing.
+// RunWithSeeds runs a series of subtests for each of the given set of seeds for deterministic simulation testing.
 func RunWithSeeds[T Tx, V SimulationApp[T]](
 	t *testing.T,
 	appFactory AppFactory[T, V],
@@ -224,13 +224,26 @@ func RunWithSeed[T Tx, V SimulationApp[T]](
 	tCfg simtypes.Config,
 	seed int64,
 	postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
+) {
+	tb.Helper()
+	RunWithRandSource(tb, appFactory, appConfigFactory, tCfg, simsxv2.NewSeededRandSource(seed), postRunActions...)
+}
+
+// RunWithRandSource initializes and executes a simulation run with the given rand source, generating blocks and transactions.
+func RunWithRandSource[T Tx, V SimulationApp[T]](
+	tb testing.TB,
+	appFactory AppFactory[T, V],
+	appConfigFactory AppConfigFactory,
+	tCfg simtypes.Config,
+	randSource simsxv2.RandSource,
+	postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
 ) {
 	tb.Helper()
 	initialBlockHeight := tCfg.InitialBlockHeight
 	require.NotEmpty(tb, initialBlockHeight, "initial block height must not be 0")
 
 	setupFn := func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account) {
-		testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, seed)
+		testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, randSource)
 		accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState(
 			testInstance.App,
 			r,
@@ -249,20 +262,21 @@ func RunWithSeed[T Tx, V SimulationApp[T]](
 
 		return testInstance, cs, accounts
 	}
-	RunWithSeedX(tb, tCfg, setupFn, seed, postRunActions...)
+	RunWithRandSourceX(tb, tCfg, setupFn, randSource, postRunActions...)
 }
 
-// RunWithSeedX entrypoint for custom chain setups.
-// The function runs the full simulation test circle for the specified seed and setup function, followed by optional post-run actions.
-func RunWithSeedX[T Tx](
+// RunWithRandSourceX entrypoint for custom chain setups.
+// The function runs the full simulation test circle for the specified random source and setup function, followed by optional post-run actions.
+// when tb implements ResetTimer, the method is called after setup, before jumping into the main loop
+func RunWithRandSourceX[T Tx](
 	tb testing.TB,
 	tCfg simtypes.Config,
 	setupChainStateFn func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account),
-	seed int64,
+	randSource rand.Source,
 	postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
 ) {
 	tb.Helper()
-	r := rand.New(rand.NewSource(seed))
+	r := rand.New(randSource)
 	rootCtx, done := context.WithCancel(context.Background())
 	defer done()
 
@@ -273,6 +287,10 @@ func RunWithSeedX[T Tx](
 	modules := testInstance.ModuleManager.Modules()
 	msgFactoriesFn := prepareSimsMsgFactories(r, modules, simsx.ParamWeightSource(emptySimParams))
 
+	if b, ok := tb.(interface{ ResetTimer() }); ok {
+		b.ResetTimer()
+	}
+
 	doMainLoop(
 		tb,
 		rootCtx,

simapp/v2/sim_test.go

@@ -25,6 +25,10 @@ func TestFullAppSimulation(t *testing.T) {
 	RunWithSeeds[Tx](t, NewSimApp[Tx], AppConfig, DefaultSeeds)
 }
 
+// Scenario:
+//
+//	Run 3 times a fresh node with the same seed,
+//	then the app hash should always be the same after n blocks
 func TestAppStateDeterminism(t *testing.T) {
 	var seeds []int64
 	if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue {
@@ -44,13 +48,14 @@ func TestAppStateDeterminism(t *testing.T) {
 		tb.Helper()
 		mx.Lock()
 		defer mx.Unlock()
-		otherHashes, ok := appHashResults[ti.Seed]
+		seed := ti.RandSource.GetSeed()
+		otherHashes, ok := appHashResults[seed]
 		if !ok {
-			appHashResults[ti.Seed] = cs.AppHash
+			appHashResults[seed] = cs.AppHash
 			return
 		}
 		if !bytes.Equal(otherHashes, cs.AppHash) {
-			tb.Fatalf("non-determinism in seed %d", ti.Seed)
+			tb.Fatalf("non-determinism in seed %d", seed)
 		}
 	}
 	// run simulations
@@ -85,7 +90,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 		chainID := SimAppChainID + "_2"
 
 		importGenesisChainStateFactory := func(ctx context.Context, r *rand.Rand) (TestInstance[Tx], ChainState[Tx], []simtypes.Account) {
-			testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.Seed)
+			testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.RandSource)
 			newCs := testInstance.InitializeChain(
 				tb,
 				ctx,
@@ -97,7 +102,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
 			return testInstance, newCs, accs
 		}
 		// run sims with new app setup from exported genesis
-		RunWithSeedX[Tx](tb, cfg, importGenesisChainStateFactory, ti.Seed)
+		RunWithRandSourceX[Tx](tb, cfg, importGenesisChainStateFactory, ti.RandSource)
 	}
 	RunWithSeeds[Tx, *SimApp[Tx]](t, appFactory, AppConfig, DefaultSeeds, exportAndStartChainFromGenesisPostAction)
 }

simsx/v2/rand_source.go

@@ -0,0 +1,89 @@
+package v2
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"math/rand"
+)
+
+const (
+	rngMax  = 1 << 63
+	rngMask = rngMax - 1
+)
+
+// RandSource defines an interface for random number sources with a method to retrieve the seed.
+type RandSource interface {
+	rand.Source
+	GetSeed() int64
+}
+
+var (
+	_ RandSource = &SeededRandomSource{}
+	_ RandSource = &ByteSource{}
+)
+
+// SeededRandomSource wraps a random source with an associated seed value for reproducible random number generation.
+// It implements the RandSource interface, allowing access to both the random source and seed.
+type SeededRandomSource struct {
+	rand.Source
+	seed int64
+}
+
+// NewSeededRandSource constructor
+func NewSeededRandSource(seed int64) *SeededRandomSource {
+	r := new(SeededRandomSource)
+	r.Seed(seed)
+	return r
+}
+
+func (r *SeededRandomSource) Seed(seed int64) {
+	r.seed = seed
+	r.Source = rand.NewSource(seed)
+}
+
+func (r SeededRandomSource) GetSeed() int64 {
+	return r.seed
+}
+
+// ByteSource offers deterministic pseudo-random numbers for math.Rand with fuzzer support.
+// The 'seed' data is read in big endian to uint64. When exhausted,
+// it falls back to a standard random number generator initialized with a specific 'seed' value.
+type ByteSource struct {
+	seed     *bytes.Reader
+	fallback *rand.Rand
+}
+
+// NewByteSource creates a new ByteSource with a specified byte slice and seed. This gives a fixed sequence of pseudo-random numbers.
+// Initially, it utilizes the byte slice. Once that's exhausted, it continues generating numbers using the provided seed.
+func NewByteSource(fuzzSeed []byte, seed int64) *ByteSource {
+	return &ByteSource{
+		seed:     bytes.NewReader(fuzzSeed),
+		fallback: rand.New(rand.NewSource(seed)),
+	}
+}
+
+func (s *ByteSource) Uint64() uint64 {
+	if s.seed.Len() < 8 {
+		return s.fallback.Uint64()
+	}
+	var b [8]byte
+	if _, err := s.seed.Read(b[:]); err != nil && err != io.EOF {
+		panic(err) // Should not happen.
+	}
+	return binary.BigEndian.Uint64(b[:])
+}
+
+func (s *ByteSource) Int63() int64 {
+	return int64(s.Uint64() & rngMask)
+}
+
+// Seed is not supported and will panic
+func (s *ByteSource) Seed(seed int64) {
+	panic("not supported")
+}
+
+// GetSeed is not supported and will panic
+func (s ByteSource) GetSeed() int64 {
+	panic("not supported")
+}

simsx/v2/rand_source_test.go

@@ -0,0 +1,53 @@
+package v2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSeededRandSource(t *testing.T) {
+	const (
+		seed1              int64 = 1
+		firstValFromSeed1  int64 = 0x4d65822107fcfd52
+		secondValFromSeed1 int64 = 0x78629a0f5f3f164f
+	)
+	src := NewSeededRandSource(seed1)
+	for _, v := range []int64{firstValFromSeed1, secondValFromSeed1} {
+		assert.Equal(t, v, src.Int63())
+	}
+	assert.Equal(t, seed1, src.GetSeed())
+}
+
+func TestByteSource(t *testing.T) {
+	const (
+		seed1              = 1
+		firstValFromSeed1  = 0x4d65822107fcfd52
+		secondValFromSeed1 = 0x78629a0f5f3f164f
+	)
+	specs := map[string]struct {
+		fuzzSeed []byte
+		exp      []uint64
+	}{
+		"fallback fuzz takes over": {
+			fuzzSeed: []byte{},
+			exp:      []uint64{firstValFromSeed1, secondValFromSeed1},
+		},
+		"fuzzSeeds served first": {
+			fuzzSeed: []byte{
+				1, 2, 3, 4, 5, 6, 7, 8,
+				9, 10, 11, 12, 13, 14, 15, 16,
+				17, 18, // incomplete uin64, should be ignored
+			},
+			exp: []uint64{0x102030405060708, 0x90a0b0c0d0e0f10, firstValFromSeed1},
+		},
+	}
+	for name, spec := range specs {
+		t.Run(name, func(t *testing.T) {
+			byteSource := NewByteSource(spec.fuzzSeed, seed1)
+			for _, v := range spec.exp {
+				assert.Equal(t, v, byteSource.Uint64())
+			}
+		})
+	}
+}