From b8638f892ce8492592067207c35749bcb6de3646 Mon Sep 17 00:00:00 2001 From: Alexander Peters Date: Fri, 17 Jan 2025 10:52:55 +0100 Subject: [PATCH] test: add more sims tests (#23421) --- scripts/build/simulations.mk | 24 ++++------ simapp/v2/.gitignore | 1 + simapp/v2/sim_bench_test.go | 17 +++++++ simapp/v2/sim_fuzz_test.go | 23 ++++++++++ simapp/v2/sim_runner.go | 40 +++++++++++----- simapp/v2/sim_test.go | 15 ++++-- simsx/v2/rand_source.go | 89 ++++++++++++++++++++++++++++++++++++ simsx/v2/rand_source_test.go | 53 +++++++++++++++++++++ 8 files changed, 230 insertions(+), 32 deletions(-) create mode 100644 simapp/v2/.gitignore create mode 100644 simapp/v2/sim_bench_test.go create mode 100644 simapp/v2/sim_fuzz_test.go create mode 100644 simsx/v2/rand_source.go create mode 100644 simsx/v2/rand_source_test.go diff --git a/scripts/build/simulations.mk b/scripts/build/simulations.mk index 6a5a2d6cfe..b452919dd0 100644 --- a/scripts/build/simulations.mk +++ b/scripts/build/simulations.mk @@ -53,12 +53,6 @@ test-sim-multi-seed-short: @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \ -NumBlocks=50 -test-sim-benchmark-invariants: - # @echo "Running simulation invariant benchmarks..." - # cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \ - # -Enabled=true -NumBlocks=1000 -BlockSize=200 \ - # -Commit=true -Seed=57 -v -timeout 24h - .PHONY: \ test-sim-nondeterminism \ test-sim-nondeterminism-streaming \ @@ -68,29 +62,27 @@ test-sim-after-import \ test-sim-custom-genesis-multi-seed \ test-sim-multi-seed-short \ test-sim-multi-seed-long \ -test-sim-benchmark-invariants SIM_NUM_BLOCKS ?= 500 SIM_BLOCK_SIZE ?= 200 -SIM_COMMIT ?= true #? test-sim-fuzz: Run fuzz test for simapp test-sim-fuzz: -# @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!" + @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!" #ld flags are a quick fix to make it work on current osx -# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -json -tags='sims' -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20 #? test-sim-benchmark: Run benchmark test for simapp test-sim-benchmark: -# @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" -# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ - -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h + @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ + -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) test-sim-profile: -# @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" -# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ - -Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out + @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" + @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \ + -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -cpuprofile cpu.out -memprofile mem.out .PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz diff --git a/simapp/v2/.gitignore b/simapp/v2/.gitignore new file mode 100644 index 0000000000..3c805e2571 --- /dev/null +++ b/simapp/v2/.gitignore @@ -0,0 +1 @@ +/simapp.test diff --git a/simapp/v2/sim_bench_test.go b/simapp/v2/sim_bench_test.go new file mode 100644 index 0000000000..b8ac3fb8fb --- /dev/null +++ b/simapp/v2/sim_bench_test.go @@ -0,0 +1,17 @@ +//go:build sims + +package simapp + +import ( + "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + "testing" +) + +func BenchmarkFullAppSimulation(b *testing.B) { + b.ReportAllocs() + cfg := cli.NewConfigFromFlags() + cfg.ChainID = SimAppChainID + for i := 0; i < b.N; i++ { + RunWithSeed[Tx](b, NewSimApp[Tx], AppConfig, cfg, 1) + } +} diff --git a/simapp/v2/sim_fuzz_test.go b/simapp/v2/sim_fuzz_test.go new file mode 100644 index 0000000000..3d34186426 --- /dev/null +++ b/simapp/v2/sim_fuzz_test.go @@ -0,0 +1,23 @@ +//go:build sims + +package simapp + +import ( + simsxv2 "github.com/cosmos/cosmos-sdk/simsx/v2" + simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + "testing" +) + +func FuzzFullAppSimulation(f *testing.F) { + cfg := simcli.NewConfigFromFlags() + cfg.ChainID = SimAppChainID + + f.Fuzz(func(t *testing.T, rawSeed []byte) { + if len(rawSeed) < 8 { + t.Skip() + return + } + randSource := simsxv2.NewByteSource(cfg.FuzzSeed, cfg.Seed) + RunWithRandSource[Tx](t, NewSimApp[Tx], AppConfig, cfg, randSource) + }) +} diff --git a/simapp/v2/sim_runner.go b/simapp/v2/sim_runner.go index 6610566e0d..318afab375 100644 --- a/simapp/v2/sim_runner.go +++ b/simapp/v2/sim_runner.go @@ -106,7 +106,7 @@ type ( // TestInstance system under test TestInstance[T Tx] struct { - Seed int64 + RandSource simsxv2.RandSource App SimulationApp[T] TxDecoder transaction.Codec[T] BankKeeper BankKeeper @@ -126,7 +126,7 @@ func SetupTestInstance[T Tx, V SimulationApp[T]]( tb testing.TB, appFactory AppFactory[T, V], appConfigFactory AppConfigFactory, - seed int64, + randSource simsxv2.RandSource, ) TestInstance[T] { tb.Helper() vp := viper.New() @@ -153,7 +153,7 @@ func SetupTestInstance[T Tx, V SimulationApp[T]]( xapp, err := appFactory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings())))) require.NoError(tb, err) return TestInstance[T]{ - Seed: seed, + RandSource: randSource, App: xapp, BankKeeper: bankKeeper, AuthKeeper: authKeeper, @@ -197,7 +197,7 @@ func (ti TestInstance[T]) InitializeChain( } } -// RunWithSeeds runs a series of subtests using the default set of random seeds for deterministic simulation testing. +// RunWithSeeds runs a series of subtests for each of the given set of seeds for deterministic simulation testing. func RunWithSeeds[T Tx, V SimulationApp[T]]( t *testing.T, appFactory AppFactory[T, V], @@ -224,13 +224,26 @@ func RunWithSeed[T Tx, V SimulationApp[T]]( tCfg simtypes.Config, seed int64, postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account), +) { + tb.Helper() + RunWithRandSource(tb, appFactory, appConfigFactory, tCfg, simsxv2.NewSeededRandSource(seed), postRunActions...) +} + +// RunWithRandSource initializes and executes a simulation run with the given rand source, generating blocks and transactions. +func RunWithRandSource[T Tx, V SimulationApp[T]]( + tb testing.TB, + appFactory AppFactory[T, V], + appConfigFactory AppConfigFactory, + tCfg simtypes.Config, + randSource simsxv2.RandSource, + postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account), ) { tb.Helper() initialBlockHeight := tCfg.InitialBlockHeight require.NotEmpty(tb, initialBlockHeight, "initial block height must not be 0") setupFn := func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account) { - testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, seed) + testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, randSource) accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState( testInstance.App, r, @@ -249,20 +262,21 @@ func RunWithSeed[T Tx, V SimulationApp[T]]( return testInstance, cs, accounts } - RunWithSeedX(tb, tCfg, setupFn, seed, postRunActions...) + RunWithRandSourceX(tb, tCfg, setupFn, randSource, postRunActions...) } -// RunWithSeedX entrypoint for custom chain setups. -// The function runs the full simulation test circle for the specified seed and setup function, followed by optional post-run actions. -func RunWithSeedX[T Tx]( +// RunWithRandSourceX entrypoint for custom chain setups. +// The function runs the full simulation test circle for the specified random source and setup function, followed by optional post-run actions. +// when tb implements ResetTimer, the method is called after setup, before jumping into the main loop +func RunWithRandSourceX[T Tx]( tb testing.TB, tCfg simtypes.Config, setupChainStateFn func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account), - seed int64, + randSource rand.Source, postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account), ) { tb.Helper() - r := rand.New(rand.NewSource(seed)) + r := rand.New(randSource) rootCtx, done := context.WithCancel(context.Background()) defer done() @@ -273,6 +287,10 @@ func RunWithSeedX[T Tx]( modules := testInstance.ModuleManager.Modules() msgFactoriesFn := prepareSimsMsgFactories(r, modules, simsx.ParamWeightSource(emptySimParams)) + if b, ok := tb.(interface{ ResetTimer() }); ok { + b.ResetTimer() + } + doMainLoop( tb, rootCtx, diff --git a/simapp/v2/sim_test.go b/simapp/v2/sim_test.go index 059e8a203f..07de5c4288 100644 --- a/simapp/v2/sim_test.go +++ b/simapp/v2/sim_test.go @@ -25,6 +25,10 @@ func TestFullAppSimulation(t *testing.T) { RunWithSeeds[Tx](t, NewSimApp[Tx], AppConfig, DefaultSeeds) } +// Scenario: +// +// Run 3 times a fresh node with the same seed, +// then the app hash should always be the same after n blocks func TestAppStateDeterminism(t *testing.T) { var seeds []int64 if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue { @@ -44,13 +48,14 @@ func TestAppStateDeterminism(t *testing.T) { tb.Helper() mx.Lock() defer mx.Unlock() - otherHashes, ok := appHashResults[ti.Seed] + seed := ti.RandSource.GetSeed() + otherHashes, ok := appHashResults[seed] if !ok { - appHashResults[ti.Seed] = cs.AppHash + appHashResults[seed] = cs.AppHash return } if !bytes.Equal(otherHashes, cs.AppHash) { - tb.Fatalf("non-determinism in seed %d", ti.Seed) + tb.Fatalf("non-determinism in seed %d", seed) } } // run simulations @@ -85,7 +90,7 @@ func TestAppSimulationAfterImport(t *testing.T) { chainID := SimAppChainID + "_2" importGenesisChainStateFactory := func(ctx context.Context, r *rand.Rand) (TestInstance[Tx], ChainState[Tx], []simtypes.Account) { - testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.Seed) + testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.RandSource) newCs := testInstance.InitializeChain( tb, ctx, @@ -97,7 +102,7 @@ func TestAppSimulationAfterImport(t *testing.T) { return testInstance, newCs, accs } // run sims with new app setup from exported genesis - RunWithSeedX[Tx](tb, cfg, importGenesisChainStateFactory, ti.Seed) + RunWithRandSourceX[Tx](tb, cfg, importGenesisChainStateFactory, ti.RandSource) } RunWithSeeds[Tx, *SimApp[Tx]](t, appFactory, AppConfig, DefaultSeeds, exportAndStartChainFromGenesisPostAction) } diff --git a/simsx/v2/rand_source.go b/simsx/v2/rand_source.go new file mode 100644 index 0000000000..d87a3d4fd8 --- /dev/null +++ b/simsx/v2/rand_source.go @@ -0,0 +1,89 @@ +package v2 + +import ( + "bytes" + "encoding/binary" + "io" + "math/rand" +) + +const ( + rngMax = 1 << 63 + rngMask = rngMax - 1 +) + +// RandSource defines an interface for random number sources with a method to retrieve the seed. +type RandSource interface { + rand.Source + GetSeed() int64 +} + +var ( + _ RandSource = &SeededRandomSource{} + _ RandSource = &ByteSource{} +) + +// SeededRandomSource wraps a random source with an associated seed value for reproducible random number generation. +// It implements the RandSource interface, allowing access to both the random source and seed. +type SeededRandomSource struct { + rand.Source + seed int64 +} + +// NewSeededRandSource constructor +func NewSeededRandSource(seed int64) *SeededRandomSource { + r := new(SeededRandomSource) + r.Seed(seed) + return r +} + +func (r *SeededRandomSource) Seed(seed int64) { + r.seed = seed + r.Source = rand.NewSource(seed) +} + +func (r SeededRandomSource) GetSeed() int64 { + return r.seed +} + +// ByteSource offers deterministic pseudo-random numbers for math.Rand with fuzzer support. +// The 'seed' data is read in big endian to uint64. When exhausted, +// it falls back to a standard random number generator initialized with a specific 'seed' value. +type ByteSource struct { + seed *bytes.Reader + fallback *rand.Rand +} + +// NewByteSource creates a new ByteSource with a specified byte slice and seed. This gives a fixed sequence of pseudo-random numbers. +// Initially, it utilizes the byte slice. Once that's exhausted, it continues generating numbers using the provided seed. +func NewByteSource(fuzzSeed []byte, seed int64) *ByteSource { + return &ByteSource{ + seed: bytes.NewReader(fuzzSeed), + fallback: rand.New(rand.NewSource(seed)), + } +} + +func (s *ByteSource) Uint64() uint64 { + if s.seed.Len() < 8 { + return s.fallback.Uint64() + } + var b [8]byte + if _, err := s.seed.Read(b[:]); err != nil && err != io.EOF { + panic(err) // Should not happen. + } + return binary.BigEndian.Uint64(b[:]) +} + +func (s *ByteSource) Int63() int64 { + return int64(s.Uint64() & rngMask) +} + +// Seed is not supported and will panic +func (s *ByteSource) Seed(seed int64) { + panic("not supported") +} + +// GetSeed is not supported and will panic +func (s ByteSource) GetSeed() int64 { + panic("not supported") +} diff --git a/simsx/v2/rand_source_test.go b/simsx/v2/rand_source_test.go new file mode 100644 index 0000000000..5d45ded94f --- /dev/null +++ b/simsx/v2/rand_source_test.go @@ -0,0 +1,53 @@ +package v2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSeededRandSource(t *testing.T) { + const ( + seed1 int64 = 1 + firstValFromSeed1 int64 = 0x4d65822107fcfd52 + secondValFromSeed1 int64 = 0x78629a0f5f3f164f + ) + src := NewSeededRandSource(seed1) + for _, v := range []int64{firstValFromSeed1, secondValFromSeed1} { + assert.Equal(t, v, src.Int63()) + } + assert.Equal(t, seed1, src.GetSeed()) +} + +func TestByteSource(t *testing.T) { + const ( + seed1 = 1 + firstValFromSeed1 = 0x4d65822107fcfd52 + secondValFromSeed1 = 0x78629a0f5f3f164f + ) + specs := map[string]struct { + fuzzSeed []byte + exp []uint64 + }{ + "fallback fuzz takes over": { + fuzzSeed: []byte{}, + exp: []uint64{firstValFromSeed1, secondValFromSeed1}, + }, + "fuzzSeeds served first": { + fuzzSeed: []byte{ + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, // incomplete uin64, should be ignored + }, + exp: []uint64{0x102030405060708, 0x90a0b0c0d0e0f10, firstValFromSeed1}, + }, + } + for name, spec := range specs { + t.Run(name, func(t *testing.T) { + byteSource := NewByteSource(spec.fuzzSeed, seed1) + for _, v := range spec.exp { + assert.Equal(t, v, byteSource.Uint64()) + } + }) + } +}