diff --git a/.pending/bugfixes/simulation/_Fix-non-determinism b/.pending/bugfixes/simulation/_Fix-non-determinism new file mode 100644 index 0000000000..7a1b606ffa --- /dev/null +++ b/.pending/bugfixes/simulation/_Fix-non-determinism @@ -0,0 +1,2 @@ +[\#4861](https://github.com/cosmos/cosmos-sdk/pull/4861) Fix non-determinism simulation +by using CLI flags as input and updating Makefile target. \ No newline at end of file diff --git a/Makefile b/Makefile index 6ca6982765..eea23bde25 100644 --- a/Makefile +++ b/Makefile @@ -86,8 +86,9 @@ test_race: @VERSION=$(VERSION) go test -mod=readonly -race $(PACKAGES_NOSIMULATION) test_sim_nondeterminism: - @echo "Running nondeterminism test..." - @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -v -timeout 10m + @echo "Running non-determinism test..." + @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \ + -NumBlocks=100 -BlockSize=200 -Commit=true -v -timeout 24h test_sim_custom_genesis_fast: @echo "Running custom genesis simulation..." @@ -125,7 +126,6 @@ test_sim_benchmark_invariants: .PHONY: test \ test_sim_nondeterminism \ test_sim_custom_genesis_fast \ -test_sim_fast \ test_sim_import_export \ test_sim_after_import \ test_sim_custom_genesis_multi_seed \ diff --git a/simapp/sim_test.go b/simapp/sim_test.go index ee828f3bae..843496e9d4 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -681,7 +681,7 @@ func TestAppSimulationAfterImport(t *testing.T) { defer func() { newDB.Close() - os.RemoveAll(newDir) + _ = os.RemoveAll(newDir) }() newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, 0, fauxMerkleModeOpt) @@ -708,21 +708,29 @@ func TestAppStateDeterminism(t *testing.T) { for i := 0; i < numSeeds; i++ { seed := rand.Int63() + for j := 0; j < numTimesToRunPerSeed; j++ { logger := log.NewNopLogger() db := dbm.NewMemDB() app := NewSimApp(logger, db, nil, true, 0) - // Run randomized simulation - simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), []sdk.Invariant{}, - 1, 50, 100, 0, "", - false, true, false, false, false, app.ModuleAccountAddrs(), + fmt.Printf( + "Running non-determinism simulation; seed: %d/%d (%d), attempt: %d/%d\n", + i+1, numSeeds, seed, j+1, numTimesToRunPerSeed, ) + + _, _, err := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app), + []sdk.Invariant{}, 1, numBlocks, exportParamsHeight, + blockSize, "", false, commit, lean, + false, false, app.ModuleAccountAddrs(), + ) + require.NoError(t, err) + appHash := app.LastCommitID().Hash appHashList[j] = appHash } + for k := 1; k < numTimesToRunPerSeed; k++ { require.Equal(t, appHashList[0], appHashList[k], "appHash list: %v", appHashList) } diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ca2b507c10..2b5857811d 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -7,7 +7,6 @@ import ( "math/rand" "os" "os/signal" - "runtime/debug" "syscall" "testing" "time" @@ -126,14 +125,12 @@ func SimulateFromSeed( if !testingMode { b.ResetTimer() } else { - // Recover logs in case of panic + // recover logs in case of panic defer func() { if r := recover(); r != nil { - fmt.Fprintf(w, "panic with err: %v\n", r) - stackTrace := string(debug.Stack()) - fmt.Println(stackTrace) + _, _ = fmt.Fprintf(w, "simulation halted due to panic on block %d; %v\n", header.Height, r) logWriter.PrintLogs() - err = fmt.Errorf("Simulation halted due to panic on block %d", header.Height) + panic(r) } }() } @@ -258,22 +255,27 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr operationQueue OperationQueue, timeOperationQueue []FutureOperation, totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn { - lastBlocksizeState := 0 // state for [4 * uniform distribution] + lastBlockSizeState := 0 // state for [4 * uniform distribution] blocksize := 0 selectOp := ops.getSelectOpFn() - return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - accounts []Account, header abci.Header) (opCount int) { + return func( + r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, header abci.Header, + ) (opCount int) { - fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", - header.Height, totalNumBlocks, opCount, blocksize) - lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize) + _, _ = fmt.Fprintf( + w, "\rSimulating... block %d/%d, operation %d/%d.", + header.Height, totalNumBlocks, opCount, blocksize, + ) + lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, avgBlockSize) type opAndR struct { op Operation rand *rand.Rand } + opAndRz := make([]opAndR, 0, blocksize) + // Predetermine the blocksize slice so that we can do things like block // out certain operations without changing the ops that follow. for i := 0; i < blocksize; i++ { diff --git a/x/simulation/util.go b/x/simulation/util.go index 5ed572b94d..28c0f7ab09 100644 --- a/x/simulation/util.go +++ b/x/simulation/util.go @@ -53,21 +53,23 @@ func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B // - "over stuffed" blocks with average size of 2 * avgblocksize, // - normal sized blocks, hitting avgBlocksize on average, // - and empty blocks, with no txs / only txs scheduled from the past. -func getBlockSize(r *rand.Rand, params Params, - lastBlockSizeState, avgBlockSize int) (state, blocksize int) { - +func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize int) (state, blockSize int) { // TODO: Make default blocksize transition matrix actually make the average // blocksize equal to avgBlockSize. state = params.BlockSizeTransitionMatrix.NextState(r, lastBlockSizeState) + switch state { case 0: - blocksize = r.Intn(avgBlockSize * 4) + blockSize = r.Intn(avgBlockSize * 4) + case 1: - blocksize = r.Intn(avgBlockSize * 2) + blockSize = r.Intn(avgBlockSize * 2) + default: - blocksize = 0 + blockSize = 0 } - return state, blocksize + + return state, blockSize } // PeriodicInvariants returns an array of wrapped Invariants. Where each