From 57cc5ae3a27393b7cc83a7bf1300a2859be344a8 Mon Sep 17 00:00:00 2001 From: Federico Kunze <31522760+fedekunze@users.noreply.github.com> Date: Thu, 15 Aug 2019 17:35:25 +0200 Subject: [PATCH] Merge PR #4906: Introduce Simulation Config --- CHANGELOG.md | 1 + simapp/sim_test.go | 278 +++++++++++++-------------------------- simapp/state.go | 142 ++++++++++++++++++++ simapp/utils.go | 134 +++++++++---------- x/simulation/config.go | 23 ++++ x/simulation/simulate.go | 76 +++++------ 6 files changed, 362 insertions(+), 292 deletions(-) create mode 100644 simapp/state.go create mode 100644 x/simulation/config.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5495e8480b..6e09fb97f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ longer panics if the store to load contains substores that we didn't explicitly * Implement `SimulationManager` for executing modules' simulation functionalities in a modularized way * Add `DecodeStore` to the `SimulationManager` for decoding each module's types * (simulation) [\#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change SimApp keepers to be public and add getter functions for keys and codec +* (simulation) [\#4906](https://github.com/cosmos/cosmos-sdk/issues/4906) Add simulation `Config` struct that wraps simulation flags * (store) [\#4792](https://github.com/cosmos/cosmos-sdk/issues/4792) panic on non-registered store * (types) [\#4821](https://github.com/cosmos/cosmos-sdk/issues/4821) types/errors package added with support for stacktraces. It is meant as a more feature-rich replacement for sdk.Errors in the mid-term. diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 180e05b83a..3557432da0 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -2,14 +2,11 @@ package simapp import ( "encoding/json" - "flag" "fmt" - "io" "io/ioutil" "math/rand" "os" "testing" - "time" "github.com/stretchr/testify/require" @@ -38,135 +35,17 @@ import ( ) func init() { - flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file") - flag.StringVar(¶msFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis") - flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON") - flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params") - flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON") - flag.StringVar(&exportStatsPath, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON") - flag.Int64Var(&seed, "Seed", 42, "simulation random seed") - flag.IntVar(&initialBlockHeight, "InitialBlockHeight", 1, "initial block to start the simulation") - flag.IntVar(&numBlocks, "NumBlocks", 500, "number of new blocks to simulate from the initial block height") - flag.IntVar(&blockSize, "BlockSize", 200, "operations per block") - flag.BoolVar(&enabled, "Enabled", false, "enable the simulation") - flag.BoolVar(&verbose, "Verbose", false, "verbose log output") - flag.BoolVar(&lean, "Lean", false, "lean simulation log output") - flag.BoolVar(&commit, "Commit", false, "have the simulation commit") - flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions") - flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation") - flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found") - flag.Int64Var(&genesisTime, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time") -} - -// helper function for populating input for SimulateFromSeed -// TODO: clean up this function along with the simulation refactor -func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) ( - testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64, - simulation.WeightedOperations, sdk.Invariants, int, int, int, int, string, - bool, bool, bool, bool, bool, map[string]bool) { - - exportParams := exportParamsPath != "" - - return tb, w, app.BaseApp, appStateFn, seed, - testAndRunTxs(app), invariants(app), - initialBlockHeight, numBlocks, exportParamsHeight, blockSize, - exportStatsPath, exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs() -} - -func appStateFn( - r *rand.Rand, accs []simulation.Account, -) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) { - - cdc := MakeCodec() - - if genesisTime == 0 { - genesisTimestamp = simulation.RandTimestamp(r) - } else { - genesisTimestamp = time.Unix(genesisTime, 0) - } - - switch { - case paramsFile != "" && genesisFile != "": - panic("cannot provide both a genesis file and a params file") - - case genesisFile != "": - appState, simAccs, chainID = AppStateFromGenesisFileFn(r, accs, genesisTimestamp) - - case paramsFile != "": - appParams := make(simulation.AppParams) - bz, err := ioutil.ReadFile(paramsFile) - if err != nil { - panic(err) - } - - cdc.MustUnmarshalJSON(bz, &appParams) - appState, simAccs, chainID = appStateRandomizedFn(r, accs, genesisTimestamp, appParams) - - default: - appParams := make(simulation.AppParams) - appState, simAccs, chainID = appStateRandomizedFn(r, accs, genesisTimestamp, appParams) - } - - return appState, simAccs, chainID, genesisTimestamp -} - -// TODO refactor out random initialization code to the modules -func appStateRandomizedFn( - r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time, appParams simulation.AppParams, -) (json.RawMessage, []simulation.Account, string) { - - cdc := MakeCodec() - genesisState := NewDefaultGenesisState() - - var ( - amount int64 - numInitiallyBonded int64 - ) - - appParams.GetOrGenerate(cdc, StakePerAccount, &amount, r, - func(r *rand.Rand) { amount = int64(r.Intn(1e12)) }) - appParams.GetOrGenerate(cdc, InitiallyBondedValidators, &amount, r, - func(r *rand.Rand) { numInitiallyBonded = int64(r.Intn(250)) }) - - numAccs := int64(len(accs)) - if numInitiallyBonded > numAccs { - numInitiallyBonded = numAccs - } - - fmt.Printf( - `Selected randomly generated parameters for simulated genesis: -{ - stake_per_account: "%v", - initially_bonded_validators: "%v" -} -`, amount, numInitiallyBonded, - ) - - GenGenesisAccounts(cdc, r, accs, genesisTimestamp, amount, numInitiallyBonded, genesisState) - GenAuthGenesisState(cdc, r, appParams, genesisState) - GenBankGenesisState(cdc, r, appParams, genesisState) - GenSupplyGenesisState(cdc, amount, numInitiallyBonded, int64(len(accs)), genesisState) - GenGovGenesisState(cdc, r, appParams, genesisState) - GenMintGenesisState(cdc, r, appParams, genesisState) - GenDistrGenesisState(cdc, r, appParams, genesisState) - stakingGen := GenStakingGenesisState(cdc, r, accs, amount, numAccs, numInitiallyBonded, appParams, genesisState) - GenSlashingGenesisState(cdc, r, stakingGen, appParams, genesisState) - - appState, err := MakeCodec().MarshalJSON(genesisState) - if err != nil { - panic(err) - } - - return appState, accs, "simulation" + GetSimulatorFlags() } // TODO: add description -func testAndRunTxs(app *SimApp) []simulation.WeightedOperation { +func testAndRunTxs(app *SimApp, config simulation.Config) []simulation.WeightedOperation { + cdc := MakeCodec() ap := make(simulation.AppParams) - if paramsFile != "" { - bz, err := ioutil.ReadFile(paramsFile) + if config.ParamsFile != "" { + bz, err := ioutil.ReadFile(config.ParamsFile) if err != nil { panic(err) } @@ -357,10 +236,10 @@ func testAndRunTxs(app *SimApp) []simulation.WeightedOperation { func invariants(app *SimApp) []sdk.Invariant { // TODO: fix PeriodicInvariants, it doesn't seem to call individual invariants for a period of 1 // Ref: https://github.com/cosmos/cosmos-sdk/issues/4631 - if period == 1 { + if flagPeriodValue == 1 { return app.CrisisKeeper.Invariants() } - return simulation.PeriodicInvariants(app.CrisisKeeper.Invariants(), period, 0) + return simulation.PeriodicInvariants(app.CrisisKeeper.Invariants(), flagPeriodValue, 0) } // Pass this in as an option to use a dbStoreAdapter instead of an IAVLStore for simulation speed. @@ -372,6 +251,7 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { // /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out func BenchmarkFullAppSimulation(b *testing.B) { logger := log.NewNopLogger() + config := NewConfigFromFlags() var db dbm.DB dir, _ := ioutil.TempDir("", "goleveldb-app-sim") @@ -384,24 +264,28 @@ func BenchmarkFullAppSimulation(b *testing.B) { // Run randomized simulation // TODO: parameterize numbers, save for a later PR - _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app)) + _, params, simErr := simulation.SimulateFromSeed( + b, os.Stdout, app.BaseApp, AppStateFn, + testAndRunTxs(app, config), invariants(app), + app.ModuleAccountAddrs(), config, + ) // export state and params before the simulation error is checked - if exportStatePath != "" { + if config.ExportStatePath != "" { fmt.Println("Exporting app state...") appState, _, err := app.ExportAppStateAndValidators(false, nil) if err != nil { fmt.Println(err) b.Fail() } - err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644) if err != nil { fmt.Println(err) b.Fail() } } - if exportParamsPath != "" { + if config.ExportParamsPath != "" { fmt.Println("Exporting simulation params...") paramsBz, err := json.MarshalIndent(params, "", " ") if err != nil { @@ -409,7 +293,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { b.Fail() } - err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644) if err != nil { fmt.Println(err) b.Fail() @@ -421,7 +305,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { b.FailNow() } - if commit { + if config.Commit { fmt.Println("\nGoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) @@ -429,13 +313,14 @@ func BenchmarkFullAppSimulation(b *testing.B) { } func TestFullAppSimulation(t *testing.T) { - if !enabled { + if !flagEnabledValue { t.Skip("Skipping application simulation") } var logger log.Logger + config := NewConfigFromFlags() - if verbose { + if flagVerboseValue { logger = log.TestingLogger() } else { logger = log.NewNopLogger() @@ -454,31 +339,35 @@ func TestFullAppSimulation(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - _, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + _, params, simErr := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, AppStateFn, + testAndRunTxs(app, config), invariants(app), + app.ModuleAccountAddrs(), config, + ) // export state and params before the simulation error is checked - if exportStatePath != "" { + if config.ExportStatePath != "" { fmt.Println("Exporting app state...") appState, _, err := app.ExportAppStateAndValidators(false, nil) require.NoError(t, err) - err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644) require.NoError(t, err) } - if exportParamsPath != "" { + if config.ExportParamsPath != "" { fmt.Println("Exporting simulation params...") fmt.Println(params) paramsBz, err := json.MarshalIndent(params, "", " ") require.NoError(t, err) - err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644) require.NoError(t, err) } require.NoError(t, simErr) - if commit { + if config.Commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) fmt.Println("\nGoLevelDB Stats") @@ -488,12 +377,14 @@ func TestFullAppSimulation(t *testing.T) { } func TestAppImportExport(t *testing.T) { - if !enabled { + if !flagEnabledValue { t.Skip("Skipping application import/export simulation") } var logger log.Logger - if verbose { + config := NewConfigFromFlags() + + if flagVerboseValue { logger = log.TestingLogger() } else { logger = log.NewNopLogger() @@ -512,30 +403,34 @@ func TestAppImportExport(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - _, simParams, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + _, simParams, simErr := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, AppStateFn, + testAndRunTxs(app, config), invariants(app), + app.ModuleAccountAddrs(), config, + ) // export state and simParams before the simulation error is checked - if exportStatePath != "" { + if config.ExportStatePath != "" { fmt.Println("Exporting app state...") appState, _, err := app.ExportAppStateAndValidators(false, nil) require.NoError(t, err) - err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644) require.NoError(t, err) } - if exportParamsPath != "" { + if config.ExportParamsPath != "" { fmt.Println("Exporting simulation params...") simParamsBz, err := json.MarshalIndent(simParams, "", " ") require.NoError(t, err) - err = ioutil.WriteFile(exportParamsPath, simParamsBz, 0644) + err = ioutil.WriteFile(config.ExportParamsPath, simParamsBz, 0644) require.NoError(t, err) } require.NoError(t, simErr) - if commit { + if config.Commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) fmt.Println("\nGoLevelDB Stats") @@ -562,9 +457,7 @@ func TestAppImportExport(t *testing.T) { var genesisState GenesisState err = app.cdc.UnmarshalJSON(appState, &genesisState) - if err != nil { - panic(err) - } + require.NoError(t, err) ctxB := newApp.NewContext(true, abci.Header{Height: app.LastBlockHeight()}) newApp.mm.InitGenesis(ctxB, genesisState) @@ -611,12 +504,14 @@ func TestAppImportExport(t *testing.T) { } func TestAppSimulationAfterImport(t *testing.T) { - if !enabled { + if !flagEnabledValue { t.Skip("Skipping application simulation after import") } var logger log.Logger - if verbose { + config := NewConfigFromFlags() + + if flagVerboseValue { logger = log.TestingLogger() } else { logger = log.NewNopLogger() @@ -634,30 +529,34 @@ func TestAppSimulationAfterImport(t *testing.T) { require.Equal(t, "SimApp", app.Name()) // Run randomized simulation - stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app)) + stopEarly, params, simErr := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, AppStateFn, + testAndRunTxs(app, config), invariants(app), + app.ModuleAccountAddrs(), config, + ) // export state and params before the simulation error is checked - if exportStatePath != "" { + if config.ExportStatePath != "" { fmt.Println("Exporting app state...") appState, _, err := app.ExportAppStateAndValidators(false, nil) require.NoError(t, err) - err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644) require.NoError(t, err) } - if exportParamsPath != "" { + if config.ExportParamsPath != "" { fmt.Println("Exporting simulation params...") paramsBz, err := json.MarshalIndent(params, "", " ") require.NoError(t, err) - err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644) require.NoError(t, err) } require.NoError(t, simErr) - if commit { + if config.Commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) fmt.Println("\nGoLevelDB Stats") @@ -674,9 +573,7 @@ func TestAppSimulationAfterImport(t *testing.T) { fmt.Printf("Exporting genesis...\n") appState, _, err := app.ExportAppStateAndValidators(true, []string{}) - if err != nil { - panic(err) - } + require.NoError(t, err) fmt.Printf("Importing genesis...\n") @@ -695,23 +592,34 @@ func TestAppSimulationAfterImport(t *testing.T) { }) // Run randomized simulation on imported app - _, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp)) - require.Nil(t, err) + _, _, err = simulation.SimulateFromSeed( + t, os.Stdout, newApp.BaseApp, AppStateFn, + testAndRunTxs(newApp, config), invariants(newApp), + newApp.ModuleAccountAddrs(), config, + ) + + require.NoError(t, err) } // TODO: Make another test for the fuzzer itself, which just has noOp txs // and doesn't depend on the application. func TestAppStateDeterminism(t *testing.T) { - if !enabled { + if !flagEnabledValue { t.Skip("Skipping application simulation") } + config := NewConfigFromFlags() + config.InitialBlockHeight = 1 + config.ExportParamsPath = "" + config.OnOperation = false + config.AllInvariants = false + numSeeds := 3 numTimesToRunPerSeed := 5 appHashList := make([]json.RawMessage, numTimesToRunPerSeed) for i := 0; i < numSeeds; i++ { - seed := rand.Int63() + config.Seed = rand.Int63() for j := 0; j < numTimesToRunPerSeed; j++ { logger := log.NewNopLogger() @@ -720,29 +628,30 @@ func TestAppStateDeterminism(t *testing.T) { fmt.Printf( "Running non-determinism simulation; seed: %d/%d (%d), attempt: %d/%d\n", - i+1, numSeeds, seed, j+1, numTimesToRunPerSeed, + i+1, numSeeds, config.Seed, j+1, numTimesToRunPerSeed, ) _, _, err := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app), - []sdk.Invariant{}, 1, numBlocks, exportParamsHeight, - blockSize, "", false, commit, lean, - false, false, app.ModuleAccountAddrs(), + t, os.Stdout, app.BaseApp, AppStateFn, + testAndRunTxs(app, config), []sdk.Invariant{}, + app.ModuleAccountAddrs(), config, ) require.NoError(t, err) appHash := app.LastCommitID().Hash appHashList[j] = appHash - } - for k := 1; k < numTimesToRunPerSeed; k++ { - require.Equal(t, appHashList[0], appHashList[k], "appHash list: %v", appHashList) + if j != 0 { + require.Equal(t, appHashList[0], appHashList[j], "appHash list: %v", appHashList) + } } } } func BenchmarkInvariants(b *testing.B) { logger := log.NewNopLogger() + config := NewConfigFromFlags() + dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench") db, _ := sdk.NewLevelDB("simulation", dir) @@ -752,31 +661,30 @@ func BenchmarkInvariants(b *testing.B) { }() app := NewSimApp(logger, db, nil, true, 0) - exportParams := exportParamsPath != "" // 2. Run parameterized simulation (w/o invariants) _, params, simErr := simulation.SimulateFromSeed( - b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app), - []sdk.Invariant{}, initialBlockHeight, numBlocks, exportParamsHeight, blockSize, - exportStatsPath, exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(), + b, ioutil.Discard, app.BaseApp, AppStateFn, + testAndRunTxs(app, config), []sdk.Invariant{}, + app.ModuleAccountAddrs(), config, ) // export state and params before the simulation error is checked - if exportStatePath != "" { + if config.ExportStatePath != "" { fmt.Println("Exporting app state...") appState, _, err := app.ExportAppStateAndValidators(false, nil) if err != nil { fmt.Println(err) b.Fail() } - err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644) + err = ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644) if err != nil { fmt.Println(err) b.Fail() } } - if exportParamsPath != "" { + if config.ExportParamsPath != "" { fmt.Println("Exporting simulation params...") paramsBz, err := json.MarshalIndent(params, "", " ") if err != nil { @@ -784,7 +692,7 @@ func BenchmarkInvariants(b *testing.B) { b.Fail() } - err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644) + err = ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644) if err != nil { fmt.Println(err) b.Fail() @@ -805,7 +713,7 @@ func BenchmarkInvariants(b *testing.B) { for _, cr := range app.CrisisKeeper.Routes() { b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) { if res, stop := cr.Invar(ctx); stop { - fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, numBlocks, res) + fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res) b.FailNow() } }) diff --git a/simapp/state.go b/simapp/state.go new file mode 100644 index 0000000000..e9405ddda8 --- /dev/null +++ b/simapp/state.go @@ -0,0 +1,142 @@ +package simapp + +// DONTCOVER + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "time" + + "github.com/tendermint/tendermint/crypto/secp256k1" + tmtypes "github.com/tendermint/tendermint/types" + + "github.com/cosmos/cosmos-sdk/x/genaccounts" + "github.com/cosmos/cosmos-sdk/x/simulation" +) + +// AppStateFn returns the initial application state using a genesis or the simulation parameters. +// It panics if the user provides files for both of them. +// If a file is not given for the genesis or the sim params, it creates a randomized one. +func AppStateFn( + r *rand.Rand, accs []simulation.Account, config simulation.Config, +) (appState json.RawMessage, simAccs []simulation.Account, chainID string, genesisTimestamp time.Time) { + + cdc := MakeCodec() + + if flagGenesisTimeValue == 0 { + genesisTimestamp = simulation.RandTimestamp(r) + } else { + genesisTimestamp = time.Unix(flagGenesisTimeValue, 0) + } + + switch { + case config.ParamsFile != "" && config.GenesisFile != "": + panic("cannot provide both a genesis file and a params file") + + case config.GenesisFile != "": + appState, simAccs, chainID = AppStateFromGenesisFileFn(r, config) + + case config.ParamsFile != "": + appParams := make(simulation.AppParams) + bz, err := ioutil.ReadFile(config.ParamsFile) + if err != nil { + panic(err) + } + + cdc.MustUnmarshalJSON(bz, &appParams) + appState, simAccs, chainID = AppStateRandomizedFn(r, accs, genesisTimestamp, appParams) + + default: + appParams := make(simulation.AppParams) + appState, simAccs, chainID = AppStateRandomizedFn(r, accs, genesisTimestamp, appParams) + } + + return appState, simAccs, chainID, genesisTimestamp +} + +// AppStateRandomizedFn creates calls each module's GenesisState generator function +// and creates +func AppStateRandomizedFn( + r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time, appParams simulation.AppParams, +) (json.RawMessage, []simulation.Account, string) { + + cdc := MakeCodec() + genesisState := NewDefaultGenesisState() + + var ( + amount int64 + numInitiallyBonded int64 + ) + + appParams.GetOrGenerate(cdc, StakePerAccount, &amount, r, + func(r *rand.Rand) { amount = int64(r.Intn(1e12)) }) + appParams.GetOrGenerate(cdc, InitiallyBondedValidators, &amount, r, + func(r *rand.Rand) { numInitiallyBonded = int64(r.Intn(250)) }) + + numAccs := int64(len(accs)) + if numInitiallyBonded > numAccs { + numInitiallyBonded = numAccs + } + + fmt.Printf( + `Selected randomly generated parameters for simulated genesis: +{ + stake_per_account: "%v", + initially_bonded_validators: "%v" +} +`, amount, numInitiallyBonded, + ) + + GenGenesisAccounts(cdc, r, accs, genesisTimestamp, amount, numInitiallyBonded, genesisState) + GenAuthGenesisState(cdc, r, appParams, genesisState) + GenBankGenesisState(cdc, r, appParams, genesisState) + GenSupplyGenesisState(cdc, amount, numInitiallyBonded, int64(len(accs)), genesisState) + GenGovGenesisState(cdc, r, appParams, genesisState) + GenMintGenesisState(cdc, r, appParams, genesisState) + GenDistrGenesisState(cdc, r, appParams, genesisState) + stakingGen := GenStakingGenesisState(cdc, r, accs, amount, numAccs, numInitiallyBonded, appParams, genesisState) + GenSlashingGenesisState(cdc, r, stakingGen, appParams, genesisState) + + appState, err := MakeCodec().MarshalJSON(genesisState) + if err != nil { + panic(err) + } + + return appState, accs, "simulation" +} + +// AppStateFromGenesisFileFn util function to generate the genesis AppState +// from a genesis.json file +func AppStateFromGenesisFileFn(r *rand.Rand, config simulation.Config) (json.RawMessage, []simulation.Account, string) { + + var genesis tmtypes.GenesisDoc + cdc := MakeCodec() + + bytes, err := ioutil.ReadFile(config.GenesisFile) + if err != nil { + panic(err) + } + + cdc.MustUnmarshalJSON(bytes, &genesis) + + var appState GenesisState + cdc.MustUnmarshalJSON(genesis.AppState, &appState) + + accounts := genaccounts.GetGenesisStateFromAppState(cdc, appState) + + var newAccs []simulation.Account + for _, acc := range accounts { + // Pick a random private key, since we don't know the actual key + // This should be fine as it's only used for mock Tendermint validators + // and these keys are never actually used to sign by mock Tendermint. + privkeySeed := make([]byte, 15) + r.Read(privkeySeed) + + privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed) + newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address}) + } + + return genesis.AppState, newAccs, genesis.ChainID +} diff --git a/simapp/utils.go b/simapp/utils.go index 90bc28176e..b0bd4f26a3 100644 --- a/simapp/utils.go +++ b/simapp/utils.go @@ -3,19 +3,13 @@ package simapp import ( "encoding/json" + "flag" "fmt" - "io" - "io/ioutil" "math/rand" "time" - "github.com/tendermint/tendermint/crypto/secp256k1" cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/log" - tmtypes "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tm-db" - "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/codec" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" @@ -30,73 +24,76 @@ import ( "github.com/cosmos/cosmos-sdk/x/supply" ) -// List of available flags for the simulator +//--------------------------------------------------------------------- +// Flags + +// List of SimApp flags for the simulator var ( - genesisFile string - paramsFile string - exportParamsPath string - exportParamsHeight int - exportStatePath string - exportStatsPath string - seed int64 - initialBlockHeight int - numBlocks int - blockSize int - enabled bool - verbose bool - lean bool - commit bool - period int - onOperation bool // TODO Remove in favor of binary search for invariant violation - allInvariants bool - genesisTime int64 + flagGenesisFileValue string + flagParamsFileValue string + flagExportParamsPathValue string + flagExportParamsHeightValue int + flagExportStatePathValue string + flagExportStatsPathValue string + flagSeedValue int64 + flagInitialBlockHeightValue int + flagNumBlocksValue int + flagBlockSizeValue int + flagLeanValue bool + flagCommitValue bool + flagOnOperationValue bool // TODO: Remove in favor of binary search for invariant violation + flagAllInvariantsValue bool + + flagEnabledValue bool + flagVerboseValue bool + flagPeriodValue int + flagGenesisTimeValue int64 ) -// NewSimAppUNSAFE is used for debugging purposes only. -// -// NOTE: to not use this function with non-test code -func NewSimAppUNSAFE(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest bool, - invCheckPeriod uint, baseAppOptions ...func(*baseapp.BaseApp), -) (gapp *SimApp, keyMain, keyStaking *sdk.KVStoreKey, stakingKeeper staking.Keeper) { +// GetSimulatorFlags gets the values of all the available simulation flags +func GetSimulatorFlags() { - gapp = NewSimApp(logger, db, traceStore, loadLatest, invCheckPeriod, baseAppOptions...) - return gapp, gapp.keys[baseapp.MainStoreKey], gapp.keys[staking.StoreKey], gapp.StakingKeeper + // Config fields + flag.StringVar(&flagGenesisFileValue, "Genesis", "", "custom simulation genesis file; cannot be used with params file") + flag.StringVar(&flagParamsFileValue, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis") + flag.StringVar(&flagExportParamsPathValue, "ExportParamsPath", "", "custom file path to save the exported params JSON") + flag.IntVar(&flagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params") + flag.StringVar(&flagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON") + flag.StringVar(&flagExportStatsPathValue, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON") + flag.Int64Var(&flagSeedValue, "Seed", 42, "simulation random seed") + flag.IntVar(&flagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation") + flag.IntVar(&flagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height") + flag.IntVar(&flagBlockSizeValue, "BlockSize", 200, "operations per block") + flag.BoolVar(&flagLeanValue, "Lean", false, "lean simulation log output") + flag.BoolVar(&flagCommitValue, "Commit", false, "have the simulation commit") + flag.BoolVar(&flagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation") + flag.BoolVar(&flagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found") + + // SimApp flags + flag.BoolVar(&flagEnabledValue, "Enabled", false, "enable the simulation") + flag.BoolVar(&flagVerboseValue, "Verbose", false, "verbose log output") + flag.IntVar(&flagPeriodValue, "Period", 1, "run slow invariants only once every period assertions") + flag.Int64Var(&flagGenesisTimeValue, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time") } -// AppStateFromGenesisFileFn util function to generate the genesis AppState -// from a genesis.json file -func AppStateFromGenesisFileFn( - r *rand.Rand, _ []simulation.Account, _ time.Time, -) (json.RawMessage, []simulation.Account, string) { - - var genesis tmtypes.GenesisDoc - cdc := MakeCodec() - - bytes, err := ioutil.ReadFile(genesisFile) - if err != nil { - panic(err) +// NewConfigFromFlags creates a simulation from the retrieved values of the flags +func NewConfigFromFlags() simulation.Config { + return simulation.Config{ + GenesisFile: flagGenesisFileValue, + ParamsFile: flagParamsFileValue, + ExportParamsPath: flagExportParamsPathValue, + ExportParamsHeight: flagExportParamsHeightValue, + ExportStatePath: flagExportStatePathValue, + ExportStatsPath: flagExportStatsPathValue, + Seed: flagSeedValue, + InitialBlockHeight: flagInitialBlockHeightValue, + NumBlocks: flagNumBlocksValue, + BlockSize: flagBlockSizeValue, + Lean: flagLeanValue, + Commit: flagCommitValue, + OnOperation: flagOnOperationValue, + AllInvariants: flagAllInvariantsValue, } - - cdc.MustUnmarshalJSON(bytes, &genesis) - - var appState GenesisState - cdc.MustUnmarshalJSON(genesis.AppState, &appState) - - accounts := genaccounts.GetGenesisStateFromAppState(cdc, appState) - - var newAccs []simulation.Account - for _, acc := range accounts { - // Pick a random private key, since we don't know the actual key - // This should be fine as it's only used for mock Tendermint validators - // and these keys are never actually used to sign by mock Tendermint. - privkeySeed := make([]byte, 15) - r.Read(privkeySeed) - - privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed) - newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address}) - } - - return genesis.AppState, newAccs, genesis.ChainID } // GenAuthGenesisState generates a random GenesisState for auth @@ -495,6 +492,9 @@ func GenStakingGenesisState( return stakingGenesis } +//--------------------------------------------------------------------- +// Simulation Utils + // GetSimulationLog unmarshals the KVPair's Value to the corresponding type based on the // each's module store key and the prefix bytes of the KVPair's key. func GetSimulationLog(storeName string, sdr sdk.StoreDecoderRegistry, cdc *codec.Codec, kvAs, kvBs []cmn.KVPair) (log string) { diff --git a/x/simulation/config.go b/x/simulation/config.go new file mode 100644 index 0000000000..616da3bcc4 --- /dev/null +++ b/x/simulation/config.go @@ -0,0 +1,23 @@ +package simulation + +// Config contains the necessary configuration flags for the simulator +type Config struct { + GenesisFile string // custom simulation genesis file; cannot be used with params file + ParamsFile string // custom simulation params file which overrides any random params; cannot be used with genesis + + ExportParamsPath string // custom file path to save the exported params JSON + ExportParamsHeight int //height to which export the randomly generated params + ExportStatePath string //custom file path to save the exported app state JSON + ExportStatsPath string // custom file path to save the exported simulation statistics JSON + + Seed int64 // simulation random seed + InitialBlockHeight int // initial block to start the simulation + NumBlocks int // number of new blocks to simulate from the initial block height + BlockSize int // operations per block + + Lean bool // lean simulation log output + Commit bool // have the simulation commit + + OnOperation bool // run slow invariants every operation + AllInvariants bool // print all failed invariants if a broken invariant is found +} \ No newline at end of file diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index 2b5857811d..e1bf1a4b4d 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -18,16 +18,17 @@ import ( ) // AppStateFn returns the app state json bytes, the genesis accounts, and the chain identifier -type AppStateFn func( - r *rand.Rand, accs []Account, -) (appState json.RawMessage, accounts []Account, chainId string, genesisTimestamp time.Time) +type AppStateFn func(r *rand.Rand, accs []Account, config Config) ( + appState json.RawMessage, accounts []Account, chainId string, genesisTimestamp time.Time, +) // initialize the chain for the simulation func initChain( - r *rand.Rand, params Params, accounts []Account, app *baseapp.BaseApp, appStateFn AppStateFn, + r *rand.Rand, params Params, accounts []Account, app *baseapp.BaseApp, + appStateFn AppStateFn, config Config, ) (mockValidators, time.Time, []Account) { - appState, accounts, chainID, genesisTimestamp := appStateFn(r, accounts) + appState, accounts, chainID, genesisTimestamp := appStateFn(r, accounts, config) req := abci.RequestInitChain{ AppStateBytes: appState, @@ -40,23 +41,19 @@ func initChain( } // SimulateFromSeed tests an application by running the provided -// operations, testing the provided invariants, but using the provided seed. +// operations, testing the provided invariants, but using the provided config.Seed. // TODO: split this monster function up func SimulateFromSeed( tb testing.TB, w io.Writer, app *baseapp.BaseApp, - appStateFn AppStateFn, seed int64, - ops WeightedOperations, invariants sdk.Invariants, - initialHeight, numBlocks, exportParamsHeight, blockSize int, - exportStatsPath string, - exportParams, commit, lean, onOperation, allInvariants bool, - blackListedAccs map[string]bool, + appStateFn AppStateFn, ops WeightedOperations, invariants sdk.Invariants, + blackListedAccs map[string]bool, config Config, ) (stopEarly bool, exportedParams Params, err error) { // in case we have to end early, don't os.Exit so that we can run cleanup code. testingMode, t, b := getTestingMode(tb) - fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(seed)) + fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with config.Seed %d\n", int(config.Seed)) - r := rand.New(rand.NewSource(seed)) + r := rand.New(rand.NewSource(config.Seed)) params := RandomParams(r) fmt.Fprintf(w, "Randomized simulation params: \n%s\n", mustMarshalJSONIndent(params)) @@ -66,7 +63,7 @@ func SimulateFromSeed( // Second variable to keep pending validator set (delayed one block since // TM 0.24) Initially this is the same as the initial validator set - validators, genesisTimestamp, accs := initChain(r, params, accs, app, appStateFn) + validators, genesisTimestamp, accs := initChain(r, params, accs, app, appStateFn, config) if len(accs) == 0 { return true, params, fmt.Errorf("must have greater than zero genesis accounts") } @@ -119,8 +116,7 @@ func SimulateFromSeed( blockSimulator := createBlockSimulator( testingMode, tb, t, w, params, eventStats.Tally, invariants, - ops, operationQueue, timeOperationQueue, - numBlocks, blockSize, logWriter, lean, onOperation, allInvariants) + ops, operationQueue, timeOperationQueue, logWriter, config) if !testingMode { b.ResetTimer() @@ -136,12 +132,12 @@ func SimulateFromSeed( } // set exported params to the initial state - if exportParams && exportParamsHeight == 0 { + if config.ExportParamsPath != "" && config.ExportParamsHeight == 0 { exportedParams = params } // TODO: split up the contents of this for loop into new functions - for height := initialHeight; height < numBlocks+initialHeight && !stopEarly; height++ { + for height := config.InitialBlockHeight; height < config.NumBlocks+config.InitialBlockHeight && !stopEarly; height++ { // Log the header time for future lookup pastTimes = append(pastTimes, header.Time) @@ -152,7 +148,7 @@ func SimulateFromSeed( app.BeginBlock(request) if testingMode { - assertAllInvariants(t, app, invariants, "BeginBlock", logWriter, allInvariants) + assertAllInvariants(t, app, invariants, "BeginBlock", logWriter, config.AllInvariants) } ctx := app.NewContext(false, header) @@ -160,21 +156,21 @@ func SimulateFromSeed( // Run queued operations. Ignores blocksize if blocksize is too small numQueuedOpsRan := runQueuedOperations( operationQueue, int(header.Height), - tb, r, app, ctx, accs, logWriter, eventStats.Tally, lean) + tb, r, app, ctx, accs, logWriter, eventStats.Tally, config.Lean) numQueuedTimeOpsRan := runQueuedTimeOperations( timeOperationQueue, int(header.Height), header.Time, - tb, r, app, ctx, accs, logWriter, eventStats.Tally, lean) + tb, r, app, ctx, accs, logWriter, eventStats.Tally, config.Lean) - if testingMode && onOperation { - assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, allInvariants) + if testingMode && config.OnOperation { + assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, config.AllInvariants) } // run standard operations operations := blockSimulator(r, app, ctx, accs, header) opCount += operations + numQueuedOpsRan + numQueuedTimeOpsRan if testingMode { - assertAllInvariants(t, app, invariants, "StandardOperations", logWriter, allInvariants) + assertAllInvariants(t, app, invariants, "StandardOperations", logWriter, config.AllInvariants) } res := app.EndBlock(abci.RequestEndBlock{}) @@ -187,9 +183,9 @@ func SimulateFromSeed( logWriter.AddEntry(EndBlockEntry(int64(height))) if testingMode { - assertAllInvariants(t, app, invariants, "EndBlock", logWriter, allInvariants) + assertAllInvariants(t, app, invariants, "EndBlock", logWriter, config.AllInvariants) } - if commit { + if config.Commit { app.Commit() } @@ -211,15 +207,15 @@ func SimulateFromSeed( validators, res.ValidatorUpdates, eventStats.Tally) // update the exported params - if exportParams && exportParamsHeight == height { + if config.ExportParamsPath != "" && config.ExportParamsHeight == height { exportedParams = params } } if stopEarly { - if exportStatsPath != "" { + if config.ExportStatsPath != "" { fmt.Println("Exporting simulation statistics...") - eventStats.ExportJSON(exportStatsPath) + eventStats.ExportJSON(config.ExportStatsPath) } else { eventStats.Print(w) } @@ -233,9 +229,9 @@ func SimulateFromSeed( header.Height, header.Time, opCount, ) - if exportStatsPath != "" { + if config.ExportStatsPath != "" { fmt.Println("Exporting simulation statistics...") - eventStats.ExportJSON(exportStatsPath) + eventStats.ExportJSON(config.ExportStatsPath) } else { eventStats.Print(w) } @@ -253,7 +249,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params, event func(route, op, evResult string), invariants sdk.Invariants, ops WeightedOperations, operationQueue OperationQueue, timeOperationQueue []FutureOperation, - totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn { + logWriter LogWriter, config Config) blockSimFn { lastBlockSizeState := 0 // state for [4 * uniform distribution] blocksize := 0 @@ -265,9 +261,9 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr _, _ = fmt.Fprintf( w, "\rSimulating... block %d/%d, operation %d/%d.", - header.Height, totalNumBlocks, opCount, blocksize, + header.Height, config.NumBlocks, opCount, blocksize, ) - lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, avgBlockSize) + lastBlockSizeState, blocksize = getBlockSize(r, params, lastBlockSizeState, config.BlockSize) type opAndR struct { op Operation @@ -291,7 +287,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr op, r2 := opAndR.op, opAndR.rand opMsg, futureOps, err := op(r2, app, ctx, accounts) opMsg.LogEvent(event) - if !lean || opMsg.OK { + if !config.Lean || opMsg.OK { logWriter.AddEntry(MsgEntry(header.Height, int64(i), opMsg)) } if err != nil { @@ -302,14 +298,14 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr queueOperations(operationQueue, timeOperationQueue, futureOps) if testingMode { - if onOperation { + if config.OnOperation { fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", - header.Height, totalNumBlocks, opCount, blocksize) + header.Height, config.NumBlocks, opCount, blocksize) eventStr := fmt.Sprintf("operation: %v", opMsg.String()) - assertAllInvariants(t, app, invariants, eventStr, logWriter, allInvariants) + assertAllInvariants(t, app, invariants, eventStr, logWriter, config.AllInvariants) } else if opCount%50 == 0 { fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ", - header.Height, totalNumBlocks, opCount, blocksize) + header.Height, config.NumBlocks, opCount, blocksize) } } opCount++