Merge PR #4731: Save sim params and export app state to file
This commit is contained in:
Federico Kunze
committed by
Alexander Bezobchuk
Alexander Bezobchuk
parent
43478167c6
commit
8af2230ee4
+49
-21
@@ -50,40 +50,68 @@ To execute a completely pseudo-random simulation:
|
||||
|
||||
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
|
||||
-run=TestFullAppSimulation \
|
||||
-SimulationEnabled=true \
|
||||
-SimulationNumBlocks=100 \
|
||||
-SimulationBlockSize=200 \
|
||||
-SimulationCommit=true \
|
||||
-SimulationSeed=99 \
|
||||
-SimulationPeriod=5 \
|
||||
-Enabled=true \
|
||||
-NumBlocks=100 \
|
||||
-BlockSize=200 \
|
||||
-Commit=true \
|
||||
-Seed=99 \
|
||||
-Period=5 \
|
||||
-v -timeout 24h
|
||||
|
||||
To execute simulation from a genesis file:
|
||||
|
||||
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
|
||||
-run=TestFullAppSimulation \
|
||||
-SimulationEnabled=true \
|
||||
-SimulationNumBlocks=100 \
|
||||
-SimulationBlockSize=200 \
|
||||
-SimulationCommit=true \
|
||||
-SimulationSeed=99 \
|
||||
-SimulationPeriod=5 \
|
||||
-SimulationGenesis=/path/to/genesis.json \
|
||||
-Enabled=true \
|
||||
-NumBlocks=100 \
|
||||
-BlockSize=200 \
|
||||
-Commit=true \
|
||||
-Seed=99 \
|
||||
-Period=5 \
|
||||
-Genesis=/path/to/genesis.json \
|
||||
-v -timeout 24h
|
||||
|
||||
To execute simulation from a params file:
|
||||
To execute simulation from a simulation params file:
|
||||
|
||||
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
|
||||
-run=TestFullAppSimulation \
|
||||
-SimulationEnabled=true \
|
||||
-SimulationNumBlocks=100 \
|
||||
-SimulationBlockSize=200 \
|
||||
-SimulationCommit=true \
|
||||
-SimulationSeed=99 \
|
||||
-SimulationPeriod=5 \
|
||||
-SimulationParams=/path/to/params.json \
|
||||
-Enabled=true \
|
||||
-NumBlocks=100 \
|
||||
-BlockSize=200 \
|
||||
-Commit=true \
|
||||
-Seed=99 \
|
||||
-Period=5 \
|
||||
-Params=/path/to/params.json \
|
||||
-v -timeout 24h
|
||||
|
||||
To export the simulation params to a file at a given block height:
|
||||
|
||||
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
|
||||
-run=TestFullAppSimulation \
|
||||
-Enabled=true \
|
||||
-NumBlocks=100 \
|
||||
-BlockSize=200 \
|
||||
-Commit=true \
|
||||
-Seed=99 \
|
||||
-Period=5 \
|
||||
-ExportParamsPath=/path/to/params.json \
|
||||
-ExportParamsHeight=50 \
|
||||
-v -timeout 24h
|
||||
|
||||
|
||||
To export the simulation app state (i.e genesis) to a file:
|
||||
|
||||
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
|
||||
-run=TestFullAppSimulation \
|
||||
-Enabled=true \
|
||||
-NumBlocks=100 \
|
||||
-BlockSize=200 \
|
||||
-Commit=true \
|
||||
-Seed=99 \
|
||||
-Period=5 \
|
||||
-ExportStatePath=/path/to/genesis.json \
|
||||
v -timeout 24h
|
||||
|
||||
Params
|
||||
|
||||
Params that are provided to simulation from a JSON file are used to used to set
|
||||
|
||||
+168
-42
@@ -31,16 +31,19 @@ import (
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&genesisFile, "SimulationGenesis", "", "custom simulation genesis file; cannot be used with params file")
|
||||
flag.StringVar(¶msFile, "SimulationParams", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
|
||||
flag.Int64Var(&seed, "SimulationSeed", 42, "simulation random seed")
|
||||
flag.IntVar(&numBlocks, "SimulationNumBlocks", 500, "number of blocks")
|
||||
flag.IntVar(&blockSize, "SimulationBlockSize", 200, "operations per block")
|
||||
flag.BoolVar(&enabled, "SimulationEnabled", false, "enable the simulation")
|
||||
flag.BoolVar(&verbose, "SimulationVerbose", false, "verbose log output")
|
||||
flag.BoolVar(&lean, "SimulationLean", false, "lean simulation log output")
|
||||
flag.BoolVar(&commit, "SimulationCommit", false, "have the simulation commit")
|
||||
flag.IntVar(&period, "SimulationPeriod", 1, "run slow invariants only once every period assertions")
|
||||
flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
|
||||
flag.StringVar(¶msFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
|
||||
flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON")
|
||||
flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params")
|
||||
flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON")
|
||||
flag.Int64Var(&seed, "Seed", 42, "simulation random seed")
|
||||
flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks")
|
||||
flag.IntVar(&blockSize, "BlockSize", 200, "operations per block")
|
||||
flag.BoolVar(&enabled, "Enabled", false, "enable the simulation")
|
||||
flag.BoolVar(&verbose, "Verbose", false, "verbose log output")
|
||||
flag.BoolVar(&lean, "Lean", false, "lean simulation log output")
|
||||
flag.BoolVar(&commit, "Commit", false, "have the simulation commit")
|
||||
flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions")
|
||||
flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation")
|
||||
flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
|
||||
}
|
||||
@@ -48,11 +51,15 @@ func init() {
|
||||
// helper function for populating input for SimulateFromSeed
|
||||
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) (
|
||||
testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
|
||||
simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool, bool, map[string]bool) {
|
||||
simulation.WeightedOperations, sdk.Invariants, int, int, int,
|
||||
bool, bool, bool, bool, bool, map[string]bool) {
|
||||
|
||||
exportParams := exportParamsPath != ""
|
||||
|
||||
return tb, w, app.BaseApp, appStateFn, seed,
|
||||
testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit,
|
||||
lean, onOperation, allInvariants, app.ModuleAccountAddrs()
|
||||
testAndRunTxs(app), invariants(app),
|
||||
numBlocks, exportParamsHeight, blockSize,
|
||||
exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
|
||||
}
|
||||
|
||||
func appStateFn(
|
||||
@@ -136,6 +143,7 @@ func appStateRandomizedFn(
|
||||
return appState, accs, "simulation"
|
||||
}
|
||||
|
||||
// TODO: add description
|
||||
func testAndRunTxs(app *SimApp) []simulation.WeightedOperation {
|
||||
cdc := MakeCodec()
|
||||
ap := make(simulation.AppParams)
|
||||
@@ -344,7 +352,7 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
|
||||
}
|
||||
|
||||
// Profile with:
|
||||
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -SimulationCommit=true -cpuprofile cpu.out
|
||||
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
|
||||
func BenchmarkFullAppSimulation(b *testing.B) {
|
||||
logger := log.NewNopLogger()
|
||||
|
||||
@@ -358,12 +366,44 @@ func BenchmarkFullAppSimulation(b *testing.B) {
|
||||
app := NewSimApp(logger, db, nil, true, 0)
|
||||
|
||||
// Run randomized simulation
|
||||
// TODO parameterize numbers, save for a later PR
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
// TODO: parameterize numbers, save for a later PR
|
||||
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
|
||||
|
||||
// export state and params before the simulation error is checked
|
||||
if exportStatePath != "" {
|
||||
fmt.Println("Exporting app state...")
|
||||
appState, _, err := app.ExportAppStateAndValidators(false, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
if exportParamsPath != "" {
|
||||
fmt.Println("Exporting simulation params...")
|
||||
paramsBz, err := json.MarshalIndent(params, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
if simErr != nil {
|
||||
fmt.Println(simErr)
|
||||
b.FailNow()
|
||||
}
|
||||
|
||||
if commit {
|
||||
fmt.Println("GoLevelDB Stats")
|
||||
fmt.Println(db.Stats()["leveldb.stats"])
|
||||
@@ -397,7 +437,30 @@ func TestFullAppSimulation(t *testing.T) {
|
||||
require.Equal(t, "SimApp", app.Name())
|
||||
|
||||
// Run randomized simulation
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
|
||||
// export state and params before the simulation error is checked
|
||||
if exportStatePath != "" {
|
||||
fmt.Println("Exporting app state...")
|
||||
appState, _, err := app.ExportAppStateAndValidators(false, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
if exportParamsPath != "" {
|
||||
fmt.Println("Exporting simulation params...")
|
||||
fmt.Println(params)
|
||||
paramsBz, err := json.MarshalIndent(params, "", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
require.NoError(t, simErr)
|
||||
|
||||
if commit {
|
||||
// for memdb:
|
||||
// fmt.Println("Database Size", db.Stats()["database.size"])
|
||||
@@ -405,8 +468,6 @@ func TestFullAppSimulation(t *testing.T) {
|
||||
fmt.Println(db.Stats()["leveldb.stats"])
|
||||
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
|
||||
}
|
||||
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestAppImportExport(t *testing.T) {
|
||||
@@ -434,7 +495,28 @@ func TestAppImportExport(t *testing.T) {
|
||||
require.Equal(t, "SimApp", app.Name())
|
||||
|
||||
// Run randomized simulation
|
||||
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
|
||||
// export state and params before the simulation error is checked
|
||||
if exportStatePath != "" {
|
||||
fmt.Println("Exporting app state...")
|
||||
appState, _, err := app.ExportAppStateAndValidators(false, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
if exportParamsPath != "" {
|
||||
fmt.Println("Exporting simulation params...")
|
||||
paramsBz, err := json.MarshalIndent(params, "", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
require.NoError(t, simErr)
|
||||
|
||||
if commit {
|
||||
// for memdb:
|
||||
@@ -444,7 +526,6 @@ func TestAppImportExport(t *testing.T) {
|
||||
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
|
||||
}
|
||||
|
||||
require.Nil(t, err)
|
||||
fmt.Printf("Exporting genesis...\n")
|
||||
|
||||
appState, _, err := app.ExportAppStateAndValidators(false, []string{})
|
||||
@@ -530,7 +611,28 @@ func TestAppSimulationAfterImport(t *testing.T) {
|
||||
require.Equal(t, "SimApp", app.Name())
|
||||
|
||||
// Run randomized simulation
|
||||
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
|
||||
|
||||
// export state and params before the simulation error is checked
|
||||
if exportStatePath != "" {
|
||||
fmt.Println("Exporting app state...")
|
||||
appState, _, err := app.ExportAppStateAndValidators(false, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
if exportParamsPath != "" {
|
||||
fmt.Println("Exporting simulation params...")
|
||||
paramsBz, err := json.MarshalIndent(params, "", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
require.NoError(t, simErr)
|
||||
|
||||
if commit {
|
||||
// for memdb:
|
||||
@@ -540,8 +642,6 @@ func TestAppSimulationAfterImport(t *testing.T) {
|
||||
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
|
||||
}
|
||||
|
||||
require.Nil(t, err)
|
||||
|
||||
if stopEarly {
|
||||
// we can't export or import a zero-validator genesis
|
||||
fmt.Printf("We can't export or import a zero-validator genesis, exiting test...\n")
|
||||
@@ -572,7 +672,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
|
||||
})
|
||||
|
||||
// Run randomized simulation on imported app
|
||||
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
|
||||
_, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
@@ -597,15 +697,9 @@ func TestAppStateDeterminism(t *testing.T) {
|
||||
// Run randomized simulation
|
||||
simulation.SimulateFromSeed(
|
||||
t, os.Stdout, app.BaseApp, appStateFn, seed,
|
||||
testAndRunTxs(app),
|
||||
[]sdk.Invariant{},
|
||||
50,
|
||||
100,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
app.ModuleAccountAddrs(),
|
||||
testAndRunTxs(app), []sdk.Invariant{},
|
||||
50, 100, 0,
|
||||
false, true, false, false, false, app.ModuleAccountAddrs(),
|
||||
)
|
||||
appHash := app.LastCommitID().Hash
|
||||
appHashList[j] = appHash
|
||||
@@ -627,15 +721,47 @@ func BenchmarkInvariants(b *testing.B) {
|
||||
}()
|
||||
|
||||
app := NewSimApp(logger, db, nil, true, 0)
|
||||
exportParams := exportParamsPath != ""
|
||||
|
||||
// 2. Run parameterized simulation (w/o invariants)
|
||||
_, err := simulation.SimulateFromSeed(
|
||||
_, params, simErr := simulation.SimulateFromSeed(
|
||||
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
|
||||
[]sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, false,
|
||||
app.ModuleAccountAddrs(),
|
||||
[]sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize,
|
||||
exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
|
||||
)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
|
||||
// export state and params before the simulation error is checked
|
||||
if exportStatePath != "" {
|
||||
fmt.Println("Exporting app state...")
|
||||
appState, _, err := app.ExportAppStateAndValidators(false, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
err = ioutil.WriteFile(exportStatePath, []byte(appState), 0644)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
if exportParamsPath != "" {
|
||||
fmt.Println("Exporting simulation params...")
|
||||
paramsBz, err := json.MarshalIndent(params, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
b.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
if simErr != nil {
|
||||
fmt.Println(simErr)
|
||||
b.FailNow()
|
||||
}
|
||||
|
||||
|
||||
+15
-12
@@ -37,18 +37,21 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
genesisFile string
|
||||
paramsFile string
|
||||
seed int64
|
||||
numBlocks int
|
||||
blockSize int
|
||||
enabled bool
|
||||
verbose bool
|
||||
lean bool
|
||||
commit bool
|
||||
period int
|
||||
onOperation bool // TODO Remove in favor of binary search for invariant violation
|
||||
allInvariants bool
|
||||
genesisFile string
|
||||
paramsFile string
|
||||
exportParamsPath string
|
||||
exportParamsHeight int
|
||||
exportStatePath string
|
||||
seed int64
|
||||
numBlocks int
|
||||
blockSize int
|
||||
enabled bool
|
||||
verbose bool
|
||||
lean bool
|
||||
commit bool
|
||||
period int
|
||||
onOperation bool // TODO Remove in favor of binary search for invariant violation
|
||||
allInvariants bool
|
||||
)
|
||||
|
||||
// NewSimAppUNSAFE is used for debugging purposes only.
|
||||
|
||||
Reference in New Issue
Block a user