LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge PR #5378: application interfaces for simulation

Browse Source
This commit is contained in:
Federico Kunze
2019-12-17 13:28:52 -05:00
committed by Alexander Bezobchuk
parent 13378bd2cf
commit 3471256785
11 changed files with 328 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+5 -2
View File
@@ -182,7 +182,7 @@ generalized genesis accounts through the `GenesisAccount` interface.
* (sdk) [\#4640](https://github.com/cosmos/cosmos-sdk/issues/4640) improve import/export simulation errors by extending `DiffKVStores` to return an array of `KVPairs` that are then compared to check for inconsistencies.
* (sdk) [\#4717](https://github.com/cosmos/cosmos-sdk/issues/4717) refactor `x/slashing` to match the new module spec
* (sdk) [\#4758](https://github.com/cosmos/cosmos-sdk/issues/4758) update `x/genaccounts` to match module spec
* (simulation) [\#4824](https://github.com/cosmos/cosmos-sdk/issues/4824) PrintAllInvariants flag will print all failed invariants
* (simulation) [\#4824](https://github.com/cosmos/cosmos-sdk/issues/4824) `PrintAllInvariants` flag will print all failed invariants
* (simulation) [\#4490](https://github.com/cosmos/cosmos-sdk/issues/4490) add `InitialBlockHeight` flag to resume a simulation from a given block
* Support exporting the simulation stats to a given JSON file
* (simulation) [\#4847](https://github.com/cosmos/cosmos-sdk/issues/4847), [\#4838](https://github.com/cosmos/cosmos-sdk/pull/4838) and [\#4869](https://github.com/cosmos/cosmos-sdk/pull/4869) `SimApp` and simulation refactors:
@@ -194,9 +194,12 @@ generalized genesis accounts through the `GenesisAccount` interface.
* Add `WeightedOperations` to the `SimulationManager` that define simulation operations (modules' `Msg`s) with their
respective weights (i.e chance of being simulated).
* Add `ProposalContents` to the `SimulationManager` to register each module's governance proposal `Content`s.
* (simulation) [\#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change SimApp keepers to be public and add getter functions for keys and codec
* (simulation) [\#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change `SimApp` keepers to be public and add getter functions for keys and codec
* (simulation) [\#4906](https://github.com/cosmos/cosmos-sdk/issues/4906) Add simulation `Config` struct that wraps simulation flags
* (simulation) [\#4935](https://github.com/cosmos/cosmos-sdk/issues/4935) Update simulation to reflect a proper `ABCI` application without bypassing `BaseApp` semantics
* (simulation) [\#5378](https://github.com/cosmos/cosmos-sdk/pull/5378) Simulation tests refactor:
* Add `App` interface for general SDK-based app's methods.
* Refactor and cleanup simulation tests into util functions to simplify their implementation for other SDK apps.
* (store) [\#4792](https://github.com/cosmos/cosmos-sdk/issues/4792) panic on non-registered store
* (types) [\#4821](https://github.com/cosmos/cosmos-sdk/issues/4821) types/errors package added with support for stacktraces. It is meant as a more feature-rich replacement for sdk.Errors in the mid-term.
* (store) [\#1947](https://github.com/cosmos/cosmos-sdk/issues/1947) Implement inter-block (persistent)
simapp/app.go
+11
View File
@@ -89,6 +89,9 @@ func MakeCodec() *codec.Codec {
return cdc
}
// Verify app interface at compile time
var _ App = (*SimApp)(nil)
// SimApp extends an ABCI application, but with most of its parameters exported.
// They are exported for convenience in creating helper functions, as object
// capabilities aren't needed for testing.
@@ -295,6 +298,9 @@ func NewSimApp(
return app
}
// Name returns the name of the App
func (app *SimApp) Name() string { return app.BaseApp.Name() }
// BeginBlocker application updates every begin block
func (app *SimApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock {
return app.mm.BeginBlock(ctx, req)
@@ -366,6 +372,11 @@ func (app *SimApp) GetSubspace(moduleName string) params.Subspace {
return app.subspaces[moduleName]
}
// SimulationManager implements the SimulationApp interface
func (app *SimApp) SimulationManager() *module.SimulationManager {
return app.sm
}
// GetMaccPerms returns a copy of the module account permissions
func GetMaccPerms() map[string][]string {
dupMaccPerms := make(map[string][]string)
simapp/app_test.go
+1 -1
View File
@@ -18,7 +18,7 @@ func TestSimAppExport(t *testing.T) {
app := NewSimApp(log.NewTMLogger(log.NewSyncWriter(os.Stdout)), db, nil, true, 0)
genesisState := NewDefaultGenesisState()
stateBytes, err := codec.MarshalJSONIndent(app.cdc, genesisState)
stateBytes, err := codec.MarshalJSONIndent(app.Codec(), genesisState)
require.NoError(t, err)
// Initialize the chain
simapp/config.go
+75
View File
@@ -0,0 +1,75 @@
package simapp
import (
"flag"
"github.com/cosmos/cosmos-sdk/x/simulation"
)
// List of available flags for the simulator
var (
FlagGenesisFileValue string
FlagParamsFileValue string
FlagExportParamsPathValue string
FlagExportParamsHeightValue int
FlagExportStatePathValue string
FlagExportStatsPathValue string
FlagSeedValue int64
FlagInitialBlockHeightValue int
FlagNumBlocksValue int
FlagBlockSizeValue int
FlagLeanValue bool
FlagCommitValue bool
FlagOnOperationValue bool // TODO: Remove in favor of binary search for invariant violation
FlagAllInvariantsValue bool
FlagEnabledValue bool
FlagVerboseValue bool
FlagPeriodValue uint
FlagGenesisTimeValue int64
)
// GetSimulatorFlags gets the values of all the available simulation flags
func GetSimulatorFlags() {
// config fields
flag.StringVar(&FlagGenesisFileValue, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&FlagParamsFileValue, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.StringVar(&FlagExportParamsPathValue, "ExportParamsPath", "", "custom file path to save the exported params JSON")
flag.IntVar(&FlagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&FlagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.StringVar(&FlagExportStatsPathValue, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
flag.Int64Var(&FlagSeedValue, "Seed", 42, "simulation random seed")
flag.IntVar(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation")
flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block")
flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output")
flag.BoolVar(&FlagCommitValue, "Commit", false, "have the simulation commit")
flag.BoolVar(&FlagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&FlagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
// simulation flags
flag.BoolVar(&FlagEnabledValue, "Enabled", false, "enable the simulation")
flag.BoolVar(&FlagVerboseValue, "Verbose", false, "verbose log output")
flag.UintVar(&FlagPeriodValue, "Period", 0, "run slow invariants only once every period assertions")
flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time")
}
// NewConfigFromFlags creates a simulation from the retrieved values of the flags.
func NewConfigFromFlags() simulation.Config {
return simulation.Config{
GenesisFile: FlagGenesisFileValue,
ParamsFile: FlagParamsFileValue,
ExportParamsPath: FlagExportParamsPathValue,
ExportParamsHeight: FlagExportParamsHeightValue,
ExportStatePath: FlagExportStatePathValue,
ExportStatsPath: FlagExportStatsPathValue,
Seed: FlagSeedValue,
InitialBlockHeight: FlagInitialBlockHeightValue,
NumBlocks: FlagNumBlocksValue,
BlockSize: FlagBlockSizeValue,
Lean: FlagLeanValue,
Commit: FlagCommitValue,
OnOperation: FlagOnOperationValue,
AllInvariants: FlagAllInvariantsValue,
}
}
simapp/params.go → simapp/params/params.go
+1 -1
View File
@@ -1,4 +1,4 @@
package simapp
package params
// Simulation parameter constants
const (
simapp/sim_bench_test.go
+38 -61
View File
@@ -2,113 +2,88 @@ package simapp
import (
"fmt"
"io/ioutil"
"os"
"testing"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/simapp/helpers"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/x/simulation"
abci "github.com/tendermint/tendermint/abci/types"
)
// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
logger := log.NewNopLogger()
config := NewConfigFromFlags()
config.ChainID = helpers.SimAppChainID
config, db, dir, logger, _, err := SetupSimulation("goleveldb-app-sim", "Simulation")
if err != nil {
b.Fatalf("simulation setup failed: %s", err.Error())
}
var db dbm.DB
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
db, _ = sdk.NewLevelDB("Simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
err = os.RemoveAll(dir)
if err != nil {
b.Fatal(err)
}
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
// Run randomized simulation
// TODO: parameterize numbers, save for a later PR
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if config.ExportParamsPath != "" {
if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
fmt.Println(err)
b.Fail()
}
// export state and simParams before the simulation error is checked
if err = CheckExportSimulation(app, config, simParams); err != nil {
b.Fatal(err)
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
b.Fatal(simErr)
}
if config.Commit {
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
PrintStats(db)
}
}
func BenchmarkInvariants(b *testing.B) {
logger := log.NewNopLogger()
config, db, dir, logger, _, err := SetupSimulation("leveldb-app-invariant-bench", "Simulation")
if err != nil {
b.Fatalf("simulation setup failed: %s", err.Error())
}
config := NewConfigFromFlags()
config.AllInvariants = false
config.ChainID = helpers.SimAppChainID
dir, _ := ioutil.TempDir("", "goleveldb-app-invariant-bench")
db, _ := sdk.NewLevelDB("simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
err = os.RemoveAll(dir)
if err != nil {
b.Fatal(err)
}
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, interBlockCacheOpt())
// 2. Run parameterized simulation (w/o invariants)
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, AppStateFn(app.Codec(), app.sm),
b, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
if err := ExportStateToJSON(app, config.ExportStatePath); err != nil {
fmt.Println(err)
b.Fail()
}
}
if config.ExportParamsPath != "" {
if err := ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil {
fmt.Println(err)
b.Fail()
}
// export state and simParams before the simulation error is checked
if err = CheckExportSimulation(app, config, simParams); err != nil {
b.Fatal(err)
}
if simErr != nil {
fmt.Println(simErr)
b.FailNow()
b.Fatal(simErr)
}
if config.Commit {
PrintStats(db)
}
ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1})
@@ -121,8 +96,10 @@ func BenchmarkInvariants(b *testing.B) {
cr := cr
b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) {
if res, stop := cr.Invar(ctx); stop {
fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res)
b.FailNow()
b.Fatalf(
"broken invariant at block %d of %d\n%s",
ctx.BlockHeight()-1, config.NumBlocks, res,
)
}
})
}
simapp/sim_test.go
+78 -152
View File
@@ -3,7 +3,6 @@ package simapp
import (
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"testing"
@@ -33,6 +32,12 @@ func init() {
GetSimulatorFlags()
}
type StoreKeysPrefixes struct {
A sdk.StoreKey
B sdk.StoreKey
Prefixes [][]byte
}
// fauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of
// an IAVLStore for faster simulation speed.
func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
@@ -46,148 +51,95 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) {
}
func TestFullAppSimulation(t *testing.T) {
if !FlagEnabledValue {
config, db, dir, logger, skip, err := SetupSimulation("leveldb-app-sim", "Simulation")
if skip {
t.Skip("skipping application simulation")
}
var logger log.Logger
config := NewConfigFromFlags()
config.ChainID = helpers.SimAppChainID
if FlagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
}
var db dbm.DB
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
db, _ = sdk.NewLevelDB("Simulation", dir)
require.NoError(t, err, "simulation setup failed")
defer func() {
db.Close()
os.RemoveAll(dir)
require.NoError(t, os.RemoveAll(dir))
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, fauxMerkleModeOpt)
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
err := ExportStateToJSON(app, config.ExportStatePath)
require.NoError(t, err)
}
if config.ExportParamsPath != "" {
err := ExportParamsToJSON(simParams, config.ExportParamsPath)
require.NoError(t, err)
}
require.NoError(t, simErr)
if config.Commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
}
func TestAppImportExport(t *testing.T) {
if !FlagEnabledValue {
t.Skip("skipping application import/export simulation")
}
var logger log.Logger
config := NewConfigFromFlags()
config.ChainID = helpers.SimAppChainID
if FlagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
}
var db dbm.DB
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
db, _ = sdk.NewLevelDB("Simulation", dir)
defer func() {
db.Close()
os.RemoveAll(dir)
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, fauxMerkleModeOpt)
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
// export state and simParams before the simulation error is checked
if config.ExportStatePath != "" {
err := ExportStateToJSON(app, config.ExportStatePath)
require.NoError(t, err)
}
if config.ExportParamsPath != "" {
err := ExportParamsToJSON(simParams, config.ExportParamsPath)
require.NoError(t, err)
}
err = CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)
if config.Commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
PrintStats(db)
}
}
func TestAppImportExport(t *testing.T) {
config, db, dir, logger, skip, err := SetupSimulation("leveldb-app-sim", "Simulation")
if skip {
t.Skip("skipping application import/export simulation")
}
require.NoError(t, err, "simulation setup failed")
defer func() {
db.Close()
require.NoError(t, os.RemoveAll(dir))
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, fauxMerkleModeOpt)
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
// export state and simParams before the simulation error is checked
err = CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)
if config.Commit {
PrintStats(db)
}
fmt.Printf("exporting genesis...\n")
appState, _, err := app.ExportAppStateAndValidators(false, []string{})
require.NoError(t, err)
fmt.Printf("importing genesis...\n")
newDir, _ := ioutil.TempDir("", "goleveldb-app-sim-2")
newDB, _ := sdk.NewLevelDB("Simulation-2", dir)
_, newDB, newDir, _, _, err := SetupSimulation("leveldb-app-sim-2", "Simulation-2")
require.NoError(t, err, "simulation setup failed")
defer func() {
newDB.Close()
_ = os.RemoveAll(newDir)
require.NoError(t, os.RemoveAll(newDir))
}()
newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, FlagPeriodValue, fauxMerkleModeOpt)
require.Equal(t, "SimApp", newApp.Name())
var genesisState GenesisState
err = app.cdc.UnmarshalJSON(appState, &genesisState)
err = app.Codec().UnmarshalJSON(appState, &genesisState)
require.NoError(t, err)
ctxA := app.NewContext(true, abci.Header{Height: app.LastBlockHeight()})
ctxB := newApp.NewContext(true, abci.Header{Height: app.LastBlockHeight()})
newApp.mm.InitGenesis(ctxB, genesisState)
fmt.Printf("comparing stores...\n")
ctxA := app.NewContext(true, abci.Header{Height: app.LastBlockHeight()})
type StoreKeysPrefixes struct {
A sdk.StoreKey
B sdk.StoreKey
Prefixes [][]byte
}
storeKeysPrefixes := []StoreKeysPrefixes{
{app.keys[baseapp.MainStoreKey], newApp.keys[baseapp.MainStoreKey], [][]byte{}},
@@ -204,43 +156,28 @@ func TestAppImportExport(t *testing.T) {
{app.keys[gov.StoreKey], newApp.keys[gov.StoreKey], [][]byte{}},
}
for _, storeKeysPrefix := range storeKeysPrefixes {
storeKeyA := storeKeysPrefix.A
storeKeyB := storeKeysPrefix.B
prefixes := storeKeysPrefix.Prefixes
for _, skp := range storeKeysPrefixes {
storeA := ctxA.KVStore(skp.A)
storeB := ctxB.KVStore(skp.B)
storeA := ctxA.KVStore(storeKeyA)
storeB := ctxB.KVStore(storeKeyB)
failedKVAs, failedKVBs := sdk.DiffKVStores(storeA, storeB, prefixes)
failedKVAs, failedKVBs := sdk.DiffKVStores(storeA, storeB, skp.Prefixes)
require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare")
fmt.Printf("compared %d key/value pairs between %s and %s\n", len(failedKVAs), storeKeyA, storeKeyB)
require.Equal(t, len(failedKVAs), 0, GetSimulationLog(storeKeyA.Name(), app.sm.StoreDecoders, app.cdc, failedKVAs, failedKVBs))
fmt.Printf("compared %d key/value pairs between %s and %s\n", len(failedKVAs), skp.A, skp.B)
require.Equal(t, len(failedKVAs), 0, GetSimulationLog(skp.A.Name(), app.SimulationManager().StoreDecoders, app.Codec(), failedKVAs, failedKVBs))
}
}
func TestAppSimulationAfterImport(t *testing.T) {
if !FlagEnabledValue {
config, db, dir, logger, skip, err := SetupSimulation("leveldb-app-sim", "Simulation")
if skip {
t.Skip("skipping application simulation after import")
}
var logger log.Logger
config := NewConfigFromFlags()
config.ChainID = helpers.SimAppChainID
if FlagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
}
dir, _ := ioutil.TempDir("", "goleveldb-app-sim")
db, _ := sdk.NewLevelDB("Simulation", dir)
require.NoError(t, err, "simulation setup failed")
defer func() {
db.Close()
os.RemoveAll(dir)
require.NoError(t, os.RemoveAll(dir))
}()
app := NewSimApp(logger, db, nil, true, FlagPeriodValue, fauxMerkleModeOpt)
@@ -248,35 +185,22 @@ func TestAppSimulationAfterImport(t *testing.T) {
// Run randomized simulation
stopEarly, simParams, simErr := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
// export state and params before the simulation error is checked
if config.ExportStatePath != "" {
err := ExportStateToJSON(app, config.ExportStatePath)
require.NoError(t, err)
}
if config.ExportParamsPath != "" {
err := ExportParamsToJSON(simParams, config.ExportParamsPath)
require.NoError(t, err)
}
// export state and simParams before the simulation error is checked
err = CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)
if config.Commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("\nGoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
PrintStats(db)
}
if stopEarly {
// we can't export or import a zero-validator genesis
fmt.Printf("we can't export or import a zero-validator genesis, exiting test...\n")
fmt.Println("can't export or import a zero-validator genesis, exiting test...")
return
}
@@ -287,12 +211,12 @@ func TestAppSimulationAfterImport(t *testing.T) {
fmt.Printf("importing genesis...\n")
newDir, _ := ioutil.TempDir("", "goleveldb-app-sim-2")
newDB, _ := sdk.NewLevelDB("Simulation-2", dir)
_, newDB, newDir, _, _, err := SetupSimulation("leveldb-app-sim-2", "Simulation-2")
require.NoError(t, err, "simulation setup failed")
defer func() {
newDB.Close()
_ = os.RemoveAll(newDir)
require.NoError(t, os.RemoveAll(newDir))
}()
newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, FlagPeriodValue, fauxMerkleModeOpt)
@@ -302,13 +226,11 @@ func TestAppSimulationAfterImport(t *testing.T) {
AppStateBytes: appState,
})
// Run randomized simulation on imported app
_, _, err = simulation.SimulateFromSeed(
t, os.Stdout, newApp.BaseApp, AppStateFn(app.Codec(), app.sm),
t, os.Stdout, newApp.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(newApp, newApp.Codec(), config),
newApp.ModuleAccountAddrs(), config,
)
require.NoError(t, err)
}
@@ -351,12 +273,16 @@ func TestAppStateDeterminism(t *testing.T) {
)
_, _, err := simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.sm),
t, os.Stdout, app.BaseApp, AppStateFn(app.Codec(), app.SimulationManager()),
SimulationOperations(app, app.Codec(), config),
app.ModuleAccountAddrs(), config,
)
require.NoError(t, err)
if config.Commit {
PrintStats(db)
}
appHash := app.LastCommitID().Hash
appHashList[j] = appHash
simapp/state.go
+3 -2
View File
@@ -12,6 +12,7 @@ import (
tmtypes "github.com/tendermint/tendermint/types"
"github.com/cosmos/cosmos-sdk/codec"
simapparams "github.com/cosmos/cosmos-sdk/simapp/params"
"github.com/cosmos/cosmos-sdk/types/module"
"github.com/cosmos/cosmos-sdk/x/auth"
"github.com/cosmos/cosmos-sdk/x/simulation"
@@ -80,11 +81,11 @@ func AppStateRandomizedFn(
// number of bonded accounts
var initialStake, numInitiallyBonded int64
appParams.GetOrGenerate(
cdc, StakePerAccount, &initialStake, r,
cdc, simapparams.StakePerAccount, &initialStake, r,
func(r *rand.Rand) { initialStake = r.Int63n(1e12) },
)
appParams.GetOrGenerate(
cdc, InitiallyBondedValidators, &numInitiallyBonded, r,
cdc, simapparams.InitiallyBondedValidators, &numInitiallyBonded, r,
func(r *rand.Rand) { numInitiallyBonded = int64(r.Intn(300)) },
)
simapp/test_helpers.go
+3 -3
View File
@@ -27,7 +27,7 @@ func Setup(isCheckTx bool) *SimApp {
if !isCheckTx {
// init chain must be called to stop deliverState from being nil
genesisState := NewDefaultGenesisState()
stateBytes, err := codec.MarshalJSONIndent(app.cdc, genesisState)
stateBytes, err := codec.MarshalJSONIndent(app.Codec(), genesisState)
if err != nil {
panic(err)
}
@@ -54,10 +54,10 @@ func SetupWithGenesisAccounts(genAccs []authexported.GenesisAccount) *SimApp {
genesisState := NewDefaultGenesisState()
authGenesis := auth.NewGenesisState(auth.DefaultParams(), genAccs)
genesisStateBz := app.cdc.MustMarshalJSON(authGenesis)
genesisStateBz := app.Codec().MustMarshalJSON(authGenesis)
genesisState[auth.ModuleName] = genesisStateBz
stateBytes, err := codec.MarshalJSONIndent(app.cdc, genesisState)
stateBytes, err := codec.MarshalJSONIndent(app.Codec(), genesisState)
if err != nil {
panic(err)
}
simapp/types.go
+46
View File
@@ -0,0 +1,46 @@
package simapp
import (
"encoding/json"
abci "github.com/tendermint/tendermint/abci/types"
tmtypes "github.com/tendermint/tendermint/types"
"github.com/cosmos/cosmos-sdk/codec"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/types/module"
)
// App implements the common methods for a Cosmos SDK-based application
// specific blockchain.
type App interface {
// The assigned name of the app.
Name() string
// The application types codec.
// NOTE: This shoult be sealed before being returned.
Codec() *codec.Codec
// Application updates every begin block.
BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock
// Application updates every end block.
EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock
// Application update at chain (i.e app) initialization.
InitChainer(ctx sdk.Context, req abci.RequestInitChain) abci.ResponseInitChain
// Loads the app at a given height.
LoadHeight(height int64) error
// Exports the state of the application for a genesis file.
ExportAppStateAndValidators(
forZeroHeight bool, jailWhiteList []string,
) (json.RawMessage, []tmtypes.GenesisValidator, error)
// All the registered module account addreses.
ModuleAccountAddrs() map[string]bool
// Helper for the simulation framework.
SimulationManager() *module.SimulationManager
}
simapp/utils.go
+67 -92
View File
@@ -2,92 +2,54 @@ package simapp
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/codec"
"github.com/cosmos/cosmos-sdk/simapp/helpers"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/types/module"
"github.com/cosmos/cosmos-sdk/x/simulation"
)
//---------------------------------------------------------------------
// Flags
// List of available flags for the simulator
var (
FlagGenesisFileValue string
FlagParamsFileValue string
FlagExportParamsPathValue string
FlagExportParamsHeightValue int
FlagExportStatePathValue string
FlagExportStatsPathValue string
FlagSeedValue int64
FlagInitialBlockHeightValue int
FlagNumBlocksValue int
FlagBlockSizeValue int
FlagLeanValue bool
FlagCommitValue bool
FlagOnOperationValue bool // TODO: Remove in favor of binary search for invariant violation
FlagAllInvariantsValue bool
FlagEnabledValue bool
FlagVerboseValue bool
FlagPeriodValue uint
FlagGenesisTimeValue int64
)
// GetSimulatorFlags gets the values of all the available simulation flags
func GetSimulatorFlags() {
// config fields
flag.StringVar(&FlagGenesisFileValue, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&FlagParamsFileValue, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.StringVar(&FlagExportParamsPathValue, "ExportParamsPath", "", "custom file path to save the exported params JSON")
flag.IntVar(&FlagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&FlagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.StringVar(&FlagExportStatsPathValue, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
flag.Int64Var(&FlagSeedValue, "Seed", 42, "simulation random seed")
flag.IntVar(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation")
flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block")
flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output")
flag.BoolVar(&FlagCommitValue, "Commit", false, "have the simulation commit")
flag.BoolVar(&FlagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&FlagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
// simulation flags
flag.BoolVar(&FlagEnabledValue, "Enabled", false, "enable the simulation")
flag.BoolVar(&FlagVerboseValue, "Verbose", false, "verbose log output")
flag.UintVar(&FlagPeriodValue, "Period", 0, "run slow invariants only once every period assertions")
flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time")
}
// NewConfigFromFlags creates a simulation from the retrieved values of the flags.
func NewConfigFromFlags() simulation.Config {
return simulation.Config{
GenesisFile: FlagGenesisFileValue,
ParamsFile: FlagParamsFileValue,
ExportParamsPath: FlagExportParamsPathValue,
ExportParamsHeight: FlagExportParamsHeightValue,
ExportStatePath: FlagExportStatePathValue,
ExportStatsPath: FlagExportStatsPathValue,
Seed: FlagSeedValue,
InitialBlockHeight: FlagInitialBlockHeightValue,
NumBlocks: FlagNumBlocksValue,
BlockSize: FlagBlockSizeValue,
Lean: FlagLeanValue,
Commit: FlagCommitValue,
OnOperation: FlagOnOperationValue,
AllInvariants: FlagAllInvariantsValue,
// SetupSimulation creates the config, db (levelDB), temporary directory and logger for
// the simulation tests. If `FlagEnabledValue` is false it skips the current test.
// Returns error on an invalid db intantiation or temp dir creation.
func SetupSimulation(dirPrefix, dbName string) (simulation.Config, dbm.DB, string, log.Logger, bool, error) {
if !FlagEnabledValue {
return simulation.Config{}, nil, "", nil, true, nil
}
config := NewConfigFromFlags()
config.ChainID = helpers.SimAppChainID
var logger log.Logger
if FlagVerboseValue {
logger = log.TestingLogger()
} else {
logger = log.NewNopLogger()
}
dir, err := ioutil.TempDir("", dirPrefix)
if err != nil {
return simulation.Config{}, nil, "", nil, false, err
}
db, err := sdk.NewLevelDB(dbName, dir)
if err != nil {
return simulation.Config{}, nil, "", nil, false, err
}
return config, db, dir, logger, false, nil
}
// SimulationOperations retrieves the simulation params from the provided file path
// and returns all the modules weighted operations
func SimulationOperations(app *SimApp, cdc *codec.Codec, config simulation.Config) []simulation.WeightedOperation {
func SimulationOperations(app App, cdc *codec.Codec, config simulation.Config) []simulation.WeightedOperation {
simState := module.SimulationState{
AppParams: make(simulation.AppParams),
Cdc: cdc,
@@ -99,37 +61,50 @@ func SimulationOperations(app *SimApp, cdc *codec.Codec, config simulation.Confi
panic(err)
}
app.cdc.MustUnmarshalJSON(bz, &simState.AppParams)
app.Codec().MustUnmarshalJSON(bz, &simState.AppParams)
}
simState.ParamChanges = app.sm.GenerateParamChanges(config.Seed)
simState.Contents = app.sm.GetProposalContents(simState)
return app.sm.WeightedOperations(simState)
simState.ParamChanges = app.SimulationManager().GenerateParamChanges(config.Seed)
simState.Contents = app.SimulationManager().GetProposalContents(simState)
return app.SimulationManager().WeightedOperations(simState)
}
//---------------------------------------------------------------------
// Simulation Utils
// CheckExportSimulation exports the app state and simulation parameters to JSON
// if the export paths are defined.
func CheckExportSimulation(
app App, config simulation.Config, params simulation.Params,
) error {
if config.ExportStatePath != "" {
fmt.Println("exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
return err
}
// ExportStateToJSON util function to export the app state to JSON
func ExportStateToJSON(app *SimApp, path string) error {
fmt.Println("exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
return err
if err := ioutil.WriteFile(config.ExportStatePath, []byte(appState), 0644); err != nil {
return err
}
}
return ioutil.WriteFile(path, []byte(appState), 0644)
if config.ExportParamsPath != "" {
fmt.Println("exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
if err != nil {
return err
}
if err := ioutil.WriteFile(config.ExportParamsPath, paramsBz, 0644); err != nil {
return err
}
}
return nil
}
// ExportParamsToJSON util function to export the simulation parameters to JSON
func ExportParamsToJSON(params simulation.Params, path string) error {
fmt.Println("exporting simulation params...")
paramsBz, err := json.MarshalIndent(params, "", " ")
if err != nil {
return err
}
return ioutil.WriteFile(path, paramsBz, 0644)
// PrintStats prints the corresponding statistics from the app DB.
func PrintStats(db dbm.DB) {
fmt.Println("\nLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("LevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}
// GetSimulationLog unmarshals the KVPair's Value to the corresponding type based on the