LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

perf: Replace runsim with Go stdlib testing (#24045)

Browse Source 
Co-authored-by: Alex | Interchain Labs <alex@interchainlabs.io>
This commit is contained in:
Alexander Peters
2025-03-21 15:48:00 +00:00
committed by GitHub
co-authored by Alex | Interchain Labs
parent b7c35f2568
commit bc57ce3ba7
26 changed files with 1041 additions and 788 deletions
Download Patch File Download Diff File Expand all files Collapse all files
simapp/sim_bench_test.go
+13 -78
View File
@@ -1,38 +1,35 @@
//go:build sims
package simapp
import (
"os"
"testing"
flag "github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
"cosmossdk.io/log"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/server"
simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
"github.com/cosmos/cosmos-sdk/testutils/sims"
simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
"github.com/cosmos/cosmos-sdk/x/simulation"
simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
)
var FlagEnableBenchStreamingValue bool
// Get flags every time the simulator is run
func init() {
flag.BoolVar(&FlagEnableBenchStreamingValue, "EnableStreaming", false, "Enable streaming service")
}
// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ cosmossdk.io/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
b.ReportAllocs()
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID
config.ChainID = sims.SimAppChainID
db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, true)
if err != nil {
b.Fatalf("simulation setup failed: %s", err.Error())
}
@@ -47,86 +44,24 @@ func BenchmarkFullAppSimulation(b *testing.B) {
}()
appOptions := viper.New()
if FlagEnableStreamingValue {
m := make(map[string]interface{})
m["streaming.abci.keys"] = []string{"*"}
m["streaming.abci.plugin"] = "abci_v1"
m["streaming.abci.stop-node-on-err"] = true
for key, value := range m {
appOptions.SetDefault(key, value)
}
}
appOptions.SetDefault(flags.FlagHome, DefaultNodeHome)
appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue)
app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID))
app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(sims.SimAppChainID))
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
simParams, simErr := simulation.SimulateFromSeedX(
b,
log.NewNopLogger(),
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(app, app.AppCodec(), config),
BlockedAddresses(),
config,
app.AppCodec(),
)
// export state and simParams before the simulation error is checked
if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil {
b.Fatal(err)
}
if simErr != nil {
b.Fatal(simErr)
}
if config.Commit {
simtestutil.PrintStats(db)
}
}
func BenchmarkInvariants(b *testing.B) {
b.ReportAllocs()
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID
db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-invariant-bench", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if err != nil {
b.Fatalf("simulation setup failed: %s", err.Error())
}
if skip {
b.Skip("skipping benchmark application simulation")
}
config.AllInvariants = false
defer func() {
require.NoError(b, db.Close())
require.NoError(b, os.RemoveAll(dir))
}()
appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue
app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID))
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
b,
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(app, app.AppCodec(), config),
simtypes.RandomAccounts,
simtestutil.BuildSimulationOperations(app, app.AppCodec(), config, app.txConfig),
BlockedAddresses(),
config,
app.AppCodec(),
&simulation.DummyLogWriter{},
)
// export state and simParams before the simulation error is checked
simapp/sim_test.go
+201 -334
View File
@@ -1,19 +1,21 @@
//go:build sims
package simapp
import (
"encoding/binary"
"encoding/json"
"flag"
"fmt"
"io"
"math/rand"
"os"
"runtime/debug"
"strings"
"sync"
"testing"
abci "github.com/cometbft/cometbft/abci/types"
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
dbm "github.com/cosmos/cosmos-db"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"cosmossdk.io/log"
@@ -22,9 +24,10 @@ import (
"cosmossdk.io/x/feegrant"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/server"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
"github.com/cosmos/cosmos-sdk/testutils/sims"
sdk "github.com/cosmos/cosmos-sdk/types"
simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
authzkeeper "github.com/cosmos/cosmos-sdk/x/authz/keeper"
"github.com/cosmos/cosmos-sdk/x/simulation"
@@ -33,9 +36,6 @@ import (
stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
)
// SimAppChainID hardcoded chainID for simulation
const SimAppChainID = "simulation-app"
var FlagEnableStreamingValue bool
// Get flags every time the simulator is run
@@ -44,12 +44,6 @@ func init() {
flag.BoolVar(&FlagEnableStreamingValue, "EnableStreaming", false, "Enable streaming service")
}
// fauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of
// an IAVLStore for faster simulation speed.
func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
bapp.SetFauxMerkleMode()
}
// interBlockCacheOpt returns a BaseApp option function that sets the persistent
// inter-block write-through cache.
func interBlockCacheOpt() func(*baseapp.BaseApp) {
@@ -57,150 +51,188 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) {
}
func TestFullAppSimulation(t *testing.T) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID
sims.Run(t, NewSimApp, setupStateFactory)
}
db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip("skipping application simulation")
}
require.NoError(t, err, "simulation setup failed")
defer func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
}()
appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue
app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())
// run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t,
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(app, app.AppCodec(), config),
BlockedAddresses(),
config,
app.AppCodec(),
)
// export state and simParams before the simulation error is checked
err = simtestutil.CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)
if config.Commit {
simtestutil.PrintStats(db)
func setupStateFactory(app *SimApp) sims.SimStateFactory {
return sims.SimStateFactory{
Codec: app.AppCodec(),
AppStateFn: simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
BlockedAddr: BlockedAddresses(),
}
}
var (
exportAllModules = []string{}
exportWithValidatorSet = []string{}
)
func TestAppImportExport(t *testing.T) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID
sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) {
t.Helper()
app := ti.App
t.Log("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules)
require.NoError(t, err)
db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip("skipping application import/export simulation")
}
require.NoError(t, err, "simulation setup failed")
defer func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
}()
appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue
app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
_, simParams, simErr := simulation.SimulateFromSeed(
t,
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(app, app.AppCodec(), config),
BlockedAddresses(),
config,
app.AppCodec(),
)
// export state and simParams before the simulation error is checked
err = simtestutil.CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)
if config.Commit {
simtestutil.PrintStats(db)
}
fmt.Printf("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{})
require.NoError(t, err)
fmt.Printf("importing genesis...\n")
newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
require.NoError(t, err, "simulation setup failed")
defer func() {
require.NoError(t, newDB.Close())
require.NoError(t, os.RemoveAll(newDir))
}()
newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
require.Equal(t, "SimApp", newApp.Name())
var genesisState GenesisState
err = json.Unmarshal(exported.AppState, &genesisState)
require.NoError(t, err)
ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
_, err = newApp.ModuleManager.InitGenesis(ctxB, app.AppCodec(), genesisState)
if err != nil {
if strings.Contains(err.Error(), "validator set is empty after InitGenesis") {
logger.Info("Skipping simulation as all validators have been unbonded")
logger.Info("err", err, "stacktrace", string(debug.Stack()))
t.Log("importing genesis...\n")
newTestInstance := sims.NewSimulationAppInstance(t, ti.Cfg, NewSimApp)
newApp := newTestInstance.App
var genesisState GenesisState
require.NoError(t, json.Unmarshal(exported.AppState, &genesisState))
ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
_, err = newApp.ModuleManager.InitGenesis(ctxB, newApp.appCodec, genesisState)
if IsEmptyValidatorSetErr(err) {
t.Skip("Skipping simulation as all validators have been unbonded")
return
}
}
require.NoError(t, err)
err = newApp.StoreConsensusParams(ctxB, exported.ConsensusParams)
require.NoError(t, err)
require.NoError(t, err)
err = newApp.StoreConsensusParams(ctxB, exported.ConsensusParams)
require.NoError(t, err)
fmt.Printf("comparing stores...\n")
t.Log("comparing stores...")
// skip certain prefixes
skipPrefixes := map[string][][]byte{
stakingtypes.StoreKey: {
stakingtypes.UnbondingQueueKey, stakingtypes.RedelegationQueueKey, stakingtypes.ValidatorQueueKey,
stakingtypes.HistoricalInfoKey, stakingtypes.UnbondingIDKey, stakingtypes.UnbondingIndexKey,
stakingtypes.UnbondingTypeKey,
stakingtypes.ValidatorUpdatesKey, // todo (Alex): double check why there is a diff with test-sim-import-export
},
authzkeeper.StoreKey: {authzkeeper.GrantQueuePrefix},
feegrant.StoreKey: {feegrant.FeeAllowanceQueueKeyPrefix},
slashingtypes.StoreKey: {slashingtypes.ValidatorMissedBlockBitmapKeyPrefix},
}
AssertEqualStores(t, app, newApp, app.SimulationManager().StoreDecoders, skipPrefixes)
})
}
// skip certain prefixes
skipPrefixes := map[string][][]byte{
stakingtypes.StoreKey: {
stakingtypes.UnbondingQueueKey, stakingtypes.RedelegationQueueKey, stakingtypes.ValidatorQueueKey,
stakingtypes.HistoricalInfoKey, stakingtypes.UnbondingIDKey, stakingtypes.UnbondingIndexKey,
stakingtypes.UnbondingTypeKey, stakingtypes.ValidatorUpdatesKey,
},
authzkeeper.StoreKey: {authzkeeper.GrantQueuePrefix},
feegrant.StoreKey: {feegrant.FeeAllowanceQueueKeyPrefix},
slashingtypes.StoreKey: {slashingtypes.ValidatorMissedBlockBitmapKeyPrefix},
// Scenario:
//
// Start a fresh node and run n blocks, export state
// set up a new node instance, Init chain from exported genesis
// run new instance for n blocks
func TestAppSimulationAfterImport(t *testing.T) {
sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) {
t.Helper()
app := ti.App
t.Log("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules)
require.NoError(t, err)
t.Log("importing genesis...\n")
newTestInstance := sims.NewSimulationAppInstance(t, ti.Cfg, NewSimApp)
newApp := newTestInstance.App
_, err = newApp.InitChain(&abci.RequestInitChain{
AppStateBytes: exported.AppState,
ChainId: sims.SimAppChainID,
})
if IsEmptyValidatorSetErr(err) {
t.Skip("Skipping simulation as all validators have been unbonded")
return
}
require.NoError(t, err)
newStateFactory := setupStateFactory(newApp)
_, err = simulation.SimulateFromSeedX(
t,
newTestInstance.AppLogger,
sims.WriteToDebugLog(newTestInstance.AppLogger),
newApp.BaseApp,
newStateFactory.AppStateFn,
simtypes.RandomAccounts,
simtestutil.BuildSimulationOperations(newApp, newApp.AppCodec(), newTestInstance.Cfg, newApp.TxConfig()),
newStateFactory.BlockedAddr,
newTestInstance.Cfg,
newStateFactory.Codec,
ti.ExecLogWriter,
)
require.NoError(t, err)
})
}
func IsEmptyValidatorSetErr(err error) bool {
return err != nil && strings.Contains(err.Error(), "validator set is empty after InitGenesis")
}
func TestAppStateDeterminism(t *testing.T) {
const numTimesToRunPerSeed = 3
var seeds []int64
if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue {
// We will be overriding the random seed and just run a single simulation on the provided seed value
for j := 0; j < numTimesToRunPerSeed; j++ { // multiple rounds
seeds = append(seeds, s)
}
} else {
// setup with 3 random seeds
for i := 0; i < 3; i++ {
seed := rand.Int63()
for j := 0; j < numTimesToRunPerSeed; j++ { // multiple rounds
seeds = append(seeds, seed)
}
}
}
// overwrite default app config
interBlockCachingAppFactory := func(logger log.Logger, db dbm.DB, traceStore io.Writer, loadLatest bool, appOpts servertypes.AppOptions, baseAppOptions ...func(*baseapp.BaseApp)) *SimApp {
if FlagEnableStreamingValue {
m := map[string]any{
"streaming.abci.keys": []string{"*"},
"streaming.abci.plugin": "abci_v1",
"streaming.abci.stop-node-on-err": true,
}
others := appOpts
appOpts = sims.AppOptionsFn(func(k string) any {
if v, ok := m[k]; ok {
return v
}
return others.Get(k)
})
}
return NewSimApp(logger, db, nil, true, appOpts, append(baseAppOptions, interBlockCacheOpt())...)
}
var mx sync.Mutex
appHashResults := make(map[int64][][]byte)
appSimLogger := make(map[int64][]simulation.LogWriter)
captureAndCheckHash := func(t *testing.T, ti sims.TestInstance[*SimApp]) {
t.Helper()
seed, appHash := ti.Cfg.Seed, ti.App.LastCommitID().Hash
mx.Lock()
otherHashes, execWriters := appHashResults[seed], appSimLogger[seed]
if len(otherHashes) < numTimesToRunPerSeed-1 {
appHashResults[seed], appSimLogger[seed] = append(otherHashes, appHash), append(execWriters, ti.ExecLogWriter)
} else { // cleanup
delete(appHashResults, seed)
delete(appSimLogger, seed)
}
mx.Unlock()
var failNow bool
// and check that all app hashes per seed are equal for each iteration
for i := 0; i < len(otherHashes); i++ {
if !assert.Equal(t, otherHashes[i], appHash) {
execWriters[i].PrintLogs()
failNow = true
}
}
if failNow {
ti.ExecLogWriter.PrintLogs()
t.Fatalf("non-determinism in seed %d", seed)
}
}
// run simulations
sims.RunWithSeeds(t, interBlockCachingAppFactory, setupStateFactory, seeds, []byte{}, captureAndCheckHash)
}
type ComparableStoreApp interface {
LastBlockHeight() int64
NewContextLegacy(isCheckTx bool, header cmtproto.Header) sdk.Context
GetKey(storeKey string) *storetypes.KVStoreKey
GetStoreKeys() []storetypes.StoreKey
}
func AssertEqualStores(t *testing.T, app, newApp ComparableStoreApp, storeDecoders simtypes.StoreDecoderRegistry, skipPrefixes map[string][][]byte) {
t.Helper()
ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
storeKeys := app.GetStoreKeys()
require.NotEmpty(t, storeKeys)
@@ -218,195 +250,30 @@ func TestAppImportExport(t *testing.T) {
storeB := ctxB.KVStore(appKeyB)
failedKVAs, failedKVBs := simtestutil.DiffKVStores(storeA, storeB, skipPrefixes[keyName])
require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare %s", keyName)
require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare %s, key stores %s and %s", keyName, appKeyA, appKeyB)
fmt.Printf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), appKeyA, appKeyB)
require.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, app.SimulationManager().StoreDecoders, failedKVAs, failedKVBs))
t.Logf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), appKeyA, appKeyB)
if !assert.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, storeDecoders, failedKVAs, failedKVBs)) {
for _, v := range failedKVAs {
t.Logf("store mismatch: %q\n", v)
}
t.FailNow()
}
}
}
func TestAppSimulationAfterImport(t *testing.T) {
config := simcli.NewConfigFromFlags()
config.ChainID = SimAppChainID
db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if skip {
t.Skip("skipping application simulation after import")
}
require.NoError(t, err, "simulation setup failed")
defer func() {
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(dir))
}()
appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue
app := NewSimApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
require.Equal(t, "SimApp", app.Name())
// Run randomized simulation
stopEarly, simParams, simErr := simulation.SimulateFromSeed(
t,
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(app, app.AppCodec(), config),
BlockedAddresses(),
config,
app.AppCodec(),
)
// export state and simParams before the simulation error is checked
err = simtestutil.CheckExportSimulation(app, config, simParams)
require.NoError(t, err)
require.NoError(t, simErr)
if config.Commit {
simtestutil.PrintStats(db)
}
if stopEarly {
fmt.Println("can't export or import a zero-validator genesis, exiting test...")
return
}
fmt.Printf("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(true, []string{}, []string{})
require.NoError(t, err)
fmt.Printf("importing genesis...\n")
newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
require.NoError(t, err, "simulation setup failed")
defer func() {
require.NoError(t, newDB.Close())
require.NoError(t, os.RemoveAll(newDir))
}()
newApp := NewSimApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(SimAppChainID))
require.Equal(t, "SimApp", newApp.Name())
_, err = newApp.InitChain(&abci.RequestInitChain{
AppStateBytes: exported.AppState,
ChainId: SimAppChainID,
func FuzzFullAppSimulation(f *testing.F) {
f.Fuzz(func(t *testing.T, rawSeed []byte) {
if len(rawSeed) < 8 {
t.Skip()
return
}
sims.RunWithSeeds(
t,
NewSimApp,
setupStateFactory,
[]int64{int64(binary.BigEndian.Uint64(rawSeed))},
rawSeed[8:],
)
})
require.NoError(t, err)
_, _, err = simulation.SimulateFromSeed(
t,
os.Stdout,
newApp.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(newApp, newApp.AppCodec(), config),
BlockedAddresses(),
config,
app.AppCodec(),
)
require.NoError(t, err)
}
// TODO: Make another test for the fuzzer itself, which just has noOp txs
// and doesn't depend on the application.
func TestAppStateDeterminism(t *testing.T) {
if !simcli.FlagEnabledValue {
t.Skip("skipping application simulation")
}
config := simcli.NewConfigFromFlags()
config.InitialBlockHeight = 1
config.ExportParamsPath = ""
config.OnOperation = false
config.AllInvariants = false
config.ChainID = SimAppChainID
numSeeds := 3
numTimesToRunPerSeed := 3 // This used to be set to 5, but we've temporarily reduced it to 3 for the sake of faster CI.
appHashList := make([]json.RawMessage, numTimesToRunPerSeed)
// We will be overriding the random seed and just run a single simulation on the provided seed value
if config.Seed != simcli.DefaultSeedValue {
numSeeds = 1
}
appOptions := viper.New()
if FlagEnableStreamingValue {
m := make(map[string]interface{})
m["streaming.abci.keys"] = []string{"*"}
m["streaming.abci.plugin"] = "abci_v1"
m["streaming.abci.stop-node-on-err"] = true
for key, value := range m {
appOptions.SetDefault(key, value)
}
}
appOptions.SetDefault(flags.FlagHome, DefaultNodeHome)
appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue)
if simcli.FlagVerboseValue {
appOptions.SetDefault(flags.FlagLogLevel, "debug")
}
for i := 0; i < numSeeds; i++ {
if config.Seed == simcli.DefaultSeedValue {
config.Seed = rand.Int63()
}
fmt.Println("config.Seed: ", config.Seed)
for j := 0; j < numTimesToRunPerSeed; j++ {
var logger log.Logger
if simcli.FlagVerboseValue {
logger = log.NewTestLogger(t)
} else {
logger = log.NewNopLogger()
}
db := dbm.NewMemDB()
app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(SimAppChainID))
if !simcli.FlagSigverifyTxValue {
app.SetNotSigverifyTx()
}
fmt.Printf(
"running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n",
config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed,
)
_, _, err := simulation.SimulateFromSeed(
t,
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1
simtestutil.SimulationOperations(app, app.AppCodec(), config),
BlockedAddresses(),
config,
app.AppCodec(),
)
require.NoError(t, err)
if config.Commit {
simtestutil.PrintStats(db)
}
appHash := app.LastCommitID().Hash
appHashList[j] = appHash
if j != 0 {
require.Equal(
t, string(appHashList[0]), string(appHashList[j]),
"non-determinism in seed %d: %d/%d, attempt: %d/%d\n", config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed,
)
}
}
}
}
simapp/test_helpers.go
-2
View File
@@ -19,7 +19,6 @@ import (
bam "github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1"
"github.com/cosmos/cosmos-sdk/server"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/testutil/mock"
"github.com/cosmos/cosmos-sdk/testutil/network"
@@ -43,7 +42,6 @@ func setup(withGenesis bool, invCheckPeriod uint) (*SimApp, GenesisState) {
appOptions := make(simtestutil.AppOptionsMap, 0)
appOptions[flags.FlagHome] = DefaultNodeHome
appOptions[server.FlagInvCheckPeriod] = invCheckPeriod
app := NewSimApp(log.NewNopLogger(), db, nil, true, appOptions)
if withGenesis {