From 24b9e84a849111e114abcd5fa72029a5c54561b4 Mon Sep 17 00:00:00 2001 From: Federico Kunze <31522760+fedekunze@users.noreply.github.com> Date: Fri, 26 Jul 2019 22:09:15 +0200 Subject: [PATCH] Merge PR #4784: JSON representation of event stats --- .clog.yaml | 1 + .../simulation/_4670-JSON-sim-stats | 2 + simapp/sim_test.go | 17 +-- simapp/utils.go | 1 + x/distribution/simulation/msgs.go | 12 +-- x/genutil/types/genesis_state_test.go | 5 +- x/gov/simulation/msgs.go | 17 ++- x/simulation/event_stats.go | 58 ++++++---- x/simulation/mock_tendermint.go | 16 +-- x/simulation/operation.go | 100 +++++++----------- x/simulation/simulate.go | 48 +++++---- x/slashing/simulation/msgs.go | 4 +- x/staking/simulation/msgs.go | 38 +++---- 13 files changed, 167 insertions(+), 152 deletions(-) create mode 100644 .pending/improvements/simulation/_4670-JSON-sim-stats diff --git a/.clog.yaml b/.clog.yaml index 08f0137919..bab256f8d9 100644 --- a/.clog.yaml +++ b/.clog.yaml @@ -11,3 +11,4 @@ tags: - rest - cli - modules + - simulation diff --git a/.pending/improvements/simulation/_4670-JSON-sim-stats b/.pending/improvements/simulation/_4670-JSON-sim-stats new file mode 100644 index 0000000000..4358d2526a --- /dev/null +++ b/.pending/improvements/simulation/_4670-JSON-sim-stats @@ -0,0 +1,2 @@ +#4670 Update simulation statistics to JSON format + - Support exporting the simulation stats to a given JSON file diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 5ea553e3e2..54b4e10340 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -36,6 +36,7 @@ func init() { flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON") flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params") flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON") + flag.StringVar(&exportStatsPath, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON") flag.Int64Var(&seed, "Seed", 42, "simulation random seed") flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks") flag.IntVar(&blockSize, "BlockSize", 200, "operations per block") @@ -52,7 +53,7 @@ func init() { // helper function for populating input for SimulateFromSeed func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) ( testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64, - simulation.WeightedOperations, sdk.Invariants, int, int, int, + simulation.WeightedOperations, sdk.Invariants, int, int, int, string, bool, bool, bool, bool, bool, map[string]bool) { exportParams := exportParamsPath != "" @@ -60,7 +61,7 @@ func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) ( return tb, w, app.BaseApp, appStateFn, seed, testAndRunTxs(app), invariants(app), numBlocks, exportParamsHeight, blockSize, - exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs() + exportStatsPath, exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs() } func appStateFn( @@ -412,7 +413,7 @@ func BenchmarkFullAppSimulation(b *testing.B) { } if commit { - fmt.Println("GoLevelDB Stats") + fmt.Println("\nGoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } @@ -471,7 +472,7 @@ func TestFullAppSimulation(t *testing.T) { if commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) - fmt.Println("GoLevelDB Stats") + fmt.Println("\nGoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } @@ -528,7 +529,7 @@ func TestAppImportExport(t *testing.T) { if commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) - fmt.Println("GoLevelDB Stats") + fmt.Println("\nGoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } @@ -644,7 +645,7 @@ func TestAppSimulationAfterImport(t *testing.T) { if commit { // for memdb: // fmt.Println("Database Size", db.Stats()["database.size"]) - fmt.Println("GoLevelDB Stats") + fmt.Println("\nGoLevelDB Stats") fmt.Println(db.Stats()["leveldb.stats"]) fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) } @@ -705,7 +706,7 @@ func TestAppStateDeterminism(t *testing.T) { simulation.SimulateFromSeed( t, os.Stdout, app.BaseApp, appStateFn, seed, testAndRunTxs(app), []sdk.Invariant{}, - 50, 100, 0, + 50, 100, 0, "", false, true, false, false, false, app.ModuleAccountAddrs(), ) appHash := app.LastCommitID().Hash @@ -734,7 +735,7 @@ func BenchmarkInvariants(b *testing.B) { _, params, simErr := simulation.SimulateFromSeed( b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app), []sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize, - exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(), + exportStatsPath, exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(), ) // export state and params before the simulation error is checked diff --git a/simapp/utils.go b/simapp/utils.go index b4271b90f5..a97e0558d7 100644 --- a/simapp/utils.go +++ b/simapp/utils.go @@ -43,6 +43,7 @@ var ( exportParamsPath string exportParamsHeight int exportStatePath string + exportStatsPath string seed int64 numBlocks int blockSize int diff --git a/x/distribution/simulation/msgs.go b/x/distribution/simulation/msgs.go index bd787b30fd..ae002010c1 100644 --- a/x/distribution/simulation/msgs.go +++ b/x/distribution/simulation/msgs.go @@ -13,7 +13,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/simulation" ) -// SimulateMsgSetWithdrawAddress +// SimulateMsgSetWithdrawAddress generates a MsgSetWithdrawAddress with random values. func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation { handler := distribution.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -24,7 +24,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper) msg := distribution.NewMsgSetWithdrawAddress(accountOrigin.Address, accountDestination.Address) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() @@ -38,7 +38,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper) } } -// SimulateMsgWithdrawDelegatorReward +// SimulateMsgWithdrawDelegatorReward generates a MsgWithdrawDelegatorReward with random values. func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation { handler := distribution.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -49,7 +49,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee msg := distribution.NewMsgWithdrawDelegatorReward(delegatorAccount.Address, sdk.ValAddress(validatorAccount.Address)) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() @@ -63,7 +63,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee } } -// SimulateMsgWithdrawValidatorCommission +// SimulateMsgWithdrawValidatorCommission generates a MsgWithdrawValidatorCommission with random values. func SimulateMsgWithdrawValidatorCommission(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation { handler := distribution.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -73,7 +73,7 @@ func SimulateMsgWithdrawValidatorCommission(m auth.AccountKeeper, k distribution msg := distribution.NewMsgWithdrawValidatorCommission(sdk.ValAddress(account.Address)) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() diff --git a/x/genutil/types/genesis_state_test.go b/x/genutil/types/genesis_state_test.go index fb15d24352..2c340a35e9 100644 --- a/x/genutil/types/genesis_state_test.go +++ b/x/genutil/types/genesis_state_test.go @@ -3,11 +3,12 @@ package types import ( "testing" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/ed25519" + sdk "github.com/cosmos/cosmos-sdk/types" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto/ed25519" ) var ( diff --git a/x/gov/simulation/msgs.go b/x/gov/simulation/msgs.go index 8b65fa3d61..0477e8273b 100644 --- a/x/gov/simulation/msgs.go +++ b/x/gov/simulation/msgs.go @@ -54,7 +54,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, contentSim Con content := contentSim(r, app, ctx, accs) msg, err := simulationCreateMsgSubmitProposal(r, content, sender) if err != nil { - return simulation.NoOpMsg(), nil, err + return simulation.NoOpMsg(gov.ModuleName), nil, err } ok := simulateHandleMsgSubmitProposal(msg, handler, ctx) @@ -66,7 +66,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, contentSim Con proposalID, err := k.GetProposalID(ctx) if err != nil { - return simulation.NoOpMsg(), nil, err + return simulation.NoOpMsg(gov.ModuleName), nil, err } proposalID = uint64(math.Max(float64(proposalID)-1, 0)) @@ -122,7 +122,7 @@ func simulationCreateMsgSubmitProposal(r *rand.Rand, c gov.Content, s simulation return } -// SimulateMsgDeposit +// SimulateMsgDeposit generates a MsgDeposit with random values. func SimulateMsgDeposit(k gov.Keeper) simulation.Operation { return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account) ( opMsg simulation.OperationMsg, fOps []simulation.FutureOperation, err error) { @@ -130,12 +130,12 @@ func SimulateMsgDeposit(k gov.Keeper) simulation.Operation { acc := simulation.RandomAcc(r, accs) proposalID, ok := randomProposalID(r, k, ctx) if !ok { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(gov.ModuleName), nil, nil } deposit := randomDeposit(r) msg := gov.NewMsgDeposit(acc.Address, proposalID, deposit) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(gov.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() ok = gov.NewHandler(k)(ctx, msg).IsOK() @@ -148,8 +148,7 @@ func SimulateMsgDeposit(k gov.Keeper) simulation.Operation { } } -// SimulateMsgVote -// nolint: unparam +// SimulateMsgVote generates a MsgVote with random values. func SimulateMsgVote(k gov.Keeper) simulation.Operation { return operationSimulateMsgVote(k, simulation.Account{}, 0) } @@ -167,14 +166,14 @@ func operationSimulateMsgVote(k gov.Keeper, acc simulation.Account, proposalID u var ok bool proposalID, ok = randomProposalID(r, k, ctx) if !ok { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(gov.ModuleName), nil, nil } } option := randomVotingOption(r) msg := gov.NewMsgVote(acc.Address, proposalID, option) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(gov.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() diff --git a/x/simulation/event_stats.go b/x/simulation/event_stats.go index ad8acddb25..875a2029d4 100644 --- a/x/simulation/event_stats.go +++ b/x/simulation/event_stats.go @@ -1,33 +1,55 @@ package simulation import ( + "encoding/json" "fmt" "io" - "sort" + "io/ioutil" ) -type eventStats map[string]uint +// EventStats defines an object that keeps a tally of each event that has occurred +// during a simulation. +type EventStats map[string]map[string]map[string]int -func newEventStats() eventStats { - events := make(map[string]uint) - return events +// NewEventStats creates a new empty EventStats object +func NewEventStats() EventStats { + return make(EventStats) } -func (es eventStats) tally(eventDesc string) { - es[eventDesc]++ -} - -// Pretty-print events as a table -func (es eventStats) Print(w io.Writer) { - var keys []string - for key := range es { - keys = append(keys, key) +// Tally increases the count of a simulation event. +func (es EventStats) Tally(route, op, evResult string) { + _, ok := es[route] + if !ok { + es[route] = make(map[string]map[string]int) } - sort.Strings(keys) - fmt.Fprintf(w, "Event statistics: \n") + _, ok = es[route][op] + if !ok { + es[route][op] = make(map[string]int) + } - for _, key := range keys { - fmt.Fprintf(w, " % 60s => %d\n", key, es[key]) + es[route][op][evResult]++ +} + +// Print the event stats in JSON format. +func (es EventStats) Print(w io.Writer) { + obj, err := json.MarshalIndent(es, "", " ") + if err != nil { + panic(err) + } + + fmt.Fprintf(w, string(obj)) +} + +// ExportJSON saves the event stats as a JSON file on a given path +func (es EventStats) ExportJSON(path string) { + bz, err := json.MarshalIndent(es, "", " ") + if err != nil { + panic(err) + } + + err = ioutil.WriteFile(path, bz, 0644) + if err != nil { + panic(err) } } diff --git a/x/simulation/mock_tendermint.go b/x/simulation/mock_tendermint.go index e241e9c672..efa84aead3 100644 --- a/x/simulation/mock_tendermint.go +++ b/x/simulation/mock_tendermint.go @@ -79,7 +79,7 @@ func (vals mockValidators) randomProposer(r *rand.Rand) cmn.HexBytes { // nolint: unparam func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[string]mockValidator, updates []abci.ValidatorUpdate, - event func(string)) map[string]mockValidator { + event func(route, op, evResult string)) map[string]mockValidator { for _, update := range updates { str := fmt.Sprintf("%v", update.PubKey) @@ -88,13 +88,13 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params, if _, ok := current[str]; !ok { tb.Fatalf("tried to delete a nonexistent validator") } - event("endblock/validatorupdates/kicked") + event("end_block", "validator_updates", "kicked") delete(current, str) } else if mVal, ok := current[str]; ok { // validator already exists mVal.val = update - event("endblock/validatorupdates/updated") + event("end_block", "validator_updates", "updated") } else { // Set this new validator @@ -102,7 +102,7 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params, update, GetMemberOfInitialState(r, params.InitialLivenessWeightings), } - event("endblock/validatorupdates/added") + event("end_block", "validator_updates", "added") } } @@ -114,7 +114,7 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params, func RandomRequestBeginBlock(r *rand.Rand, params Params, validators mockValidators, pastTimes []time.Time, pastVoteInfos [][]abci.VoteInfo, - event func(string), header abci.Header) abci.RequestBeginBlock { + event func(route, op, evResult string), header abci.Header) abci.RequestBeginBlock { if len(validators) == 0 { return abci.RequestBeginBlock{ @@ -139,9 +139,9 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, } if signed { - event("beginblock/signing/signed") + event("begin_block", "signing", "signed") } else { - event("beginblock/signing/missed") + event("begin_block", "signing", "missed") } pubkey, err := tmtypes.PB2TM.PubKey(mVal.val.PubKey) @@ -197,7 +197,7 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, TotalVotingPower: totalVotingPower, }, ) - event("beginblock/evidence") + event("begin_block", "evidence", "ok") } return abci.RequestBeginBlock{ diff --git a/x/simulation/operation.go b/x/simulation/operation.go index d894067af7..e833807dbc 100644 --- a/x/simulation/operation.go +++ b/x/simulation/operation.go @@ -2,7 +2,6 @@ package simulation import ( "encoding/json" - "fmt" "math/rand" "sort" "time" @@ -26,10 +25,10 @@ type Operation func(r *rand.Rand, app *baseapp.BaseApp, // entry kinds for use within OperationEntry const ( - BeginBlockEntryKind = "begin_block" - EndBlockEntryKind = "end_block" - MsgEntryKind = "msg" - QueuedsgMsgEntryKind = "queued_msg" + BeginBlockEntryKind = "begin_block" + EndBlockEntryKind = "end_block" + MsgEntryKind = "msg" + QueuedMsgEntryKind = "queued_msg" ) // OperationEntry - an operation entry for logging (ex. BeginBlock, EndBlock, XxxMsg, etc) @@ -40,47 +39,37 @@ type OperationEntry struct { Operation json.RawMessage `json:"operation" yaml:"operation"` } +// NewOperationEntry creates a new OperationEntry instance +func NewOperationEntry(entry string, height, order int64, op json.RawMessage) OperationEntry { + return OperationEntry{ + EntryKind: entry, + Height: height, + Order: order, + Operation: op, + } +} + // BeginBlockEntry - operation entry for begin block func BeginBlockEntry(height int64) OperationEntry { - return OperationEntry{ - EntryKind: BeginBlockEntryKind, - Height: height, - Order: -1, - Operation: nil, - } + return NewOperationEntry(BeginBlockEntryKind, height, -1, nil) } // EndBlockEntry - operation entry for end block func EndBlockEntry(height int64) OperationEntry { - return OperationEntry{ - EntryKind: EndBlockEntryKind, - Height: height, - Order: -1, - Operation: nil, - } + return NewOperationEntry(EndBlockEntryKind, height, -1, nil) } // MsgEntry - operation entry for standard msg -func MsgEntry(height int64, opMsg OperationMsg, order int64) OperationEntry { - return OperationEntry{ - EntryKind: MsgEntryKind, - Height: height, - Order: order, - Operation: opMsg.MustMarshal(), - } +func MsgEntry(height, order int64, opMsg OperationMsg) OperationEntry { + return NewOperationEntry(MsgEntryKind, height, order, opMsg.MustMarshal()) } -// MsgEntry - operation entry for queued msg +// QueuedMsgEntry creates an operation entry for a given queued message. func QueuedMsgEntry(height int64, opMsg OperationMsg) OperationEntry { - return OperationEntry{ - EntryKind: QueuedsgMsgEntryKind, - Height: height, - Order: -1, - Operation: opMsg.MustMarshal(), - } + return NewOperationEntry(QueuedMsgEntryKind, height, -1, opMsg.MustMarshal()) } -// OperationEntry - log entry text for this operation entry +// MustMarshal marshals the operation entry, panic on error. func (oe OperationEntry) MustMarshal() json.RawMessage { out, err := json.Marshal(oe) if err != nil { @@ -100,19 +89,7 @@ type OperationMsg struct { Msg json.RawMessage `json:"msg" yaml:"msg"` } -// OperationMsg - create a new operation message from sdk.Msg -func NewOperationMsg(msg sdk.Msg, ok bool, comment string) OperationMsg { - - return OperationMsg{ - Route: msg.Route(), - Name: msg.Type(), - Comment: comment, - OK: ok, - Msg: msg.GetSignBytes(), - } -} - -// OperationMsg - create a new operation message from raw input +// NewOperationMsgBasic creates a new operation message from raw input. func NewOperationMsgBasic(route, name, comment string, ok bool, msg []byte) OperationMsg { return OperationMsg{ Route: route, @@ -123,15 +100,14 @@ func NewOperationMsgBasic(route, name, comment string, ok bool, msg []byte) Oper } } +// NewOperationMsg - create a new operation message from sdk.Msg +func NewOperationMsg(msg sdk.Msg, ok bool, comment string) OperationMsg { + return NewOperationMsgBasic(msg.Route(), msg.Type(), comment, ok, msg.GetSignBytes()) +} + // NoOpMsg - create a no-operation message -func NoOpMsg() OperationMsg { - return OperationMsg{ - Route: "", - Name: "no-operation", - Comment: "", - OK: false, - Msg: nil, - } +func NoOpMsg(route string) OperationMsg { + return NewOperationMsgBasic(route, "no-operation", "", false, nil) } // log entry text for this operation msg @@ -143,7 +119,7 @@ func (om OperationMsg) String() string { return string(out) } -// Marshal the operation msg, panic on error +// MustMarshal Marshals the operation msg, panic on error func (om OperationMsg) MustMarshal() json.RawMessage { out, err := json.Marshal(om) if err != nil { @@ -152,24 +128,24 @@ func (om OperationMsg) MustMarshal() json.RawMessage { return out } -// add event for event stats -func (om OperationMsg) LogEvent(eventLogger func(string)) { +// LogEvent adds an event for the events stats +func (om OperationMsg) LogEvent(eventLogger func(route, op, evResult string)) { pass := "ok" if !om.OK { pass = "failure" } - eventLogger(fmt.Sprintf("%v/%v/%v", om.Route, om.Name, pass)) + eventLogger(om.Route, om.Name, pass) } -// queue of operations +// OperationQueue defines an object for a queue of operations type OperationQueue map[int][]Operation -func newOperationQueue() OperationQueue { - operationQueue := make(OperationQueue) - return operationQueue +// NewOperationQueue creates a new OperationQueue instance. +func NewOperationQueue() OperationQueue { + return make(OperationQueue) } -// adds all future operations into the operation queue. +// queueOperations adds all future operations into the operation queue. func queueOperations(queuedOps OperationQueue, queuedTimeOps []FutureOperation, futureOps []FutureOperation) { diff --git a/x/simulation/simulate.go b/x/simulation/simulate.go index ae0c83fb97..59c24a3959 100644 --- a/x/simulation/simulate.go +++ b/x/simulation/simulate.go @@ -45,9 +45,10 @@ func initChain( // TODO: split this monster function up func SimulateFromSeed( tb testing.TB, w io.Writer, app *baseapp.BaseApp, - appStateFn AppStateFn, seed int64, ops WeightedOperations, - invariants sdk.Invariants, + appStateFn AppStateFn, seed int64, + ops WeightedOperations, invariants sdk.Invariants, numBlocks, exportParamsHeight, blockSize int, + exportStatsPath string, exportParams, commit, lean, onOperation, allInvariants bool, blackListedAccs map[string]bool, ) (stopEarly bool, exportedParams Params, err error) { @@ -62,7 +63,7 @@ func SimulateFromSeed( timeDiff := maxTimePerBlock - minTimePerBlock accs := RandomAccounts(r, params.NumKeys) - eventStats := newEventStats() + eventStats := NewEventStats() // Second variable to keep pending validator set (delayed one block since // TM 0.24) Initially this is the same as the initial validator set @@ -109,16 +110,16 @@ func SimulateFromSeed( var pastVoteInfos [][]abci.VoteInfo request := RandomRequestBeginBlock(r, params, - validators, pastTimes, pastVoteInfos, eventStats.tally, header) + validators, pastTimes, pastVoteInfos, eventStats.Tally, header) // These are operations which have been queued by previous operations - operationQueue := newOperationQueue() + operationQueue := NewOperationQueue() timeOperationQueue := []FutureOperation{} logWriter := NewLogWriter(testingMode) blockSimulator := createBlockSimulator( - testingMode, tb, t, w, params, eventStats.tally, invariants, + testingMode, tb, t, w, params, eventStats.Tally, invariants, ops, operationQueue, timeOperationQueue, numBlocks, blockSize, logWriter, lean, onOperation, allInvariants) @@ -160,11 +161,11 @@ func SimulateFromSeed( // Run queued operations. Ignores blocksize if blocksize is too small numQueuedOpsRan := runQueuedOperations( operationQueue, int(header.Height), - tb, r, app, ctx, accs, logWriter, eventStats.tally, lean) + tb, r, app, ctx, accs, logWriter, eventStats.Tally, lean) numQueuedTimeOpsRan := runQueuedTimeOperations( timeOperationQueue, int(header.Height), header.Time, - tb, r, app, ctx, accs, logWriter, eventStats.tally, lean) + tb, r, app, ctx, accs, logWriter, eventStats.Tally, lean) if testingMode && onOperation { assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, allInvariants) @@ -202,13 +203,13 @@ func SimulateFromSeed( // Generate a random RequestBeginBlock with the current validator set // for the next block request = RandomRequestBeginBlock(r, params, validators, - pastTimes, pastVoteInfos, eventStats.tally, header) + pastTimes, pastVoteInfos, eventStats.Tally, header) // Update the validator set, which will be reflected in the application // on the next block validators = nextValidators nextValidators = updateValidators(tb, r, params, - validators, res.ValidatorUpdates, eventStats.tally) + validators, res.ValidatorUpdates, eventStats.Tally) // update the exported params if exportParams && exportParamsHeight == height { @@ -217,7 +218,13 @@ func SimulateFromSeed( } if stopEarly { - eventStats.Print(w) + if exportStatsPath != "" { + fmt.Println("Exporting simulation statistics...") + eventStats.ExportJSON(exportStatsPath) + } else { + eventStats.Print(w) + } + return true, exportedParams, err } @@ -227,7 +234,12 @@ func SimulateFromSeed( header.Height, header.Time, opCount, ) - eventStats.Print(w) + if exportStatsPath != "" { + fmt.Println("Exporting simulation statistics...") + eventStats.ExportJSON(exportStatsPath) + } else { + eventStats.Print(w) + } return false, exportedParams, nil } @@ -240,7 +252,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, // Returns a function to simulate blocks. Written like this to avoid constant // parameters being passed everytime, to minimize memory overhead. func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params, - event func(string), invariants sdk.Invariants, ops WeightedOperations, + event func(route, op, evResult string), invariants sdk.Invariants, ops WeightedOperations, operationQueue OperationQueue, timeOperationQueue []FutureOperation, totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn { @@ -276,7 +288,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr opMsg, futureOps, err := op(r2, app, ctx, accounts) opMsg.LogEvent(event) if !lean || opMsg.OK { - logWriter.AddEntry(MsgEntry(header.Height, opMsg, int64(i))) + logWriter.AddEntry(MsgEntry(header.Height, int64(i), opMsg)) } if err != nil { logWriter.PrintLogs() @@ -305,7 +317,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr // nolint: errcheck func runQueuedOperations(queueOps map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, - ctx sdk.Context, accounts []Account, logWriter LogWriter, tallyEvent func(string), lean bool) (numOpsRan int) { + ctx sdk.Context, accounts []Account, logWriter LogWriter, event func(route, op, evResult string), lean bool) (numOpsRan int) { queuedOp, ok := queueOps[height] if !ok { @@ -319,7 +331,7 @@ func runQueuedOperations(queueOps map[int][]Operation, // If a need arises for us to support queued messages to queue more messages, this can // be changed. opMsg, _, err := queuedOp[i](r, app, ctx, accounts) - opMsg.LogEvent(tallyEvent) + opMsg.LogEvent(event) if !lean || opMsg.OK { logWriter.AddEntry((QueuedMsgEntry(int64(height), opMsg))) } @@ -335,7 +347,7 @@ func runQueuedOperations(queueOps map[int][]Operation, func runQueuedTimeOperations(queueOps []FutureOperation, height int, currentTime time.Time, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, - logWriter LogWriter, tallyEvent func(string), lean bool) (numOpsRan int) { + logWriter LogWriter, event func(route, op, evResult string), lean bool) (numOpsRan int) { numOpsRan = 0 for len(queueOps) > 0 && currentTime.After(queueOps[0].BlockTime) { @@ -344,7 +356,7 @@ func runQueuedTimeOperations(queueOps []FutureOperation, // If a need arises for us to support queued messages to queue more messages, this can // be changed. opMsg, _, err := queueOps[0].Op(r, app, ctx, accounts) - opMsg.LogEvent(tallyEvent) + opMsg.LogEvent(event) if !lean || opMsg.OK { logWriter.AddEntry(QueuedMsgEntry(int64(height), opMsg)) } diff --git a/x/slashing/simulation/msgs.go b/x/slashing/simulation/msgs.go index 9aaa8382d2..01acf4a819 100644 --- a/x/slashing/simulation/msgs.go +++ b/x/slashing/simulation/msgs.go @@ -10,7 +10,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/slashing" ) -// SimulateMsgUnjail +// SimulateMsgUnjail generates a MsgUnjail with random values func SimulateMsgUnjail(k slashing.Keeper) simulation.Operation { return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account) (opMsg simulation.OperationMsg, fOps []simulation.FutureOperation, err error) { @@ -19,7 +19,7 @@ func SimulateMsgUnjail(k slashing.Keeper) simulation.Operation { address := sdk.ValAddress(acc.Address) msg := slashing.NewMsgUnjail(address) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(slashing.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() ok := slashing.NewHandler(k)(ctx, msg).IsOK() diff --git a/x/staking/simulation/msgs.go b/x/staking/simulation/msgs.go index 2cd3ce0db6..c16d4075cb 100644 --- a/x/staking/simulation/msgs.go +++ b/x/staking/simulation/msgs.go @@ -12,7 +12,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/staking/keeper" ) -// SimulateMsgCreateValidator +// SimulateMsgCreateValidator generates a MsgCreateValidator with random values func SimulateMsgCreateValidator(m auth.AccountKeeper, k staking.Keeper) simulation.Operation { handler := staking.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -39,7 +39,7 @@ func SimulateMsgCreateValidator(m auth.AccountKeeper, k staking.Keeper) simulati } if amount.Equal(sdk.ZeroInt()) { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } selfDelegation := sdk.NewCoin(denom, amount) @@ -47,7 +47,7 @@ func SimulateMsgCreateValidator(m auth.AccountKeeper, k staking.Keeper) simulati selfDelegation, description, commission, sdk.OneInt()) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(staking.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() @@ -61,7 +61,7 @@ func SimulateMsgCreateValidator(m auth.AccountKeeper, k staking.Keeper) simulati } } -// SimulateMsgEditValidator +// SimulateMsgEditValidator generates a MsgEditValidator with random values func SimulateMsgEditValidator(k staking.Keeper) simulation.Operation { handler := staking.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -75,7 +75,7 @@ func SimulateMsgEditValidator(k staking.Keeper) simulation.Operation { } if len(k.GetAllValidators(ctx)) == 0 { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } val := keeper.RandomValidator(r, k, ctx) address := val.GetOperator() @@ -84,7 +84,7 @@ func SimulateMsgEditValidator(k staking.Keeper) simulation.Operation { msg := staking.NewMsgEditValidator(address, description, &newCommissionRate, nil) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(staking.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() ok := handler(ctx, msg).IsOK() @@ -96,7 +96,7 @@ func SimulateMsgEditValidator(k staking.Keeper) simulation.Operation { } } -// SimulateMsgDelegate +// SimulateMsgDelegate generates a MsgDelegate with random values func SimulateMsgDelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Operation { handler := staking.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -104,7 +104,7 @@ func SimulateMsgDelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Oper denom := k.GetParams(ctx).BondDenom if len(k.GetAllValidators(ctx)) == 0 { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } val := keeper.RandomValidator(r, k, ctx) validatorAddress := val.GetOperator() @@ -115,14 +115,14 @@ func SimulateMsgDelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Oper amount = simulation.RandomAmount(r, amount) } if amount.Equal(sdk.ZeroInt()) { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } msg := staking.NewMsgDelegate( delegatorAddress, validatorAddress, sdk.NewCoin(denom, amount)) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(staking.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext() ok := handler(ctx, msg).IsOK() @@ -134,7 +134,7 @@ func SimulateMsgDelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Oper } } -// SimulateMsgUndelegate +// SimulateMsgUndelegate generates a MsgUndelegate with random values func SimulateMsgUndelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Operation { handler := staking.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -144,26 +144,26 @@ func SimulateMsgUndelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Op delegatorAddress := delegatorAcc.Address delegations := k.GetAllDelegatorDelegations(ctx, delegatorAddress) if len(delegations) == 0 { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } delegation := delegations[r.Intn(len(delegations))] validator, found := k.GetValidator(ctx, delegation.GetValidatorAddr()) if !found { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } totalBond := validator.TokensFromShares(delegation.GetShares()).TruncateInt() unbondAmt := simulation.RandomAmount(r, totalBond) if unbondAmt.Equal(sdk.ZeroInt()) { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } msg := staking.NewMsgUndelegate( delegatorAddress, delegation.ValidatorAddress, sdk.NewCoin(k.GetParams(ctx).BondDenom, unbondAmt), ) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s, got error %v", + return simulation.NoOpMsg(staking.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s, got error %v", msg.GetSignBytes(), msg.ValidateBasic()) } @@ -178,7 +178,7 @@ func SimulateMsgUndelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Op } } -// SimulateMsgBeginRedelegate +// SimulateMsgBeginRedelegate generates a MsgBeginRedelegate with random values func SimulateMsgBeginRedelegate(m auth.AccountKeeper, k staking.Keeper) simulation.Operation { handler := staking.NewHandler(k) return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, @@ -186,7 +186,7 @@ func SimulateMsgBeginRedelegate(m auth.AccountKeeper, k staking.Keeper) simulati denom := k.GetParams(ctx).BondDenom if len(k.GetAllValidators(ctx)) == 0 { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } srcVal := keeper.RandomValidator(r, k, ctx) srcValidatorAddress := srcVal.GetOperator() @@ -200,14 +200,14 @@ func SimulateMsgBeginRedelegate(m auth.AccountKeeper, k staking.Keeper) simulati amount = simulation.RandomAmount(r, amount) } if amount.Equal(sdk.ZeroInt()) { - return simulation.NoOpMsg(), nil, nil + return simulation.NoOpMsg(staking.ModuleName), nil, nil } msg := staking.NewMsgBeginRedelegate( delegatorAddress, srcValidatorAddress, destValidatorAddress, sdk.NewCoin(denom, amount), ) if msg.ValidateBasic() != nil { - return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + return simulation.NoOpMsg(staking.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) } ctx, write := ctx.CacheContext()