From 46bbada4eea507426a9909491279ceb5564522a5 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Wed, 29 Aug 2018 23:02:15 -0700 Subject: [PATCH 01/15] simulation: Add benchmarking --- PENDING.md | 1 + cmd/gaia/app/sim_test.go | 21 +++ x/bank/simulation/msgs.go | 18 +-- x/gov/simulation/msgs.go | 50 +++--- x/mock/simulation/random_simulate_blocks.go | 166 +++++++++++++++----- x/mock/simulation/types.go | 2 +- x/slashing/simulation/msgs.go | 8 +- x/stake/simulation/msgs.go | 44 ++++-- 8 files changed, 220 insertions(+), 90 deletions(-) diff --git a/PENDING.md b/PENDING.md index f45541ccdc..2e862ead6a 100644 --- a/PENDING.md +++ b/PENDING.md @@ -45,6 +45,7 @@ FEATURES * SDK * [querier] added custom querier functionality, so ABCI query requests can be handled by keepers * [simulation] \#1924 allow operations to specify future operations + * [simulation] Add benchmarking capabilities * Tendermint diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 0ca936bdbf..2bad6aba23 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -112,6 +112,27 @@ func invariants(app *GaiaApp) []simulation.Invariant { } } +// Profile with: +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$ -cpuprofile cpu.out +func BenchmarkFullGaiaSimulation(b *testing.B) { + // Setup Gaia application + var logger log.Logger + logger = log.NewNopLogger() + db := dbm.NewMemDB() + app := NewGaiaApp(logger, db, nil) + + // Run randomized simulation + // TODO parameterize numbers, save for a later PR + simulation.BenchmarkSimulationFromSeed( + b, app.BaseApp, appStateFn, seed, + testAndRunTxs(app), + []simulation.RandSetup{}, + 10, + 100, + false, + ) +} + func TestFullGaiaSimulation(t *testing.T) { if !enabled { t.Skip("Skipping Gaia simulation") diff --git a/x/bank/simulation/msgs.go b/x/bank/simulation/msgs.go index e3f3ab77b8..c53916a613 100644 --- a/x/bank/simulation/msgs.go +++ b/x/bank/simulation/msgs.go @@ -21,7 +21,7 @@ import ( // SimulateSingleInputMsgSend tests and runs a single msg send, with one input and one output, where both // accounts already exist. func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { fromKey := simulation.RandomKey(r, keys) fromAddr := sdk.AccAddress(fromKey.PubKey().Address()) toKey := simulation.RandomKey(r, keys) @@ -58,7 +58,7 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation Inputs: []bank.Input{bank.NewInput(fromAddr, coins)}, Outputs: []bank.Output{bank.NewOutput(toAddr, coins)}, } - sendAndVerifyMsgSend(t, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey}) + sendAndVerifyMsgSend(tb, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey}) event("bank/sendAndVerifyMsgSend/ok") return action, nil, nil @@ -66,7 +66,7 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation } // Sends and verifies the transition of a msg send. This fails if there are repeated inputs or outputs -func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) { +func sendAndVerifyMsgSend(tb testing.TB, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) { initialInputAddrCoins := make([]sdk.Coins, len(msg.Inputs)) initialOutputAddrCoins := make([]sdk.Coins, len(msg.Outputs)) AccountNumbers := make([]int64, len(msg.Inputs)) @@ -91,12 +91,12 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun // TODO: Do this in a more 'canonical' way fmt.Println(res) fmt.Println(log) - t.FailNow() + tb.FailNow() } for i := 0; i < len(msg.Inputs); i++ { terminalInputCoins := mapper.GetAccount(ctx, msg.Inputs[i].Address).GetCoins() - require.Equal(t, + require.Equal(tb, initialInputAddrCoins[i].Minus(msg.Inputs[i].Coins), terminalInputCoins, fmt.Sprintf("Input #%d had an incorrect amount of coins\n%s", i, log), @@ -104,11 +104,9 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun } for i := 0; i < len(msg.Outputs); i++ { terminalOutputCoins := mapper.GetAccount(ctx, msg.Outputs[i].Address).GetCoins() - require.Equal(t, - initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins), - terminalOutputCoins, - fmt.Sprintf("Output #%d had an incorrect amount of coins\n%s", i, log), - ) + if !terminalOutputCoins.IsEqual(initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins)) { + tb.Fatalf("Output #%d had an incorrect amount of coins\n%s", i, log) + } } } diff --git a/x/gov/simulation/msgs.go b/x/gov/simulation/msgs.go index 0b1530138f..eca8accae0 100644 --- a/x/gov/simulation/msgs.go +++ b/x/gov/simulation/msgs.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto" "github.com/cosmos/cosmos-sdk/baseapp" @@ -23,24 +21,15 @@ const ( // SimulateMsgSubmitProposal simulates a msg Submit Proposal // Note: Currently doesn't ensure that the proposal txt is in JSON form func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { - key := simulation.RandomKey(r, keys) - addr := sdk.AccAddress(key.PubKey().Address()) - deposit := randomDeposit(r) - msg := gov.NewMsgSubmitProposal( - simulation.RandStringOfLength(r, 5), - simulation.RandStringOfLength(r, 5), - gov.ProposalTypeText, - addr, - deposit, - ) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + handler := gov.NewHandler(k) + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) { + msg := simulationCreateMsgSubmitProposal(tb, r, keys, log) ctx, write := ctx.CacheContext() - result := gov.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { // Update pool to keep invariants pool := sk.GetPool(ctx) - pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(deposit.AmountOf(denom))) + pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(msg.InitialDeposit.AmountOf(denom))) sk.SetPool(ctx, pool) write() } @@ -50,9 +39,26 @@ func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operati } } +func simulationCreateMsgSubmitProposal(tb testing.TB, r *rand.Rand, keys []crypto.PrivKey, log string) gov.MsgSubmitProposal { + key := simulation.RandomKey(r, keys) + addr := sdk.AccAddress(key.PubKey().Address()) + deposit := randomDeposit(r) + msg := gov.NewMsgSubmitProposal( + simulation.RandStringOfLength(r, 5), + simulation.RandStringOfLength(r, 5), + gov.ProposalTypeText, + addr, + deposit, + ) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } + return msg +} + // SimulateMsgDeposit func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { key := simulation.RandomKey(r, keys) addr := sdk.AccAddress(key.PubKey().Address()) proposalID, ok := randomProposalID(r, k, ctx) @@ -61,7 +67,9 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation { } deposit := randomDeposit(r) msg := gov.NewMsgDeposit(addr, proposalID, deposit) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := gov.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -79,7 +87,7 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation { // SimulateMsgVote func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { key := simulation.RandomKey(r, keys) addr := sdk.AccAddress(key.PubKey().Address()) proposalID, ok := randomProposalID(r, k, ctx) @@ -88,7 +96,9 @@ func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation { } option := randomVotingOption(r) msg := gov.NewMsgVote(addr, proposalID, option) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := gov.NewHandler(k)(ctx, msg) if result.IsOK() { diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index 995013ef8f..aa4df613dd 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "math/rand" + "os" "sort" "testing" "time" @@ -28,6 +29,26 @@ func Simulate( SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit) } +func initChain(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress, setups []RandSetup, app *baseapp.BaseApp, + appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage) (validators map[string]mockValidator) { + res := app.InitChain(abci.RequestInitChain{AppStateBytes: appStateFn(r, keys, accs)}) + validators = make(map[string]mockValidator) + for _, validator := range res.Validators { + validators[string(validator.Address)] = mockValidator{validator, GetMemberOfInitialState(r, initialLivenessWeightings)} + } + + for i := 0; i < len(setups); i++ { + setups[i](r, keys) + } + + return +} + +func randTimestamp(r *rand.Rand) time.Time { + unixTime := r.Int63n(int64(math.Pow(2, 40))) + return time.Unix(unixTime, 0) +} + // SimulateFromSeed tests an application by running the provided // operations, testing the provided invariants, but using the provided seed. func SimulateFromSeed( @@ -36,11 +57,7 @@ func SimulateFromSeed( ) { log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed)) r := rand.New(rand.NewSource(seed)) - - unixTime := r.Int63n(int64(math.Pow(2, 40))) - - // Set the timestamp for simulation - timestamp := time.Unix(unixTime, 0) + timestamp := randTimestamp(r) log = fmt.Sprintf("%s\nStarting the simulation from time %v, unixtime %v", log, timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) fmt.Printf("%s\n", log) timeDiff := maxTimePerBlock - minTimePerBlock @@ -54,30 +71,18 @@ func SimulateFromSeed( events[what]++ } - res := app.InitChain(abci.RequestInitChain{AppStateBytes: appStateFn(r, keys, accs)}) - validators := make(map[string]mockValidator) - for _, validator := range res.Validators { - validators[string(validator.Address)] = mockValidator{validator, GetMemberOfInitialState(r, initialLivenessWeightings)} - } - - for i := 0; i < len(setups); i++ { - setups[i](r, keys) - } + validators := initChain(r, keys, accs, setups, app, appStateFn) header := abci.Header{Height: 0, Time: timestamp} opCount := 0 request := abci.RequestBeginBlock{Header: header} - var pastTimes []time.Time + var lastHeaderTime time.Time // These are operations which have been queued by previous operations operationQueue := make(map[int][]Operation) for i := 0; i < numBlocks; i++ { - - // Log the header time for future lookup - pastTimes = append(pastTimes, header.Time) - // Run the BeginBlock handler app.BeginBlock(request) @@ -88,16 +93,8 @@ func SimulateFromSeed( ctx := app.NewContext(false, header) - var thisBlockSize int - load := r.Float64() - switch { - case load < 0.33: - thisBlockSize = 0 - case load < 0.66: - thisBlockSize = r.Intn(blockSize * 2) - default: - thisBlockSize = r.Intn(blockSize * 4) - } + thisBlockSize := getBlockSize(r, blockSize) + // Run queued operations. Ignores blocksize if blocksize is too small log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), t, r, app, ctx, keys, log, event) opCount += numQueuedOpsRan @@ -119,6 +116,7 @@ func SimulateFromSeed( res := app.EndBlock(abci.RequestEndBlock{}) header.Height++ + lastHeaderTime = header.Time header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) log += "\nEndBlock" @@ -131,7 +129,7 @@ func SimulateFromSeed( } // Generate a random RequestBeginBlock with the current validator set for the next block - request = RandomRequestBeginBlock(t, r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, event, header, log) + request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, lastHeaderTime, event, header, log) // Update the validator set validators = updateValidators(t, r, validators, res.ValidatorUpdates, event) @@ -141,6 +139,84 @@ func SimulateFromSeed( DisplayEvents(events) } +func getBlockSize(r *rand.Rand, blockSize int) int { + load := r.Float64() + switch { + case load < 0.33: + return 0 + case load < 0.66: + return r.Intn(blockSize * 2) + default: + return r.Intn(blockSize * 4) + } +} + +// Simulate from seed, benchmarks +func BenchmarkSimulationFromSeed(b *testing.B, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, + seed int64, ops []Operation, setups []RandSetup, numBlocks int, blockSize int, commit bool) { + r := rand.New(rand.NewSource(seed)) + timestamp := randTimestamp(r) + timeDiff := maxTimePerBlock - minTimePerBlock + keys, accs := mock.GeneratePrivKeyAddressPairsFromRand(r, numKeys) + + // Setup event stats + events := make(map[string]uint) + event := func(what string) { + events[what]++ + } + + validators := initChain(r, keys, accs, setups, app, appStateFn) + + header := abci.Header{Height: 0, Time: timestamp} + opCount := 0 + + request := abci.RequestBeginBlock{Header: header} + + var lastHeaderTime time.Time + // These are operations which have been queued by previous operations + operationQueue := make(map[int][]Operation) + b.ResetTimer() + + for i := 0; i < numBlocks; i++ { + + // Run the BeginBlock handler + app.BeginBlock(request) + + ctx := app.NewContext(false, header) + + thisBlockSize := getBlockSize(r, blockSize) + + // Run queued operations. Ignores blocksize if blocksize is too small + log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), b, r, app, ctx, keys, "", event) + opCount += numQueuedOpsRan + thisBlockSize -= numQueuedOpsRan + for j := 0; j < thisBlockSize; j++ { + _, futureOps, err := ops[r.Intn(len(ops))](b, r, app, ctx, keys, "", event) + queueOperations(operationQueue, futureOps) + if err != nil { + b.Fatalf("error on operation %d, %v", opCount, err) + } + opCount++ + } + + res := app.EndBlock(abci.RequestEndBlock{}) + header.Height++ + lastHeaderTime = header.Time + header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) + if commit { + app.Commit() + } + + // Generate a random RequestBeginBlock with the current validator set for the next block + request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, lastHeaderTime, event, header, log) + + // Update the validator set + validators = updateValidators(b, r, validators, res.ValidatorUpdates, event) + } + DisplayEvents(events) + fmt.Printf("Benchmark simulation ran %d operations\n", opCount) +} + // adds all future operations into the operation queue. func queueOperations(queuedOperations map[int][]Operation, futureOperations []FutureOperation) { if futureOperations == nil { @@ -155,7 +231,7 @@ func queueOperations(queuedOperations map[int][]Operation, futureOperations []Fu } } -func runQueuedOperations(queueOperations map[int][]Operation, height int, t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, +func runQueuedOperations(queueOperations map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, event func(string)) (updatedLog string, numOpsRan int) { updatedLog = log if queuedOps, ok := queueOperations[height]; ok { @@ -164,9 +240,12 @@ func runQueuedOperations(queueOperations map[int][]Operation, height int, t *tes // For now, queued operations cannot queue more operations. // If a need arises for us to support queued messages to queue more messages, this can // be changed. - logUpdate, _, err := queuedOps[i](t, r, app, ctx, privKeys, updatedLog, event) - updatedLog += "\n" + logUpdate - require.Nil(t, err, updatedLog) + logUpdate, _, err := queuedOps[i](tb, r, app, ctx, privKeys, updatedLog, event) + updatedLog = fmt.Sprintf("%s\n%s", updatedLog, logUpdate) + if err != nil { + fmt.Fprint(os.Stderr, updatedLog) + tb.FailNow() + } } delete(queueOperations, height) return updatedLog, numOps @@ -186,8 +265,8 @@ func getKeys(validators map[string]mockValidator) []string { } // RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction -func RandomRequestBeginBlock(t *testing.T, r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64, - pastTimes []time.Time, event func(string), header abci.Header, log string) abci.RequestBeginBlock { +func RandomRequestBeginBlock(r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64, + lastHeaderTime time.Time, event func(string), header abci.Header, log string) abci.RequestBeginBlock { if len(validators) == 0 { return abci.RequestBeginBlock{Header: header} } @@ -219,13 +298,14 @@ func RandomRequestBeginBlock(t *testing.T, r *rand.Rand, validators map[string]m } i++ } + // TODO: Determine capacity before allocation evidence := make([]abci.Evidence, 0) for r.Float64() < evidenceFraction { height := header.Height time := header.Time if r.Float64() < pastEvidenceFraction { height = int64(r.Intn(int(header.Height))) - time = pastTimes[height] + time = lastHeaderTime } validator := signingValidators[r.Intn(len(signingValidators))].Validator var currentTotalVotingPower int64 @@ -258,11 +338,19 @@ func AssertAllInvariants(t *testing.T, app *baseapp.BaseApp, tests []Invariant, } // updateValidators mimicks Tendermint's update logic -func updateValidators(t *testing.T, r *rand.Rand, current map[string]mockValidator, updates []abci.Validator, event func(string)) map[string]mockValidator { +func updateValidators(tb testing.TB, r *rand.Rand, current map[string]mockValidator, updates []abci.Validator, event func(string)) map[string]mockValidator { for _, update := range updates { switch { case update.Power == 0: - require.NotNil(t, current[string(update.PubKey.Data)], "tried to delete a nonexistent validator") + // // TEMPORARY DEBUG CODE TO PROVE THAT THE OLD METHOD WAS BROKEN + // // (i.e. didn't catch in the event of problem) + // if val, ok := tb.(*testing.T); ok { + // require.NotNil(val, current[string(update.PubKey.Data)]) + // } + // // CORRECT CHECK + // if _, ok := current[string(update.PubKey.Data)]; !ok { + // tb.Fatalf("tried to delete a nonexistent validator") + // } event("endblock/validatorupdates/kicked") delete(current, string(update.PubKey.Data)) default: diff --git a/x/mock/simulation/types.go b/x/mock/simulation/types.go index 2516e07ae5..2f91a4f263 100644 --- a/x/mock/simulation/types.go +++ b/x/mock/simulation/types.go @@ -23,7 +23,7 @@ type ( // Operations can optionally provide a list of "FutureOperations" to run later // These will be ran at the beginning of the corresponding block. Operation func( - t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, event func(string), ) (action string, futureOperations []FutureOperation, err sdk.Error) diff --git a/x/slashing/simulation/msgs.go b/x/slashing/simulation/msgs.go index e4900889e3..7b5490e423 100644 --- a/x/slashing/simulation/msgs.go +++ b/x/slashing/simulation/msgs.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto" "github.com/cosmos/cosmos-sdk/baseapp" @@ -17,11 +15,13 @@ import ( // SimulateMsgUnjail func SimulateMsgUnjail(k slashing.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { key := simulation.RandomKey(r, keys) address := sdk.AccAddress(key.PubKey().Address()) msg := slashing.NewMsgUnjail(address) - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := slashing.NewHandler(k)(ctx, msg) if result.IsOK() { diff --git a/x/stake/simulation/msgs.go b/x/stake/simulation/msgs.go index 4280e70c13..bc1063494d 100644 --- a/x/stake/simulation/msgs.go +++ b/x/stake/simulation/msgs.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/stretchr/testify/require" - "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" @@ -19,7 +17,7 @@ import ( // SimulateMsgCreateValidator func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom description := stake.Description{ Moniker: simulation.RandStringOfLength(r, 10), @@ -41,7 +39,9 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation PubKey: pubkey, Delegation: sdk.NewCoin(denom, amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -56,7 +56,7 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation // SimulateMsgEditValidator func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { description := stake.Description{ Moniker: simulation.RandStringOfLength(r, 10), Identity: simulation.RandStringOfLength(r, 10), @@ -70,7 +70,9 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { Description: description, ValidatorAddr: address, } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -84,7 +86,7 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { // SimulateMsgDelegate func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom validatorKey := simulation.RandomKey(r, keys) validatorAddress := sdk.AccAddress(validatorKey.PubKey().Address()) @@ -102,7 +104,9 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat ValidatorAddr: validatorAddress, Delegation: sdk.NewCoin(denom, amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -116,7 +120,7 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat // SimulateMsgBeginUnbonding func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom validatorKey := simulation.RandomKey(r, keys) validatorAddress := sdk.AccAddress(validatorKey.PubKey().Address()) @@ -134,7 +138,9 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation. ValidatorAddr: validatorAddress, SharesAmount: sdk.NewDecFromInt(amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -148,7 +154,7 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation. // SimulateMsgCompleteUnbonding func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { validatorKey := simulation.RandomKey(r, keys) validatorAddress := sdk.AccAddress(validatorKey.PubKey().Address()) delegatorKey := simulation.RandomKey(r, keys) @@ -157,7 +163,9 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { DelegatorAddr: delegatorAddress, ValidatorAddr: validatorAddress, } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -171,7 +179,7 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { // SimulateMsgBeginRedelegate func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom sourceValidatorKey := simulation.RandomKey(r, keys) sourceValidatorAddress := sdk.AccAddress(sourceValidatorKey.PubKey().Address()) @@ -193,7 +201,9 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation ValidatorDstAddr: destValidatorAddress, SharesAmount: sdk.NewDecFromInt(amount), } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { @@ -207,7 +217,7 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation // SimulateMsgCompleteRedelegate func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation { - return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { + return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { validatorSrcKey := simulation.RandomKey(r, keys) validatorSrcAddress := sdk.AccAddress(validatorSrcKey.PubKey().Address()) validatorDstKey := simulation.RandomKey(r, keys) @@ -219,7 +229,9 @@ func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation { ValidatorSrcAddr: validatorSrcAddress, ValidatorDstAddr: validatorDstAddress, } - require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes()) + if msg.ValidateBasic() != nil { + tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) + } ctx, write := ctx.CacheContext() result := stake.NewHandler(k)(ctx, msg) if result.IsOK() { From 31ca2e058b9d666ac3ea60ad5b399f3f9da28da2 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Wed, 29 Aug 2018 23:11:14 -0700 Subject: [PATCH 02/15] Update PENDING --- PENDING.md | 2 +- x/mock/simulation/random_simulate_blocks.go | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/PENDING.md b/PENDING.md index 2e862ead6a..53c7a6b343 100644 --- a/PENDING.md +++ b/PENDING.md @@ -45,7 +45,7 @@ FEATURES * SDK * [querier] added custom querier functionality, so ABCI query requests can be handled by keepers * [simulation] \#1924 allow operations to specify future operations - * [simulation] Add benchmarking capabilities + * [simulation] \#1924 Add benchmarking capabilities * Tendermint diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index aa4df613dd..ea0078853e 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -85,14 +85,12 @@ func SimulateFromSeed( for i := 0; i < numBlocks; i++ { // Run the BeginBlock handler app.BeginBlock(request) - log += "\nBeginBlock" // Make sure invariants hold at beginning of block AssertAllInvariants(t, app, invariants, log) ctx := app.NewContext(false, header) - thisBlockSize := getBlockSize(r, blockSize) // Run queued operations. Ignores blocksize if blocksize is too small @@ -102,9 +100,10 @@ func SimulateFromSeed( for j := 0; j < thisBlockSize; j++ { logUpdate, futureOps, err := ops[r.Intn(len(ops))](t, r, app, ctx, keys, log, event) log += "\n" + logUpdate + require.Nil(t, err, log) + queueOperations(operationQueue, futureOps) - require.Nil(t, err, log) if onOperation { AssertAllInvariants(t, app, invariants, log) } @@ -118,12 +117,10 @@ func SimulateFromSeed( header.Height++ lastHeaderTime = header.Time header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) - log += "\nEndBlock" // Make sure invariants hold at end of block AssertAllInvariants(t, app, invariants, log) - if commit { app.Commit() } @@ -178,12 +175,9 @@ func BenchmarkSimulationFromSeed(b *testing.B, app *baseapp.BaseApp, appStateFn b.ResetTimer() for i := 0; i < numBlocks; i++ { - // Run the BeginBlock handler app.BeginBlock(request) - ctx := app.NewContext(false, header) - thisBlockSize := getBlockSize(r, blockSize) // Run queued operations. Ignores blocksize if blocksize is too small From 03d2f7331b418d68f0b3e4f5448754e4c27771f5 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Thu, 30 Aug 2018 00:13:31 -0700 Subject: [PATCH 03/15] Remove code duplication --- cmd/gaia/app/sim_test.go | 3 +- x/mock/simulation/random_simulate_blocks.go | 125 +++++++------------- x/stake/simulation/msgs.go | 21 ++-- 3 files changed, 61 insertions(+), 88 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 2bad6aba23..1fd07b3302 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -123,10 +123,11 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { // Run randomized simulation // TODO parameterize numbers, save for a later PR - simulation.BenchmarkSimulationFromSeed( + simulation.SimulateFromSeed( b, app.BaseApp, appStateFn, seed, testAndRunTxs(app), []simulation.RandSetup{}, + invariants(app), // these shouldn't get ran 10, 100, false, diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index ea0078853e..6376d53dcb 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -17,7 +17,6 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/mock" - "github.com/stretchr/testify/require" ) // Simulate tests application by sending random messages. @@ -52,13 +51,22 @@ func randTimestamp(r *rand.Rand) time.Time { // SimulateFromSeed tests an application by running the provided // operations, testing the provided invariants, but using the provided seed. func SimulateFromSeed( - t *testing.T, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup, + tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup, invariants []Invariant, numBlocks int, blockSize int, commit bool, ) { + var t *testing.T + var b *testing.B + testingmode := false + if _t, ok := tb.(*testing.T); ok { + t = _t + testingmode = true + } else { + b = tb.(*testing.B) + } log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed)) r := rand.New(rand.NewSource(seed)) timestamp := randTimestamp(r) - log = fmt.Sprintf("%s\nStarting the simulation from time %v, unixtime %v", log, timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) + log = updateLog(testingmode, log, "Starting the simulation from time %v, unixtime %v", timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) fmt.Printf("%s\n", log) timeDiff := maxTimePerBlock - minTimePerBlock @@ -67,7 +75,7 @@ func SimulateFromSeed( // Setup event stats events := make(map[string]uint) event := func(what string) { - log += "\nevent - " + what + log = updateLog(testingmode, log, "event - "+what) events[what]++ } @@ -82,32 +90,40 @@ func SimulateFromSeed( // These are operations which have been queued by previous operations operationQueue := make(map[int][]Operation) + if !testingmode { + b.ResetTimer() + } + for i := 0; i < numBlocks; i++ { // Run the BeginBlock handler app.BeginBlock(request) - log += "\nBeginBlock" + log = updateLog(testingmode, log, "BeginBlock") - // Make sure invariants hold at beginning of block - AssertAllInvariants(t, app, invariants, log) + if testingmode { + // Make sure invariants hold at beginning of block + AssertAllInvariants(t, app, invariants, log) + } ctx := app.NewContext(false, header) thisBlockSize := getBlockSize(r, blockSize) // Run queued operations. Ignores blocksize if blocksize is too small - log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), t, r, app, ctx, keys, log, event) + log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, keys, log, event) opCount += numQueuedOpsRan thisBlockSize -= numQueuedOpsRan for j := 0; j < thisBlockSize; j++ { - logUpdate, futureOps, err := ops[r.Intn(len(ops))](t, r, app, ctx, keys, log, event) - log += "\n" + logUpdate - require.Nil(t, err, log) + logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event) + log = updateLog(testingmode, log, logUpdate) + if err != nil { + tb.Fatalf("error on operation %d, %v", opCount, err) + } queueOperations(operationQueue, futureOps) - if onOperation { + if onOperation && testingmode { AssertAllInvariants(t, app, invariants, log) } - if opCount%200 == 0 { + if testingmode && opCount%200 == 0 { fmt.Printf("\rSimulating... block %d/%d, operation %d.", header.Height, numBlocks, opCount) } opCount++ @@ -119,8 +135,10 @@ func SimulateFromSeed( header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) log += "\nEndBlock" - // Make sure invariants hold at end of block - AssertAllInvariants(t, app, invariants, log) + if testingmode { + // Make sure invariants hold at end of block + AssertAllInvariants(t, app, invariants, log) + } if commit { app.Commit() } @@ -129,13 +147,23 @@ func SimulateFromSeed( request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, lastHeaderTime, event, header, log) // Update the validator set - validators = updateValidators(t, r, validators, res.ValidatorUpdates, event) + validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event) } - fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds): %v\n", header.Height, header.Time) + if testingmode { + fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds): %v\n", header.Height, header.Time) + } DisplayEvents(events) } +func updateLog(testingmode bool, log string, update string, args ...interface{}) (updatedLog string) { + if testingmode == true { + update = fmt.Sprintf(update, args) + return fmt.Sprintf("%s\n%s", log, update) + } + return "" +} + func getBlockSize(r *rand.Rand, blockSize int) int { load := r.Float64() switch { @@ -148,69 +176,6 @@ func getBlockSize(r *rand.Rand, blockSize int) int { } } -// Simulate from seed, benchmarks -func BenchmarkSimulationFromSeed(b *testing.B, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, - seed int64, ops []Operation, setups []RandSetup, numBlocks int, blockSize int, commit bool) { - r := rand.New(rand.NewSource(seed)) - timestamp := randTimestamp(r) - timeDiff := maxTimePerBlock - minTimePerBlock - keys, accs := mock.GeneratePrivKeyAddressPairsFromRand(r, numKeys) - - // Setup event stats - events := make(map[string]uint) - event := func(what string) { - events[what]++ - } - - validators := initChain(r, keys, accs, setups, app, appStateFn) - - header := abci.Header{Height: 0, Time: timestamp} - opCount := 0 - - request := abci.RequestBeginBlock{Header: header} - - var lastHeaderTime time.Time - // These are operations which have been queued by previous operations - operationQueue := make(map[int][]Operation) - b.ResetTimer() - - for i := 0; i < numBlocks; i++ { - // Run the BeginBlock handler - app.BeginBlock(request) - ctx := app.NewContext(false, header) - thisBlockSize := getBlockSize(r, blockSize) - - // Run queued operations. Ignores blocksize if blocksize is too small - log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), b, r, app, ctx, keys, "", event) - opCount += numQueuedOpsRan - thisBlockSize -= numQueuedOpsRan - for j := 0; j < thisBlockSize; j++ { - _, futureOps, err := ops[r.Intn(len(ops))](b, r, app, ctx, keys, "", event) - queueOperations(operationQueue, futureOps) - if err != nil { - b.Fatalf("error on operation %d, %v", opCount, err) - } - opCount++ - } - - res := app.EndBlock(abci.RequestEndBlock{}) - header.Height++ - lastHeaderTime = header.Time - header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) - if commit { - app.Commit() - } - - // Generate a random RequestBeginBlock with the current validator set for the next block - request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, lastHeaderTime, event, header, log) - - // Update the validator set - validators = updateValidators(b, r, validators, res.ValidatorUpdates, event) - } - DisplayEvents(events) - fmt.Printf("Benchmark simulation ran %d operations\n", opCount) -} - // adds all future operations into the operation queue. func queueOperations(queuedOperations map[int][]Operation, futureOperations []FutureOperation) { if futureOperations == nil { diff --git a/x/stake/simulation/msgs.go b/x/stake/simulation/msgs.go index bc1063494d..033487254f 100644 --- a/x/stake/simulation/msgs.go +++ b/x/stake/simulation/msgs.go @@ -17,6 +17,7 @@ import ( // SimulateMsgCreateValidator func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom description := stake.Description{ @@ -43,7 +44,7 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -56,6 +57,7 @@ func SimulateMsgCreateValidator(m auth.AccountMapper, k stake.Keeper) simulation // SimulateMsgEditValidator func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { description := stake.Description{ Moniker: simulation.RandStringOfLength(r, 10), @@ -74,7 +76,7 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -86,6 +88,7 @@ func SimulateMsgEditValidator(k stake.Keeper) simulation.Operation { // SimulateMsgDelegate func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom validatorKey := simulation.RandomKey(r, keys) @@ -108,7 +111,7 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -120,6 +123,7 @@ func SimulateMsgDelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operat // SimulateMsgBeginUnbonding func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom validatorKey := simulation.RandomKey(r, keys) @@ -142,7 +146,7 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation. tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -154,6 +158,7 @@ func SimulateMsgBeginUnbonding(m auth.AccountMapper, k stake.Keeper) simulation. // SimulateMsgCompleteUnbonding func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { validatorKey := simulation.RandomKey(r, keys) validatorAddress := sdk.AccAddress(validatorKey.PubKey().Address()) @@ -167,7 +172,7 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -179,6 +184,7 @@ func SimulateMsgCompleteUnbonding(k stake.Keeper) simulation.Operation { // SimulateMsgBeginRedelegate func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { denom := k.GetParams(ctx).BondDenom sourceValidatorKey := simulation.RandomKey(r, keys) @@ -205,7 +211,7 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } @@ -217,6 +223,7 @@ func SimulateMsgBeginRedelegate(m auth.AccountMapper, k stake.Keeper) simulation // SimulateMsgCompleteRedelegate func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation { + handler := stake.NewHandler(k) return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) { validatorSrcKey := simulation.RandomKey(r, keys) validatorSrcAddress := sdk.AccAddress(validatorSrcKey.PubKey().Address()) @@ -233,7 +240,7 @@ func SimulateMsgCompleteRedelegate(k stake.Keeper) simulation.Operation { tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log) } ctx, write := ctx.CacheContext() - result := stake.NewHandler(k)(ctx, msg) + result := handler(ctx, msg) if result.IsOK() { write() } From b3d08bcb239198709191895ee15c86a09434ed08 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Thu, 30 Aug 2018 00:35:02 -0700 Subject: [PATCH 04/15] minor cleanup --- x/mock/simulation/random_simulate_blocks.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index 6376d53dcb..e609d3c5f9 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -75,7 +75,7 @@ func SimulateFromSeed( // Setup event stats events := make(map[string]uint) event := func(what string) { - log = updateLog(testingmode, log, "event - "+what) + log = updateLog(testingmode, log, "event - %s", what) events[what]++ } @@ -119,12 +119,13 @@ func SimulateFromSeed( } queueOperations(operationQueue, futureOps) - - if onOperation && testingmode { - AssertAllInvariants(t, app, invariants, log) - } - if testingmode && opCount%200 == 0 { - fmt.Printf("\rSimulating... block %d/%d, operation %d.", header.Height, numBlocks, opCount) + if testingmode { + if onOperation { + AssertAllInvariants(t, app, invariants, log) + } + if opCount%200 == 0 { + fmt.Printf("\rSimulating... block %d/%d, operation %d.", header.Height, numBlocks, opCount) + } } opCount++ } @@ -133,7 +134,7 @@ func SimulateFromSeed( header.Height++ lastHeaderTime = header.Time header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) - log += "\nEndBlock" + log = updateLog(testingmode, log, "EndBlock") if testingmode { // Make sure invariants hold at end of block @@ -152,13 +153,15 @@ func SimulateFromSeed( if testingmode { fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds): %v\n", header.Height, header.Time) + } else { + fmt.Printf("%d operations ran\n", opCount) } DisplayEvents(events) } func updateLog(testingmode bool, log string, update string, args ...interface{}) (updatedLog string) { if testingmode == true { - update = fmt.Sprintf(update, args) + update = fmt.Sprintf(update, args...) return fmt.Sprintf("%s\n%s", log, update) } return "" From d1a5808a7518f1244a1716877580c4df9c5ceb12 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Thu, 30 Aug 2018 09:32:15 -0700 Subject: [PATCH 05/15] Address Anton's comment --- x/mock/simulation/random_simulate_blocks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index e609d3c5f9..e0ab9ba586 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -115,7 +115,7 @@ func SimulateFromSeed( logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event) log = updateLog(testingmode, log, logUpdate) if err != nil { - tb.Fatalf("error on operation %d, %v", opCount, err) + tb.Fatalf("error on operation %d, %v, log %s", opCount, err, log) } queueOperations(operationQueue, futureOps) @@ -160,7 +160,7 @@ func SimulateFromSeed( } func updateLog(testingmode bool, log string, update string, args ...interface{}) (updatedLog string) { - if testingmode == true { + if testingmode { update = fmt.Sprintf(update, args...) return fmt.Sprintf("%s\n%s", log, update) } From 4be69077fe262524527a1e265d4c1032877946ff Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Thu, 30 Aug 2018 11:43:56 -0700 Subject: [PATCH 06/15] Fix cyclomatic complexity, add ability to use GoLevelDB --- cmd/gaia/app/sim_test.go | 19 ++++-- server/export_test.go | 16 ++--- server/mock/app.go | 2 +- x/mock/simulation/random_simulate_blocks.go | 76 ++++++++++++--------- 4 files changed, 66 insertions(+), 47 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 1fd07b3302..6568f124b0 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "flag" "math/rand" + "os" "testing" "github.com/stretchr/testify/require" @@ -23,11 +24,12 @@ import ( ) var ( - seed int64 - numBlocks int - blockSize int - enabled bool - verbose bool + seed int64 + numBlocks int + blockSize int + enabled bool + verbose bool + usegoleveldb bool ) func init() { @@ -36,6 +38,7 @@ func init() { flag.IntVar(&blockSize, "SimulationBlockSize", 200, "Operations per block") flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation") flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output") + flag.BoolVar(&usegoleveldb, "SimulationGoLevelDB", false, "Use GoLevelDB instead of memdb") } func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage { @@ -118,7 +121,11 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { // Setup Gaia application var logger log.Logger logger = log.NewNopLogger() - db := dbm.NewMemDB() + var db dbm.DB + db = dbm.NewMemDB() + if usegoleveldb { + db, _ = dbm.NewGoLevelDB("Simulation", os.TempDir()) + } app := NewGaiaApp(logger, db, nil) // Run randomized simulation diff --git a/server/export_test.go b/server/export_test.go index 358f72cf60..488c55bbf6 100644 --- a/server/export_test.go +++ b/server/export_test.go @@ -1,16 +1,16 @@ package server import ( - "testing" - "github.com/stretchr/testify/require" - "github.com/cosmos/cosmos-sdk/wire" - "github.com/tendermint/tendermint/libs/log" - tcmd "github.com/tendermint/tendermint/cmd/tendermint/commands" - "os" "bytes" + "github.com/cosmos/cosmos-sdk/server/mock" + "github.com/cosmos/cosmos-sdk/wire" + "github.com/stretchr/testify/require" + tcmd "github.com/tendermint/tendermint/cmd/tendermint/commands" + "github.com/tendermint/tendermint/libs/log" "io" - "github.com/cosmos/cosmos-sdk/server/mock" - ) + "os" + "testing" +) func TestEmptyState(t *testing.T) { defer setupViper(t)() diff --git a/server/mock/app.go b/server/mock/app.go index eb2dfc3cc3..3c6ad3ec27 100644 --- a/server/mock/app.go +++ b/server/mock/app.go @@ -129,7 +129,7 @@ func AppGenStateEmpty(_ *wire.Codec, _ []json.RawMessage) (appState json.RawMess // Return a validator, not much else func AppGenTx(_ *wire.Codec, pk crypto.PubKey, genTxConfig gc.GenTx) ( - appGenTx, cliPrint json.RawMessage, validator tmtypes.GenesisValidator, err error) { + appGenTx, cliPrint json.RawMessage, validator tmtypes.GenesisValidator, err error) { validator = tmtypes.GenesisValidator{ PubKey: pk, diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index e0ab9ba586..42a616331d 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -54,15 +54,7 @@ func SimulateFromSeed( tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup, invariants []Invariant, numBlocks int, blockSize int, commit bool, ) { - var t *testing.T - var b *testing.B - testingmode := false - if _t, ok := tb.(*testing.T); ok { - t = _t - testingmode = true - } else { - b = tb.(*testing.B) - } + testingmode, t, b := getTestingMode(tb) log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed)) r := rand.New(rand.NewSource(seed)) timestamp := randTimestamp(r) @@ -93,6 +85,7 @@ func SimulateFromSeed( if !testingmode { b.ResetTimer() } + blockSimulator := createBlockSimulator(testingmode, tb, t, event, invariants, ops, operationQueue, numBlocks) for i := 0; i < numBlocks; i++ { // Run the BeginBlock handler @@ -111,24 +104,8 @@ func SimulateFromSeed( log, numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, keys, log, event) opCount += numQueuedOpsRan thisBlockSize -= numQueuedOpsRan - for j := 0; j < thisBlockSize; j++ { - logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event) - log = updateLog(testingmode, log, logUpdate) - if err != nil { - tb.Fatalf("error on operation %d, %v, log %s", opCount, err, log) - } - - queueOperations(operationQueue, futureOps) - if testingmode { - if onOperation { - AssertAllInvariants(t, app, invariants, log) - } - if opCount%200 == 0 { - fmt.Printf("\rSimulating... block %d/%d, operation %d.", header.Height, numBlocks, opCount) - } - } - opCount++ - } + log, operations := blockSimulator(thisBlockSize, r, app, ctx, keys, log, header) + opCount += operations res := app.EndBlock(abci.RequestEndBlock{}) header.Height++ @@ -151,14 +128,49 @@ func SimulateFromSeed( validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event) } - if testingmode { - fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds): %v\n", header.Height, header.Time) - } else { - fmt.Printf("%d operations ran\n", opCount) - } + fmt.Printf("\nSimulation complete. Final height (blocks): %d, final time (seconds), : %v, operations ran %d\n", header.Height, header.Time, opCount) DisplayEvents(events) } +// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize +// memory overhead +func createBlockSimulator(testingmode bool, tb testing.TB, t *testing.T, event func(string), invariants []Invariant, ops []Operation, operationQueue map[int][]Operation, totalNumBlocks int) func( + blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) { + return func(blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + keys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) { + for j := 0; j < blocksize; j++ { + logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event) + log = updateLog(testingmode, log, logUpdate) + if err != nil { + tb.Fatalf("error on operation %d within block %d, %v, log %s", header.Height, opCount, err, log) + } + + queueOperations(operationQueue, futureOps) + if testingmode { + if onOperation { + AssertAllInvariants(t, app, invariants, log) + } + if opCount%50 == 0 { + fmt.Printf("\rSimulating... block %d/%d, operation %d/%d.", header.Height, totalNumBlocks, opCount, blocksize) + } + } + opCount++ + } + return log, opCount + } +} + +func getTestingMode(tb testing.TB) (testingmode bool, t *testing.T, b *testing.B) { + testingmode = false + if _t, ok := tb.(*testing.T); ok { + t = _t + testingmode = true + } else { + b = tb.(*testing.B) + } + return +} + func updateLog(testingmode bool, log string, update string, args ...interface{}) (updatedLog string) { if testingmode { update = fmt.Sprintf(update, args...) From a69725d40d5ceaf2b27c0f5fba8e4e3c47029703 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Thu, 30 Aug 2018 11:47:32 -0700 Subject: [PATCH 07/15] Add method to benchmark with commits --- cmd/gaia/app/sim_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 6568f124b0..c7452ae533 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -30,6 +30,7 @@ var ( enabled bool verbose bool usegoleveldb bool + commit bool ) func init() { @@ -39,6 +40,7 @@ func init() { flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation") flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output") flag.BoolVar(&usegoleveldb, "SimulationGoLevelDB", false, "Use GoLevelDB instead of memdb") + flag.BoolVar(&commit, "SimulationCommit", false, "Have the simulation commit") } func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage { @@ -137,7 +139,7 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { invariants(app), // these shouldn't get ran 10, 100, - false, + commit, ) } @@ -165,7 +167,7 @@ func TestFullGaiaSimulation(t *testing.T) { invariants(app), numBlocks, blockSize, - false, + commit, ) } From e64c6da6f223019a5db6fbcb51c8d51745e42e44 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Thu, 30 Aug 2018 23:42:12 -0700 Subject: [PATCH 08/15] cleanup goleveldb dirs --- cmd/gaia/app/sim_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index c7452ae533..38dd352bde 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -124,10 +124,14 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { var logger log.Logger logger = log.NewNopLogger() var db dbm.DB - db = dbm.NewMemDB() if usegoleveldb { - db, _ = dbm.NewGoLevelDB("Simulation", os.TempDir()) + dir := os.TempDir() + db, _ = dbm.NewGoLevelDB("Simulation", dir) + defer os.RemoveAll(dir) + } else { + db = dbm.NewMemDB() } + defer db.Close() app := NewGaiaApp(logger, db, nil) // Run randomized simulation From 75025720a3f63ae162ebe9ae48aa789b81287958 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Fri, 31 Aug 2018 21:57:33 -0700 Subject: [PATCH 09/15] simulation: Initialize governance properly --- cmd/gaia/app/sim_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 38dd352bde..be42511d05 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -16,6 +16,7 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" banksim "github.com/cosmos/cosmos-sdk/x/bank/simulation" + "github.com/cosmos/cosmos-sdk/x/gov" govsim "github.com/cosmos/cosmos-sdk/x/gov/simulation" "github.com/cosmos/cosmos-sdk/x/mock/simulation" slashingsim "github.com/cosmos/cosmos-sdk/x/slashing/simulation" @@ -54,7 +55,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json Coins: coins, }) } - + govGenesis := gov.DefaultGenesisState() // Default genesis state stakeGenesis := stake.DefaultGenesisState() var validators []stake.Validator @@ -78,6 +79,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json genesis := GenesisState{ Accounts: genesisAccounts, StakeData: stakeGenesis, + GovData: govGenesis, } // Marshal genesis @@ -141,8 +143,8 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { testAndRunTxs(app), []simulation.RandSetup{}, invariants(app), // these shouldn't get ran - 10, - 100, + 210, + blockSize, commit, ) } From 3c2100793eb8ba584a5dff16706acee39463b76b Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Sat, 1 Sep 2018 12:32:24 -0700 Subject: [PATCH 10/15] Address @cwgoes comments --- cmd/gaia/app/sim_test.go | 28 ++++++------- x/mock/simulation/random_simulate_blocks.go | 46 +++++++++++---------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index be42511d05..68f67709a3 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -25,13 +25,12 @@ import ( ) var ( - seed int64 - numBlocks int - blockSize int - enabled bool - verbose bool - usegoleveldb bool - commit bool + seed int64 + numBlocks int + blockSize int + enabled bool + verbose bool + commit bool ) func init() { @@ -40,7 +39,6 @@ func init() { flag.IntVar(&blockSize, "SimulationBlockSize", 200, "Operations per block") flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation") flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output") - flag.BoolVar(&usegoleveldb, "SimulationGoLevelDB", false, "Use GoLevelDB instead of memdb") flag.BoolVar(&commit, "SimulationCommit", false, "Have the simulation commit") } @@ -126,14 +124,12 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { var logger log.Logger logger = log.NewNopLogger() var db dbm.DB - if usegoleveldb { - dir := os.TempDir() - db, _ = dbm.NewGoLevelDB("Simulation", dir) - defer os.RemoveAll(dir) - } else { - db = dbm.NewMemDB() - } - defer db.Close() + dir := os.TempDir() + db, _ = dbm.NewGoLevelDB("Simulation", dir) + defer func() { + db.Close() + os.RemoveAll(dir) + }() app := NewGaiaApp(logger, db, nil) // Run randomized simulation diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index 42a616331d..be1963a9e8 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -54,11 +54,11 @@ func SimulateFromSeed( tb testing.TB, app *baseapp.BaseApp, appStateFn func(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage, seed int64, ops []Operation, setups []RandSetup, invariants []Invariant, numBlocks int, blockSize int, commit bool, ) { - testingmode, t, b := getTestingMode(tb) + testingMode, t, b := getTestingMode(tb) log := fmt.Sprintf("Starting SimulateFromSeed with randomness created with seed %d", int(seed)) r := rand.New(rand.NewSource(seed)) timestamp := randTimestamp(r) - log = updateLog(testingmode, log, "Starting the simulation from time %v, unixtime %v", timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) + log = updateLog(testingMode, log, "Starting the simulation from time %v, unixtime %v", timestamp.UTC().Format(time.UnixDate), timestamp.Unix()) fmt.Printf("%s\n", log) timeDiff := maxTimePerBlock - minTimePerBlock @@ -67,7 +67,7 @@ func SimulateFromSeed( // Setup event stats events := make(map[string]uint) event := func(what string) { - log = updateLog(testingmode, log, "event - %s", what) + log = updateLog(testingMode, log, "event - %s", what) events[what]++ } @@ -78,21 +78,24 @@ func SimulateFromSeed( request := abci.RequestBeginBlock{Header: header} - var lastHeaderTime time.Time + var pastTimes []time.Time // These are operations which have been queued by previous operations operationQueue := make(map[int][]Operation) - if !testingmode { + if !testingMode { b.ResetTimer() } - blockSimulator := createBlockSimulator(testingmode, tb, t, event, invariants, ops, operationQueue, numBlocks) + blockSimulator := createBlockSimulator(testingMode, tb, t, event, invariants, ops, operationQueue, numBlocks) for i := 0; i < numBlocks; i++ { + // Log the header time for future lookup + pastTimes = append(pastTimes, header.Time) + // Run the BeginBlock handler app.BeginBlock(request) - log = updateLog(testingmode, log, "BeginBlock") + log = updateLog(testingMode, log, "BeginBlock") - if testingmode { + if testingMode { // Make sure invariants hold at beginning of block AssertAllInvariants(t, app, invariants, log) } @@ -109,11 +112,10 @@ func SimulateFromSeed( res := app.EndBlock(abci.RequestEndBlock{}) header.Height++ - lastHeaderTime = header.Time header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) - log = updateLog(testingmode, log, "EndBlock") + log = updateLog(testingMode, log, "EndBlock") - if testingmode { + if testingMode { // Make sure invariants hold at end of block AssertAllInvariants(t, app, invariants, log) } @@ -122,7 +124,7 @@ func SimulateFromSeed( } // Generate a random RequestBeginBlock with the current validator set for the next block - request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, lastHeaderTime, event, header, log) + request = RandomRequestBeginBlock(r, validators, livenessTransitionMatrix, evidenceFraction, pastTimes, event, header, log) // Update the validator set validators = updateValidators(tb, r, validators, res.ValidatorUpdates, event) @@ -134,19 +136,19 @@ func SimulateFromSeed( // Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize // memory overhead -func createBlockSimulator(testingmode bool, tb testing.TB, t *testing.T, event func(string), invariants []Invariant, ops []Operation, operationQueue map[int][]Operation, totalNumBlocks int) func( +func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, event func(string), invariants []Invariant, ops []Operation, operationQueue map[int][]Operation, totalNumBlocks int) func( blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, privKeys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) { return func(blocksize int, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, header abci.Header) (updatedLog string, opCount int) { for j := 0; j < blocksize; j++ { logUpdate, futureOps, err := ops[r.Intn(len(ops))](tb, r, app, ctx, keys, log, event) - log = updateLog(testingmode, log, logUpdate) + log = updateLog(testingMode, log, logUpdate) if err != nil { tb.Fatalf("error on operation %d within block %d, %v, log %s", header.Height, opCount, err, log) } queueOperations(operationQueue, futureOps) - if testingmode { + if testingMode { if onOperation { AssertAllInvariants(t, app, invariants, log) } @@ -160,19 +162,19 @@ func createBlockSimulator(testingmode bool, tb testing.TB, t *testing.T, event f } } -func getTestingMode(tb testing.TB) (testingmode bool, t *testing.T, b *testing.B) { - testingmode = false +func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B) { + testingMode = false if _t, ok := tb.(*testing.T); ok { t = _t - testingmode = true + testingMode = true } else { b = tb.(*testing.B) } return } -func updateLog(testingmode bool, log string, update string, args ...interface{}) (updatedLog string) { - if testingmode { +func updateLog(testingMode bool, log string, update string, args ...interface{}) (updatedLog string) { + if testingMode { update = fmt.Sprintf(update, args...) return fmt.Sprintf("%s\n%s", log, update) } @@ -240,7 +242,7 @@ func getKeys(validators map[string]mockValidator) []string { // RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction func RandomRequestBeginBlock(r *rand.Rand, validators map[string]mockValidator, livenessTransitions TransitionMatrix, evidenceFraction float64, - lastHeaderTime time.Time, event func(string), header abci.Header, log string) abci.RequestBeginBlock { + pastTimes []time.Time, event func(string), header abci.Header, log string) abci.RequestBeginBlock { if len(validators) == 0 { return abci.RequestBeginBlock{Header: header} } @@ -279,7 +281,7 @@ func RandomRequestBeginBlock(r *rand.Rand, validators map[string]mockValidator, time := header.Time if r.Float64() < pastEvidenceFraction { height = int64(r.Intn(int(header.Height))) - time = lastHeaderTime + time = pastTimes[height] } validator := signingValidators[r.Intn(len(signingValidators))].Validator var currentTotalVotingPower int64 From a991a2e1c4b328cb70af86fb5b794a1967813c5a Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Fri, 31 Aug 2018 00:37:25 -0700 Subject: [PATCH 11/15] Improve GetValidator speed In simulation, this was shown to cause a 4x speedup. There are no safety concerns here, as amino encoding is deterministic. --- PENDING.md | 2 +- x/stake/keeper/validator.go | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/PENDING.md b/PENDING.md index 0aef9fd193..5557a8771e 100644 --- a/PENDING.md +++ b/PENDING.md @@ -78,7 +78,7 @@ IMPROVEMENTS * [x/stake] [#2023](https://github.com/cosmos/cosmos-sdk/pull/2023) Terminate iteration loop in `UpdateBondedValidators` and `UpdateBondedValidatorsFull` when the first revoked validator is encountered and perform a sanity check. * [x/auth] Signature verification's gas cost now accounts for pubkey type. [#2046](https://github.com/tendermint/tendermint/pull/2046) * [x/stake] [x/slashing] Ensure delegation invariants to jailed validators [#1883](https://github.com/cosmos/cosmos-sdk/issues/1883). - + * [x/stake] Improve speed of GetValidator, which was shown to be a performance bottleneck. [#2046](https://github.com/tendermint/tendermint/pull/2200) * SDK * [tools] Make get_vendor_deps deletes `.vendor-new` directories, in case scratch files are present. * [cli] \#1632 Add integration tests to ensure `basecoind init && basecoind` start sequences run successfully for both `democoin` and `basecoin` examples. diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index 0ea24e6396..5dc075742e 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -2,6 +2,7 @@ package keeper import ( "bytes" + "container/list" "fmt" abci "github.com/tendermint/tendermint/abci/types" @@ -11,6 +12,14 @@ import ( "github.com/cosmos/cosmos-sdk/x/stake/types" ) +type cachedValidator struct { + val types.Validator + marshalled string +} + +var validatorCache = make(map[string]cachedValidator, 1000) +var validatorCacheList = list.New() + // get a single validator func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator types.Validator, found bool) { store := ctx.KVStore(k.storeKey) @@ -18,6 +27,23 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty if value == nil { return validator, false } + // return cached validator + strValue := string(value) + if val, ok := validatorCache[strValue]; ok { + valToReturn := val.val + // Doesn't mutate the cache's value + valToReturn.Operator = addr + return valToReturn, true + } else { // get validator from cache + validator = types.MustUnmarshalValidator(k.cdc, addr, value) + cachedVal := cachedValidator{validator, strValue} + validatorCache[strValue] = cachedValidator{validator, strValue} + validatorCacheList.PushBack(cachedVal) + if validatorCacheList.Len() > 500 { + valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator) + delete(validatorCache, valToRemove.marshalled) + } + } validator = types.MustUnmarshalValidator(k.cdc, addr, value) return validator, true } From 3b4caa5dd2bf218f24fd53616a41aefea79c9f20 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Fri, 31 Aug 2018 08:29:32 -0700 Subject: [PATCH 12/15] fix lint --- x/stake/keeper/validator.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index 5dc075742e..a12801dc9e 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -34,16 +34,17 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty // Doesn't mutate the cache's value valToReturn.Operator = addr return valToReturn, true - } else { // get validator from cache - validator = types.MustUnmarshalValidator(k.cdc, addr, value) - cachedVal := cachedValidator{validator, strValue} - validatorCache[strValue] = cachedValidator{validator, strValue} - validatorCacheList.PushBack(cachedVal) - if validatorCacheList.Len() > 500 { - valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator) - delete(validatorCache, valToRemove.marshalled) - } } + // get validator from cache + validator = types.MustUnmarshalValidator(k.cdc, addr, value) + cachedVal := cachedValidator{validator, strValue} + validatorCache[strValue] = cachedValidator{validator, strValue} + validatorCacheList.PushBack(cachedVal) + if validatorCacheList.Len() > 500 { + valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator) + delete(validatorCache, valToRemove.marshalled) + } + validator = types.MustUnmarshalValidator(k.cdc, addr, value) return validator, true } From f29fdcafddd84e1faf290305b840aca1cddd7ec2 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Sat, 1 Sep 2018 13:21:42 -0700 Subject: [PATCH 13/15] Add comments --- x/stake/keeper/validator.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index a12801dc9e..d5fba14337 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -12,12 +12,16 @@ import ( "github.com/cosmos/cosmos-sdk/x/stake/types" ) +// Cache the amino decoding of validators, as it can be the case that repeated slashing calls +// cause many calls to GetValidator, which were shown to throttle the state machine in our +// simulation. Note this is quite biased though, as the simulator does more slashes than a +// live chain should, however we require the slashing to be fast as noone pays gas for it. type cachedValidator struct { val types.Validator marshalled string } -var validatorCache = make(map[string]cachedValidator, 1000) +var validatorCache = make(map[string]cachedValidator, 500) var validatorCacheList = list.New() // get a single validator @@ -27,7 +31,7 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty if value == nil { return validator, false } - // return cached validator + // If these amino encoded bytes are in the cache, return the cached validator strValue := string(value) if val, ok := validatorCache[strValue]; ok { valToReturn := val.val @@ -35,11 +39,12 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty valToReturn.Operator = addr return valToReturn, true } - // get validator from cache + // amino bytes weren't found in cache, so amino unmarshal and add it to the cache validator = types.MustUnmarshalValidator(k.cdc, addr, value) cachedVal := cachedValidator{validator, strValue} validatorCache[strValue] = cachedValidator{validator, strValue} validatorCacheList.PushBack(cachedVal) + // if the cache is too big, pop off the last element from it if validatorCacheList.Len() > 500 { valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator) delete(validatorCache, valToRemove.marshalled) From 8a452b940a38c799ebb432361a0daeeec833fe63 Mon Sep 17 00:00:00 2001 From: ValarDragon Date: Sat, 1 Sep 2018 19:04:44 -0700 Subject: [PATCH 14/15] simulation: display db size at end of simulation, add makefile entries --- Makefile | 11 +++++++++++ PENDING.md | 2 +- cmd/gaia/app/sim_test.go | 14 +++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 983234dbc3..e363891e2a 100644 --- a/Makefile +++ b/Makefile @@ -163,6 +163,17 @@ test_sim_gaia_slow: @echo "Running full Gaia simulation. This may take awhile!" @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationVerbose=true -v -timeout 24h +SIM_NUM_BLOCKS ?= 210 +SIM_BLOCK_SIZE ?= 200 +SIM_COMMIT ?= true +test_sim_gaia_benchmark: + @echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" + @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h + +test_sim_gaia_profile: + @echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" + @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out + test_cover: @bash tests/test_cover.sh diff --git a/PENDING.md b/PENDING.md index 5557a8771e..34e202999d 100644 --- a/PENDING.md +++ b/PENDING.md @@ -58,7 +58,7 @@ FEATURES * SDK * [querier] added custom querier functionality, so ABCI query requests can be handled by keepers * [simulation] \#1924 allow operations to specify future operations - * [simulation] \#1924 Add benchmarking capabilities + * [simulation] \#1924 Add benchmarking capabilities, with makefile commands "test_sim_gaia_benchmark, test_sim_gaia_profile" * Tendermint diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 58d7ca18d3..3878166b3a 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -3,6 +3,7 @@ package app import ( "encoding/json" "flag" + "fmt" "math/rand" "os" "testing" @@ -118,7 +119,7 @@ func invariants(app *GaiaApp) []simulation.Invariant { } // Profile with: -// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$ -cpuprofile cpu.out +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$ -SimulationCommit=true -cpuprofile cpu.out func BenchmarkFullGaiaSimulation(b *testing.B) { // Setup Gaia application var logger log.Logger @@ -139,10 +140,15 @@ func BenchmarkFullGaiaSimulation(b *testing.B) { testAndRunTxs(app), []simulation.RandSetup{}, invariants(app), // these shouldn't get ran - 210, + numBlocks, blockSize, commit, ) + if commit { + fmt.Println("GoLevelDB Stats") + fmt.Println(db.Stats()["leveldb.stats"]) + fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) + } } func TestFullGaiaSimulation(t *testing.T) { @@ -171,7 +177,9 @@ func TestFullGaiaSimulation(t *testing.T) { blockSize, commit, ) - + if commit { + fmt.Println("Database Size", db.Stats()["database.size"]) + } } // TODO: Make another test for the fuzzer itself, which just has noOp txs From 2c66ba0bd4e47a4692f55105b0c405a63b004875 Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Sun, 2 Sep 2018 15:42:25 -0400 Subject: [PATCH 15/15] extra comment on cache key usage --- x/stake/keeper/validator.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x/stake/keeper/validator.go b/x/stake/keeper/validator.go index d5fba14337..2d8f947fc0 100644 --- a/x/stake/keeper/validator.go +++ b/x/stake/keeper/validator.go @@ -18,9 +18,10 @@ import ( // live chain should, however we require the slashing to be fast as noone pays gas for it. type cachedValidator struct { val types.Validator - marshalled string + marshalled string // marshalled amino bytes for the validator object (not operator address) } +// validatorCache-key: validator amino bytes var validatorCache = make(map[string]cachedValidator, 500) var validatorCacheList = list.New() @@ -31,6 +32,7 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty if value == nil { return validator, false } + // If these amino encoded bytes are in the cache, return the cached validator strValue := string(value) if val, ok := validatorCache[strValue]; ok { @@ -39,11 +41,13 @@ func (k Keeper) GetValidator(ctx sdk.Context, addr sdk.ValAddress) (validator ty valToReturn.Operator = addr return valToReturn, true } + // amino bytes weren't found in cache, so amino unmarshal and add it to the cache validator = types.MustUnmarshalValidator(k.cdc, addr, value) cachedVal := cachedValidator{validator, strValue} validatorCache[strValue] = cachedValidator{validator, strValue} validatorCacheList.PushBack(cachedVal) + // if the cache is too big, pop off the last element from it if validatorCacheList.Len() > 500 { valToRemove := validatorCacheList.Remove(validatorCacheList.Front()).(cachedValidator)