diff --git a/x/mock/simulation/doc.go b/x/mock/simulation/doc.go index 8b9a3f6932..2febd9e471 100644 --- a/x/mock/simulation/doc.go +++ b/x/mock/simulation/doc.go @@ -2,26 +2,25 @@ Package simulation implements a simulation framework for any state machine built on the SDK which utilizes auth. -It is primarily intended for fuzz testing the integration of modules. -It will test that the provided operations are interoperable, -and that the desired invariants hold. -It can additionally be used to detect what the performance benchmarks in the -system are, by using benchmarking mode and cpu / mem profiling. -If it detects a failure, it provides the entire log of what was ran, +It is primarily intended for fuzz testing the integration of modules. It will +test that the provided operations are interoperable, and that the desired +invariants hold. It can additionally be used to detect what the performance +benchmarks in the system are, by using benchmarking mode and cpu / mem +profiling. If it detects a failure, it provides the entire log of what was +ran, -The simulator takes as input: a random seed, the set of operations to run, -the invariants to test, and additional parameters to configure how long to run, -and misc. parameters that affect simulation speed. +The simulator takes as input: a random seed, the set of operations to run, the +invariants to test, and additional parameters to configure how long to run, and +misc. parameters that affect simulation speed. -It is intended that every module provides a list of Operations which will randomly -create and run a message / tx in a manner that is interesting to fuzz, and verify that -the state transition was executed as expected. -Each module should additionally provide methods to assert that the desired invariants hold. +It is intended that every module provides a list of Operations which will +randomly create and run a message / tx in a manner that is interesting to fuzz, +and verify that the state transition was executed as expected. Each module +should additionally provide methods to assert that the desired invariants hold. Then to perform a randomized simulation, select the set of desired operations, -the weightings for each, the invariants you want to test, and how long to run it for. -Then run simulation.Simulate! -The simulator will handle things like ensuring that validators periodically double signing, -or go offline. +the weightings for each, the invariants you want to test, and how long to run +it for. Then run simulation.Simulate! The simulator will handle things like +ensuring that validators periodically double signing, or go offline. */ package simulation diff --git a/x/mock/simulation/params.go b/x/mock/simulation/params.go index 404a85e544..8499e6c118 100644 --- a/x/mock/simulation/params.go +++ b/x/mock/simulation/params.go @@ -15,15 +15,18 @@ const ( onOperation bool = false ) +// TODO explain transitional matrix usage var ( - // Currently there are 3 different liveness types, fully online, spotty connection, offline. + // Currently there are 3 different liveness types, + // fully online, spotty connection, offline. defaultLivenessTransitionMatrix, _ = CreateTransitionMatrix([][]int{ {90, 20, 1}, {10, 50, 5}, {0, 10, 1000}, }) - // 3 states: rand in range [0, 4*provided blocksize], rand in range [0, 2 * provided blocksize], 0 + // 3 states: rand in range [0, 4*provided blocksize], + // rand in range [0, 2 * provided blocksize], 0 defaultBlockSizeTransitionMatrix, _ = CreateTransitionMatrix([][]int{ {85, 5, 0}, {15, 92, 1}, diff --git a/x/mock/simulation/random_simulate_blocks.go b/x/mock/simulation/random_simulate_blocks.go index a568997e62..17a26f7626 100644 --- a/x/mock/simulation/random_simulate_blocks.go +++ b/x/mock/simulation/random_simulate_blocks.go @@ -31,8 +31,11 @@ func Simulate(t *testing.T, app *baseapp.BaseApp, return SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit) } -func initChain(r *rand.Rand, params Params, accounts []Account, setups []RandSetup, app *baseapp.BaseApp, - appStateFn func(r *rand.Rand, accounts []Account) json.RawMessage) (validators map[string]mockValidator) { +func initChain(r *rand.Rand, params Params, + accounts []Account, setups []RandSetup, app *baseapp.BaseApp, + appStateFn func(r *rand.Rand, accounts []Account) json.RawMessage) ( + validators map[string]mockValidator) { + res := app.InitChain(abci.RequestInitChain{AppStateBytes: appStateFn(r, accounts)}) validators = make(map[string]mockValidator) for _, validator := range res.Validators { @@ -101,6 +104,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, var pastVoteInfos [][]abci.VoteInfo request := RandomRequestBeginBlock(r, params, validators, pastTimes, pastVoteInfos, event, header) + // These are operations which have been queued by previous operations operationQueue := make(map[int][]Operation) timeOperationQueue := []FutureOperation{} @@ -110,7 +114,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, blockLogBuilders = make([]*strings.Builder, numBlocks) } displayLogs := logPrinter(testingMode, blockLogBuilders) - blockSimulator := createBlockSimulator(testingMode, tb, t, params, event, invariants, ops, operationQueue, timeOperationQueue, numBlocks, blockSize, displayLogs) + blockSimulator := createBlockSimulator( + testingMode, tb, t, params, event, invariants, + ops, operationQueue, timeOperationQueue, + numBlocks, blockSize, displayLogs) + if !testingMode { b.ResetTimer() } else { @@ -147,8 +155,14 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, // Run queued operations. Ignores blocksize if blocksize is too small logWriter("Queued operations") - numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, accs, logWriter, displayLogs, event) - numQueuedTimeOpsRan := runQueuedTimeOperations(timeOperationQueue, header.Time, tb, r, app, ctx, accs, logWriter, displayLogs, event) + numQueuedOpsRan := runQueuedOperations( + operationQueue, int(header.Height), + tb, r, app, ctx, accs, logWriter, + displayLogs, event) + numQueuedTimeOpsRan := runQueuedTimeOperations( + timeOperationQueue, header.Time, + tb, r, app, ctx, accs, + logWriter, displayLogs, event) if testingMode && onOperation { // Make sure invariants hold at end of queued operations assertAllInvariants(t, app, invariants, "QueuedOperations", displayLogs) @@ -164,7 +178,10 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, res := app.EndBlock(abci.RequestEndBlock{}) header.Height++ - header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) + header.Time = header.Time.Add( + time.Duration(minTimePerBlock) * time.Second) + header.Time = header.Time.Add( + time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second) header.ProposerAddress = randomProposer(r, validators) logWriter("EndBlock") @@ -198,16 +215,14 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, return nil } -type blockSimFn func( - r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - accounts []Account, header abci.Header, logWriter func(string), -) (opCount int) +type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + accounts []Account, header abci.Header, logWriter func(string)) (opCount int) // Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize // memory overhead func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params, - event func(string), invariants []Invariant, - ops []WeightedOperation, operationQueue map[int][]Operation, timeOperationQueue []FutureOperation, + event func(string), invariants []Invariant, ops []WeightedOperation, + operationQueue map[int][]Operation, timeOperationQueue []FutureOperation, totalNumBlocks int, avgBlockSize int, displayLogs func()) blockSimFn { var ( @@ -233,23 +248,29 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, header abci.Header, logWriter func(string)) (opCount int) { - fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize) + + fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", + header.Height, totalNumBlocks, opCount, blocksize) lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize) + for j := 0; j < blocksize; j++ { logUpdate, futureOps, err := selectOp(r)(r, app, ctx, accounts, event) logWriter(logUpdate) if err != nil { displayLogs() - tb.Fatalf("error on operation %d within block %d, %v", header.Height, opCount, err) + tb.Fatalf("error on operation %d within block %d, %v", + header.Height, opCount, err) } queueOperations(operationQueue, timeOperationQueue, futureOps) if testingMode { if onOperation { - assertAllInvariants(t, app, invariants, fmt.Sprintf("operation: %v", logUpdate), displayLogs) + assertAllInvariants(t, app, invariants, + fmt.Sprintf("operation: %v", logUpdate), displayLogs) } if opCount%50 == 0 { - fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize) + fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", + header.Height, totalNumBlocks, opCount, blocksize) } } opCount++ @@ -272,10 +293,11 @@ func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B // getBlockSize returns a block size as determined from the transition matrix. // It targets making average block size the provided parameter. The three // states it moves between are: -// "over stuffed" blocks with average size of 2 * avgblocksize, -// normal sized blocks, hitting avgBlocksize on average, -// and empty blocks, with no txs / only txs scheduled from the past. -func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize int) (state, blocksize int) { +// - "over stuffed" blocks with average size of 2 * avgblocksize, +// - normal sized blocks, hitting avgBlocksize on average, +// - and empty blocks, with no txs / only txs scheduled from the past. +func getBlockSize(r *rand.Rand, params Params, + lastBlockSizeState, avgBlockSize int) (state, blocksize int) { // TODO: Make default blocksize transition matrix actually make the average // blocksize equal to avgBlockSize. state = params.BlockSizeTransitionMatrix.NextState(r, lastBlockSizeState) @@ -290,7 +312,10 @@ func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize } // adds all future operations into the operation queue. -func queueOperations(queuedOperations map[int][]Operation, queuedTimeOperations []FutureOperation, futureOperations []FutureOperation) { +func queueOperations(queuedOperations map[int][]Operation, + queuedTimeOperations []FutureOperation, + futureOperations []FutureOperation) { + if futureOperations == nil { return } @@ -312,8 +337,11 @@ func queueOperations(queuedOperations map[int][]Operation, queuedTimeOperations } // nolint: errcheck -func runQueuedOperations(queueOperations map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - accounts []Account, logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) { +func runQueuedOperations(queueOperations map[int][]Operation, + height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + accounts []Account, logWriter func(string), + displayLogs func(), event func(string)) (numOpsRan int) { + if queuedOps, ok := queueOperations[height]; ok { numOps := len(queuedOps) for i := 0; i < numOps; i++ { @@ -333,8 +361,10 @@ func runQueuedOperations(queueOperations map[int][]Operation, height int, tb tes return 0 } -func runQueuedTimeOperations(queueOperations []FutureOperation, currentTime time.Time, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - accounts []Account, logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) { +func runQueuedTimeOperations(queueOperations []FutureOperation, + currentTime time.Time, tb testing.TB, r *rand.Rand, + app *baseapp.BaseApp, ctx sdk.Context, accounts []Account, + logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) { numOpsRan = 0 for len(queueOperations) > 0 && currentTime.After(queueOperations[0].BlockTime) { @@ -379,10 +409,13 @@ func randomProposer(r *rand.Rand, validators map[string]mockValidator) cmn.HexBy return pk.Address() } -// RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction -// nolint: unparam -func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]mockValidator, - pastTimes []time.Time, pastVoteInfos [][]abci.VoteInfo, event func(string), header abci.Header) abci.RequestBeginBlock { +// RandomRequestBeginBlock generates a list of signing validators according to +// the provided list of validators, signing fraction, and evidence fraction +func RandomRequestBeginBlock(r *rand.Rand, params Params, + validators map[string]mockValidator, pastTimes []time.Time, + pastVoteInfos [][]abci.VoteInfo, + event func(string), header abci.Header) abci.RequestBeginBlock { + if len(validators) == 0 { return abci.RequestBeginBlock{Header: header} } @@ -459,7 +492,9 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string] // updateValidators mimicks Tendermint's update logic // nolint: unparam -func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[string]mockValidator, updates []abci.ValidatorUpdate, event func(string)) map[string]mockValidator { +func updateValidators(tb testing.TB, r *rand.Rand, params Params, + current map[string]mockValidator, updates []abci.ValidatorUpdate, + event func(string)) map[string]mockValidator { for _, update := range updates { str := fmt.Sprintf("%v", update.PubKey) @@ -478,7 +513,10 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[st event("endblock/validatorupdates/updated") } else { // Set this new validator - current[str] = mockValidator{update, GetMemberOfInitialState(r, params.InitialLivenessWeightings)} + current[str] = mockValidator{ + update, + GetMemberOfInitialState(r, params.InitialLivenessWeightings), + } event("endblock/validatorupdates/added") } } diff --git a/x/mock/simulation/transition_matrix.go b/x/mock/simulation/transition_matrix.go index 39bdb1e4f9..97cd307e56 100644 --- a/x/mock/simulation/transition_matrix.go +++ b/x/mock/simulation/transition_matrix.go @@ -5,12 +5,11 @@ import ( "math/rand" ) -// TransitionMatrix is _almost_ a left stochastic matrix. -// It is technically not one due to not normalizing the column values. -// In the future, if we want to find the steady state distribution, -// it will be quite easy to normalize these values to get a stochastic matrix. -// Floats aren't currently used as the default due to non-determinism across -// architectures +// TransitionMatrix is _almost_ a left stochastic matrix. It is technically +// not one due to not normalizing the column values. In the future, if we want +// to find the steady state distribution, it will be quite easy to normalize +// these values to get a stochastic matrix. Floats aren't currently used as +// the default due to non-determinism across architectures type TransitionMatrix struct { weights [][]int // total in each column @@ -24,7 +23,8 @@ func CreateTransitionMatrix(weights [][]int) (TransitionMatrix, error) { n := len(weights) for i := 0; i < n; i++ { if len(weights[i]) != n { - return TransitionMatrix{}, fmt.Errorf("Transition Matrix: Non-square matrix provided, error on row %d", i) + return TransitionMatrix{}, + fmt.Errorf("Transition Matrix: Non-square matrix provided, error on row %d", i) } } totals := make([]int, n) @@ -36,8 +36,8 @@ func CreateTransitionMatrix(weights [][]int) (TransitionMatrix, error) { return TransitionMatrix{weights, totals, n}, nil } -// NextState returns the next state randomly chosen using r, and the weightings provided -// in the transition matrix. +// NextState returns the next state randomly chosen using r, and the weightings +// provided in the transition matrix. func (t TransitionMatrix) NextState(r *rand.Rand, i int) int { randNum := r.Intn(t.totals[i]) for row := 0; row < t.n; row++ { @@ -51,7 +51,7 @@ func (t TransitionMatrix) NextState(r *rand.Rand, i int) int { } // GetMemberOfInitialState takes an initial array of weights, of size n. -// It returns a weighted random number in [0,n). +// It returns a weighted random number in [0,n]. func GetMemberOfInitialState(r *rand.Rand, weights []int) int { n := len(weights) total := 0 diff --git a/x/mock/simulation/types.go b/x/mock/simulation/types.go index e601f2e1f9..198e2a4ff8 100644 --- a/x/mock/simulation/types.go +++ b/x/mock/simulation/types.go @@ -10,68 +10,71 @@ import ( "github.com/tendermint/tendermint/crypto" ) -type ( - // Operation runs a state machine transition, - // and ensures the transition happened as expected. - // The operation could be running and testing a fuzzed transaction, - // or doing the same for a message. - // - // For ease of debugging, - // an operation returns a descriptive message "action", - // which details what this fuzzed state machine transition actually did. - // - // Operations can optionally provide a list of "FutureOperations" to run later - // These will be ran at the beginning of the corresponding block. - Operation func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, - accounts []Account, event func(string), - ) (action string, futureOperations []FutureOperation, err error) +// Operation runs a state machine transition, +// and ensures the transition happened as expected. +// The operation could be running and testing a fuzzed transaction, +// or doing the same for a message. +// +// For ease of debugging, +// an operation returns a descriptive message "action", +// which details what this fuzzed state machine transition actually did. +// +// Operations can optionally provide a list of "FutureOperations" to run later +// These will be ran at the beginning of the corresponding block. +type Operation func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, + accounts []Account, event func(string)) ( + action string, futureOperations []FutureOperation, err error) - // RandSetup performs the random setup the mock module needs. - RandSetup func(r *rand.Rand, accounts []Account) +// RandSetup performs the random setup the mock module needs. +type RandSetup func(r *rand.Rand, accounts []Account) - // An Invariant is a function which tests a particular invariant. - // If the invariant has been broken, it should return an error - // containing a descriptive message about what happened. - // The simulator will then halt and print the logs. - Invariant func(app *baseapp.BaseApp) error +// An Invariant is a function which tests a particular invariant. +// If the invariant has been broken, it should return an error +// containing a descriptive message about what happened. +// The simulator will then halt and print the logs. +type Invariant func(app *baseapp.BaseApp) error - // Account contains a privkey, pubkey, address tuple - // eventually more useful data can be placed in here. - // (e.g. number of coins) - Account struct { - PrivKey crypto.PrivKey - PubKey crypto.PubKey - Address sdk.AccAddress - } +// Account contains a privkey, pubkey, address tuple +// eventually more useful data can be placed in here. +// (e.g. number of coins) +type Account struct { + PrivKey crypto.PrivKey + PubKey crypto.PubKey + Address sdk.AccAddress +} - mockValidator struct { - val abci.ValidatorUpdate - livenessState int - } +// are two accounts equal +func (acc Account) Equals(acc2 Account) bool { + return acc.Address.Equals(acc2.Address) +} - // FutureOperation is an operation which will be ran at the - // beginning of the provided BlockHeight. - // If both a BlockHeight and BlockTime are specified, it will use the BlockHeight. - // In the (likely) event that multiple operations are queued at the same - // block height, they will execute in a FIFO pattern. - FutureOperation struct { - BlockHeight int - BlockTime time.Time - Op Operation - } +type mockValidator struct { + val abci.ValidatorUpdate + livenessState int +} - // WeightedOperation is an operation with associated weight. - // This is used to bias the selection operation within the simulator. - WeightedOperation struct { - Weight int - Op Operation - } -) +// FutureOperation is an operation which will be ran at the +// beginning of the provided BlockHeight. +// If both a BlockHeight and BlockTime are specified, it will use the BlockHeight. +// In the (likely) event that multiple operations are queued at the same +// block height, they will execute in a FIFO pattern. +type FutureOperation struct { + BlockHeight int + BlockTime time.Time + Op Operation +} + +// WeightedOperation is an operation with associated weight. +// This is used to bias the selection operation within the simulator. +type WeightedOperation struct { + Weight int + Op Operation +} // TODO remove? not being called anywhere -// PeriodicInvariant returns an Invariant function closure that asserts -// a given invariant if the mock application's last block modulo the given -// period is congruent to the given offset. +// PeriodicInvariant returns an Invariant function closure that asserts a given +// invariant if the mock application's last block modulo the given period is +// congruent to the given offset. func PeriodicInvariant(invariant Invariant, period int, offset int) Invariant { return func(app *baseapp.BaseApp) error { if int(app.LastBlockHeight())%period == offset { @@ -80,8 +83,3 @@ func PeriodicInvariant(invariant Invariant, period int, offset int) Invariant { return nil } } - -// nolint -func (acc Account) Equals(acc2 Account) bool { - return acc.Address.Equals(acc2.Address) -} diff --git a/x/mock/simulation/util.go b/x/mock/simulation/util.go index 62b253a258..bb974b65f1 100644 --- a/x/mock/simulation/util.go +++ b/x/mock/simulation/util.go @@ -21,8 +21,8 @@ import ( // shamelessly copied from https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang#31832326 // TODO we should probably move this to tendermint/libs/common/random.go -const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" const ( + letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" letterIdxBits = 6 // 6 bits to represent a letter index letterIdxMask = 1< 10 { - fileName := fmt.Sprintf("simulation_log_%s.txt", time.Now().Format("2006-01-02 15:04:05")) + + fileName := fmt.Sprintf("simulation_log_%s.txt", + time.Now().Format("2006-01-02 15:04:05")) + fmt.Printf("Too many logs to display, instead writing to %s\n", fileName) f, _ = os.Create(fileName) }