whitespacing

This commit is contained in:
rigelrozanski 2018-11-07 10:28:18 -05:00
parent 8f690b5b6c
commit 74b2a90087
6 changed files with 164 additions and 122 deletions

View File

@ -2,26 +2,25 @@
Package simulation implements a simulation framework for any state machine
built on the SDK which utilizes auth.
It is primarily intended for fuzz testing the integration of modules.
It will test that the provided operations are interoperable,
and that the desired invariants hold.
It can additionally be used to detect what the performance benchmarks in the
system are, by using benchmarking mode and cpu / mem profiling.
If it detects a failure, it provides the entire log of what was ran,
It is primarily intended for fuzz testing the integration of modules. It will
test that the provided operations are interoperable, and that the desired
invariants hold. It can additionally be used to detect what the performance
benchmarks in the system are, by using benchmarking mode and cpu / mem
profiling. If it detects a failure, it provides the entire log of what was
ran,
The simulator takes as input: a random seed, the set of operations to run,
the invariants to test, and additional parameters to configure how long to run,
and misc. parameters that affect simulation speed.
The simulator takes as input: a random seed, the set of operations to run, the
invariants to test, and additional parameters to configure how long to run, and
misc. parameters that affect simulation speed.
It is intended that every module provides a list of Operations which will randomly
create and run a message / tx in a manner that is interesting to fuzz, and verify that
the state transition was executed as expected.
Each module should additionally provide methods to assert that the desired invariants hold.
It is intended that every module provides a list of Operations which will
randomly create and run a message / tx in a manner that is interesting to fuzz,
and verify that the state transition was executed as expected. Each module
should additionally provide methods to assert that the desired invariants hold.
Then to perform a randomized simulation, select the set of desired operations,
the weightings for each, the invariants you want to test, and how long to run it for.
Then run simulation.Simulate!
The simulator will handle things like ensuring that validators periodically double signing,
or go offline.
the weightings for each, the invariants you want to test, and how long to run
it for. Then run simulation.Simulate! The simulator will handle things like
ensuring that validators periodically double signing, or go offline.
*/
package simulation

View File

@ -15,15 +15,18 @@ const (
onOperation bool = false
)
// TODO explain transitional matrix usage
var (
// Currently there are 3 different liveness types, fully online, spotty connection, offline.
// Currently there are 3 different liveness types,
// fully online, spotty connection, offline.
defaultLivenessTransitionMatrix, _ = CreateTransitionMatrix([][]int{
{90, 20, 1},
{10, 50, 5},
{0, 10, 1000},
})
// 3 states: rand in range [0, 4*provided blocksize], rand in range [0, 2 * provided blocksize], 0
// 3 states: rand in range [0, 4*provided blocksize],
// rand in range [0, 2 * provided blocksize], 0
defaultBlockSizeTransitionMatrix, _ = CreateTransitionMatrix([][]int{
{85, 5, 0},
{15, 92, 1},

View File

@ -31,8 +31,11 @@ func Simulate(t *testing.T, app *baseapp.BaseApp,
return SimulateFromSeed(t, app, appStateFn, time, ops, setups, invariants, numBlocks, blockSize, commit)
}
func initChain(r *rand.Rand, params Params, accounts []Account, setups []RandSetup, app *baseapp.BaseApp,
appStateFn func(r *rand.Rand, accounts []Account) json.RawMessage) (validators map[string]mockValidator) {
func initChain(r *rand.Rand, params Params,
accounts []Account, setups []RandSetup, app *baseapp.BaseApp,
appStateFn func(r *rand.Rand, accounts []Account) json.RawMessage) (
validators map[string]mockValidator) {
res := app.InitChain(abci.RequestInitChain{AppStateBytes: appStateFn(r, accounts)})
validators = make(map[string]mockValidator)
for _, validator := range res.Validators {
@ -101,6 +104,7 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
var pastVoteInfos [][]abci.VoteInfo
request := RandomRequestBeginBlock(r, params, validators, pastTimes, pastVoteInfos, event, header)
// These are operations which have been queued by previous operations
operationQueue := make(map[int][]Operation)
timeOperationQueue := []FutureOperation{}
@ -110,7 +114,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
blockLogBuilders = make([]*strings.Builder, numBlocks)
}
displayLogs := logPrinter(testingMode, blockLogBuilders)
blockSimulator := createBlockSimulator(testingMode, tb, t, params, event, invariants, ops, operationQueue, timeOperationQueue, numBlocks, blockSize, displayLogs)
blockSimulator := createBlockSimulator(
testingMode, tb, t, params, event, invariants,
ops, operationQueue, timeOperationQueue,
numBlocks, blockSize, displayLogs)
if !testingMode {
b.ResetTimer()
} else {
@ -147,8 +155,14 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
// Run queued operations. Ignores blocksize if blocksize is too small
logWriter("Queued operations")
numQueuedOpsRan := runQueuedOperations(operationQueue, int(header.Height), tb, r, app, ctx, accs, logWriter, displayLogs, event)
numQueuedTimeOpsRan := runQueuedTimeOperations(timeOperationQueue, header.Time, tb, r, app, ctx, accs, logWriter, displayLogs, event)
numQueuedOpsRan := runQueuedOperations(
operationQueue, int(header.Height),
tb, r, app, ctx, accs, logWriter,
displayLogs, event)
numQueuedTimeOpsRan := runQueuedTimeOperations(
timeOperationQueue, header.Time,
tb, r, app, ctx, accs,
logWriter, displayLogs, event)
if testingMode && onOperation {
// Make sure invariants hold at end of queued operations
assertAllInvariants(t, app, invariants, "QueuedOperations", displayLogs)
@ -164,7 +178,10 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
res := app.EndBlock(abci.RequestEndBlock{})
header.Height++
header.Time = header.Time.Add(time.Duration(minTimePerBlock) * time.Second).Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second)
header.Time = header.Time.Add(
time.Duration(minTimePerBlock) * time.Second)
header.Time = header.Time.Add(
time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second)
header.ProposerAddress = randomProposer(r, validators)
logWriter("EndBlock")
@ -198,16 +215,14 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp,
return nil
}
type blockSimFn func(
r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, header abci.Header, logWriter func(string),
) (opCount int)
type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, header abci.Header, logWriter func(string)) (opCount int)
// Returns a function to simulate blocks. Written like this to avoid constant parameters being passed everytime, to minimize
// memory overhead
func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params Params,
event func(string), invariants []Invariant,
ops []WeightedOperation, operationQueue map[int][]Operation, timeOperationQueue []FutureOperation,
event func(string), invariants []Invariant, ops []WeightedOperation,
operationQueue map[int][]Operation, timeOperationQueue []FutureOperation,
totalNumBlocks int, avgBlockSize int, displayLogs func()) blockSimFn {
var (
@ -233,23 +248,29 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, params
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, header abci.Header, logWriter func(string)) (opCount int) {
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize)
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, totalNumBlocks, opCount, blocksize)
lastBlocksizeState, blocksize = getBlockSize(r, params, lastBlocksizeState, avgBlockSize)
for j := 0; j < blocksize; j++ {
logUpdate, futureOps, err := selectOp(r)(r, app, ctx, accounts, event)
logWriter(logUpdate)
if err != nil {
displayLogs()
tb.Fatalf("error on operation %d within block %d, %v", header.Height, opCount, err)
tb.Fatalf("error on operation %d within block %d, %v",
header.Height, opCount, err)
}
queueOperations(operationQueue, timeOperationQueue, futureOps)
if testingMode {
if onOperation {
assertAllInvariants(t, app, invariants, fmt.Sprintf("operation: %v", logUpdate), displayLogs)
assertAllInvariants(t, app, invariants,
fmt.Sprintf("operation: %v", logUpdate), displayLogs)
}
if opCount%50 == 0 {
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ", header.Height, totalNumBlocks, opCount, blocksize)
fmt.Printf("\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, totalNumBlocks, opCount, blocksize)
}
}
opCount++
@ -272,10 +293,11 @@ func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B
// getBlockSize returns a block size as determined from the transition matrix.
// It targets making average block size the provided parameter. The three
// states it moves between are:
// "over stuffed" blocks with average size of 2 * avgblocksize,
// normal sized blocks, hitting avgBlocksize on average,
// and empty blocks, with no txs / only txs scheduled from the past.
func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize int) (state, blocksize int) {
// - "over stuffed" blocks with average size of 2 * avgblocksize,
// - normal sized blocks, hitting avgBlocksize on average,
// - and empty blocks, with no txs / only txs scheduled from the past.
func getBlockSize(r *rand.Rand, params Params,
lastBlockSizeState, avgBlockSize int) (state, blocksize int) {
// TODO: Make default blocksize transition matrix actually make the average
// blocksize equal to avgBlockSize.
state = params.BlockSizeTransitionMatrix.NextState(r, lastBlockSizeState)
@ -290,7 +312,10 @@ func getBlockSize(r *rand.Rand, params Params, lastBlockSizeState, avgBlockSize
}
// adds all future operations into the operation queue.
func queueOperations(queuedOperations map[int][]Operation, queuedTimeOperations []FutureOperation, futureOperations []FutureOperation) {
func queueOperations(queuedOperations map[int][]Operation,
queuedTimeOperations []FutureOperation,
futureOperations []FutureOperation) {
if futureOperations == nil {
return
}
@ -312,8 +337,11 @@ func queueOperations(queuedOperations map[int][]Operation, queuedTimeOperations
}
// nolint: errcheck
func runQueuedOperations(queueOperations map[int][]Operation, height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) {
func runQueuedOperations(queueOperations map[int][]Operation,
height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, logWriter func(string),
displayLogs func(), event func(string)) (numOpsRan int) {
if queuedOps, ok := queueOperations[height]; ok {
numOps := len(queuedOps)
for i := 0; i < numOps; i++ {
@ -333,8 +361,10 @@ func runQueuedOperations(queueOperations map[int][]Operation, height int, tb tes
return 0
}
func runQueuedTimeOperations(queueOperations []FutureOperation, currentTime time.Time, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) {
func runQueuedTimeOperations(queueOperations []FutureOperation,
currentTime time.Time, tb testing.TB, r *rand.Rand,
app *baseapp.BaseApp, ctx sdk.Context, accounts []Account,
logWriter func(string), displayLogs func(), event func(string)) (numOpsRan int) {
numOpsRan = 0
for len(queueOperations) > 0 && currentTime.After(queueOperations[0].BlockTime) {
@ -379,10 +409,13 @@ func randomProposer(r *rand.Rand, validators map[string]mockValidator) cmn.HexBy
return pk.Address()
}
// RandomRequestBeginBlock generates a list of signing validators according to the provided list of validators, signing fraction, and evidence fraction
// nolint: unparam
func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]mockValidator,
pastTimes []time.Time, pastVoteInfos [][]abci.VoteInfo, event func(string), header abci.Header) abci.RequestBeginBlock {
// RandomRequestBeginBlock generates a list of signing validators according to
// the provided list of validators, signing fraction, and evidence fraction
func RandomRequestBeginBlock(r *rand.Rand, params Params,
validators map[string]mockValidator, pastTimes []time.Time,
pastVoteInfos [][]abci.VoteInfo,
event func(string), header abci.Header) abci.RequestBeginBlock {
if len(validators) == 0 {
return abci.RequestBeginBlock{Header: header}
}
@ -459,7 +492,9 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params, validators map[string]
// updateValidators mimicks Tendermint's update logic
// nolint: unparam
func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[string]mockValidator, updates []abci.ValidatorUpdate, event func(string)) map[string]mockValidator {
func updateValidators(tb testing.TB, r *rand.Rand, params Params,
current map[string]mockValidator, updates []abci.ValidatorUpdate,
event func(string)) map[string]mockValidator {
for _, update := range updates {
str := fmt.Sprintf("%v", update.PubKey)
@ -478,7 +513,10 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params, current map[st
event("endblock/validatorupdates/updated")
} else {
// Set this new validator
current[str] = mockValidator{update, GetMemberOfInitialState(r, params.InitialLivenessWeightings)}
current[str] = mockValidator{
update,
GetMemberOfInitialState(r, params.InitialLivenessWeightings),
}
event("endblock/validatorupdates/added")
}
}

View File

@ -5,12 +5,11 @@ import (
"math/rand"
)
// TransitionMatrix is _almost_ a left stochastic matrix.
// It is technically not one due to not normalizing the column values.
// In the future, if we want to find the steady state distribution,
// it will be quite easy to normalize these values to get a stochastic matrix.
// Floats aren't currently used as the default due to non-determinism across
// architectures
// TransitionMatrix is _almost_ a left stochastic matrix. It is technically
// not one due to not normalizing the column values. In the future, if we want
// to find the steady state distribution, it will be quite easy to normalize
// these values to get a stochastic matrix. Floats aren't currently used as
// the default due to non-determinism across architectures
type TransitionMatrix struct {
weights [][]int
// total in each column
@ -24,7 +23,8 @@ func CreateTransitionMatrix(weights [][]int) (TransitionMatrix, error) {
n := len(weights)
for i := 0; i < n; i++ {
if len(weights[i]) != n {
return TransitionMatrix{}, fmt.Errorf("Transition Matrix: Non-square matrix provided, error on row %d", i)
return TransitionMatrix{},
fmt.Errorf("Transition Matrix: Non-square matrix provided, error on row %d", i)
}
}
totals := make([]int, n)
@ -36,8 +36,8 @@ func CreateTransitionMatrix(weights [][]int) (TransitionMatrix, error) {
return TransitionMatrix{weights, totals, n}, nil
}
// NextState returns the next state randomly chosen using r, and the weightings provided
// in the transition matrix.
// NextState returns the next state randomly chosen using r, and the weightings
// provided in the transition matrix.
func (t TransitionMatrix) NextState(r *rand.Rand, i int) int {
randNum := r.Intn(t.totals[i])
for row := 0; row < t.n; row++ {
@ -51,7 +51,7 @@ func (t TransitionMatrix) NextState(r *rand.Rand, i int) int {
}
// GetMemberOfInitialState takes an initial array of weights, of size n.
// It returns a weighted random number in [0,n).
// It returns a weighted random number in [0,n].
func GetMemberOfInitialState(r *rand.Rand, weights []int) int {
n := len(weights)
total := 0

View File

@ -10,68 +10,71 @@ import (
"github.com/tendermint/tendermint/crypto"
)
type (
// Operation runs a state machine transition,
// and ensures the transition happened as expected.
// The operation could be running and testing a fuzzed transaction,
// or doing the same for a message.
//
// For ease of debugging,
// an operation returns a descriptive message "action",
// which details what this fuzzed state machine transition actually did.
//
// Operations can optionally provide a list of "FutureOperations" to run later
// These will be ran at the beginning of the corresponding block.
Operation func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, event func(string),
) (action string, futureOperations []FutureOperation, err error)
// Operation runs a state machine transition,
// and ensures the transition happened as expected.
// The operation could be running and testing a fuzzed transaction,
// or doing the same for a message.
//
// For ease of debugging,
// an operation returns a descriptive message "action",
// which details what this fuzzed state machine transition actually did.
//
// Operations can optionally provide a list of "FutureOperations" to run later
// These will be ran at the beginning of the corresponding block.
type Operation func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
accounts []Account, event func(string)) (
action string, futureOperations []FutureOperation, err error)
// RandSetup performs the random setup the mock module needs.
RandSetup func(r *rand.Rand, accounts []Account)
// RandSetup performs the random setup the mock module needs.
type RandSetup func(r *rand.Rand, accounts []Account)
// An Invariant is a function which tests a particular invariant.
// If the invariant has been broken, it should return an error
// containing a descriptive message about what happened.
// The simulator will then halt and print the logs.
Invariant func(app *baseapp.BaseApp) error
// An Invariant is a function which tests a particular invariant.
// If the invariant has been broken, it should return an error
// containing a descriptive message about what happened.
// The simulator will then halt and print the logs.
type Invariant func(app *baseapp.BaseApp) error
// Account contains a privkey, pubkey, address tuple
// eventually more useful data can be placed in here.
// (e.g. number of coins)
Account struct {
PrivKey crypto.PrivKey
PubKey crypto.PubKey
Address sdk.AccAddress
}
// Account contains a privkey, pubkey, address tuple
// eventually more useful data can be placed in here.
// (e.g. number of coins)
type Account struct {
PrivKey crypto.PrivKey
PubKey crypto.PubKey
Address sdk.AccAddress
}
mockValidator struct {
val abci.ValidatorUpdate
livenessState int
}
// are two accounts equal
func (acc Account) Equals(acc2 Account) bool {
return acc.Address.Equals(acc2.Address)
}
// FutureOperation is an operation which will be ran at the
// beginning of the provided BlockHeight.
// If both a BlockHeight and BlockTime are specified, it will use the BlockHeight.
// In the (likely) event that multiple operations are queued at the same
// block height, they will execute in a FIFO pattern.
FutureOperation struct {
BlockHeight int
BlockTime time.Time
Op Operation
}
type mockValidator struct {
val abci.ValidatorUpdate
livenessState int
}
// WeightedOperation is an operation with associated weight.
// This is used to bias the selection operation within the simulator.
WeightedOperation struct {
Weight int
Op Operation
}
)
// FutureOperation is an operation which will be ran at the
// beginning of the provided BlockHeight.
// If both a BlockHeight and BlockTime are specified, it will use the BlockHeight.
// In the (likely) event that multiple operations are queued at the same
// block height, they will execute in a FIFO pattern.
type FutureOperation struct {
BlockHeight int
BlockTime time.Time
Op Operation
}
// WeightedOperation is an operation with associated weight.
// This is used to bias the selection operation within the simulator.
type WeightedOperation struct {
Weight int
Op Operation
}
// TODO remove? not being called anywhere
// PeriodicInvariant returns an Invariant function closure that asserts
// a given invariant if the mock application's last block modulo the given
// period is congruent to the given offset.
// PeriodicInvariant returns an Invariant function closure that asserts a given
// invariant if the mock application's last block modulo the given period is
// congruent to the given offset.
func PeriodicInvariant(invariant Invariant, period int, offset int) Invariant {
return func(app *baseapp.BaseApp) error {
if int(app.LastBlockHeight())%period == offset {
@ -80,8 +83,3 @@ func PeriodicInvariant(invariant Invariant, period int, offset int) Invariant {
return nil
}
}
// nolint
func (acc Account) Equals(acc2 Account) bool {
return acc.Address.Equals(acc2.Address)
}

View File

@ -21,8 +21,8 @@ import (
// shamelessly copied from https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang#31832326
// TODO we should probably move this to tendermint/libs/common/random.go
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
@ -108,7 +108,8 @@ func addLogMessage(testingmode bool, blockLogBuilders []*strings.Builder, height
return func(x string) {}
}
// assertAllInvariants asserts a list of provided invariants against application state
// assertAllInvariants asserts a list of provided invariants against
// application state
func assertAllInvariants(t *testing.T, app *baseapp.BaseApp,
invariants []Invariant, where string, displayLogs func()) {
@ -146,7 +147,10 @@ func logPrinter(testingmode bool, logs []*strings.Builder) func() {
}
var f *os.File
if numLoggers > 10 {
fileName := fmt.Sprintf("simulation_log_%s.txt", time.Now().Format("2006-01-02 15:04:05"))
fileName := fmt.Sprintf("simulation_log_%s.txt",
time.Now().Format("2006-01-02 15:04:05"))
fmt.Printf("Too many logs to display, instead writing to %s\n", fileName)
f, _ = os.Create(fileName)
}