Merge pull request #1175: Randomized Module Testing

* WIP, ammend this later

* Add randomized testing suite

* Fix linting

* Auth invariant check, method to take in seed, way to run invariant check less frequently

* Fix merge conflicts

* Update bank

* Fix error on zero input by skipping it

* Add PeriodicInvariant Function

* Abstract verification / send functionality

* Fix liniting errors (PeriodicInvariant godoc)

* Update formatting and docs of randomization

* Minor refactor, update godocs

* Update godoc for mock

* Export TestAndRunTx

* fix cyclic dependencies

* Address PR most pr comments

* Fix merge conflict: Bring back codec.seal

* remove debug code, fix linting

* Fix merge conflicts

x/bank/app_test.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"math/rand"
+
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/auth"
 	"github.com/cosmos/cosmos-sdk/x/mock"
@@ -81,17 +83,19 @@ func getMockApp(t *testing.T) *mock.App {
 	return mapp
 }
 
-// getBenchmarkMockApp initializes a mock application for this module, for purposes of benchmarking
-// Any long term API support commitments do not apply to this function.
-func getBenchmarkMockApp() (*mock.App, error) {
-	mapp := mock.NewApp()
+func TestBankWithRandomMessages(t *testing.T) {
+	mapp := getMockApp(t)
+	setup := func(r *rand.Rand, keys []crypto.PrivKey) {
+		return
+	}
 
-	RegisterWire(mapp.Cdc)
-	coinKeeper := NewKeeper(mapp.AccountMapper)
-	mapp.Router().AddRoute("bank", NewHandler(coinKeeper))
-
-	err := mapp.CompleteSetup([]*sdk.KVStoreKey{})
-	return mapp, err
+	mapp.RandomizedTesting(
+		t,
+		[]mock.TestAndRunTx{TestAndRunSingleInputMsgSend},
+		[]mock.RandSetup{setup},
+		[]mock.Invariant{ModuleInvariants},
+		100, 30, 30,
+	)
 }
 
 func TestMsgSendWithAccounts(t *testing.T) {

x/bank/bench_test.go

@@ -10,6 +10,19 @@ import (
 	abci "github.com/tendermint/tendermint/abci/types"
 )
 
+// getBenchmarkMockApp initializes a mock application for this module, for purposes of benchmarking
+// Any long term API support commitments do not apply to this function.
+func getBenchmarkMockApp() (*mock.App, error) {
+	mapp := mock.NewApp()
+
+	RegisterWire(mapp.Cdc)
+	coinKeeper := NewKeeper(mapp.AccountMapper)
+	mapp.Router().AddRoute("bank", NewHandler(coinKeeper))
+
+	err := mapp.CompleteSetup([]*sdk.KVStoreKey{})
+	return mapp, err
+}
+
 func BenchmarkOneBankSendTxPerBlock(b *testing.B) {
 	benchmarkApp, _ := getBenchmarkMockApp()
 

x/bank/test_helpers.go

@@ -0,0 +1,150 @@
+package bank
+
+import (
+	"errors"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"testing"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/x/auth"
+	"github.com/cosmos/cosmos-sdk/x/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto"
+)
+
+// ModuleInvariants runs all invariants of the bank module.
+// Currently runs non-negative balance invariant and TotalCoinsInvariant
+func ModuleInvariants(t *testing.T, app *mock.App, log string) {
+	NonnegativeBalanceInvariant(t, app, log)
+	TotalCoinsInvariant(t, app, log)
+}
+
+// NonnegativeBalanceInvariant checks that all accounts in the application have non-negative balances
+func NonnegativeBalanceInvariant(t *testing.T, app *mock.App, log string) {
+	ctx := app.NewContext(false, abci.Header{})
+	accts := mock.GetAllAccounts(app.AccountMapper, ctx)
+	for _, acc := range accts {
+		coins := acc.GetCoins()
+		assert.True(t, coins.IsNotNegative(),
+			fmt.Sprintf("%s has a negative denomination of %s\n%s",
+				acc.GetAddress().String(),
+				coins.String(),
+				log),
+		)
+	}
+}
+
+// TotalCoinsInvariant checks that the sum of the coins across all accounts
+// is what is expected
+func TotalCoinsInvariant(t *testing.T, app *mock.App, log string) {
+	ctx := app.BaseApp.NewContext(false, abci.Header{})
+	totalCoins := sdk.Coins{}
+
+	chkAccount := func(acc auth.Account) bool {
+		coins := acc.GetCoins()
+		totalCoins = totalCoins.Plus(coins)
+		return false
+	}
+
+	app.AccountMapper.IterateAccounts(ctx, chkAccount)
+	require.Equal(t, app.TotalCoinsSupply, totalCoins, log)
+}
+
+// TestAndRunSingleInputMsgSend tests and runs a single msg send, with one input and one output, where both
+// accounts already exist.
+func TestAndRunSingleInputMsgSend(t *testing.T, r *rand.Rand, app *mock.App, ctx sdk.Context, keys []crypto.PrivKey, log string) (action string, err sdk.Error) {
+	fromKey := keys[r.Intn(len(keys))]
+	fromAddr := fromKey.PubKey().Address()
+	toKey := keys[r.Intn(len(keys))]
+	// Disallow sending money to yourself
+	for {
+		if !fromKey.Equals(toKey) {
+			break
+		}
+		toKey = keys[r.Intn(len(keys))]
+	}
+	toAddr := toKey.PubKey().Address()
+	initFromCoins := app.AccountMapper.GetAccount(ctx, fromAddr).GetCoins()
+
+	denomIndex := r.Intn(len(initFromCoins))
+	amt, goErr := randPositiveInt(r, initFromCoins[denomIndex].Amount)
+	if goErr != nil {
+		return "skipping bank send due to account having no coins of denomination " + initFromCoins[denomIndex].Denom, nil
+	}
+
+	action = fmt.Sprintf("%s is sending %s %s to %s",
+		fromAddr.String(),
+		amt.String(),
+		initFromCoins[denomIndex].Denom,
+		toAddr.String(),
+	)
+	log = fmt.Sprintf("%s\n%s", log, action)
+
+	coins := sdk.Coins{{initFromCoins[denomIndex].Denom, amt}}
+	var msg = MsgSend{
+		Inputs:  []Input{NewInput(fromAddr, coins)},
+		Outputs: []Output{NewOutput(toAddr, coins)},
+	}
+	sendAndVerifyMsgSend(t, app, msg, ctx, log, []crypto.PrivKey{fromKey})
+
+	return action, nil
+}
+
+// Sends and verifies the transition of a msg send. This fails if there are repeated inputs or outputs
+func sendAndVerifyMsgSend(t *testing.T, app *mock.App, msg MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) {
+	initialInputAddrCoins := make([]sdk.Coins, len(msg.Inputs))
+	initialOutputAddrCoins := make([]sdk.Coins, len(msg.Outputs))
+	AccountNumbers := make([]int64, len(msg.Inputs))
+	SequenceNumbers := make([]int64, len(msg.Inputs))
+
+	for i := 0; i < len(msg.Inputs); i++ {
+		acc := app.AccountMapper.GetAccount(ctx, msg.Inputs[i].Address)
+		AccountNumbers[i] = acc.GetAccountNumber()
+		SequenceNumbers[i] = acc.GetSequence()
+		initialInputAddrCoins[i] = acc.GetCoins()
+	}
+	for i := 0; i < len(msg.Outputs); i++ {
+		acc := app.AccountMapper.GetAccount(ctx, msg.Outputs[i].Address)
+		initialOutputAddrCoins[i] = acc.GetCoins()
+	}
+	tx := mock.GenTx([]sdk.Msg{msg},
+		AccountNumbers,
+		SequenceNumbers,
+		privkeys...)
+	res := app.Deliver(tx)
+	if !res.IsOK() {
+		// TODO: Do this in a more 'canonical' way
+		fmt.Println(res)
+		fmt.Println(log)
+		t.FailNow()
+	}
+
+	for i := 0; i < len(msg.Inputs); i++ {
+		terminalInputCoins := app.AccountMapper.GetAccount(ctx, msg.Inputs[i].Address).GetCoins()
+		require.Equal(t,
+			initialInputAddrCoins[i].Minus(msg.Inputs[i].Coins),
+			terminalInputCoins,
+			fmt.Sprintf("Input #%d had an incorrect amount of coins\n%s", i, log),
+		)
+	}
+	for i := 0; i < len(msg.Outputs); i++ {
+		terminalOutputCoins := app.AccountMapper.GetAccount(ctx, msg.Outputs[i].Address).GetCoins()
+		require.Equal(t,
+			initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins),
+			terminalOutputCoins,
+			fmt.Sprintf("Output #%d had an incorrect amount of coins\n%s", i, log),
+		)
+	}
+}
+
+func randPositiveInt(r *rand.Rand, max sdk.Int) (sdk.Int, error) {
+	if !max.GT(sdk.OneInt()) {
+		return sdk.Int{}, errors.New("max too small")
+	}
+	max = max.Sub(sdk.OneInt())
+	return sdk.NewIntFromBigInt(new(big.Int).Rand(r, max.BigInt())).Add(sdk.OneInt()), nil
+}

x/mock/app.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"math/rand"
 	"os"
 
 	bam "github.com/cosmos/cosmos-sdk/baseapp"
@@ -15,7 +16,9 @@ import (
 
 const chainID = ""
 
-// App extends an ABCI application.
+// App extends an ABCI application, but with most of its parameters exported.
+// They are exported for convenience in creating helper functions, as object
+// capabilities aren't needed for testing.
 type App struct {
 	*bam.BaseApp
 	Cdc        *wire.Codec // Cdc is public since the codec is passed into the module anyways
@@ -26,7 +29,8 @@ type App struct {
 	AccountMapper       auth.AccountMapper
 	FeeCollectionKeeper auth.FeeCollectionKeeper
 
-	GenesisAccounts []auth.Account
+	GenesisAccounts  []auth.Account
+	TotalCoinsSupply sdk.Coins
 }
 
 // NewApp partially constructs a new app on the memstore for module and genesis
@@ -43,10 +47,11 @@ func NewApp() *App {
 
 	// Create your application object
 	app := &App{
-		BaseApp:    bam.NewBaseApp("mock", cdc, logger, db),
-		Cdc:        cdc,
-		KeyMain:    sdk.NewKVStoreKey("main"),
-		KeyAccount: sdk.NewKVStoreKey("acc"),
+		BaseApp:          bam.NewBaseApp("mock", cdc, logger, db),
+		Cdc:              cdc,
+		KeyMain:          sdk.NewKVStoreKey("main"),
+		KeyAccount:       sdk.NewKVStoreKey("acc"),
+		TotalCoinsSupply: sdk.Coins{},
 	}
 
 	// Define the accountMapper
@@ -124,8 +129,8 @@ func SetGenesis(app *App, accs []auth.Account) {
 func GenTx(msgs []sdk.Msg, accnums []int64, seq []int64, priv ...crypto.PrivKey) auth.StdTx {
 	// Make the transaction free
 	fee := auth.StdFee{
-		sdk.Coins{sdk.NewCoin("foocoin", 0)},
-		100000,
+		Amount: sdk.Coins{sdk.NewCoin("foocoin", 0)},
+		Gas:    100000,
 	}
 
 	sigs := make([]auth.StdSignature, len(priv))
@@ -148,6 +153,70 @@ func GenTx(msgs []sdk.Msg, accnums []int64, seq []int64, priv ...crypto.PrivKey)
 	return auth.NewStdTx(msgs, fee, sigs, memo)
 }
 
+// GeneratePrivKeys generates a total n Ed25519 private keys.
+func GeneratePrivKeys(n int) (keys []crypto.PrivKey) {
+	// TODO: Randomize this between ed25519 and secp256k1
+	keys = make([]crypto.PrivKey, n, n)
+	for i := 0; i < n; i++ {
+		keys[i] = crypto.GenPrivKeyEd25519()
+	}
+
+	return
+}
+
+// GeneratePrivKeyAddressPairs generates a total of n private key, address
+// pairs.
+func GeneratePrivKeyAddressPairs(n int) (keys []crypto.PrivKey, addrs []sdk.Address) {
+	keys = make([]crypto.PrivKey, n, n)
+	addrs = make([]sdk.Address, n, n)
+	for i := 0; i < n; i++ {
+		keys[i] = crypto.GenPrivKeyEd25519()
+		addrs[i] = keys[i].PubKey().Address()
+	}
+	return
+}
+
+// RandomSetGenesis set genesis accounts with random coin values using the
+// provided addresses and coin denominations.
+func RandomSetGenesis(r *rand.Rand, app *App, addrs []sdk.Address, denoms []string) {
+	accts := make([]auth.Account, len(addrs), len(addrs))
+	randCoinIntervals := []BigInterval{
+		{sdk.NewIntWithDecimal(1, 0), sdk.NewIntWithDecimal(1, 1)},
+		{sdk.NewIntWithDecimal(1, 2), sdk.NewIntWithDecimal(1, 3)},
+		{sdk.NewIntWithDecimal(1, 40), sdk.NewIntWithDecimal(1, 50)},
+	}
+
+	for i := 0; i < len(accts); i++ {
+		coins := make([]sdk.Coin, len(denoms), len(denoms))
+
+		// generate a random coin for each denomination
+		for j := 0; j < len(denoms); j++ {
+			coins[j] = sdk.Coin{Denom: denoms[j],
+				Amount: RandFromBigInterval(r, randCoinIntervals),
+			}
+		}
+
+		app.TotalCoinsSupply = app.TotalCoinsSupply.Plus(coins)
+		baseAcc := auth.NewBaseAccountWithAddress(addrs[i])
+
+		(&baseAcc).SetCoins(coins)
+		accts[i] = &baseAcc
+	}
+
+	SetGenesis(app, accts)
+}
+
+// GetAllAccounts returns all accounts in the accountMapper.
+func GetAllAccounts(mapper auth.AccountMapper, ctx sdk.Context) []auth.Account {
+	accounts := []auth.Account{}
+	appendAccount := func(acc auth.Account) (stop bool) {
+		accounts = append(accounts, acc)
+		return false
+	}
+	mapper.IterateAccounts(ctx, appendAccount)
+	return accounts
+}
+
 // GenSequenceOfTxs generates a set of signed transactions of messages, such
 // that they differ only by having the sequence numbers incremented between
 // every transaction.

x/mock/doc.go

@@ -0,0 +1,15 @@
+/*
+Package mock provides functions for creating applications for testing.
+
+This module also features randomized testing, so that various modules can test
+that their operations are interoperable.
+
+The intended method of using this randomized testing framework is that every
+module provides TestAndRunTx methods for each of its desired methods of fuzzing
+its own txs, and it also provides the invariants that it assumes to be true.
+You then pick and choose from these tx types and invariants. To pick and choose
+these, you first build a mock app with the correct keepers. Then you call the
+app.RandomizedTesting method with the set of desired txs, invariants, along
+with the setups each module requires.
+*/
+package mock

x/mock/random_simulate_blocks.go

@@ -0,0 +1,95 @@
+package mock
+
+import (
+	"fmt"
+	"math/big"
+	"math/rand"
+	"testing"
+	"time"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/stretchr/testify/require"
+	abci "github.com/tendermint/tendermint/abci/types"
+)
+
+// RandomizedTesting tests application by sending random messages.
+func (app *App) RandomizedTesting(
+	t *testing.T, ops []TestAndRunTx, setups []RandSetup,
+	invariants []Invariant, numKeys int, numBlocks int, blockSize int,
+) {
+	time := time.Now().UnixNano()
+	app.RandomizedTestingFromSeed(t, time, ops, setups, invariants, numKeys, numBlocks, blockSize)
+}
+
+// RandomizedTestingFromSeed tests an application by running the provided
+// operations, testing the provided invariants, but using the provided seed.
+func (app *App) RandomizedTestingFromSeed(
+	t *testing.T, seed int64, ops []TestAndRunTx, setups []RandSetup,
+	invariants []Invariant, numKeys int, numBlocks int, blockSize int,
+) {
+	log := fmt.Sprintf("Starting SingleModuleTest with randomness created with seed %d", int(seed))
+	keys, addrs := GeneratePrivKeyAddressPairs(numKeys)
+	r := rand.New(rand.NewSource(seed))
+
+	for i := 0; i < len(setups); i++ {
+		setups[i](r, keys)
+	}
+
+	RandomSetGenesis(r, app, addrs, []string{"foocoin"})
+	header := abci.Header{Height: 0}
+
+	for i := 0; i < numBlocks; i++ {
+		app.BeginBlock(abci.RequestBeginBlock{})
+
+		// Make sure invariants hold at beginning of block and when nothing was
+		// done.
+		app.assertAllInvariants(t, invariants, log)
+
+		ctx := app.NewContext(false, header)
+
+		// TODO: Add modes to simulate "no load", "medium load", and
+		// "high load" blocks.
+		for j := 0; j < blockSize; j++ {
+			logUpdate, err := ops[r.Intn(len(ops))](t, r, app, ctx, keys, log)
+			log += "\n" + logUpdate
+
+			require.Nil(t, err, log)
+			app.assertAllInvariants(t, invariants, log)
+		}
+
+		app.EndBlock(abci.RequestEndBlock{})
+		header.Height++
+	}
+}
+
+func (app *App) assertAllInvariants(t *testing.T, tests []Invariant, log string) {
+	for i := 0; i < len(tests); i++ {
+		tests[i](t, app, log)
+	}
+}
+
+// BigInterval is a representation of the interval [lo, hi), where
+// lo and hi are both of type sdk.Int
+type BigInterval struct {
+	lo sdk.Int
+	hi sdk.Int
+}
+
+// RandFromBigInterval chooses an interval uniformly from the provided list of
+// BigIntervals, and then chooses an element from an interval uniformly at random.
+func RandFromBigInterval(r *rand.Rand, intervals []BigInterval) sdk.Int {
+	if len(intervals) == 0 {
+		return sdk.ZeroInt()
+	}
+
+	interval := intervals[r.Intn(len(intervals))]
+
+	lo := interval.lo
+	hi := interval.hi
+
+	diff := hi.Sub(lo)
+	result := sdk.NewIntFromBigInt(new(big.Int).Rand(r, diff.BigInt()))
+	result = result.Add(lo)
+
+	return result
+}

x/mock/types.go

@@ -0,0 +1,38 @@
+package mock
+
+import (
+	"math/rand"
+	"testing"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/tendermint/tendermint/crypto"
+)
+
+type (
+	// TestAndRunTx produces a fuzzed transaction, and ensures the state
+	// transition was as expected. It returns a descriptive message "action"
+	// about what this fuzzed tx actually did, for ease of debugging.
+	TestAndRunTx func(
+		t *testing.T, r *rand.Rand, app *App, ctx sdk.Context,
+		privKeys []crypto.PrivKey, log string,
+	) (action string, err sdk.Error)
+
+	// RandSetup performs the random setup the mock module needs.
+	RandSetup func(r *rand.Rand, privKeys []crypto.PrivKey)
+
+	// An Invariant is a function which tests a particular invariant.
+	// If the invariant has been broken, the function should halt the
+	// test and output the log.
+	Invariant func(t *testing.T, app *App, log string)
+)
+
+// PeriodicInvariant returns an Invariant function closure that asserts
+// a given invariant if the mock application's last block modulo the given
+// period is congruent to the given offset.
+func PeriodicInvariant(invariant Invariant, period int, offset int) Invariant {
+	return func(t *testing.T, app *App, log string) {
+		if int(app.LastBlockHeight())%period == offset {
+			invariant(t, app, log)
+		}
+	}
+}