collect all invariants for a block before stopping (#4707)

x/simulation/simulate.go

@@ -46,7 +46,7 @@ func SimulateFromSeed(
 	tb testing.TB, w io.Writer, app *baseapp.BaseApp,
 	appStateFn AppStateFn, seed int64, ops WeightedOperations,
 	invariants sdk.Invariants,
-	numBlocks, blockSize int, commit, lean, onOperation bool,
+	numBlocks, blockSize int, commit, lean, onOperation, allInvariants bool,
 ) (stopEarly bool, simError error) {
 
 	// in case we have to end early, don't os.Exit so that we can run cleanup code.
@@ -108,7 +108,7 @@ func SimulateFromSeed(
 	blockSimulator := createBlockSimulator(
 		testingMode, tb, t, w, params, eventStats.tally, invariants,
 		ops, operationQueue, timeOperationQueue,
-		numBlocks, blockSize, logWriter, lean, onOperation)
+		numBlocks, blockSize, logWriter, lean, onOperation, allInvariants)
 
 	if !testingMode {
 		b.ResetTimer()
@@ -137,7 +137,7 @@ func SimulateFromSeed(
 		app.BeginBlock(request)
 
 		if testingMode {
-			assertAllInvariants(t, app, invariants, "BeginBlock", logWriter)
+			assertAllInvariants(t, app, invariants, "BeginBlock", logWriter, allInvariants)
 		}
 
 		ctx := app.NewContext(false, header)
@@ -152,14 +152,14 @@ func SimulateFromSeed(
 			tb, r, app, ctx, accs, logWriter, eventStats.tally, lean)
 
 		if testingMode && onOperation {
-			assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter)
+			assertAllInvariants(t, app, invariants, "QueuedOperations", logWriter, allInvariants)
 		}
 
 		// run standard operations
 		operations := blockSimulator(r, app, ctx, accs, header)
 		opCount += operations + numQueuedOpsRan + numQueuedTimeOpsRan
 		if testingMode {
-			assertAllInvariants(t, app, invariants, "StandardOperations", logWriter)
+			assertAllInvariants(t, app, invariants, "StandardOperations", logWriter, allInvariants)
 		}
 
 		res := app.EndBlock(abci.RequestEndBlock{})
@@ -172,7 +172,7 @@ func SimulateFromSeed(
 		logWriter.AddEntry(EndBlockEntry(int64(height)))
 
 		if testingMode {
-			assertAllInvariants(t, app, invariants, "EndBlock", logWriter)
+			assertAllInvariants(t, app, invariants, "EndBlock", logWriter, allInvariants)
 		}
 		if commit {
 			app.Commit()
@@ -221,7 +221,7 @@ type blockSimFn func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
 func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Writer, params Params,
 	event func(string), invariants sdk.Invariants, ops WeightedOperations,
 	operationQueue OperationQueue, timeOperationQueue []FutureOperation,
-	totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation bool) blockSimFn {
+	totalNumBlocks, avgBlockSize int, logWriter LogWriter, lean, onOperation, allInvariants bool) blockSimFn {
 
 	lastBlocksizeState := 0 // state for [4 * uniform distribution]
 	blocksize := 0
@@ -269,7 +269,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, t *testing.T, w io.Wr
 					fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
 						header.Height, totalNumBlocks, opCount, blocksize)
 					eventStr := fmt.Sprintf("operation: %v", opMsg.String())
-					assertAllInvariants(t, app, invariants, eventStr, logWriter)
+					assertAllInvariants(t, app, invariants, eventStr, logWriter, allInvariants)
 				} else if opCount%50 == 0 {
 					fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
 						header.Height, totalNumBlocks, opCount, blocksize)

x/simulation/util.go

@@ -14,17 +14,30 @@ import (
 
 // assertAll asserts the all invariants against application state
 func assertAllInvariants(t *testing.T, app *baseapp.BaseApp, invs sdk.Invariants,
-	event string, logWriter LogWriter) {
+	event string, logWriter LogWriter, allInvariants bool) {
 
 	ctx := app.NewContext(false, abci.Header{Height: app.LastBlockHeight() + 1})
 
+	var broken bool
+	var invariantResults []string
 	for i := 0; i < len(invs); i++ {
-		if err := invs[i](ctx); err != nil {
-			fmt.Printf("Invariants broken after %s\n%s\n", event, err.Error())
-			logWriter.PrintLogs()
-			t.Fatal()
+		res, stop := invs[i](ctx)
+		if stop {
+			broken = true
+			invariantResults = append(invariantResults, res)
+		} else if allInvariants {
+			invariantResults = append(invariantResults, res)
 		}
 	}
+
+	if broken {
+		fmt.Printf("Invariants broken after %s\n\n", event)
+		for _, res := range invariantResults {
+			fmt.Printf("%s\n", res)
+		}
+		logWriter.PrintLogs()
+		t.Fatal()
+	}
 }
 
 func getTestingMode(tb testing.TB) (testingMode bool, t *testing.T, b *testing.B) {
@@ -66,11 +79,11 @@ func getBlockSize(r *rand.Rand, params Params,
 func PeriodicInvariants(invariants []sdk.Invariant, period, offset int) []sdk.Invariant {
 	var outInvariants []sdk.Invariant
 	for _, invariant := range invariants {
-		outInvariant := func(ctx sdk.Context) error {
+		outInvariant := func(ctx sdk.Context) (string, bool) {
 			if int(ctx.BlockHeight())%period == offset {
 				return invariant(ctx)
 			}
-			return nil
+			return "", false
 		}
 		outInvariants = append(outInvariants, outInvariant)
 	}