perf: Replace runsim with Go stdlib testing (#24045)

Co-authored-by: Alex | Interchain Labs <alex@interchainlabs.io>
This commit is contained in:
Alexander Peters
2025-03-21 15:48:00 +00:00
committed by GitHub
co-authored by Alex | Interchain Labs
parent b7c35f2568
commit bc57ce3ba7
26 changed files with 1041 additions and 788 deletions
+38 -13
View File
@@ -2,6 +2,7 @@ package cli
import (
"flag"
"time"
"github.com/cosmos/cosmos-sdk/types/simulation"
)
@@ -22,15 +23,21 @@ var (
FlagBlockSizeValue int
FlagLeanValue bool
FlagCommitValue bool
FlagOnOperationValue bool // TODO: Remove in favor of binary search for invariant violation
FlagAllInvariantsValue bool
FlagDBBackendValue string
FlagEnabledValue bool
FlagVerboseValue bool
FlagPeriodValue uint
FlagGenesisTimeValue int64
FlagSigverifyTxValue bool
FlagFauxMerkle bool
// Deprecated: This flag is unused and will be removed in a future release.
FlagPeriodValue uint
// Deprecated: This flag is unused and will be removed in a future release.
FlagEnabledValue bool
// Deprecated: This flag is unused and will be removed in a future release.
FlagOnOperationValue bool
// Deprecated: This flag is unused and will be removed in a future release.
FlagAllInvariantsValue bool
)
// GetSimulatorFlags gets the values of all the available simulation flags
@@ -47,17 +54,19 @@ func GetSimulatorFlags() {
flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block")
flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output")
flag.BoolVar(&FlagCommitValue, "Commit", false, "have the simulation commit")
flag.BoolVar(&FlagOnOperationValue, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&FlagAllInvariantsValue, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
flag.StringVar(&FlagDBBackendValue, "DBBackend", "goleveldb", "custom db backend type")
flag.BoolVar(&FlagCommitValue, "Commit", true, "have the simulation commit")
flag.StringVar(&FlagDBBackendValue, "DBBackend", "memdb", "custom db backend type: goleveldb, pebbledb, memdb")
// simulation flags
flag.BoolVar(&FlagEnabledValue, "Enabled", false, "enable the simulation")
flag.BoolVar(&FlagVerboseValue, "Verbose", false, "verbose log output")
flag.UintVar(&FlagPeriodValue, "Period", 0, "run slow invariants only once every period assertions")
flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", 0, "override genesis UNIX time instead of using a random UNIX time")
flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", time.Now().Unix(), "use current time as genesis UNIX time for default")
flag.BoolVar(&FlagSigverifyTxValue, "SigverifyTx", true, "whether to sigverify check for transaction ")
flag.BoolVar(&FlagFauxMerkle, "FauxMerkle", false, "use faux merkle instead of iavl")
flag.UintVar(&FlagPeriodValue, "Period", 0, "This parameter is unused and will be removed")
flag.BoolVar(&FlagEnabledValue, "Enabled", false, "This parameter is unused and will be removed")
flag.BoolVar(&FlagOnOperationValue, "SimulateEveryOperation", false, "This parameter is unused and will be removed")
flag.BoolVar(&FlagAllInvariantsValue, "PrintAllInvariants", false, "This parameter is unused and will be removed")
}
// NewConfigFromFlags creates a simulation from the retrieved values of the flags.
@@ -71,12 +80,28 @@ func NewConfigFromFlags() simulation.Config {
ExportStatsPath: FlagExportStatsPathValue,
Seed: FlagSeedValue,
InitialBlockHeight: FlagInitialBlockHeightValue,
GenesisTime: FlagGenesisTimeValue,
NumBlocks: FlagNumBlocksValue,
BlockSize: FlagBlockSizeValue,
Lean: FlagLeanValue,
Commit: FlagCommitValue,
OnOperation: FlagOnOperationValue,
AllInvariants: FlagAllInvariantsValue,
DBBackend: FlagDBBackendValue,
}
}
// GetDeprecatedFlagUsed return list of deprecated flag names that are being used.
// This function is for internal usage only and may be removed with the deprecated fields.
func GetDeprecatedFlagUsed() []string {
var usedFlags []string
for _, flagName := range []string{
"Enabled",
"SimulateEveryOperation",
"PrintAllInvariants",
"Period",
} {
if flag.Lookup(flagName) != nil {
usedFlags = append(usedFlags, flagName)
}
}
return usedFlags
}
+16 -21
View File
@@ -46,70 +46,65 @@ others state execution outcome.
# Usage
Switch to `simapp/` directory:
$ cd simapp/
To execute a completely pseudo-random simulation:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
$ go test -mod=readonly . \
-tags='sims' \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-v -timeout 24h
To execute simulation from a genesis file:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
$ go test -mod=readonly . \
-tags='sims' \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-Genesis=/path/to/genesis.json \
-v -timeout 24h
To execute simulation from a simulation params file:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
$ go test -mod=readonly . \
-tags='sims' \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-Params=/path/to/params.json \
-v -timeout 24h
To export the simulation params to a file at a given block height:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
$ go test -mod=readonly . \
-tags='sims' \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-ExportParamsPath=/path/to/params.json \
-ExportParamsHeight=50 \
-v -timeout 24h
-v -timeout 24h
To export the simulation app state (i.e genesis) to a file:
$ go test -mod=readonly github.com/cosmos/cosmos-sdk/simapp \
$ go test -mod=readonly . \
-tags='sims' \
-run=TestFullAppSimulation \
-Enabled=true \
-NumBlocks=100 \
-BlockSize=200 \
-Commit=true \
-Seed=99 \
-Period=5 \
-ExportStatePath=/path/to/genesis.json \
v -timeout 24h
-v -timeout 24h
# Params
+18 -4
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path"
"sync"
"time"
)
@@ -24,7 +25,11 @@ func NewLogWriter(testingmode bool) LogWriter {
// log writter
type StandardLogWriter struct {
Seed int64
OpEntries []OperationEntry `json:"op_entries" yaml:"op_entries"`
wMtx sync.Mutex
written bool
}
// add an entry to the log writter
@@ -34,7 +39,12 @@ func (lw *StandardLogWriter) AddEntry(opEntry OperationEntry) {
// PrintLogs - print the logs to a simulation file
func (lw *StandardLogWriter) PrintLogs() {
f := createLogFile()
lw.wMtx.Lock()
defer lw.wMtx.Unlock()
if lw.written { // print once only
return
}
f := createLogFile(lw.Seed)
defer f.Close()
for i := 0; i < len(lw.OpEntries); i++ {
@@ -44,12 +54,16 @@ func (lw *StandardLogWriter) PrintLogs() {
panic("Failed to write logs to file")
}
}
lw.written = true
}
func createLogFile() *os.File {
func createLogFile(seed int64) *os.File {
var f *os.File
fileName := fmt.Sprintf("%d.log", time.Now().UnixMilli())
var prefix string
if seed != 0 {
prefix = fmt.Sprintf("seed_%10d", seed)
}
fileName := fmt.Sprintf("%s--%d.log", prefix, time.Now().UnixNano())
folderPath := path.Join(os.ExpandEnv("$HOME"), ".simapp", "simulations")
filePath := path.Join(folderPath, fileName)
-8
View File
@@ -2,7 +2,6 @@ package simulation
import (
"encoding/json"
"fmt"
"math/rand"
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
@@ -197,12 +196,5 @@ func randomConsensusParams(r *rand.Rand, appState json.RawMessage, cdc codec.JSO
MaxAgeDuration: stakingGenesisState.Params.UnbondingTime,
},
}
bz, err := json.MarshalIndent(&consensusParams, "", " ")
if err != nil {
panic(err)
}
fmt.Printf("Selected randomly generated consensus parameters:\n%s\n", bz)
return consensusParams
}
+116 -67
View File
@@ -1,18 +1,21 @@
package simulation
import (
"bytes"
"encoding/binary"
"encoding/hex"
"fmt"
"io"
"math/rand"
"os"
"os/signal"
"syscall"
"testing"
"time"
abci "github.com/cometbft/cometbft/abci/types"
cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
"cosmossdk.io/core/header"
"cosmossdk.io/log"
"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/codec"
sdk "github.com/cosmos/cosmos-sdk/types"
@@ -65,14 +68,37 @@ func SimulateFromSeed(
config simulation.Config,
cdc codec.JSONCodec,
) (stopEarly bool, exportedParams Params, err error) {
tb.Helper()
mode, _, _ := getTestingMode(tb)
expParams, err := SimulateFromSeedX(tb, log.NewTestLogger(tb), w, app, appStateFn, randAccFn, ops, blockedAddrs, config, cdc, NewLogWriter(mode))
return false, expParams, err
}
// SimulateFromSeedX tests an application by running the provided
// operations, testing the provided invariants, but using the provided config.Seed.
func SimulateFromSeedX(
tb testing.TB,
logger log.Logger,
w io.Writer,
app *baseapp.BaseApp,
appStateFn simulation.AppStateFn,
randAccFn simulation.RandomAccountFn,
ops WeightedOperations,
blockedAddrs map[string]bool,
config simulation.Config,
cdc codec.JSONCodec,
logWriter LogWriter,
) (exportedParams Params, err error) {
tb.Helper()
// in case we have to end early, don't os.Exit so that we can run cleanup code.
testingMode, _, b := getTestingMode(tb)
r := rand.New(rand.NewSource(config.Seed))
r := rand.New(newByteSource(config.FuzzSeed, config.Seed))
params := RandomParams(r)
fmt.Fprintf(w, "Starting SimulateFromSeed with randomness created with seed %d\n", int(config.Seed))
fmt.Fprintf(w, "Randomized simulation params: \n%s\n", mustMarshalJSONIndent(params))
startTime := time.Now()
logger.Info("Starting SimulateFromSeed with randomness", "time", startTime)
logger.Debug("Randomized simulation setup", "params", mustMarshalJSONIndent(params))
timeDiff := maxTimePerBlock - minTimePerBlock
accs := randAccFn(r, params.NumKeys())
@@ -84,16 +110,11 @@ func SimulateFromSeed(
// At least 2 accounts must be added here, otherwise when executing SimulateMsgSend
// two accounts will be selected to meet the conditions from != to and it will fall into an infinite loop.
if len(accs) <= 1 {
return true, params, fmt.Errorf("at least two genesis accounts are required")
return params, fmt.Errorf("at least two genesis accounts are required")
}
config.ChainID = chainID
fmt.Printf(
"Starting the simulation from time %v (unixtime %v)\n",
blockTime.UTC().Format(time.UnixDate), blockTime.Unix(),
)
// remove module account address if they exist in accs
var tmpAccs []simulation.Account
@@ -107,7 +128,7 @@ func SimulateFromSeed(
nextValidators := validators
if len(nextValidators) == 0 {
tb.Skip("skipping: empty validator set in genesis")
return true, params, nil
return params, nil
}
var (
@@ -120,17 +141,6 @@ func SimulateFromSeed(
opCount = 0
)
// Setup code to catch SIGTERM's
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
go func() {
receivedSignal := <-c
fmt.Fprintf(w, "\nExiting early due to %s, on block %d, operation %d\n", receivedSignal, blockHeight, opCount)
err = fmt.Errorf("exited due to %s", receivedSignal)
stopEarly = true
}()
finalizeBlockReq := RandomRequestFinalizeBlock(
r,
params,
@@ -145,11 +155,10 @@ func SimulateFromSeed(
// These are operations which have been queued by previous operations
operationQueue := NewOperationQueue()
logWriter := NewLogWriter(testingMode)
blockSimulator := createBlockSimulator(
testingMode,
tb,
testingMode,
w,
params,
eventStats.Tally,
@@ -166,7 +175,7 @@ func SimulateFromSeed(
// recover logs in case of panic
defer func() {
if r := recover(); r != nil {
_, _ = fmt.Fprintf(w, "simulation halted due to panic on block %d\n", blockHeight)
logger.Error("simulation halted due to panic", "height", blockHeight)
logWriter.PrintLogs()
panic(r)
}
@@ -178,7 +187,7 @@ func SimulateFromSeed(
exportedParams = params
}
for blockHeight < int64(config.NumBlocks+config.InitialBlockHeight) && !stopEarly {
for blockHeight < int64(config.NumBlocks+config.InitialBlockHeight) {
pastTimes = append(pastTimes, blockTime)
pastVoteInfos = append(pastVoteInfos, finalizeBlockReq.DecidedLastCommit.Votes)
@@ -187,7 +196,7 @@ func SimulateFromSeed(
res, err := app.FinalizeBlock(finalizeBlockReq)
if err != nil {
return true, params, err
return params, fmt.Errorf("block finalization failed at height %d: %w", blockHeight, err)
}
ctx := app.NewContextLegacy(false, cmtproto.Header{
@@ -195,17 +204,21 @@ func SimulateFromSeed(
Time: blockTime,
ProposerAddress: proposerAddress,
ChainID: config.ChainID,
}).WithHeaderInfo(header.Info{
Height: blockHeight,
Time: blockTime,
ChainID: config.ChainID,
})
// run queued operations; ignores block size if block size is too small
numQueuedOpsRan, futureOps := runQueuedOperations(
operationQueue, int(blockHeight), tb, r, app, ctx, accs, logWriter,
tb, operationQueue, blockTime, int(blockHeight), r, app, ctx, accs, logWriter,
eventStats.Tally, config.Lean, config.ChainID,
)
numQueuedTimeOpsRan, timeFutureOps := runQueuedTimeOperations(
numQueuedTimeOpsRan, timeFutureOps := runQueuedTimeOperations(tb,
timeOperationQueue, int(blockHeight), blockTime,
tb, r, app, ctx, accs, logWriter, eventStats.Tally,
r, app, ctx, accs, logWriter, eventStats.Tally,
config.Lean, config.ChainID,
)
@@ -223,19 +236,20 @@ func SimulateFromSeed(
blockHeight++
logWriter.AddEntry(EndBlockEntry(blockHeight))
blockTime = blockTime.Add(time.Duration(minTimePerBlock) * time.Second)
blockTime = blockTime.Add(time.Duration(int64(r.Intn(int(timeDiff)))) * time.Second)
proposerAddress = validators.randomProposer(r)
logWriter.AddEntry(EndBlockEntry(blockHeight))
if config.Commit {
app.Commit()
if _, err := app.Commit(); err != nil {
return params, fmt.Errorf("commit failed at height %d: %w", blockHeight, err)
}
}
if proposerAddress == nil {
fmt.Fprintf(w, "\nSimulation stopped early as all validators have been unbonded; nobody left to propose a block!\n")
stopEarly = true
logger.Info("Simulation stopped early as all validators have been unbonded; nobody left to propose a block", "height", blockHeight)
break
}
@@ -249,7 +263,7 @@ func SimulateFromSeed(
nextValidators = updateValidators(tb, r, params, validators, res.ValidatorUpdates, eventStats.Tally)
if len(nextValidators) == 0 {
tb.Skip("skipping: empty validator set")
return true, params, nil
return params, nil
}
// update the exported params
@@ -258,22 +272,8 @@ func SimulateFromSeed(
}
}
if stopEarly {
if config.ExportStatsPath != "" {
fmt.Println("Exporting simulation statistics...")
eventStats.ExportJSON(config.ExportStatsPath)
} else {
eventStats.Print(w)
}
return true, exportedParams, err
}
fmt.Fprintf(
w,
"\nSimulation complete; Final height (blocks): %d, final time (seconds): %v, operations ran: %d\n",
blockHeight, blockTime, opCount,
)
logger.Info("Simulation complete", "height", blockHeight, "block-time", blockTime, "opsCount", opCount,
"run-time", time.Since(startTime), "app-hash", hex.EncodeToString(app.LastCommitID().Hash))
if config.ExportStatsPath != "" {
fmt.Println("Exporting simulation statistics...")
@@ -281,8 +281,7 @@ func SimulateFromSeed(
} else {
eventStats.Print(w)
}
return false, exportedParams, nil
return exportedParams, err
}
type blockSimFn func(
@@ -294,12 +293,13 @@ type blockSimFn func(
) (opCount int)
// Returns a function to simulate blocks. Written like this to avoid constant
// parameters being passed everytime, to minimize memory overhead.
func createBlockSimulator(testingMode bool, tb testing.TB, w io.Writer, params Params,
// parameters being passed every time, to minimize memory overhead.
func createBlockSimulator(tb testing.TB, printProgress bool, w io.Writer, params Params,
event func(route, op, evResult string), ops WeightedOperations,
operationQueue OperationQueue, timeOperationQueue []simulation.FutureOperation,
logWriter LogWriter, config simulation.Config,
) blockSimFn {
tb.Helper()
lastBlockSizeState := 0 // state for [4 * uniform distribution]
blocksize := 0
selectOp := ops.getSelectOpFn()
@@ -325,7 +325,7 @@ func createBlockSimulator(testingMode bool, tb testing.TB, w io.Writer, params P
for i := 0; i < blocksize; i++ {
opAndRz = append(opAndRz, opAndR{
op: selectOp(r),
rand: simulation.DeriveRand(r),
rand: r,
})
}
@@ -350,8 +350,8 @@ Comment: %s`,
queueOperations(operationQueue, timeOperationQueue, futureOps)
if testingMode && opCount%50 == 0 {
fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
if printProgress && opCount%50 == 0 {
_, _ = fmt.Fprintf(w, "\rSimulating... block %d/%d, operation %d/%d. ",
header.Height, config.NumBlocks, opCount, blocksize)
}
@@ -362,11 +362,21 @@ Comment: %s`,
}
}
func runQueuedOperations(queueOps map[int][]simulation.Operation,
height int, tb testing.TB, r *rand.Rand, app *baseapp.BaseApp,
ctx sdk.Context, accounts []simulation.Account, logWriter LogWriter,
event func(route, op, evResult string), lean bool, chainID string,
func runQueuedOperations(
tb testing.TB,
queueOps map[int][]simulation.Operation,
blockTime time.Time,
height int,
r *rand.Rand,
app *baseapp.BaseApp,
ctx sdk.Context,
accounts []simulation.Account,
logWriter LogWriter,
event func(route, op, evResult string),
lean bool,
chainID string,
) (numOpsRan int, allFutureOps []simulation.FutureOperation) {
tb.Helper()
queuedOp, ok := queueOps[height]
if !ok {
return 0, nil
@@ -398,12 +408,13 @@ func runQueuedOperations(queueOps map[int][]simulation.Operation,
return numOpsRan, allFutureOps
}
func runQueuedTimeOperations(queueOps []simulation.FutureOperation,
height int, currentTime time.Time, tb testing.TB, r *rand.Rand,
func runQueuedTimeOperations(tb testing.TB, queueOps []simulation.FutureOperation,
height int, currentTime time.Time, r *rand.Rand,
app *baseapp.BaseApp, ctx sdk.Context, accounts []simulation.Account,
logWriter LogWriter, event func(route, op, evResult string),
lean bool, chainID string,
) (numOpsRan int, allFutureOps []simulation.FutureOperation) {
tb.Helper()
// Keep all future operations
allFutureOps = make([]simulation.FutureOperation, 0)
@@ -432,3 +443,41 @@ func runQueuedTimeOperations(queueOps []simulation.FutureOperation,
return numOpsRan, allFutureOps
}
const (
rngMax = 1 << 63
rngMask = rngMax - 1
)
// byteSource offers deterministic pseudo-random numbers for math.Rand with fuzzer support.
// The 'seed' data is read in big endian to uint64. When exhausted,
// it falls back to a standard random number generator initialized with a specific 'seed' value.
type byteSource struct {
seed *bytes.Reader
fallback *rand.Rand
}
// newByteSource creates a new byteSource with a specified byte slice and seed. This gives a fixed sequence of pseudo-random numbers.
// Initially, it utilizes the byte slice. Once that's exhausted, it continues generating numbers using the provided seed.
func newByteSource(fuzzSeed []byte, seed int64) *byteSource {
return &byteSource{
seed: bytes.NewReader(fuzzSeed),
fallback: rand.New(rand.NewSource(seed)),
}
}
func (s *byteSource) Uint64() uint64 {
if s.seed.Len() < 8 {
return s.fallback.Uint64()
}
var b [8]byte
if _, err := s.seed.Read(b[:]); err != nil && err != io.EOF {
panic(err) // Should not happen.
}
return binary.BigEndian.Uint64(b[:])
}
func (s *byteSource) Int63() int64 {
return int64(s.Uint64() & rngMask)
}
func (s *byteSource) Seed(seed int64) {}