From b9f3db1be8b9978094265407396fc864e19c00cd Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Thu, 11 Mar 2021 07:59:13 -0800 Subject: [PATCH] all: skip noisy/faulty benchmarks + add b.ReportAllocs for every benchmark (#8856) * Skips very noisy benchmarks that end up running only for b.N=1 because their entire time is spent in setup, and varying parameters doesn't change much given that the number of stores is what dominates the expense. To ensure we can provide reliable benchmarks, progressively for the project, skip these until there is a proper re-work of what the benchmarks need to do * Previously sub-benchmarks: b.Run(...) did not b.ReportAllocs() due to a faulty assumption that invoking b.ReportAllocs() at the top would be inherited by all sub-benchmarks. This change fixes that Fixes #8779 Fixes #8855 Co-authored-by: Alessio Treglia Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- crypto/armor_test.go | 3 +-- crypto/types/compact_bit_array_test.go | 1 + store/rootmulti/store_test.go | 4 ++++ types/coin_benchmark_test.go | 2 ++ x/auth/ante/sigverify_benchmark_test.go | 2 ++ x/auth/keeper/keeper_bench_test.go | 1 + x/bank/bench_test.go | 1 + 7 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crypto/armor_test.go b/crypto/armor_test.go index fb80b5266a..8c7c0c5257 100644 --- a/crypto/armor_test.go +++ b/crypto/armor_test.go @@ -158,12 +158,11 @@ func TestUnarmorInfoBytesErrors(t *testing.T) { } func BenchmarkBcryptGenerateFromPassword(b *testing.B) { - b.ReportAllocs() - passphrase := []byte("passphrase") for securityParam := 9; securityParam < 16; securityParam++ { param := securityParam b.Run(fmt.Sprintf("benchmark-security-param-%d", param), func(b *testing.B) { + b.ReportAllocs() saltBytes := tmcrypto.CRandBytes(16) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/crypto/types/compact_bit_array_test.go b/crypto/types/compact_bit_array_test.go index be4b87a23c..44f97d6f40 100644 --- a/crypto/types/compact_bit_array_test.go +++ b/crypto/types/compact_bit_array_test.go @@ -233,6 +233,7 @@ func BenchmarkNumTrueBitsBefore(b *testing.B) { ba, _ := randCompactBitArray(100) b.Run("new", func(b *testing.B) { + b.ReportAllocs() for i := 0; i < b.N; i++ { ba.NumTrueBitsBefore(90) } diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index eafc0d6bb8..4b4028c272 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -689,6 +689,8 @@ func BenchmarkMultistoreSnapshotRestore1M(b *testing.B) { } func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) { + b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.") + b.ReportAllocs() b.StopTimer() source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys) @@ -717,6 +719,8 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) { } func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys uint64) { + b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.") + b.ReportAllocs() b.StopTimer() source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys) diff --git a/types/coin_benchmark_test.go b/types/coin_benchmark_test.go index c6e0401121..8c8088923e 100644 --- a/types/coin_benchmark_test.go +++ b/types/coin_benchmark_test.go @@ -13,6 +13,7 @@ func BenchmarkCoinsAdditionIntersect(b *testing.B) { b.ReportAllocs() benchmarkingFunc := func(numCoinsA int, numCoinsB int) func(b *testing.B) { return func(b *testing.B) { + b.ReportAllocs() coinsA := Coins(make([]Coin, numCoinsA)) coinsB := Coins(make([]Coin, numCoinsB)) @@ -43,6 +44,7 @@ func BenchmarkCoinsAdditionNoIntersect(b *testing.B) { b.ReportAllocs() benchmarkingFunc := func(numCoinsA int, numCoinsB int) func(b *testing.B) { return func(b *testing.B) { + b.ReportAllocs() coinsA := Coins(make([]Coin, numCoinsA)) coinsB := Coins(make([]Coin, numCoinsB)) diff --git a/x/auth/ante/sigverify_benchmark_test.go b/x/auth/ante/sigverify_benchmark_test.go index 683cc261a9..56e596fa6b 100644 --- a/x/auth/ante/sigverify_benchmark_test.go +++ b/x/auth/ante/sigverify_benchmark_test.go @@ -27,6 +27,7 @@ func BenchmarkSig(b *testing.B) { b.ResetTimer() b.Run("secp256k1", func(b *testing.B) { + b.ReportAllocs() for i := 0; i < b.N; i++ { ok := pkK.VerifySignature(msg, sigK) require.True(ok) @@ -34,6 +35,7 @@ func BenchmarkSig(b *testing.B) { }) b.Run("secp256r1", func(b *testing.B) { + b.ReportAllocs() for i := 0; i < b.N; i++ { ok := pkR.VerifySignature(msg, sigR) require.True(ok) diff --git a/x/auth/keeper/keeper_bench_test.go b/x/auth/keeper/keeper_bench_test.go index 1a18dff845..3e15783d96 100644 --- a/x/auth/keeper/keeper_bench_test.go +++ b/x/auth/keeper/keeper_bench_test.go @@ -26,6 +26,7 @@ func BenchmarkAccountMapperGetAccountFound(b *testing.B) { } func BenchmarkAccountMapperSetAccount(b *testing.B) { + b.ReportAllocs() app, ctx := createTestApp(false) b.ResetTimer() diff --git a/x/bank/bench_test.go b/x/bank/bench_test.go index c7da72c199..7de0682ee5 100644 --- a/x/bank/bench_test.go +++ b/x/bank/bench_test.go @@ -60,6 +60,7 @@ func BenchmarkOneBankSendTxPerBlock(b *testing.B) { } func BenchmarkOneBankMultiSendTxPerBlock(b *testing.B) { + b.ReportAllocs() // Add an account at genesis acc := authtypes.BaseAccount{ Address: addr1.String(),