From b3838364185dfa0a48f8f5491bd84769f5895513 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Wed, 9 Jun 2021 02:30:06 +0000
Subject: [PATCH] Modify Malloc Tuning (#2398)

## Issue Addressed

NA

## Proposed Changes

I've noticed some of the SigP Prater nodes struggling on v1.4.0-rc.0. I suspect this is due to the changes in #2296. Specifically, the trade-off which lowered the memory footprint whilst increasing runtime on some functions.

Presently, this PR is documenting my testing on Prater.

## Additional Info

NA
---
 Cargo.lock                       |  1 -
 common/malloc_utils/Cargo.toml   |  1 -
 common/malloc_utils/src/glibc.rs | 49 --------------------------------
 3 files changed, 51 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9f9924308..83643ca07 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3884,7 +3884,6 @@ dependencies = [
  "lazy_static",
  "libc",
  "lighthouse_metrics",
- "num_cpus",
  "parking_lot",
 ]
 
diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml
index 89cc0d362..685c52421 100644
--- a/common/malloc_utils/Cargo.toml
+++ b/common/malloc_utils/Cargo.toml
@@ -11,4 +11,3 @@ lighthouse_metrics = { path = "../lighthouse_metrics" }
 lazy_static = "1.4.0"
 libc = "0.2.79"
 parking_lot = "0.11.0"
-num_cpus = "1.13.0"
diff --git a/common/malloc_utils/src/glibc.rs b/common/malloc_utils/src/glibc.rs
index 9cefc6441..861deec4d 100644
--- a/common/malloc_utils/src/glibc.rs
+++ b/common/malloc_utils/src/glibc.rs
@@ -21,37 +21,20 @@ use std::result::Result;
 /// NODES_PER_VALIDATOR * VALIDATORS_PER_ARENA * 32 = 15 * 4096 * 32 = 1.875 MiB
 const OPTIMAL_MMAP_THRESHOLD: c_int = 2 * 1_024 * 1_024;
 
-/// The maximum number of arenas allowed to be created by malloc.
-///
-/// See `ArenaMaxSetting` docs for details.
-const OPTIMAL_ARENA_MAX: ArenaMaxSetting = ArenaMaxSetting::NumCpus;
-
 /// Constants used to configure malloc internals.
 ///
 /// Source:
 ///
 /// https://github.com/lattera/glibc/blob/895ef79e04a953cac1493863bcae29ad85657ee1/malloc/malloc.h#L115-L123
 const M_MMAP_THRESHOLD: c_int = -4;
-const M_ARENA_MAX: c_int = -8;
 
 /// Environment variables used to configure malloc.
 ///
 /// Source:
 ///
 /// https://man7.org/linux/man-pages/man3/mallopt.3.html
-const ENV_VAR_ARENA_MAX: &str = "MALLOC_ARENA_MAX";
 const ENV_VAR_MMAP_THRESHOLD: &str = "MALLOC_MMAP_THRESHOLD_";
 
-#[allow(dead_code)]
-enum ArenaMaxSetting {
-    /// Do not set any value for MALLOC_ARENA_MAX, leave it as default.
-    DoNotSet,
-    /// Set a fixed value.
-    Fixed(c_int),
-    /// Read the number of CPUs at runtime and use that value.
-    NumCpus,
-}
-
 lazy_static! {
     pub static ref GLOBAL_LOCK: Mutex<()> = <_>::default();
 }
@@ -123,20 +106,6 @@ pub fn scrape_mallinfo_metrics() {
 
 /// Perform all configuration routines.
 pub fn configure_glibc_malloc() -> Result<(), String> {
-    if !env_var_present(ENV_VAR_ARENA_MAX) {
-        let arena_max = match OPTIMAL_ARENA_MAX {
-            ArenaMaxSetting::DoNotSet => None,
-            ArenaMaxSetting::Fixed(n) => Some(n),
-            ArenaMaxSetting::NumCpus => Some(num_cpus::get() as c_int),
-        };
-
-        if let Some(max) = arena_max {
-            if let Err(e) = malloc_arena_max(max) {
-                return Err(format!("failed (code {}) to set malloc max arena count", e));
-            }
-        }
-    }
-
     if !env_var_present(ENV_VAR_MMAP_THRESHOLD) {
         if let Err(e) = malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD) {
             return Err(format!("failed (code {}) to set malloc mmap threshold", e));
@@ -151,19 +120,6 @@ fn env_var_present(name: &str) -> bool {
     env::var(name) != Err(env::VarError::NotPresent)
 }
 
-/// Uses `mallopt` to set the `M_ARENA_MAX` value, specifying the number of memory arenas to be
-/// created by malloc.
-///
-/// Generally speaking, a smaller arena count reduces memory fragmentation at the cost of memory contention
-/// between threads.
-///
-/// ## Resources
-///
-/// - https://man7.org/linux/man-pages/man3/mallopt.3.html
-fn malloc_arena_max(num_arenas: c_int) -> Result<(), c_int> {
-    into_result(mallopt(M_ARENA_MAX, num_arenas))
-}
-
 /// Uses `mallopt` to set the `M_MMAP_THRESHOLD` value, specifying the threshold where objects of this
 /// size or larger are allocated via an `mmap`.
 ///
@@ -198,11 +154,6 @@ fn into_result(result: c_int) -> Result<(), c_int> {
 mod tests {
     use super::*;
 
-    #[test]
-    fn malloc_arena_max_does_not_panic() {
-        malloc_arena_max(2).unwrap();
-    }
-
     #[test]
     fn malloc_mmap_threshold_does_not_panic() {
         malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD).unwrap();