cerc-io/lighthouse
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Modify Malloc Tuning (#2398)

Browse Source 
## Issue Addressed

NA

## Proposed Changes

I've noticed some of the SigP Prater nodes struggling on v1.4.0-rc.0. I suspect this is due to the changes in #2296. Specifically, the trade-off which lowered the memory footprint whilst increasing runtime on some functions.

Presently, this PR is documenting my testing on Prater.

## Additional Info

NA
This commit is contained in:
Paul Hauner 2021-06-09 02:30:06 +00:00
parent 4a6f2fac81
commit b383836418
3 changed files with 0 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -3884,7 +3884,6 @@ dependencies = [
"lazy_static", "lazy_static",
"libc", "libc",
"lighthouse_metrics", "lighthouse_metrics",
"num_cpus",
"parking_lot", "parking_lot",
] ]

1
common/malloc_utils/Cargo.toml
View File

@ -11,4 +11,3 @@ lighthouse_metrics = { path = "../lighthouse_metrics" }
lazy_static = "1.4.0" lazy_static = "1.4.0"
libc = "0.2.79" libc = "0.2.79"
parking_lot = "0.11.0" parking_lot = "0.11.0"
num_cpus = "1.13.0"

49
common/malloc_utils/src/glibc.rs
View File

@ -21,37 +21,20 @@ use std::result::Result;
/// NODES_PER_VALIDATOR * VALIDATORS_PER_ARENA * 32 = 15 * 4096 * 32 = 1.875 MiB /// NODES_PER_VALIDATOR * VALIDATORS_PER_ARENA * 32 = 15 * 4096 * 32 = 1.875 MiB
const OPTIMAL_MMAP_THRESHOLD: c_int = 2 * 1_024 * 1_024; const OPTIMAL_MMAP_THRESHOLD: c_int = 2 * 1_024 * 1_024;
/// The maximum number of arenas allowed to be created by malloc.
///
/// See `ArenaMaxSetting` docs for details.
const OPTIMAL_ARENA_MAX: ArenaMaxSetting = ArenaMaxSetting::NumCpus;
/// Constants used to configure malloc internals. /// Constants used to configure malloc internals.
/// ///
/// Source: /// Source:
/// ///
/// https://github.com/lattera/glibc/blob/895ef79e04a953cac1493863bcae29ad85657ee1/malloc/malloc.h#L115-L123 /// https://github.com/lattera/glibc/blob/895ef79e04a953cac1493863bcae29ad85657ee1/malloc/malloc.h#L115-L123
const M_MMAP_THRESHOLD: c_int = -4; const M_MMAP_THRESHOLD: c_int = -4;
const M_ARENA_MAX: c_int = -8;
/// Environment variables used to configure malloc. /// Environment variables used to configure malloc.
/// ///
/// Source: /// Source:
/// ///
/// https://man7.org/linux/man-pages/man3/mallopt.3.html /// https://man7.org/linux/man-pages/man3/mallopt.3.html
const ENV_VAR_ARENA_MAX: &str = "MALLOC_ARENA_MAX";
const ENV_VAR_MMAP_THRESHOLD: &str = "MALLOC_MMAP_THRESHOLD_"; const ENV_VAR_MMAP_THRESHOLD: &str = "MALLOC_MMAP_THRESHOLD_";
#[allow(dead_code)]
enum ArenaMaxSetting {
/// Do not set any value for MALLOC_ARENA_MAX, leave it as default.
DoNotSet,
/// Set a fixed value.
Fixed(c_int),
/// Read the number of CPUs at runtime and use that value.
NumCpus,
}
lazy_static! { lazy_static! {
pub static ref GLOBAL_LOCK: Mutex<()> = <_>::default(); pub static ref GLOBAL_LOCK: Mutex<()> = <_>::default();
} }
@ -123,20 +106,6 @@ pub fn scrape_mallinfo_metrics() {
/// Perform all configuration routines. /// Perform all configuration routines.
pub fn configure_glibc_malloc() -> Result<(), String> { pub fn configure_glibc_malloc() -> Result<(), String> {
if !env_var_present(ENV_VAR_ARENA_MAX) {
let arena_max = match OPTIMAL_ARENA_MAX {
ArenaMaxSetting::DoNotSet => None,
ArenaMaxSetting::Fixed(n) => Some(n),
ArenaMaxSetting::NumCpus => Some(num_cpus::get() as c_int),
};
if let Some(max) = arena_max {
if let Err(e) = malloc_arena_max(max) {
return Err(format!("failed (code {}) to set malloc max arena count", e));
}
}
}
if !env_var_present(ENV_VAR_MMAP_THRESHOLD) { if !env_var_present(ENV_VAR_MMAP_THRESHOLD) {
if let Err(e) = malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD) { if let Err(e) = malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD) {
return Err(format!("failed (code {}) to set malloc mmap threshold", e)); return Err(format!("failed (code {}) to set malloc mmap threshold", e));
@ -151,19 +120,6 @@ fn env_var_present(name: &str) -> bool {
env::var(name) != Err(env::VarError::NotPresent) env::var(name) != Err(env::VarError::NotPresent)
} }
/// Uses `mallopt` to set the `M_ARENA_MAX` value, specifying the number of memory arenas to be
/// created by malloc.
///
/// Generally speaking, a smaller arena count reduces memory fragmentation at the cost of memory contention
/// between threads.
///
/// ## Resources
///
/// - https://man7.org/linux/man-pages/man3/mallopt.3.html
fn malloc_arena_max(num_arenas: c_int) -> Result<(), c_int> {
into_result(mallopt(M_ARENA_MAX, num_arenas))
}
/// Uses `mallopt` to set the `M_MMAP_THRESHOLD` value, specifying the threshold where objects of this /// Uses `mallopt` to set the `M_MMAP_THRESHOLD` value, specifying the threshold where objects of this
/// size or larger are allocated via an `mmap`. /// size or larger are allocated via an `mmap`.
/// ///
@ -198,11 +154,6 @@ fn into_result(result: c_int) -> Result<(), c_int> {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn malloc_arena_max_does_not_panic() {
malloc_arena_max(2).unwrap();
}
#[test] #[test]
fn malloc_mmap_threshold_does_not_panic() { fn malloc_mmap_threshold_does_not_panic() {
malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD).unwrap(); malloc_mmap_threshold(OPTIMAL_MMAP_THRESHOLD).unwrap();