cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Switch allocator to jemalloc (#3697)

Browse Source 
## Proposed Changes

Another `tree-states` motivated PR, this adds `jemalloc` as the default allocator, with an option to use the system allocator by compiling with `FEATURES="" make`.

- [x] Metrics
- [x] Test on Windows
- [x] Test on macOS
- [x] Test with `musl`
- [x] Metrics dashboard on `lighthouse-metrics` (https://github.com/sigp/lighthouse-metrics/pull/37)


Co-authored-by: Michael Sproul <micsproul@gmail.com>
This commit is contained in:
Michael Sproul 2023-01-20 04:19:29 +00:00
parent f8a3b3b95a
commit 4deab888c9
13 changed files with 175 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.cargo/config.toml Normal file
View File

@ -0,0 +1,4 @@
[env]
# Set the number of arenas to 16 when using jemalloc.
JEMALLOC_SYS_WITH_MALLOC_CONF = "abort_conf:true,narenas:16"

14
.github/workflows/test-suite.yml vendored
View File

@ -306,16 +306,6 @@ jobs:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Typecheck benchmark code without running it
run: make check-benches
check-consensus:
name: check-consensus
runs-on: ubuntu-latest
needs: cargo-fmt
steps:
- uses: actions/checkout@v3
- name: Get latest version of stable Rust
run: rustup update stable
- name: Typecheck consensus code in strict mode
run: make check-consensus
clippy:
name: clippy
runs-on: ubuntu-latest
@ -382,14 +372,12 @@ jobs:
- uses: actions/checkout@v3
- name: Install Rust (${{ env.PINNED_NIGHTLY }})
run: rustup toolchain install $PINNED_NIGHTLY
# NOTE: cargo-udeps version is pinned until this issue is resolved:
# https://github.com/est31/cargo-udeps/issues/135
- name: Install Protoc
uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install cargo-udeps
run: cargo install cargo-udeps --locked --force --version 0.1.30
run: cargo install cargo-udeps --locked --force
- name: Create Cargo config dir
run: mkdir -p .cargo
- name: Install custom Cargo config

41
Cargo.lock generated
View File

@ -2695,6 +2695,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "fs_extra"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
[[package]]
name = "funty"
version = "1.1.0"
@ -3595,6 +3601,38 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
[[package]]
name = "jemalloc-ctl"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1891c671f3db85d8ea8525dd43ab147f9977041911d24a03e5a36187a7bfde9"
dependencies = [
"jemalloc-sys",
"libc",
"paste",
]
[[package]]
name = "jemalloc-sys"
version = "0.5.2+5.3.0-patched"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8"
dependencies = [
"cc",
"fs_extra",
"libc",
]
[[package]]
name = "jemallocator"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6"
dependencies = [
"jemalloc-sys",
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.60"
@ -3703,6 +3741,7 @@ dependencies = [
"lighthouse_network",
"lighthouse_version",
"log",
"malloc_utils",
"sensitive_url",
"serde",
"serde_json",
@ -4510,6 +4549,8 @@ dependencies = [
name = "malloc_utils"
version = "0.1.0"
dependencies = [
"jemalloc-ctl",
"jemallocator",
"lazy_static",
"libc",
"lighthouse_metrics",

1
Cargo.toml
View File

@ -88,6 +88,7 @@ members = [
"validator_client",
"validator_client/slashing_protection",
]
resolver = "2"
[patch]
[patch.crates-io]

14
Makefile
View File

@ -14,8 +14,16 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
PINNED_NIGHTLY ?= nightly
CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19
# List of features to use when building natively. Can be overriden via the environment.
# No jemalloc on Windows
ifeq ($(OS),Windows_NT)
FEATURES?=
else
FEATURES?=jemalloc
endif
# List of features to use when cross-compiling. Can be overridden via the environment.
CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx
CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,jemalloc
# Cargo profile for Cross builds. Default is for local builds, CI uses an override.
CROSS_PROFILE ?= release
@ -101,10 +109,6 @@ cargo-fmt:
check-benches:
cargo check --workspace --benches
# Typechecks consensus code *without* allowing deprecated legacy arithmetic or metrics.
check-consensus:
cargo check -p state_processing --no-default-features
# Runs only the ef-test vectors.
run-ef-tests:
rm -rf $(EF_TESTS)/.accessed_file_log.txt

7
book/src/installation-source.md
View File

@ -64,6 +64,7 @@ choco install protoc
These dependencies are for compiling Lighthouse natively on Windows. Lighthouse can also run
successfully under the [Windows Subsystem for Linux (WSL)][WSL]. If using Ubuntu under WSL, you
should follow the instructions for Ubuntu listed in the [Dependencies (Ubuntu)](#ubuntu) section.
[WSL]: https://docs.microsoft.com/en-us/windows/wsl/about
## Build Lighthouse
@ -128,8 +129,12 @@ Commonly used features include:
* `gnosis`: support for the Gnosis Beacon Chain.
* `portable`: support for legacy hardware.
* `modern`: support for exclusively modern hardware.
* `slasher-mdbx`: support for the MDBX slasher backend (enabled by default).
* `slasher-mdbx`: support for the MDBX slasher backend. Enabled by default.
* `slasher-lmdb`: support for the LMDB slasher backend.
* `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS.
Not supported on Windows.
[jemalloc]: https://jemalloc.net/
## Compilation Profiles

1
bors.toml
View File

@ -10,7 +10,6 @@ status = [
"merge-transition-ubuntu",
"no-eth1-simulator-ubuntu",
"check-benchmarks",
"check-consensus",
"clippy",
"arbitrary-check",
"cargo-audit",

12
common/malloc_utils/Cargo.toml
View File

@ -4,13 +4,21 @@ version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
lighthouse_metrics = { path = "../lighthouse_metrics" }
lazy_static = "1.4.0"
libc = "0.2.79"
parking_lot = "0.12.0"
jemalloc-ctl = { version = "0.5.0", optional = true }
# Jemalloc's background_threads feature requires Linux (pthreads).
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = { version = "0.5.0", optional = true, features = ["stats", "background_threads"] }
[target.'cfg(not(target_os = "linux"))'.dependencies]
jemallocator = { version = "0.5.0", optional = true, features = ["stats"] }
[features]
mallinfo2 = []
jemalloc = ["jemallocator", "jemalloc-ctl"]
jemalloc-profiling = ["jemallocator/profiling"]

52
common/malloc_utils/src/jemalloc.rs Normal file
View File

@ -0,0 +1,52 @@
//! Set the allocator to `jemalloc`.
//!
//! Due to `jemalloc` requiring configuration at compile time or immediately upon runtime
//! initialisation it is configured via a Cargo config file in `.cargo/config.toml`.
//!
//! The `jemalloc` tuning can be overriden by:
//!
//! A) `JEMALLOC_SYS_WITH_MALLOC_CONF` at compile-time.
//! B) `_RJEM_MALLOC_CONF` at runtime.
use jemalloc_ctl::{arenas, epoch, stats, Error};
use lazy_static::lazy_static;
use lighthouse_metrics::{set_gauge, try_create_int_gauge, IntGauge};
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
// Metrics for jemalloc.
lazy_static! {
pub static ref NUM_ARENAS: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_num_arenas", "The number of arenas in use");
pub static ref BYTES_ALLOCATED: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_bytes_allocated", "Equivalent to stats.allocated");
pub static ref BYTES_ACTIVE: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_bytes_active", "Equivalent to stats.active");
pub static ref BYTES_MAPPED: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_bytes_mapped", "Equivalent to stats.mapped");
pub static ref BYTES_METADATA: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_bytes_metadata", "Equivalent to stats.metadata");
pub static ref BYTES_RESIDENT: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_bytes_resident", "Equivalent to stats.resident");
pub static ref BYTES_RETAINED: lighthouse_metrics::Result<IntGauge> =
try_create_int_gauge("jemalloc_bytes_retained", "Equivalent to stats.retained");
}
pub fn scrape_jemalloc_metrics() {
scrape_jemalloc_metrics_fallible().unwrap()
}
pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> {
// Advance the epoch so that the underlying statistics are updated.
epoch::advance()?;
set_gauge(&NUM_ARENAS, arenas::narenas::read()? as i64);
set_gauge(&BYTES_ALLOCATED, stats::allocated::read()? as i64);
set_gauge(&BYTES_ACTIVE, stats::active::read()? as i64);
set_gauge(&BYTES_MAPPED, stats::mapped::read()? as i64);
set_gauge(&BYTES_METADATA, stats::metadata::read()? as i64);
set_gauge(&BYTES_RESIDENT, stats::resident::read()? as i64);
set_gauge(&BYTES_RETAINED, stats::retained::read()? as i64);
Ok(())
}

44
common/malloc_utils/src/lib.rs
View File

@ -2,18 +2,18 @@
//!
//! ## Conditional Compilation
//!
//! Presently, only configuration for "The GNU Allocator" from `glibc` is supported. All other
//! allocators are ignored.
//! This crate can be compiled with different feature flags to support different allocators:
//!
//! It is assumed that if the following two statements are correct then we should expect to
//! configure `glibc`:
//! - Jemalloc, via the `jemalloc` feature.
//! - GNU malloc, if no features are set and the system supports it.
//! - The system allocator, if no features are set and the allocator is not GNU malloc.
//!
//! It is assumed that if Jemalloc is not in use, and the following two statements are correct then
//! we should expect to configure `glibc`:
//!
//! - `target_os = linux`
//! - `target_env != musl`
//!
//! In all other cases this library will not attempt to do anything (i.e., all functions are
//! no-ops).
//!
//! If the above conditions are fulfilled but `glibc` still isn't present at runtime then a panic
//! may be triggered. It is understood that there's no way to be certain that a compatible `glibc`
//! is present: https://github.com/rust-lang/rust/issues/33244.
@ -24,18 +24,42 @@
//! detecting `glibc` are best-effort. If this crate throws errors about undefined external
//! functions, then try to compile with the `not_glibc_interface` module.
#[cfg(all(target_os = "linux", not(target_env = "musl")))]
#[cfg(all(
target_os = "linux",
not(target_env = "musl"),
not(feature = "jemalloc")
))]
mod glibc;
#[cfg(feature = "jemalloc")]
mod jemalloc;
pub use interface::*;
#[cfg(all(target_os = "linux", not(target_env = "musl")))]
#[cfg(all(
target_os = "linux",
not(target_env = "musl"),
not(feature = "jemalloc")
))]
mod interface {
pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator;
pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics;
}
#[cfg(any(not(target_os = "linux"), target_env = "musl"))]
#[cfg(feature = "jemalloc")]
mod interface {
#[allow(dead_code)]
pub fn configure_memory_allocator() -> Result<(), String> {
Ok(())
}
pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics;
}
#[cfg(all(
any(not(target_os = "linux"), target_env = "musl"),
not(feature = "jemalloc")
))]
mod interface {
#[allow(dead_code, clippy::unnecessary_wraps)]
pub fn configure_memory_allocator() -> Result<(), String> {

5
lcli/Cargo.toml
View File

@ -8,6 +8,7 @@ edition = "2021"
[features]
portable = ["bls/supranational-portable"]
fake_crypto = ['bls/fake_crypto']
jemalloc = ["malloc_utils/jemalloc"]
[dependencies]
bls = { path = "../crypto/bls" }
@ -40,3 +41,7 @@ eth2 = { path = "../common/eth2" }
snap = "1.0.1"
beacon_chain = { path = "../beacon_node/beacon_chain" }
store = { path = "../beacon_node/store" }
malloc_utils = { path = "../common/malloc_utils" }
[package.metadata.cargo-udeps.ignore]
normal = ["malloc_utils"]

2
lighthouse/Cargo.toml
View File

@ -24,6 +24,8 @@ gnosis = []
slasher-mdbx = ["slasher/mdbx"]
# Support slasher LMDB backend.
slasher-lmdb = ["slasher/lmdb"]
# Use jemalloc.
jemalloc = ["malloc_utils/jemalloc"]
[dependencies]
beacon_node = { "path" = "../beacon_node" }

10
lighthouse/src/main.rs
View File

@ -31,6 +31,14 @@ fn bls_library_name() -> &'static str {
}
}
fn allocator_name() -> &'static str {
if cfg!(feature = "jemalloc") {
"jemalloc"
} else {
"system"
}
}
fn main() {
// Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
if std::env::var("RUST_BACKTRACE").is_err() {
@ -51,10 +59,12 @@ fn main() {
"{}\n\
BLS library: {}\n\
SHA256 hardware acceleration: {}\n\
Allocator: {}\n\
Specs: mainnet (true), minimal ({}), gnosis ({})",
VERSION.replace("Lighthouse/", ""),
bls_library_name(),
have_sha_extensions(),
allocator_name(),
cfg!(feature = "spec-minimal"),
cfg!(feature = "gnosis"),
).as_str()