From f77e3bc0add947ed9a04d553734bfb6a01f46f92 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 29 Sep 2022 06:13:33 +0000 Subject: [PATCH] Add maxperf build profile (#3608) ## Proposed Changes Add a new Cargo compilation profile called `maxperf` which enables more aggressive compiler optimisations at the expense of compilation time. Some rough initial benchmarks show that this can provide up to a 25% reduction to run time for CPU bound tasks like block processing: https://docs.google.com/spreadsheets/d/15jHuZe7lLHhZq9Nw8kc6EL0Qh_N_YAYqkW2NQ_Afmtk/edit The numbers in that spreadsheet compare the `consensus-context` branch from #3604 to the same branch compiled with the `maxperf` profile using: ``` PROFILE=maxperf make install-lcli ``` ## Additional Info The downsides of the maxperf profile are: - It increases compile times substantially, which will particularly impact low-spec hardware. Compiling `lcli` is about 3x slower. Compiling Lighthouse is about 5x slower on my 5950X: 17m 38s rather than 3m 28s. As a result I think we should not enable this everywhere by default. - **Option 1**: enable by default for our released binaries. This gives the majority of users the fastest version of `lighthouse` possible, at the expense of slowing down our release CI. Source builds will continue to use the default `release` profile unless users opt-in to `maxperf`. - **Option 2**: enable by default for source builds. This gives users building from source an edge, but makes them pay for it with compilation time. I think I would prefer Option 1. I'll try doing some benchmarking to see how long a maxperf build of Lighthouse would take on GitHub actions. Credit to Nicholas Nethercote for documenting these options in the Rust Performance Book: https://nnethercote.github.io/perf-book/build-configuration.html. --- .github/workflows/docker.yml | 9 ++++++--- .github/workflows/release.yml | 30 +++++++++++++++++++----------- Cargo.toml | 6 ++++++ Makefile | 18 ++++++++++++------ book/src/cross-compiling.md | 6 ++++++ book/src/installation-source.md | 21 ++++++++++++++++++++- 6 files changed, 69 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b07f2ad3d..8d72319c6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -51,6 +51,9 @@ jobs: aarch64-portable, x86_64, x86_64-portable] + include: + - profile: maxperf + needs: [extract-version] env: # We need to enable experimental docker features in order to use `docker buildx` @@ -67,17 +70,17 @@ jobs: - name: Cross build Lighthouse binary run: | cargo install cross - make build-${{ matrix.binary }} + env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.binary }} - name: Move cross-built binary into Docker scope (if ARM) if: startsWith(matrix.binary, 'aarch64') run: | mkdir ./bin; - mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin; + mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin; - name: Move cross-built binary into Docker scope (if x86_64) if: startsWith(matrix.binary, 'x86_64') run: | mkdir ./bin; - mv ./target/x86_64-unknown-linux-gnu/release/lighthouse ./bin; + mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin; - name: Map aarch64 to arm64 short arch if: startsWith(matrix.binary, 'aarch64') run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 341e53354..158cbaa08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,20 +35,28 @@ jobs: include: - arch: aarch64-unknown-linux-gnu platform: ubuntu-latest + profile: maxperf - arch: aarch64-unknown-linux-gnu-portable platform: ubuntu-latest + profile: maxperf - arch: x86_64-unknown-linux-gnu platform: ubuntu-latest + profile: maxperf - arch: x86_64-unknown-linux-gnu-portable platform: ubuntu-latest + profile: maxperf - arch: x86_64-apple-darwin platform: macos-latest + profile: maxperf - arch: x86_64-apple-darwin-portable platform: macos-latest + profile: maxperf - arch: x86_64-windows platform: windows-2019 + profile: maxperf - arch: x86_64-windows-portable platform: windows-2019 + profile: maxperf runs-on: ${{ matrix.platform }} needs: extract-version @@ -83,49 +91,49 @@ jobs: if: matrix.arch == 'aarch64-unknown-linux-gnu-portable' run: | cargo install cross - make build-aarch64-portable + env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64-portable - name: Build Lighthouse for aarch64-unknown-linux-gnu if: matrix.arch == 'aarch64-unknown-linux-gnu' run: | cargo install cross - make build-aarch64 + env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64 - name: Build Lighthouse for x86_64-unknown-linux-gnu-portable if: matrix.arch == 'x86_64-unknown-linux-gnu-portable' run: | cargo install cross - make build-x86_64-portable + env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64-portable - name: Build Lighthouse for x86_64-unknown-linux-gnu if: matrix.arch == 'x86_64-unknown-linux-gnu' run: | cargo install cross - make build-x86_64 + env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 - name: Move cross-compiled binary if: startsWith(matrix.arch, 'aarch64') - run: mv target/aarch64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse + run: mv target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse - name: Move cross-compiled binary if: startsWith(matrix.arch, 'x86_64-unknown-linux-gnu') - run: mv target/x86_64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse + run: mv target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse - name: Build Lighthouse for x86_64-apple-darwin portable if: matrix.arch == 'x86_64-apple-darwin-portable' - run: cargo install --path lighthouse --force --locked --features portable,gnosis + run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }} - name: Build Lighthouse for x86_64-apple-darwin modern if: matrix.arch == 'x86_64-apple-darwin' - run: cargo install --path lighthouse --force --locked --features modern,gnosis + run: cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }} - name: Build Lighthouse for Windows portable if: matrix.arch == 'x86_64-windows-portable' - run: cargo install --path lighthouse --force --locked --features portable,gnosis + run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }} - name: Build Lighthouse for Windows modern if: matrix.arch == 'x86_64-windows' - run: cargo install --path lighthouse --force --locked --features modern,gnosis + run: cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }} - name: Configure GPG and create artifacts if: startsWith(matrix.arch, 'x86_64-windows') != true @@ -212,7 +220,7 @@ jobs: ## Testing Checklist (DELETE ME) - + - [ ] Run on synced Prater Sigma Prime nodes. - [ ] Run on synced Canary (mainnet) Sigma Prime nodes. - [ ] Resync a Prater node. diff --git a/Cargo.toml b/Cargo.toml index 415c721d9..27120e217 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,3 +100,9 @@ eth2_hashing = { path = "crypto/eth2_hashing" } tree_hash = { path = "consensus/tree_hash" } tree_hash_derive = { path = "consensus/tree_hash_derive" } eth2_serde_utils = { path = "consensus/serde_utils" } + +[profile.maxperf] +inherits = "release" +lto = "fat" +codegen-units = 1 +incremental = false diff --git a/Makefile b/Makefile index 3bf23a4ce..33077a6c9 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,12 @@ CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19 # List of features to use when cross-compiling. Can be overridden via the environment. CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx +# Cargo profile for Cross builds. Default is for local builds, CI uses an override. +CROSS_PROFILE ?= release + +# Cargo profile for regular builds. +PROFILE ?= release + # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. FORKS=phase0 altair merge @@ -25,11 +31,11 @@ FORKS=phase0 altair merge # # Binaries will most likely be found in `./target/release` install: - cargo install --path lighthouse --force --locked --features "$(FEATURES)" + cargo install --path lighthouse --force --locked --features "$(FEATURES)" --profile "$(PROFILE)" # Builds the lcli binary in release (optimized). install-lcli: - cargo install --path lcli --force --locked --features "$(FEATURES)" + cargo install --path lcli --force --locked --features "$(FEATURES)" --profile "$(PROFILE)" # The following commands use `cross` to build a cross-compile. # @@ -45,13 +51,13 @@ install-lcli: # optimized CPU functions that may not be available on some systems. This # results in a more portable binary with ~20% slower BLS verification. build-x86_64: - cross build --release --bin lighthouse --target x86_64-unknown-linux-gnu --features "modern,$(CROSS_FEATURES)" + cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "modern,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" build-x86_64-portable: - cross build --release --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" + cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" build-aarch64: - cross build --release --bin lighthouse --target aarch64-unknown-linux-gnu --features "$(CROSS_FEATURES)" + cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" build-aarch64-portable: - cross build --release --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" + cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary diff --git a/book/src/cross-compiling.md b/book/src/cross-compiling.md index 8ccf23da9..7cf7f4feb 100644 --- a/book/src/cross-compiling.md +++ b/book/src/cross-compiling.md @@ -44,3 +44,9 @@ in `lighthouse/target/aarch64-unknown-linux-gnu/release`. When using the makefile the set of features used for building can be controlled with the environment variable `CROSS_FEATURES`. See [Feature Flags](./installation-source.md#feature-flags) for available features. + +## Compilation Profiles + +When using the makefile the build profile can be controlled with the environment variable +`CROSS_PROFILE`. See [Compilation Profiles](./installation-source.md#compilation-profiles) for +available profiles. diff --git a/book/src/installation-source.md b/book/src/installation-source.md index 61eb20642..b3d83ef9f 100644 --- a/book/src/installation-source.md +++ b/book/src/installation-source.md @@ -120,7 +120,7 @@ You can customise the features that Lighthouse is built with using the `FEATURES variable. E.g. ``` -env FEATURES="gnosis,slasher-lmdb" make +FEATURES=gnosis,slasher-lmdb make ``` Commonly used features include: @@ -131,6 +131,25 @@ Commonly used features include: * `slasher-mdbx`: support for the MDBX slasher backend (enabled by default). * `slasher-lmdb`: support for the LMDB slasher backend. +## Compilation Profiles + +You can customise the compiler settings used to compile Lighthouse via +[Cargo profiles](https://doc.rust-lang.org/cargo/reference/profiles.html). + +Lighthouse includes several profiles which can be selected via the `PROFILE` environment variable. + +* `release`: default for source builds, enables most optimisations while not taking too long to + compile. +* `maxperf`: default for binary releases, enables aggressive optimisations including full LTO. + Although compiling with this profile improves some benchmarks by around 20% compared to `release`, + it imposes a _significant_ cost at compile time and is only recommended if you have a fast CPU. + +To compile with `maxperf`: + +``` +PROFILE=maxperf make +``` + ## Troubleshooting ### Command is not found