Update dependencies to get rid of the yanked deps (#4994)

* Initial attempt to upgrade hashbrown to latest to get rid of the crate warnings.

* Replace `.expect()` usage with use of `const`.

* Update ahash 0.7 as well

* Remove unsafe code.

* Update `lru` to 0.12 and fix release test errors.

* Set non-blocking socket

* Bump testcontainers to 0.15.

* Fix lint

---------

Co-authored-by: Michael Sproul <michael@sigmaprime.io>
This commit is contained in:
Jimmy Chen 2023-12-15 18:31:59 +11:00 committed by GitHub
parent f1113540d8
commit b0c374c1ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 290 additions and 136 deletions

243
Cargo.lock generated
View File

@ -130,9 +130,9 @@ dependencies = [
[[package]]
name = "ahash"
version = "0.7.6"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd"
dependencies = [
"getrandom",
"once_cell",
@ -141,13 +141,14 @@ dependencies = [
[[package]]
name = "ahash"
version = "0.8.3"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
dependencies = [
"cfg-if",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
@ -355,7 +356,7 @@ version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247"
dependencies = [
"http",
"http 0.2.9",
"log",
"url",
"wildmatch",
@ -367,7 +368,7 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2"
dependencies = [
"http",
"http 0.2.9",
"log",
"url",
]
@ -403,18 +404,19 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
version = "0.6.20"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d"
dependencies = [
"async-trait",
"axum-core",
"bitflags 1.3.2",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"http 1.0.0",
"http-body 1.0.0",
"http-body-util",
"hyper 1.0.1",
"hyper-util",
"itoa",
"matchit",
"memchr",
@ -435,17 +437,20 @@ dependencies = [
[[package]]
name = "axum-core"
version = "0.3.4"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"http 1.0.0",
"http-body 1.0.0",
"http-body-util",
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"tower-layer",
"tower-service",
]
@ -528,7 +533,7 @@ dependencies = [
"lazy_static",
"lighthouse_metrics",
"logging",
"lru 0.7.8",
"lru 0.12.0",
"maplit",
"merkle_proof",
"oneshot_broadcast",
@ -580,7 +585,7 @@ dependencies = [
"genesis",
"hex",
"http_api",
"hyper",
"hyper 1.0.1",
"lighthouse_network",
"lighthouse_version",
"monitoring_api",
@ -2474,7 +2479,7 @@ dependencies = [
"getrandom",
"hashers",
"hex",
"http",
"http 0.2.9",
"once_cell",
"parking_lot 0.11.2",
"pin-project",
@ -2548,7 +2553,7 @@ dependencies = [
"kzg",
"lazy_static",
"lighthouse_metrics",
"lru 0.7.8",
"lru 0.12.0",
"parking_lot 0.12.1",
"pretty_reqwest_error",
"rand",
@ -3052,7 +3057,7 @@ dependencies = [
"futures-core",
"futures-sink",
"futures-util",
"http",
"http 0.2.9",
"indexmap 1.9.3",
"slab",
"tokio",
@ -3060,6 +3065,25 @@ dependencies = [
"tracing",
]
[[package]]
name = "h2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d308f63daf4181410c242d34c11f928dcb3aa105852019e043c9d1f4e4368a"
dependencies = [
"bytes",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http 1.0.0",
"indexmap 2.0.2",
"slab",
"tokio",
"tokio-util 0.7.9",
"tracing",
]
[[package]]
name = "half"
version = "1.8.2"
@ -3087,7 +3111,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [
"ahash 0.7.6",
"ahash 0.7.7",
]
[[package]]
@ -3096,16 +3120,16 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash 0.7.6",
"ahash 0.7.7",
]
[[package]]
name = "hashbrown"
version = "0.14.1"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
dependencies = [
"ahash 0.8.3",
"ahash 0.8.6",
"allocator-api2",
]
@ -3133,7 +3157,7 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
dependencies = [
"hashbrown 0.14.1",
"hashbrown 0.14.3",
]
[[package]]
@ -3145,7 +3169,7 @@ dependencies = [
"base64 0.21.5",
"bytes",
"headers-core",
"http",
"http 0.2.9",
"httpdate",
"mime",
"sha1",
@ -3157,7 +3181,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
dependencies = [
"http",
"http 0.2.9",
]
[[package]]
@ -3319,6 +3343,17 @@ dependencies = [
"itoa",
]
[[package]]
name = "http"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.5"
@ -3326,7 +3361,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
dependencies = [
"bytes",
"http",
"http 0.2.9",
"pin-project-lite",
]
[[package]]
name = "http-body"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
dependencies = [
"bytes",
"http 1.0.0",
]
[[package]]
name = "http-body-util"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840"
dependencies = [
"bytes",
"futures-util",
"http 1.0.0",
"http-body 1.0.0",
"pin-project-lite",
]
@ -3353,7 +3411,7 @@ dependencies = [
"lighthouse_network",
"lighthouse_version",
"logging",
"lru 0.7.8",
"lru 0.12.0",
"network",
"operation_pool",
"parking_lot 0.12.1",
@ -3426,9 +3484,9 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"h2 0.3.21",
"http 0.2.9",
"http-body 0.4.5",
"httparse",
"httpdate",
"itoa",
@ -3440,6 +3498,25 @@ dependencies = [
"want",
]
[[package]]
name = "hyper"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"h2 0.4.0",
"http 1.0.0",
"http-body 1.0.0",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"tokio",
]
[[package]]
name = "hyper-rustls"
version = "0.24.1"
@ -3447,8 +3524,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
dependencies = [
"futures-util",
"http",
"hyper",
"http 0.2.9",
"hyper 0.14.27",
"rustls",
"tokio",
"tokio-rustls",
@ -3461,12 +3538,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes",
"hyper",
"hyper 0.14.27",
"native-tls",
"tokio",
"tokio-native-tls",
]
[[package]]
name = "hyper-util"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http 1.0.0",
"http-body 1.0.0",
"hyper 1.0.1",
"pin-project-lite",
"socket2 0.5.5",
"tokio",
"tower",
"tower-service",
"tracing",
]
[[package]]
name = "iana-time-zone"
version = "0.1.58"
@ -3579,8 +3676,8 @@ dependencies = [
"attohttpc 0.24.1",
"bytes",
"futures",
"http",
"hyper",
"http 0.2.9",
"hyper 0.14.27",
"log",
"rand",
"tokio",
@ -3667,7 +3764,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
dependencies = [
"equivalent",
"hashbrown 0.14.1",
"hashbrown 0.14.3",
]
[[package]]
@ -4585,7 +4682,7 @@ dependencies = [
"libp2p-mplex",
"lighthouse_metrics",
"lighthouse_version",
"lru 0.7.8",
"lru 0.12.0",
"lru_cache",
"parking_lot 0.12.1",
"prometheus-client",
@ -4728,7 +4825,7 @@ version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60"
dependencies = [
"hashbrown 0.14.1",
"hashbrown 0.14.3",
]
[[package]]
@ -5158,7 +5255,7 @@ dependencies = [
"lighthouse_metrics",
"lighthouse_network",
"logging",
"lru 0.7.8",
"lru 0.12.0",
"lru_cache",
"matches",
"num_cpus",
@ -6406,10 +6503,10 @@ dependencies = [
"encoding_rs",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"h2 0.3.21",
"http 0.2.9",
"http-body 0.4.5",
"hyper 0.14.27",
"hyper-rustls",
"hyper-tls",
"ipnet",
@ -6986,14 +7083,15 @@ dependencies = [
[[package]]
name = "serde_yaml"
version = "0.8.26"
version = "0.9.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b"
checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
dependencies = [
"indexmap 1.9.3",
"indexmap 2.0.2",
"itoa",
"ryu",
"serde",
"yaml-rust",
"unsafe-libyaml",
]
[[package]]
@ -7158,7 +7256,7 @@ dependencies = [
"lmdb-rkv",
"lmdb-rkv-sys",
"logging",
"lru 0.7.8",
"lru 0.12.0",
"maplit",
"parking_lot 0.12.1",
"rand",
@ -7478,7 +7576,7 @@ dependencies = [
"lazy_static",
"leveldb",
"lighthouse_metrics",
"lru 0.7.8",
"lru 0.12.0",
"parking_lot 0.12.1",
"serde",
"slog",
@ -7740,8 +7838,9 @@ dependencies = [
[[package]]
name = "testcontainers"
version = "0.14.0"
source = "git+https://github.com/testcontainers/testcontainers-rs/?rev=0f2c9851#0f2c985160e51a200cfc847097c15b8d85ed7df1"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d2931d7f521af5bae989f716c3fa43a6af9af7ec7a5e21b59ae40878cec00"
dependencies = [
"bollard-stubs",
"futures",
@ -8354,6 +8453,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "unsafe-libyaml"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
[[package]]
name = "unsigned-varint"
version = "0.6.0"
@ -8436,7 +8541,7 @@ dependencies = [
"filesystem",
"futures",
"hex",
"hyper",
"hyper 1.0.1",
"itertools",
"lazy_static",
"libsecp256k1",
@ -8579,8 +8684,8 @@ dependencies = [
"futures-channel",
"futures-util",
"headers",
"http",
"hyper",
"http 0.2.9",
"hyper 0.14.27",
"log",
"mime",
"mime_guess",
@ -8733,7 +8838,7 @@ dependencies = [
"eth2",
"hex",
"http_api",
"hyper",
"hyper 1.0.1",
"log",
"logging",
"network",
@ -9158,6 +9263,26 @@ dependencies = [
"time",
]
[[package]]
name = "zerocopy"
version = "0.7.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
]
[[package]]
name = "zeroize"
version = "1.6.0"

View File

@ -120,12 +120,12 @@ fnv = "1"
fs2 = "0.4"
futures = "0.3"
hex = "0.4"
hyper = "0.14"
hyper = "1"
itertools = "0.10"
lazy_static = "1"
libsecp256k1 = "0.7"
log = "0.4"
lru = "0.7"
lru = "0.12"
maplit = "1"
num_cpus = "1"
parking_lot = "0.12"
@ -143,7 +143,7 @@ rusqlite = { version = "0.28", features = ["bundled"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_repr = "0.1"
serde_yaml = "0.8"
serde_yaml = "0.9"
sha2 = "0.9"
slog = { version = "2", features = ["max_level_trace", "release_max_level_trace", "nested-values"] }
slog-async = "2"
@ -220,7 +220,7 @@ swap_or_not_shuffle = { path = "consensus/swap_or_not_shuffle" }
task_executor = { path = "common/task_executor" }
types = { path = "consensus/types" }
unused_port = { path = "common/unused_port" }
validator_client = { path = "validator_client/" }
validator_client = { path = "validator_client" }
validator_dir = { path = "common/validator_dir" }
warp_utils = { path = "common/warp_utils" }

View File

@ -14,13 +14,15 @@ use lru::LruCache;
use smallvec::SmallVec;
use state_processing::state_advance::partial_state_advance;
use std::cmp::Ordering;
use std::num::NonZeroUsize;
use types::non_zero_usize::new_non_zero_usize;
use types::{
BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, Fork, Hash256, Slot,
Unsigned,
};
/// The number of sets of proposer indices that should be cached.
const CACHE_SIZE: usize = 16;
const CACHE_SIZE: NonZeroUsize = new_non_zero_usize(16);
/// This value is fairly unimportant, it's used to avoid heap allocations. The result of it being
/// incorrect is non-substantial from a consensus perspective (and probably also from a

View File

@ -17,6 +17,7 @@ use slog::{debug, error, Logger};
use slot_clock::SlotClock;
use std::fmt;
use std::fmt::Debug;
use std::num::NonZeroUsize;
use std::sync::Arc;
use task_executor::TaskExecutor;
use types::beacon_block_body::{KzgCommitmentOpts, KzgCommitments};
@ -32,15 +33,17 @@ mod processing_cache;
mod state_lru_cache;
pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory};
use types::non_zero_usize::new_non_zero_usize;
/// The LRU Cache stores `PendingComponents` which can store up to
/// `MAX_BLOBS_PER_BLOCK = 6` blobs each. A `BlobSidecar` is 0.131256 MB. So
/// the maximum size of a `PendingComponents` is ~ 0.787536 MB. Setting this
/// to 1024 means the maximum size of the cache is ~ 0.8 GB. But the cache
/// will target a size of less than 75% of capacity.
pub const OVERFLOW_LRU_CAPACITY: usize = 1024;
pub const OVERFLOW_LRU_CAPACITY: NonZeroUsize = new_non_zero_usize(1024);
/// Until tree-states is implemented, we can't store very many states in memory :(
pub const STATE_LRU_CAPACITY: usize = 2;
pub const STATE_LRU_CAPACITY_NON_ZERO: NonZeroUsize = new_non_zero_usize(2);
pub const STATE_LRU_CAPACITY: usize = STATE_LRU_CAPACITY_NON_ZERO.get();
/// This includes a cache for any blocks or blobs that have been received over gossip or RPC
/// and are awaiting more components before they can be imported. Additionally the

View File

@ -42,6 +42,7 @@ use parking_lot::{Mutex, RwLock, RwLockUpgradableReadGuard};
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use ssz_types::{FixedVector, VariableList};
use std::num::NonZeroUsize;
use std::{collections::HashSet, sync::Arc};
use types::blob_sidecar::BlobIdentifier;
use types::{BlobSidecar, ChainSpec, Epoch, EthSpec, Hash256};
@ -288,7 +289,7 @@ struct Critical<T: BeaconChainTypes> {
}
impl<T: BeaconChainTypes> Critical<T> {
pub fn new(capacity: usize) -> Self {
pub fn new(capacity: NonZeroUsize) -> Self {
Self {
in_memory: LruCache::new(capacity),
store_keys: HashSet::new(),
@ -329,7 +330,7 @@ impl<T: BeaconChainTypes> Critical<T> {
pending_components: PendingComponents<T::EthSpec>,
overflow_store: &OverflowStore<T>,
) -> Result<(), AvailabilityCheckError> {
if self.in_memory.len() == self.in_memory.cap() {
if self.in_memory.len() == self.in_memory.cap().get() {
// cache will overflow, must write lru entry to disk
if let Some((lru_key, lru_value)) = self.in_memory.pop_lru() {
overflow_store.persist_pending_components(lru_key, lru_value)?;
@ -377,12 +378,12 @@ pub struct OverflowLRUCache<T: BeaconChainTypes> {
/// Mutex to guard maintenance methods which move data between disk and memory
maintenance_lock: Mutex<()>,
/// The capacity of the LRU cache
capacity: usize,
capacity: NonZeroUsize,
}
impl<T: BeaconChainTypes> OverflowLRUCache<T> {
pub fn new(
capacity: usize,
capacity: NonZeroUsize,
beacon_store: BeaconStore<T>,
spec: ChainSpec,
) -> Result<Self, AvailabilityCheckError> {
@ -514,7 +515,7 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
/// maintain the cache
pub fn do_maintenance(&self, cutoff_epoch: Epoch) -> Result<(), AvailabilityCheckError> {
// ensure memory usage is below threshold
let threshold = self.capacity * 3 / 4;
let threshold = self.capacity.get() * 3 / 4;
self.maintain_threshold(threshold, cutoff_epoch)?;
// clean up any keys on the disk that shouldn't be there
self.prune_disk(cutoff_epoch)?;
@ -753,6 +754,7 @@ mod test {
use std::ops::AddAssign;
use store::{HotColdDB, ItemStore, LevelDB, StoreConfig};
use tempfile::{tempdir, TempDir};
use types::non_zero_usize::new_non_zero_usize;
use types::{ChainSpec, ExecPayload, MinimalEthSpec};
const LOW_VALIDATOR_COUNT: usize = 32;
@ -974,8 +976,9 @@ mod test {
let harness = get_deneb_chain(log.clone(), &chain_db_path).await;
let spec = harness.spec.clone();
let test_store = harness.chain.store.clone();
let capacity_non_zero = new_non_zero_usize(capacity);
let cache = Arc::new(
OverflowLRUCache::<T>::new(capacity, test_store, spec.clone())
OverflowLRUCache::<T>::new(capacity_non_zero, test_store, spec.clone())
.expect("should create cache"),
);
(harness, cache, chain_db_path)
@ -1477,7 +1480,7 @@ mod test {
// create a new cache with the same store
let recovered_cache = OverflowLRUCache::<T>::new(
capacity,
new_non_zero_usize(capacity),
harness.chain.store.clone(),
harness.chain.spec.clone(),
)

View File

@ -1,7 +1,7 @@
use crate::block_verification_types::AsBlock;
use crate::{
block_verification_types::BlockImportData,
data_availability_checker::{AvailabilityCheckError, STATE_LRU_CAPACITY},
data_availability_checker::{AvailabilityCheckError, STATE_LRU_CAPACITY_NON_ZERO},
eth1_finalization_cache::Eth1FinalizationData,
AvailabilityPendingExecutedBlock, BeaconChainTypes, BeaconStore, PayloadVerificationOutcome,
};
@ -61,7 +61,7 @@ pub struct StateLRUCache<T: BeaconChainTypes> {
impl<T: BeaconChainTypes> StateLRUCache<T> {
pub fn new(store: BeaconStore<T>, spec: ChainSpec) -> Self {
Self {
states: RwLock::new(LruCache::new(STATE_LRU_CAPACITY)),
states: RwLock::new(LruCache::new(STATE_LRU_CAPACITY_NON_ZERO)),
store,
spec,
}

View File

@ -3,11 +3,13 @@ use itertools::process_results;
use lru::LruCache;
use parking_lot::Mutex;
use slog::debug;
use std::num::NonZeroUsize;
use std::time::Duration;
use types::non_zero_usize::new_non_zero_usize;
use types::Hash256;
const BLOCK_ROOT_CACHE_LIMIT: usize = 512;
const LOOKUP_LIMIT: usize = 8;
const BLOCK_ROOT_CACHE_LIMIT: NonZeroUsize = new_non_zero_usize(512);
const LOOKUP_LIMIT: NonZeroUsize = new_non_zero_usize(8);
const METRICS_TIMEOUT: Duration = Duration::from_millis(100);
/// Cache for rejecting attestations to blocks from before finalization.
@ -78,7 +80,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// 3. Check the network with a single block lookup.
cache.in_progress_lookups.put(block_root, ());
if cache.in_progress_lookups.len() == LOOKUP_LIMIT {
if cache.in_progress_lookups.len() == LOOKUP_LIMIT.get() {
// NOTE: we expect this to occur sometimes if a lot of blocks that we look up fail to be
// imported for reasons other than being pre-finalization. The cache will eventually
// self-repair in this case by replacing old entries with new ones until all the failed

View File

@ -8,17 +8,19 @@ use crate::HttpJsonRpc;
use lru::LruCache;
use slog::{debug, error, info, warn, Logger};
use std::future::Future;
use std::num::NonZeroUsize;
use std::sync::Arc;
use std::time::Duration;
use task_executor::TaskExecutor;
use tokio::sync::{watch, Mutex, RwLock};
use tokio_stream::wrappers::WatchStream;
use types::non_zero_usize::new_non_zero_usize;
use types::ExecutionBlockHash;
/// The number of payload IDs that will be stored for each `Engine`.
///
/// Since the size of each value is small (~800 bytes) a large number is used for safety.
const PAYLOAD_ID_LRU_CACHE_SIZE: usize = 512;
const PAYLOAD_ID_LRU_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(512);
const CACHED_ENGINE_CAPABILITIES_AGE_LIMIT: Duration = Duration::from_secs(900); // 15 minutes
/// Stores the remembered state of a engine.

View File

@ -29,6 +29,7 @@ use std::collections::HashMap;
use std::fmt;
use std::future::Future;
use std::io::Write;
use std::num::NonZeroUsize;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
@ -42,6 +43,7 @@ use tokio_stream::wrappers::WatchStream;
use tree_hash::TreeHash;
use types::beacon_block_body::KzgCommitments;
use types::builder_bid::BuilderBid;
use types::non_zero_usize::new_non_zero_usize;
use types::payload::BlockProductionVersion;
use types::{
AbstractExecPayload, BlobsList, ExecutionPayloadDeneb, KzgProofs, SignedBlindedBeaconBlock,
@ -68,7 +70,7 @@ pub const DEFAULT_JWT_FILE: &str = "jwt.hex";
/// Each time the `ExecutionLayer` retrieves a block from an execution node, it stores that block
/// in an LRU cache to avoid redundant lookups. This is the size of that cache.
const EXECUTION_BLOCKS_LRU_CACHE_SIZE: usize = 128;
const EXECUTION_BLOCKS_LRU_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(128);
/// A fee recipient address for use during block production. Only used as a very last resort if
/// there is no address provided by the user.

View File

@ -1,10 +1,12 @@
use eth2::types::FullPayloadContents;
use lru::LruCache;
use parking_lot::Mutex;
use std::num::NonZeroUsize;
use tree_hash::TreeHash;
use types::non_zero_usize::new_non_zero_usize;
use types::{EthSpec, Hash256};
pub const DEFAULT_PAYLOAD_CACHE_SIZE: usize = 10;
pub const DEFAULT_PAYLOAD_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(10);
/// A cache mapping execution payloads by tree hash roots.
pub struct PayloadCache<T: EthSpec> {

View File

@ -3,13 +3,13 @@ use eth2::lighthouse::{BlockReward, BlockRewardsQuery};
use lru::LruCache;
use slog::{debug, warn, Logger};
use state_processing::BlockReplayer;
use std::num::NonZeroUsize;
use std::sync::Arc;
use types::beacon_block::BlindedBeaconBlock;
use warp_utils::reject::{
beacon_chain_error, beacon_state_error, custom_bad_request, custom_server_error,
};
use types::non_zero_usize::new_non_zero_usize;
use warp_utils::reject::{beacon_chain_error, beacon_state_error, custom_bad_request};
const STATE_CACHE_SIZE: usize = 2;
const STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(2);
/// Fetch block rewards for blocks from the canonical chain.
pub fn get_block_rewards<T: BeaconChainTypes>(
@ -164,11 +164,7 @@ pub fn compute_block_rewards<T: BeaconChainTypes>(
.build_all_committee_caches(&chain.spec)
.map_err(beacon_state_error)?;
state_cache
.get_or_insert((parent_root, block.slot()), || state)
.ok_or_else(|| {
custom_server_error("LRU cache insert should always succeed".into())
})?
state_cache.get_or_insert((parent_root, block.slot()), || state)
};
// Compute block reward.

View File

@ -35,6 +35,7 @@ pub use libp2p::{
use lru::LruCache;
use slog::{crit, debug, error, info, trace, warn};
use ssz::Encode;
use std::num::NonZeroUsize;
use std::{
collections::{HashMap, VecDeque},
net::{IpAddr, SocketAddr},
@ -49,6 +50,7 @@ use types::{EnrForkId, EthSpec};
mod subnet_predicate;
pub use subnet_predicate::subnet_predicate;
use types::non_zero_usize::new_non_zero_usize;
/// Local ENR storage filename.
pub const ENR_FILENAME: &str = "enr.dat";
@ -70,6 +72,8 @@ const MAX_SUBNETS_IN_QUERY: usize = 3;
pub const FIND_NODE_QUERY_CLOSEST_PEERS: usize = 16;
/// The threshold for updating `min_ttl` on a connected peer.
const DURATION_DIFFERENCE: Duration = Duration::from_millis(1);
/// The capacity of the Discovery ENR cache.
const ENR_CACHE_CAPACITY: NonZeroUsize = new_non_zero_usize(50);
/// A query has completed. This result contains a mapping of discovered peer IDs to the `min_ttl`
/// of the peer if it is specified.
@ -318,7 +322,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
};
Ok(Self {
cached_enrs: LruCache::new(50),
cached_enrs: LruCache::new(ENR_CACHE_CAPACITY),
network_globals,
find_peer_active: false,
queued_queries: VecDeque::with_capacity(10),

View File

@ -2,12 +2,14 @@ use crate::{DBColumn, Error, StoreItem};
use serde::{Deserialize, Serialize};
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use std::num::NonZeroUsize;
use types::non_zero_usize::new_non_zero_usize;
use types::{EthSpec, MinimalEthSpec};
pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048;
pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192;
pub const DEFAULT_BLOCK_CACHE_SIZE: usize = 5;
pub const DEFAULT_HISTORIC_STATE_CACHE_SIZE: usize = 1;
pub const DEFAULT_BLOCK_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(5);
pub const DEFAULT_HISTORIC_STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(1);
pub const DEFAULT_EPOCHS_PER_BLOB_PRUNE: u64 = 1;
pub const DEFAULT_BLOB_PUNE_MARGIN_EPOCHS: u64 = 0;
@ -19,9 +21,9 @@ pub struct StoreConfig {
/// Flag indicating whether the `slots_per_restore_point` was set explicitly by the user.
pub slots_per_restore_point_set_explicitly: bool,
/// Maximum number of blocks to store in the in-memory block cache.
pub block_cache_size: usize,
pub block_cache_size: NonZeroUsize,
/// Maximum number of states from freezer database to store in the in-memory state cache.
pub historic_state_cache_size: usize,
pub historic_state_cache_size: NonZeroUsize,
/// Whether to compact the database on initialization.
pub compact_on_init: bool,
/// Whether to compact the database during database pruning.

View File

@ -35,6 +35,7 @@ use state_processing::{
use std::cmp::min;
use std::convert::TryInto;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
@ -85,7 +86,7 @@ struct BlockCache<E: EthSpec> {
}
impl<E: EthSpec> BlockCache<E> {
pub fn new(size: usize) -> Self {
pub fn new(size: NonZeroUsize) -> Self {
Self {
block_cache: LruCache::new(size),
blob_cache: LruCache::new(size),

View File

@ -367,7 +367,8 @@ impl ValidatorDefinitions {
pub fn save<P: AsRef<Path>>(&self, validators_dir: P) -> Result<(), Error> {
let config_path = validators_dir.as_ref().join(CONFIG_FILENAME);
let temp_path = validators_dir.as_ref().join(CONFIG_TEMP_FILENAME);
let bytes = serde_yaml::to_vec(self).map_err(Error::UnableToEncodeFile)?;
let mut bytes = vec![];
serde_yaml::to_writer(&mut bytes, self).map_err(Error::UnableToEncodeFile)?;
write_file_via_temporary(&config_path, &temp_path, &bytes)
.map_err(Error::UnableToWriteFile)?;

View File

@ -100,6 +100,7 @@ pub mod sqlite;
pub mod blob_sidecar;
pub mod light_client_header;
pub mod non_zero_usize;
use ethereum_types::{H160, H256};

View File

@ -0,0 +1,8 @@
use std::num::NonZeroUsize;
pub const fn new_non_zero_usize(x: usize) -> NonZeroUsize {
match NonZeroUsize::new(x) {
Some(n) => n,
None => panic!("Expected a non zero usize."),
}
}

View File

@ -18,6 +18,7 @@ use std::str::FromStr;
use std::string::ToString;
use std::time::Duration;
use tempfile::TempDir;
use types::non_zero_usize::new_non_zero_usize;
use types::{
Address, Checkpoint, Epoch, ExecutionBlockHash, ForkName, Hash256, MainnetEthSpec,
ProgressiveBalancesMode,
@ -1769,14 +1770,19 @@ fn block_cache_size_flag() {
CommandLineTest::new()
.flag("block-cache-size", Some("4"))
.run_with_zero_port()
.with_config(|config| assert_eq!(config.store.block_cache_size, 4_usize));
.with_config(|config| assert_eq!(config.store.block_cache_size, new_non_zero_usize(4)));
}
#[test]
fn historic_state_cache_size_flag() {
CommandLineTest::new()
.flag("historic-state-cache-size", Some("4"))
.run_with_zero_port()
.with_config(|config| assert_eq!(config.store.historic_state_cache_size, 4_usize));
.with_config(|config| {
assert_eq!(
config.store.historic_state_cache_size,
new_non_zero_usize(4)
)
});
}
#[test]
fn historic_state_cache_size_default() {
@ -2001,7 +2007,10 @@ fn slasher_attestation_cache_size_flag() {
.slasher
.as_ref()
.expect("Unable to parse Slasher config");
assert_eq!(slasher_config.attestation_root_cache_size, 10000);
assert_eq!(
slasher_config.attestation_root_cache_size,
new_non_zero_usize(10000)
);
});
}
#[test]

View File

@ -1,7 +1,9 @@
use crate::Error;
use serde::{Deserialize, Serialize};
use std::num::NonZeroUsize;
use std::path::PathBuf;
use strum::{Display, EnumString, EnumVariantNames};
use types::non_zero_usize::new_non_zero_usize;
use types::{Epoch, EthSpec, IndexedAttestation};
pub const DEFAULT_CHUNK_SIZE: usize = 16;
@ -10,7 +12,7 @@ pub const DEFAULT_HISTORY_LENGTH: usize = 4096;
pub const DEFAULT_UPDATE_PERIOD: u64 = 12;
pub const DEFAULT_SLOT_OFFSET: f64 = 10.5;
pub const DEFAULT_MAX_DB_SIZE: usize = 256 * 1024; // 256 GiB
pub const DEFAULT_ATTESTATION_ROOT_CACHE_SIZE: usize = 100_000;
pub const DEFAULT_ATTESTATION_ROOT_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(100_000);
pub const DEFAULT_BROADCAST: bool = false;
#[cfg(all(feature = "mdbx", not(feature = "lmdb")))]
@ -38,7 +40,7 @@ pub struct Config {
/// Maximum size of the database in megabytes.
pub max_db_size_mbs: usize,
/// Maximum size of the in-memory cache for attestation roots.
pub attestation_root_cache_size: usize,
pub attestation_root_cache_size: NonZeroUsize,
/// Whether to broadcast slashings found to the network.
pub broadcast: bool,
/// Database backend to use.

View File

@ -21,7 +21,7 @@ types = { workspace = true }
eth2 = { workspace = true }
beacon_node = { workspace = true }
tokio = { workspace = true }
axum = "0.6.18"
axum = "0.7"
hyper = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
@ -41,8 +41,7 @@ tokio-postgres = "0.7.5"
http_api = { workspace = true }
beacon_chain = { workspace = true }
network = { workspace = true }
# TODO: update to 0.15 when released: https://github.com/testcontainers/testcontainers-rs/issues/497
testcontainers = { git = "https://github.com/testcontainers/testcontainers-rs/", rev = "0f2c9851" }
testcontainers = "0.15"
unused_port = { workspace = true }
task_executor = { workspace = true }
logging = { workspace = true }

View File

@ -1,6 +1,5 @@
use crate::{config::Config, logger, server, updater};
use clap::{App, Arg};
use tokio::sync::oneshot;
pub const SERVE: &str = "serve";
pub const RUN_UPDATER: &str = "run-updater";
@ -44,12 +43,9 @@ pub async fn run() -> Result<(), String> {
(RUN_UPDATER, Some(_)) => updater::run_updater(config)
.await
.map_err(|e| format!("Failure: {:?}", e)),
(SERVE, Some(_)) => {
let (_shutdown_tx, shutdown_rx) = oneshot::channel();
server::serve(config, shutdown_rx)
(SERVE, Some(_)) => server::serve(config)
.await
.map_err(|e| format!("Failure: {:?}", e))
}
.map_err(|e| format!("Failure: {:?}", e)),
_ => Err("Unsupported subcommand. See --help".into()),
}
}

View File

@ -3,12 +3,14 @@ use axum::Error as AxumError;
use axum::{http::StatusCode, response::IntoResponse, Json};
use hyper::Error as HyperError;
use serde_json::json;
use std::io::Error as IoError;
#[derive(Debug)]
pub enum Error {
Axum(AxumError),
Hyper(HyperError),
Database(DbError),
IoError(IoError),
BadRequest,
NotFound,
Other(String),
@ -43,6 +45,12 @@ impl From<DbError> for Error {
}
}
impl From<IoError> for Error {
fn from(e: IoError) -> Self {
Error::IoError(e)
}
}
impl From<String> for Error {
fn from(e: String) -> Self {
Error::Other(e)

View File

@ -11,9 +11,8 @@ use axum::{
};
use eth2::types::ErrorMessage;
use log::info;
use std::future::Future;
use std::net::SocketAddr;
use tokio::sync::oneshot;
use std::future::{Future, IntoFuture};
use std::net::{SocketAddr, TcpListener};
pub use config::Config;
pub use error::Error;
@ -22,7 +21,7 @@ mod config;
mod error;
mod handler;
pub async fn serve(config: FullConfig, shutdown: oneshot::Receiver<()>) -> Result<(), Error> {
pub async fn serve(config: FullConfig) -> Result<(), Error> {
let db = database::build_connection_pool(&config.database)?;
let (_, slots_per_epoch) = database::get_active_config(&mut database::get_connection(&db)?)?
.ok_or_else(|| {
@ -32,9 +31,7 @@ pub async fn serve(config: FullConfig, shutdown: oneshot::Receiver<()>) -> Resul
)
})?;
let server = start_server(&config, slots_per_epoch as u64, db, async {
let _ = shutdown.await;
})?;
let server = start_server(&config, slots_per_epoch as u64, db)?;
server.await?;
@ -61,8 +58,7 @@ pub fn start_server(
config: &FullConfig,
slots_per_epoch: u64,
pool: PgPool,
shutdown: impl Future<Output = ()> + Send + Sync + 'static,
) -> Result<impl Future<Output = Result<(), hyper::Error>> + 'static, Error> {
) -> Result<impl Future<Output = Result<(), std::io::Error>> + 'static, Error> {
let mut routes = Router::new()
.route("/v1/slots", get(handler::get_slots_by_range))
.route("/v1/slots/:slot", get(handler::get_slot))
@ -108,16 +104,13 @@ pub fn start_server(
.layer(Extension(slots_per_epoch));
let addr = SocketAddr::new(config.server.listen_addr, config.server.listen_port);
let server = axum::Server::try_bind(&addr)?.serve(app.into_make_service());
let server = server.with_graceful_shutdown(async {
shutdown.await;
});
let listener = TcpListener::bind(addr)?;
listener.set_nonblocking(true)?;
let serve = axum::serve(tokio::net::TcpListener::from_std(listener)?, app);
info!("HTTP server listening on {}", addr);
Ok(server)
Ok(serve.into_future())
}
// The default route indicating that no available routes matched the request.

View File

@ -17,7 +17,6 @@ use std::env;
use std::net::SocketAddr;
use std::time::Duration;
use testcontainers::{clients::Cli, core::WaitFor, Image, RunnableImage};
use tokio::sync::oneshot;
use tokio::{runtime, task::JoinHandle};
use tokio_postgres::{config::Config as PostgresConfig, Client, NoTls};
use types::{Hash256, MainnetEthSpec, Slot};
@ -188,11 +187,7 @@ impl TesterBuilder {
/*
* Spawn a Watch HTTP API.
*/
let (_watch_shutdown_tx, watch_shutdown_rx) = oneshot::channel();
let watch_server = start_server(&self.config, SLOTS_PER_EPOCH, pool, async {
let _ = watch_shutdown_rx.await;
})
.unwrap();
let watch_server = start_server(&self.config, SLOTS_PER_EPOCH, pool).unwrap();
tokio::spawn(watch_server);
let addr = SocketAddr::new(
@ -228,7 +223,6 @@ impl TesterBuilder {
config: self.config,
updater,
_bn_network_rx: self._bn_network_rx,
_watch_shutdown_tx,
}
}
async fn initialize_database(&self) -> PgPool {
@ -245,7 +239,6 @@ struct Tester {
pub config: Config,
pub updater: UpdateHandler<E>,
_bn_network_rx: NetworkReceivers<E>,
_watch_shutdown_tx: oneshot::Sender<()>,
}
impl Tester {