diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 23c4977b7..e09d2621f 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -68,6 +68,7 @@ pub struct ClientBuilder { gossipsub_registry: Option, db_path: Option, freezer_db_path: Option, + blobs_freezer_db_path: Option, http_api_config: http_api::Config, http_metrics_config: http_metrics::Config, slasher: Option>>, @@ -100,6 +101,7 @@ where gossipsub_registry: None, db_path: None, freezer_db_path: None, + blobs_freezer_db_path: None, http_api_config: <_>::default(), http_metrics_config: <_>::default(), slasher: None, @@ -892,6 +894,7 @@ where mut self, hot_path: &Path, cold_path: &Path, + cold_blobs_path: Option, config: StoreConfig, log: Logger, ) -> Result { @@ -907,6 +910,7 @@ where self.db_path = Some(hot_path.into()); self.freezer_db_path = Some(cold_path.into()); + self.blobs_freezer_db_path = cold_blobs_path; let inner_spec = spec.clone(); let deposit_contract_deploy_block = context @@ -929,6 +933,7 @@ where let store = HotColdDB::open( hot_path, cold_path, + cold_blobs_path, schema_upgrade, config, spec, diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 6c3a98a46..598945923 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -49,6 +49,9 @@ pub struct Config { pub db_name: String, /// Path where the freezer database will be located. pub freezer_db_path: Option, + /// Path where the blobs freezer database will be located if it should be separate from the + /// historical state freezer. + pub blobs_freezer_db_path: Option, pub log_file: PathBuf, /// If true, the node will use co-ordinated junk for eth1 values. /// @@ -89,6 +92,7 @@ impl Default for Config { data_dir: PathBuf::from(DEFAULT_ROOT_DIR), db_name: "chain_db".to_string(), freezer_db_path: None, + blobs_freezer_db_path: None, log_file: PathBuf::from(""), genesis: <_>::default(), store: <_>::default(), @@ -149,11 +153,28 @@ impl Config { .unwrap_or_else(|| self.default_freezer_db_path()) } + /// Returns the path to which the client may initialize the on-disk blobs freezer database. + /// + /// Will attempt to use the user-supplied path from e.g. the CLI, or will default + /// to None. + pub fn get_blobs_freezer_db_path(&self) -> Option { + self.blobs_freezer_db_path.clone() + } + /// Get the freezer DB path, creating it if necessary. pub fn create_freezer_db_path(&self) -> Result { ensure_dir_exists(self.get_freezer_db_path()) } + /// Get the blobs freezer DB path, creating it if necessary. + pub fn create_blobs_freezer_db_path(&self) -> Result, String> { + if let Some(blobs_freezer_path) = self.get_blobs_freezer_db_path() { + Ok(Some(ensure_dir_exists(blobs_freezer_path)?)) + } else { + Ok(None) + } + } + /// Returns the "modern" path to the data_dir. /// /// See `Self::get_data_dir` documentation for more info. diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index e711dfca9..a3e4dbcc8 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -28,6 +28,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Data directory for the freezer database.") .takes_value(true) ) + .arg( + Arg::with_name("blobs-freezer-dir") + .long("blobs-freezer-dir") + .value_name("DIR") + .help("Data directory for the blobs freezer database.") + .takes_value(true) + ) /* * Network parameters. */ diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 7ced91274..4fa916dc3 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -390,6 +390,10 @@ pub fn get_config( client_config.freezer_db_path = Some(PathBuf::from(freezer_dir)); } + if let Some(blobs_freezer_dir) = cli_args.value_of("blobs-freezer-dir") { + client_config.blobs_freezer_db_path = Some(PathBuf::from(blobs_freezer_dir)); + } + let (sprp, sprp_explicit) = get_slots_per_restore_point::(cli_args)?; client_config.store.slots_per_restore_point = sprp; client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit; diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index 650763dca..38e4854a5 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -64,6 +64,12 @@ impl ProductionBeaconNode { let _datadir = client_config.create_data_dir()?; let db_path = client_config.create_db_path()?; let freezer_db_path = client_config.create_freezer_db_path()?; + let blobs_freezer_db_path = + if let Some(path) = client_config.create_blobs_freezer_db_path()? { + Some(*path.as_path().clone()) + } else { + None + }; let executor = context.executor.clone(); if let Some(legacy_dir) = client_config.get_existing_legacy_data_dir() { @@ -84,7 +90,13 @@ impl ProductionBeaconNode { .runtime_context(context) .chain_spec(spec) .http_api_config(client_config.http_api.clone()) - .disk_store(&db_path, &freezer_db_path, store_config, log.clone())?; + .disk_store( + &db_path, + &freezer_db_path, + blobs_freezer_db_path, + store_config, + log.clone(), + )?; let builder = if let Some(slasher_config) = client_config.slasher.clone() { let slasher = Arc::new( diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 99b516ee9..951dbd2ca 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -35,7 +35,7 @@ use state_processing::{ use std::cmp::min; use std::convert::TryInto; use std::marker::PhantomData; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; @@ -59,6 +59,8 @@ pub struct HotColdDB, Cold: ItemStore> { pub(crate) config: StoreConfig, /// Cold database containing compact historical data. pub cold_db: Cold, + /// Cold database containing blob data with slots less than `split.slot`. + pub cold_blobs_db: Option, /// Hot database containing duplicated but quick-to-access recent data. /// /// The hot database also contains all blocks. @@ -92,6 +94,7 @@ pub enum HotColdDBError { MissingRestorePointHash(u64), MissingRestorePoint(Hash256), MissingColdStateSummary(Hash256), + MissingColdBlobs(Hash256), MissingHotStateSummary(Hash256), MissingEpochBoundaryState(Hash256), MissingSplitState(Hash256, Slot), @@ -134,6 +137,7 @@ impl HotColdDB, MemoryStore> { anchor_info: RwLock::new(None), blob_info: RwLock::new(BlobInfo::default()), cold_db: MemoryStore::open(), + cold_blobs_db: Some(MemoryStore::open()), hot_db: MemoryStore::open(), block_cache: Mutex::new(LruCache::new(config.block_cache_size)), blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)), @@ -157,6 +161,7 @@ impl HotColdDB, LevelDB> { pub fn open( hot_path: &Path, cold_path: &Path, + cold_blobs_path: Option, migrate_schema: impl FnOnce(Arc, SchemaVersion, SchemaVersion) -> Result<(), Error>, config: StoreConfig, spec: ChainSpec, @@ -164,11 +169,18 @@ impl HotColdDB, LevelDB> { ) -> Result, Error> { Self::verify_slots_per_restore_point(config.slots_per_restore_point)?; + let cold_blobs_db = if let Some(path) = cold_blobs_path { + Some(LevelDB::open(path.as_path())?) + } else { + None + }; + let mut db = HotColdDB { split: RwLock::new(Split::default()), anchor_info: RwLock::new(None), blob_info: RwLock::new(BlobInfo::default()), cold_db: LevelDB::open(cold_path)?, + cold_blobs_db, hot_db: LevelDB::open(hot_path)?, block_cache: Mutex::new(LruCache::new(config.block_cache_size)), blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)), @@ -532,7 +544,19 @@ impl, Cold: ItemStore> HotColdDB self.blob_cache.lock().put(*block_root, ret.clone()); Ok(Some(ret)) } else { - Ok(None) + let blobs_freezer = if let Some(ref cold_blobs_db) = self.cold_blobs_db { + cold_blobs_db + } else { + &self.cold_db + }; + + if let Some(ref blobs_bytes) = + blobs_freezer.get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())? + { + Ok(Some(BlobsSidecar::from_ssz_bytes(blobs_bytes)?)) + } else { + Ok(None) + } } } @@ -1918,6 +1942,13 @@ pub fn migrate_database, Cold: ItemStore>( } let mut hot_db_ops: Vec> = Vec::new(); + let mut cold_blobs_db_ops: Vec> = Vec::new(); + + let blobs_freezer = if let Some(ref cold_blobs_db) = store.cold_blobs_db { + cold_blobs_db + } else { + &store.cold_db + }; // 1. Copy all of the states between the head and the split slot, from the hot DB // to the cold DB. Delete the execution payloads of these now-finalized blocks. @@ -1961,8 +1992,17 @@ pub fn migrate_database, Cold: ItemStore>( if store.config.prune_payloads { hot_db_ops.push(StoreOp::DeleteExecutionPayload(block_root)); } + + // Prepare migration of blobs to freezer. + if let Some(blobs) = store.get_blobs(&block_root)? { + hot_db_ops.push(StoreOp::DeleteBlobs(block_root)); + cold_blobs_db_ops.push(StoreOp::PutBlobs(block_root, Arc::new(blobs))); + } } + // Migrate blobs to freezer. + blobs_freezer.do_atomically(store.convert_to_kv_batch(cold_blobs_db_ops)?)?; + // Warning: Critical section. We have to take care not to put any of the two databases in an // inconsistent state if the OS process dies at any point during the freezing // procedure. @@ -1975,6 +2015,9 @@ pub fn migrate_database, Cold: ItemStore>( // Flush to disk all the states that have just been migrated to the cold store. store.cold_db.sync()?; + if let Some(ref cold_blobs_db) = store.cold_blobs_db { + cold_blobs_db.sync()?; + } { let mut split_guard = store.split.write(); diff --git a/database_manager/src/lib.rs b/database_manager/src/lib.rs index a33e6c149..93377c60e 100644 --- a/database_manager/src/lib.rs +++ b/database_manager/src/lib.rs @@ -103,6 +103,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { ) .takes_value(true) .default_value("0"), + Arg::with_name("blobs-freezer-dir") + .long("blobs-freezer-dir") + .value_name("DIR") + .help("Data directory for the blobs freezer database.") + .takes_value(true), ) .subcommand(migrate_cli_app()) .subcommand(version_cli_app()) @@ -123,6 +128,10 @@ fn parse_client_config( client_config.freezer_db_path = Some(freezer_dir); } + if let Some(blobs_freezer_dir) = clap_utils::parse_optional(cli_args, "blobs-freezer-dir")? { + client_config.blobs_freezer_db_path = Some(blobs_freezer_dir); + } + let (sprp, sprp_explicit) = get_slots_per_restore_point::(cli_args)?; client_config.store.slots_per_restore_point = sprp; client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit; @@ -144,11 +153,13 @@ pub fn display_db_version( let spec = runtime_context.eth2_config.spec.clone(); let hot_path = client_config.get_db_path(); let cold_path = client_config.get_freezer_db_path(); + let cold_blobs_path = client_config.get_blobs_freezer_db_path(); let mut version = CURRENT_SCHEMA_VERSION; HotColdDB::, LevelDB>::open( &hot_path, &cold_path, + &cold_blobs_path, |_, from, _| { version = from; Ok(()) @@ -200,10 +211,12 @@ pub fn inspect_db( let spec = runtime_context.eth2_config.spec.clone(); let hot_path = client_config.get_db_path(); let cold_path = client_config.get_freezer_db_path(); + let cold_blobs_path = client_config.get_blobs_freezer_db_path(); let db = HotColdDB::, LevelDB>::open( &hot_path, &cold_path, + &cold_blobs_path, |_, _, _| Ok(()), client_config.store, spec, @@ -254,12 +267,14 @@ pub fn migrate_db( let spec = &runtime_context.eth2_config.spec; let hot_path = client_config.get_db_path(); let cold_path = client_config.get_freezer_db_path(); + let cold_blobs_path = client_config.get_blobs_freezer_db_path(); let mut from = CURRENT_SCHEMA_VERSION; let to = migrate_config.to; let db = HotColdDB::, LevelDB>::open( &hot_path, &cold_path, + &cold_blobs_path, |_, db_initial_version, _| { from = db_initial_version; Ok(()) @@ -294,10 +309,12 @@ pub fn prune_payloads( let spec = &runtime_context.eth2_config.spec; let hot_path = client_config.get_db_path(); let cold_path = client_config.get_freezer_db_path(); + let cold_blobs_path = client_config.get_blobs_freezer_db_path(); let db = HotColdDB::, LevelDB>::open( &hot_path, &cold_path, + &cold_blobs_path, |_, _, _| Ok(()), client_config.store, spec.clone(),