Store blobs in separate freezer or historical state freezer

beacon_node/client/src/builder.rs

@@ -68,6 +68,7 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     gossipsub_registry: Option<Registry>,
     db_path: Option<PathBuf>,
     freezer_db_path: Option<PathBuf>,
+    blobs_freezer_db_path: Option<PathBuf>,
     http_api_config: http_api::Config,
     http_metrics_config: http_metrics::Config,
     slasher: Option<Arc<Slasher<T::EthSpec>>>,
@@ -100,6 +101,7 @@ where
             gossipsub_registry: None,
             db_path: None,
             freezer_db_path: None,
+            blobs_freezer_db_path: None,
             http_api_config: <_>::default(),
             http_metrics_config: <_>::default(),
             slasher: None,
@@ -892,6 +894,7 @@ where
         mut self,
         hot_path: &Path,
         cold_path: &Path,
+        cold_blobs_path: Option<PathBuf>,
         config: StoreConfig,
         log: Logger,
     ) -> Result<Self, String> {
@@ -907,6 +910,7 @@ where
 
         self.db_path = Some(hot_path.into());
         self.freezer_db_path = Some(cold_path.into());
+        self.blobs_freezer_db_path = cold_blobs_path;
 
         let inner_spec = spec.clone();
         let deposit_contract_deploy_block = context
@@ -929,6 +933,7 @@ where
         let store = HotColdDB::open(
             hot_path,
             cold_path,
+            cold_blobs_path,
             schema_upgrade,
             config,
             spec,

beacon_node/client/src/config.rs

@@ -49,6 +49,9 @@ pub struct Config {
     pub db_name: String,
     /// Path where the freezer database will be located.
     pub freezer_db_path: Option<PathBuf>,
+    /// Path where the blobs freezer database will be located if it should be separate from the
+    /// historical state freezer.
+    pub blobs_freezer_db_path: Option<PathBuf>,
     pub log_file: PathBuf,
     /// If true, the node will use co-ordinated junk for eth1 values.
     ///
@@ -89,6 +92,7 @@ impl Default for Config {
             data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
+            blobs_freezer_db_path: None,
             log_file: PathBuf::from(""),
             genesis: <_>::default(),
             store: <_>::default(),
@@ -149,11 +153,28 @@ impl Config {
             .unwrap_or_else(|| self.default_freezer_db_path())
     }
 
+    /// Returns the path to which the client may initialize the on-disk blobs freezer database.
+    ///
+    /// Will attempt to use the user-supplied path from e.g. the CLI, or will default
+    /// to None.
+    pub fn get_blobs_freezer_db_path(&self) -> Option<PathBuf> {
+        self.blobs_freezer_db_path.clone()
+    }
+
     /// Get the freezer DB path, creating it if necessary.
     pub fn create_freezer_db_path(&self) -> Result<PathBuf, String> {
         ensure_dir_exists(self.get_freezer_db_path())
     }
 
+    /// Get the blobs freezer DB path, creating it if necessary.
+    pub fn create_blobs_freezer_db_path(&self) -> Result<Option<PathBuf>, String> {
+        if let Some(blobs_freezer_path) = self.get_blobs_freezer_db_path() {
+            Ok(Some(ensure_dir_exists(blobs_freezer_path)?))
+        } else {
+            Ok(None)
+        }
+    }
+
     /// Returns the "modern" path to the data_dir.
     ///
     /// See `Self::get_data_dir` documentation for more info.

beacon_node/src/cli.rs

@@ -28,6 +28,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .help("Data directory for the freezer database.")
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name("blobs-freezer-dir")
+                .long("blobs-freezer-dir")
+                .value_name("DIR")
+                .help("Data directory for the blobs freezer database.")
+                .takes_value(true)
+        )
         /*
          * Network parameters.
          */

beacon_node/src/config.rs

@@ -390,6 +390,10 @@ pub fn get_config<E: EthSpec>(
         client_config.freezer_db_path = Some(PathBuf::from(freezer_dir));
     }
 
+    if let Some(blobs_freezer_dir) = cli_args.value_of("blobs-freezer-dir") {
+        client_config.blobs_freezer_db_path = Some(PathBuf::from(blobs_freezer_dir));
+    }
+
     let (sprp, sprp_explicit) = get_slots_per_restore_point::<E>(cli_args)?;
     client_config.store.slots_per_restore_point = sprp;
     client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit;

beacon_node/src/lib.rs

@@ -64,6 +64,12 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
         let _datadir = client_config.create_data_dir()?;
         let db_path = client_config.create_db_path()?;
         let freezer_db_path = client_config.create_freezer_db_path()?;
+        let blobs_freezer_db_path =
+            if let Some(path) = client_config.create_blobs_freezer_db_path()? {
+                Some(*path.as_path().clone())
+            } else {
+                None
+            };
         let executor = context.executor.clone();
 
         if let Some(legacy_dir) = client_config.get_existing_legacy_data_dir() {
@@ -84,7 +90,13 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
             .runtime_context(context)
             .chain_spec(spec)
             .http_api_config(client_config.http_api.clone())
-            .disk_store(&db_path, &freezer_db_path, store_config, log.clone())?;
+            .disk_store(
+                &db_path,
+                &freezer_db_path,
+                blobs_freezer_db_path,
+                store_config,
+                log.clone(),
+            )?;
 
         let builder = if let Some(slasher_config) = client_config.slasher.clone() {
             let slasher = Arc::new(

beacon_node/store/src/hot_cold_store.rs

@@ -35,7 +35,7 @@ use state_processing::{
 use std::cmp::min;
 use std::convert::TryInto;
 use std::marker::PhantomData;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS;
@@ -59,6 +59,8 @@ pub struct HotColdDB<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     pub(crate) config: StoreConfig,
     /// Cold database containing compact historical data.
     pub cold_db: Cold,
+    /// Cold database containing blob data with slots less than `split.slot`.
+    pub cold_blobs_db: Option<Cold>,
     /// Hot database containing duplicated but quick-to-access recent data.
     ///
     /// The hot database also contains all blocks.
@@ -92,6 +94,7 @@ pub enum HotColdDBError {
     MissingRestorePointHash(u64),
     MissingRestorePoint(Hash256),
     MissingColdStateSummary(Hash256),
+    MissingColdBlobs(Hash256),
     MissingHotStateSummary(Hash256),
     MissingEpochBoundaryState(Hash256),
     MissingSplitState(Hash256, Slot),
@@ -134,6 +137,7 @@ impl<E: EthSpec> HotColdDB<E, MemoryStore<E>, MemoryStore<E>> {
             anchor_info: RwLock::new(None),
             blob_info: RwLock::new(BlobInfo::default()),
             cold_db: MemoryStore::open(),
+            cold_blobs_db: Some(MemoryStore::open()),
             hot_db: MemoryStore::open(),
             block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
             blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)),
@@ -157,6 +161,7 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
     pub fn open(
         hot_path: &Path,
         cold_path: &Path,
+        cold_blobs_path: Option<PathBuf>,
         migrate_schema: impl FnOnce(Arc<Self>, SchemaVersion, SchemaVersion) -> Result<(), Error>,
         config: StoreConfig,
         spec: ChainSpec,
@@ -164,11 +169,18 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
     ) -> Result<Arc<Self>, Error> {
         Self::verify_slots_per_restore_point(config.slots_per_restore_point)?;
 
+        let cold_blobs_db = if let Some(path) = cold_blobs_path {
+            Some(LevelDB::open(path.as_path())?)
+        } else {
+            None
+        };
+
         let mut db = HotColdDB {
             split: RwLock::new(Split::default()),
             anchor_info: RwLock::new(None),
             blob_info: RwLock::new(BlobInfo::default()),
             cold_db: LevelDB::open(cold_path)?,
+            cold_blobs_db,
             hot_db: LevelDB::open(hot_path)?,
             block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
             blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)),
@@ -531,10 +543,22 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
             let ret = BlobsSidecar::from_ssz_bytes(&bytes)?;
             self.blob_cache.lock().put(*block_root, ret.clone());
             Ok(Some(ret))
+        } else {
+            let blobs_freezer = if let Some(ref cold_blobs_db) = self.cold_blobs_db {
+                cold_blobs_db
+            } else {
+                &self.cold_db
+            };
+
+            if let Some(ref blobs_bytes) =
+                blobs_freezer.get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())?
+            {
+                Ok(Some(BlobsSidecar::from_ssz_bytes(blobs_bytes)?))
             } else {
                 Ok(None)
             }
         }
+    }
 
     pub fn blobs_as_kv_store_ops(
         &self,
@@ -1918,6 +1942,13 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
     }
 
     let mut hot_db_ops: Vec<StoreOp<E>> = Vec::new();
+    let mut cold_blobs_db_ops: Vec<StoreOp<E>> = Vec::new();
+
+    let blobs_freezer = if let Some(ref cold_blobs_db) = store.cold_blobs_db {
+        cold_blobs_db
+    } else {
+        &store.cold_db
+    };
 
     // 1. Copy all of the states between the head and the split slot, from the hot DB
     // to the cold DB. Delete the execution payloads of these now-finalized blocks.
@@ -1961,7 +1992,16 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
         if store.config.prune_payloads {
             hot_db_ops.push(StoreOp::DeleteExecutionPayload(block_root));
         }
+
+        // Prepare migration of blobs to freezer.
+        if let Some(blobs) = store.get_blobs(&block_root)? {
+            hot_db_ops.push(StoreOp::DeleteBlobs(block_root));
+            cold_blobs_db_ops.push(StoreOp::PutBlobs(block_root, Arc::new(blobs)));
         }
+    }
+
+    // Migrate blobs to freezer.
+    blobs_freezer.do_atomically(store.convert_to_kv_batch(cold_blobs_db_ops)?)?;
 
     // Warning: Critical section.  We have to take care not to put any of the two databases in an
     //          inconsistent state if the OS process dies at any point during the freezing
@@ -1975,6 +2015,9 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
 
     // Flush to disk all the states that have just been migrated to the cold store.
     store.cold_db.sync()?;
+    if let Some(ref cold_blobs_db) = store.cold_blobs_db {
+        cold_blobs_db.sync()?;
+    }
 
     {
         let mut split_guard = store.split.write();

database_manager/src/lib.rs

@@ -103,6 +103,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 )
                 .takes_value(true)
                 .default_value("0"),
+            Arg::with_name("blobs-freezer-dir")
+                .long("blobs-freezer-dir")
+                .value_name("DIR")
+                .help("Data directory for the blobs freezer database.")
+                .takes_value(true),
         )
         .subcommand(migrate_cli_app())
         .subcommand(version_cli_app())
@@ -123,6 +128,10 @@ fn parse_client_config<E: EthSpec>(
         client_config.freezer_db_path = Some(freezer_dir);
     }
 
+    if let Some(blobs_freezer_dir) = clap_utils::parse_optional(cli_args, "blobs-freezer-dir")? {
+        client_config.blobs_freezer_db_path = Some(blobs_freezer_dir);
+    }
+
     let (sprp, sprp_explicit) = get_slots_per_restore_point::<E>(cli_args)?;
     client_config.store.slots_per_restore_point = sprp;
     client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit;
@@ -144,11 +153,13 @@ pub fn display_db_version<E: EthSpec>(
     let spec = runtime_context.eth2_config.spec.clone();
     let hot_path = client_config.get_db_path();
     let cold_path = client_config.get_freezer_db_path();
+    let cold_blobs_path = client_config.get_blobs_freezer_db_path();
 
     let mut version = CURRENT_SCHEMA_VERSION;
     HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
         &hot_path,
         &cold_path,
+        &cold_blobs_path,
         |_, from, _| {
             version = from;
             Ok(())
@@ -200,10 +211,12 @@ pub fn inspect_db<E: EthSpec>(
     let spec = runtime_context.eth2_config.spec.clone();
     let hot_path = client_config.get_db_path();
     let cold_path = client_config.get_freezer_db_path();
+    let cold_blobs_path = client_config.get_blobs_freezer_db_path();
 
     let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
         &hot_path,
         &cold_path,
+        &cold_blobs_path,
         |_, _, _| Ok(()),
         client_config.store,
         spec,
@@ -254,12 +267,14 @@ pub fn migrate_db<E: EthSpec>(
     let spec = &runtime_context.eth2_config.spec;
     let hot_path = client_config.get_db_path();
     let cold_path = client_config.get_freezer_db_path();
+    let cold_blobs_path = client_config.get_blobs_freezer_db_path();
 
     let mut from = CURRENT_SCHEMA_VERSION;
     let to = migrate_config.to;
     let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
         &hot_path,
         &cold_path,
+        &cold_blobs_path,
         |_, db_initial_version, _| {
             from = db_initial_version;
             Ok(())
@@ -294,10 +309,12 @@ pub fn prune_payloads<E: EthSpec>(
     let spec = &runtime_context.eth2_config.spec;
     let hot_path = client_config.get_db_path();
     let cold_path = client_config.get_freezer_db_path();
+    let cold_blobs_path = client_config.get_blobs_freezer_db_path();
 
     let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
         &hot_path,
         &cold_path,
+        &cold_blobs_path,
         |_, _, _| Ok(()),
         client_config.store,
         spec.clone(),