cerc-io/lighthouse
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Clean up database metrics, add freezer DB size (#715)

Browse Source 
* Clean up database metrics, add freezer DB size

* Address review comments
This commit is contained in:
Michael Sproul 2019-12-13 13:30:58 +11:00 committed by Paul Hauner
parent b9d00ee8b8
commit 5e7803f00b
7 changed files with 57 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@ -1209,14 +1209,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}); });
} }
if self.store.exists::<BeaconBlock<T::EthSpec>>(&block_root)? {
return Ok(BlockProcessingOutcome::BlockIsAlreadyKnown);
}
// Records the time taken to load the block and state from the database during block // Records the time taken to load the block and state from the database during block
// processing. // processing.
let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ); let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ);
if self.store.exists::<BeaconBlock<T::EthSpec>>(&block_root)? {
return Ok(BlockProcessingOutcome::BlockIsAlreadyKnown);
}
// Load the blocks parent block from the database, returning invalid if that block is not // Load the blocks parent block from the database, returning invalid if that block is not
// found. // found.
let parent_block: BeaconBlock<T::EthSpec> = let parent_block: BeaconBlock<T::EthSpec> =

5
beacon_node/client/src/builder.rs
View File

@ -314,7 +314,10 @@ where
network_info, network_info,
client_config client_config
.create_db_path() .create_db_path()
.expect("unable to read datadir"), .map_err(|_| "unable to read data dir")?,
client_config
.create_freezer_db_path()
.map_err(|_| "unable to read freezer DB dir")?,
eth2_config.clone(), eth2_config.clone(),
context.log, context.log,
) )

3
beacon_node/rest_api/src/lib.rs
View File

@ -56,6 +56,7 @@ pub fn start_server<T: BeaconChainTypes>(
beacon_chain: Arc<BeaconChain<T>>, beacon_chain: Arc<BeaconChain<T>>,
network_info: NetworkInfo<T>, network_info: NetworkInfo<T>,
db_path: PathBuf, db_path: PathBuf,
freezer_db_path: PathBuf,
eth2_config: Eth2Config, eth2_config: Eth2Config,
log: slog::Logger, log: slog::Logger,
) -> Result<(exit_future::Signal, SocketAddr), hyper::Error> { ) -> Result<(exit_future::Signal, SocketAddr), hyper::Error> {
@ -70,6 +71,7 @@ pub fn start_server<T: BeaconChainTypes>(
let network_service = network_info.network_service.clone(); let network_service = network_info.network_service.clone();
let network_channel = Arc::new(RwLock::new(network_info.network_chan.clone())); let network_channel = Arc::new(RwLock::new(network_info.network_chan.clone()));
let db_path = db_path.clone(); let db_path = db_path.clone();
let freezer_db_path = freezer_db_path.clone();
service_fn(move |req: Request<Body>| { service_fn(move |req: Request<Body>| {
router::route( router::route(
@ -80,6 +82,7 @@ pub fn start_server<T: BeaconChainTypes>(
eth2_config.clone(), eth2_config.clone(),
log.clone(), log.clone(),
db_path.clone(), db_path.clone(),
freezer_db_path.clone(),
) )
}) })
}); });

3
beacon_node/rest_api/src/metrics.rs
View File

@ -32,6 +32,7 @@ pub fn get_prometheus<T: BeaconChainTypes>(
req: Request<Body>, req: Request<Body>,
beacon_chain: Arc<BeaconChain<T>>, beacon_chain: Arc<BeaconChain<T>>,
db_path: PathBuf, db_path: PathBuf,
freezer_db_path: PathBuf,
) -> ApiResult { ) -> ApiResult {
let mut buffer = vec![]; let mut buffer = vec![];
let encoder = TextEncoder::new(); let encoder = TextEncoder::new();
@ -53,7 +54,7 @@ pub fn get_prometheus<T: BeaconChainTypes>(
// a string that can be returned via HTTP. // a string that can be returned via HTTP.
slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock); slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock);
store::scrape_for_metrics(&db_path); store::scrape_for_metrics(&db_path, &freezer_db_path);
beacon_chain::scrape_for_metrics(&beacon_chain); beacon_chain::scrape_for_metrics(&beacon_chain);
encoder encoder

10
beacon_node/rest_api/src/router.rs
View File

@ -27,6 +27,7 @@ pub fn route<T: BeaconChainTypes>(
eth2_config: Arc<Eth2Config>, eth2_config: Arc<Eth2Config>,
local_log: slog::Logger, local_log: slog::Logger,
db_path: PathBuf, db_path: PathBuf,
freezer_db_path: PathBuf,
) -> impl Future<Item = Response<Body>, Error = Error> { ) -> impl Future<Item = Response<Body>, Error = Error> {
metrics::inc_counter(&metrics::REQUEST_COUNT); metrics::inc_counter(&metrics::REQUEST_COUNT);
let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME);
@ -143,9 +144,12 @@ pub fn route<T: BeaconChainTypes>(
into_boxfut(spec::get_eth2_config::<T>(req, eth2_config)) into_boxfut(spec::get_eth2_config::<T>(req, eth2_config))
} }
(&Method::GET, "/metrics") => { (&Method::GET, "/metrics") => into_boxfut(metrics::get_prometheus::<T>(
into_boxfut(metrics::get_prometheus::<T>(req, beacon_chain, db_path)) req,
} beacon_chain,
db_path,
freezer_db_path,
)),
_ => Box::new(futures::future::err(ApiError::NotFound( _ => Box::new(futures::future::err(ApiError::NotFound(
"Request path and/or method not found.".to_owned(), "Request path and/or method not found.".to_owned(),

13
beacon_node/store/src/impls/beacon_state.rs
View File

@ -9,12 +9,15 @@ pub fn store_full_state<S: Store<E>, E: EthSpec>(
state_root: &Hash256, state_root: &Hash256,
state: &BeaconState<E>, state: &BeaconState<E>,
) -> Result<(), Error> { ) -> Result<(), Error> {
let timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES); let total_timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES);
let overhead_timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_OVERHEAD_TIMES);
let bytes = StorageContainer::new(state).as_ssz_bytes(); let bytes = StorageContainer::new(state).as_ssz_bytes();
metrics::stop_timer(overhead_timer);
let result = store.put_bytes(DBColumn::BeaconState.into(), state_root.as_bytes(), &bytes); let result = store.put_bytes(DBColumn::BeaconState.into(), state_root.as_bytes(), &bytes);
metrics::stop_timer(timer); metrics::stop_timer(total_timer);
metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT); metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT);
metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as i64); metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as i64);
@ -25,13 +28,15 @@ pub fn get_full_state<S: Store<E>, E: EthSpec>(
store: &S, store: &S,
state_root: &Hash256, state_root: &Hash256,
) -> Result<Option<BeaconState<E>>, Error> { ) -> Result<Option<BeaconState<E>>, Error> {
let timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES); let total_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES);
match store.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())? { match store.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())? {
Some(bytes) => { Some(bytes) => {
let overhead_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_OVERHEAD_TIMES);
let container = StorageContainer::from_ssz_bytes(&bytes)?; let container = StorageContainer::from_ssz_bytes(&bytes)?;
metrics::stop_timer(timer); metrics::stop_timer(overhead_timer);
metrics::stop_timer(total_timer);
metrics::inc_counter(&metrics::BEACON_STATE_READ_COUNT); metrics::inc_counter(&metrics::BEACON_STATE_READ_COUNT);
metrics::inc_counter_by(&metrics::BEACON_STATE_READ_BYTES, bytes.len() as i64); metrics::inc_counter_by(&metrics::BEACON_STATE_READ_BYTES, bytes.len() as i64);

40
beacon_node/store/src/metrics.rs
View File

@ -1,29 +1,31 @@
pub use lighthouse_metrics::{set_gauge, try_create_int_gauge, *}; pub use lighthouse_metrics::{set_gauge, try_create_int_gauge, *};
use std::fs; use std::fs;
use std::path::PathBuf; use std::path::Path;
lazy_static! { lazy_static! {
/* /*
* General * General
*/ */
pub static ref DISK_DB_SIZE: Result<IntGauge> = pub static ref DISK_DB_SIZE: Result<IntGauge> =
try_create_int_gauge("store_disk_db_size", "Size of the on-disk database (bytes)"); try_create_int_gauge("store_disk_db_size", "Size of the hot on-disk database (bytes)");
pub static ref FREEZER_DB_SIZE: Result<IntGauge> =
try_create_int_gauge("store_freezer_db_size", "Size of the on-disk freezer database (bytes)");
pub static ref DISK_DB_WRITE_BYTES: Result<IntCounter> = try_create_int_counter( pub static ref DISK_DB_WRITE_BYTES: Result<IntCounter> = try_create_int_counter(
"store_disk_db_write_bytes_total", "store_disk_db_write_bytes_total",
"Number of bytes attempted to be written to the on-disk DB" "Number of bytes attempted to be written to the hot on-disk DB"
); );
pub static ref DISK_DB_READ_BYTES: Result<IntCounter> = try_create_int_counter( pub static ref DISK_DB_READ_BYTES: Result<IntCounter> = try_create_int_counter(
"store_disk_db_read_bytes_total", "store_disk_db_read_bytes_total",
"Number of bytes read from the on-disk DB" "Number of bytes read from the hot on-disk DB"
); );
pub static ref DISK_DB_READ_COUNT: Result<IntCounter> = try_create_int_counter( pub static ref DISK_DB_READ_COUNT: Result<IntCounter> = try_create_int_counter(
"store_disk_db_read_count_total", "store_disk_db_read_count_total",
"Total number of reads to the on-disk DB" "Total number of reads to the hot on-disk DB"
); );
pub static ref DISK_DB_WRITE_COUNT: Result<IntCounter> = try_create_int_counter( pub static ref DISK_DB_WRITE_COUNT: Result<IntCounter> = try_create_int_counter(
"store_disk_db_write_count_total", "store_disk_db_write_count_total",
"Total number of writes to the on-disk DB" "Total number of writes to the hot on-disk DB"
); );
pub static ref DISK_DB_READ_TIMES: Result<Histogram> = try_create_histogram( pub static ref DISK_DB_READ_TIMES: Result<Histogram> = try_create_histogram(
"store_disk_db_read_seconds", "store_disk_db_read_seconds",
@ -35,16 +37,20 @@ lazy_static! {
); );
pub static ref DISK_DB_EXISTS_COUNT: Result<IntCounter> = try_create_int_counter( pub static ref DISK_DB_EXISTS_COUNT: Result<IntCounter> = try_create_int_counter(
"store_disk_db_exists_count_total", "store_disk_db_exists_count_total",
"Total number of checks if a key is in the on-disk DB" "Total number of checks if a key is in the hot on-disk DB"
); );
pub static ref DISK_DB_DELETE_COUNT: Result<IntCounter> = try_create_int_counter( pub static ref DISK_DB_DELETE_COUNT: Result<IntCounter> = try_create_int_counter(
"store_disk_db_delete_count_total", "store_disk_db_delete_count_total",
"Total number of deletions from the on-disk DB" "Total number of deletions from the hot on-disk DB"
); );
/* /*
* Beacon State * Beacon State
*/ */
pub static ref BEACON_STATE_READ_TIMES: Result<Histogram> = try_create_histogram( pub static ref BEACON_STATE_READ_TIMES: Result<Histogram> = try_create_histogram(
"store_beacon_state_read_seconds",
"Total time required to read a BeaconState from the database"
);
pub static ref BEACON_STATE_READ_OVERHEAD_TIMES: Result<Histogram> = try_create_histogram(
"store_beacon_state_read_overhead_seconds", "store_beacon_state_read_overhead_seconds",
"Overhead on reading a beacon state from the DB (e.g., decoding)" "Overhead on reading a beacon state from the DB (e.g., decoding)"
); );
@ -57,6 +63,10 @@ lazy_static! {
"Total number of beacon state bytes read from the DB" "Total number of beacon state bytes read from the DB"
); );
pub static ref BEACON_STATE_WRITE_TIMES: Result<Histogram> = try_create_histogram( pub static ref BEACON_STATE_WRITE_TIMES: Result<Histogram> = try_create_histogram(
"store_beacon_state_write_seconds",
"Total time required to write a BeaconState to the database"
);
pub static ref BEACON_STATE_WRITE_OVERHEAD_TIMES: Result<Histogram> = try_create_histogram(
"store_beacon_state_write_overhead_seconds", "store_beacon_state_write_overhead_seconds",
"Overhead on writing a beacon state to the DB (e.g., encoding)" "Overhead on writing a beacon state to the DB (e.g., encoding)"
); );
@ -98,15 +108,21 @@ lazy_static! {
} }
/// Updates the global metrics registry with store-related information. /// Updates the global metrics registry with store-related information.
pub fn scrape_for_metrics(db_path: &PathBuf) { pub fn scrape_for_metrics(db_path: &Path, freezer_db_path: &Path) {
let db_size = if let Ok(iter) = fs::read_dir(db_path) { let db_size = size_of_dir(db_path);
set_gauge(&DISK_DB_SIZE, db_size as i64);
let freezer_db_size = size_of_dir(freezer_db_path);
set_gauge(&FREEZER_DB_SIZE, freezer_db_size as i64);
}
fn size_of_dir(path: &Path) -> u64 {
if let Ok(iter) = fs::read_dir(path) {
iter.filter_map(std::result::Result::ok) iter.filter_map(std::result::Result::ok)
.map(size_of_dir_entry) .map(size_of_dir_entry)
.sum() .sum()
} else { } else {
0 0
}; }
set_gauge(&DISK_DB_SIZE, db_size as i64);
} }
fn size_of_dir_entry(dir: fs::DirEntry) -> u64 { fn size_of_dir_entry(dir: fs::DirEntry) -> u64 {