From 8aa9f83c48a28681dc5e6db861b2a98dcb20c405 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 4 Jan 2019 17:55:15 +1100 Subject: [PATCH] Implement new API for db stores --- .../db/src/stores/beacon_block_store.rs | 176 ++++-------------- .../db/src/stores/beacon_state_store.rs | 57 ++++-- lighthouse/db/src/stores/macros.rs | 102 ++++++++++ lighthouse/db/src/stores/mod.rs | 9 +- lighthouse/db/src/stores/pow_chain_store.rs | 4 +- 5 files changed, 187 insertions(+), 161 deletions(-) create mode 100644 lighthouse/db/src/stores/macros.rs diff --git a/lighthouse/db/src/stores/beacon_block_store.rs b/lighthouse/db/src/stores/beacon_block_store.rs index 5faa208df..ff7247b1c 100644 --- a/lighthouse/db/src/stores/beacon_block_store.rs +++ b/lighthouse/db/src/stores/beacon_block_store.rs @@ -1,12 +1,9 @@ use super::BLOCKS_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; -use ssz::{Decodable, DecodeError}; +use ssz::Decodable; use std::sync::Arc; use types::{readers::BeaconBlockReader, BeaconBlock, Hash256}; -type BeaconBlockHash = Vec; -type BeaconBlockSsz = Vec; - #[derive(Clone, Debug, PartialEq)] pub enum BeaconBlockAtSlotError { UnknownBeaconBlock, @@ -21,34 +18,25 @@ where db: Arc, } +// Implements `put`, `get`, `exists` and `delete` for the store. +impl_crud_for_store!(BeaconBlockStore, DB_COLUMN); + impl BeaconBlockStore { pub fn new(db: Arc) -> Self { Self { db } } - pub fn put(&self, hash: &[u8], ssz: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, ssz) - } - - pub fn get(&self, hash: &[u8]) -> Result>, DBError> { - self.db.get(DB_COLUMN, hash) - } - - pub fn exists(&self, hash: &[u8]) -> Result { - self.db.exists(DB_COLUMN, hash) - } - - pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.delete(DB_COLUMN, hash) - } - - /// Retuns a fully de-serialized `BeaconBlock` (or `None` if hash not known). - pub fn get_deserialized(&self, hash: &[u8]) -> Result, DBError> { + /// Retuns an object implementing `BeaconBlockReader`, or `None` (if hash not known). + /// + /// Note: Presently, this function fully deserializes a `BeaconBlock` and returns that. In the + /// future, it would be ideal to return an object capable of reading directly from serialized + /// SSZ bytes. + pub fn get_reader(&self, hash: &Hash256) -> Result, DBError> { match self.get(&hash)? { None => Ok(None), Some(ssz) => { let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { - message: "Bad Block SSZ.".to_string(), + message: "Bad BeaconBlock SSZ.".to_string(), })?; Ok(Some(block)) } @@ -66,35 +54,25 @@ impl BeaconBlockStore { /// If a block is found, a tuple of (block_hash, serialized_block) is returned. pub fn block_at_slot( &self, - head_hash: &[u8], + head_hash: &Hash256, slot: u64, - ) -> Result, BeaconBlockAtSlotError> { - match self.get(head_hash)? { + ) -> Result, BeaconBlockAtSlotError> { + match self.get_reader(head_hash)? { None => Err(BeaconBlockAtSlotError::UnknownBeaconBlock), - Some(ssz) => { - let (retrieved_slot, parent_hash) = slot_and_parent_from_block_ssz(&ssz, 0) - .map_err(|_| BeaconBlockAtSlotError::InvalidBeaconBlock)?; - match retrieved_slot { - s if s == slot => Ok(Some((head_hash.to_vec(), ssz.to_vec()))), - s if s < slot => Ok(None), - _ => self.block_at_slot(&parent_hash, slot), + Some(block_reader) => match block_reader.slot() { + s if s == slot => { + let block = block_reader + .into_beacon_block() + .ok_or(BeaconBlockAtSlotError::InvalidBeaconBlock)?; + Ok(Some((*head_hash, block))) } - } + s if s < slot => Ok(None), + _ => self.block_at_slot(&block_reader.parent_root(), slot), + }, } } } -/// Read `block.slot` and `block.parent_root` from a SSZ-encoded block bytes. -/// -/// Assumes the block starts at byte `i`. -fn slot_and_parent_from_block_ssz(ssz: &[u8], i: usize) -> Result<(u64, Hash256), DecodeError> { - // Assuming the slot is the first field on a block. - let (slot, i) = u64::ssz_decode(&ssz, i)?; - // Assuming the parent has is the second field on a block. - let (parent_root, _) = Hash256::ssz_decode(&ssz, i)?; - Ok((slot, parent_root)) -} - impl From for BeaconBlockAtSlotError { fn from(e: DBError) -> Self { BeaconBlockAtSlotError::DBError(e.message) @@ -114,82 +92,7 @@ mod tests { use types::BeaconBlock; use types::Hash256; - #[test] - fn test_put() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - store.put(hash, ssz).unwrap(); - assert_eq!(db.get(DB_COLUMN, hash).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert_eq!(store.get(hash).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get_unknown() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, other_hash, ssz).unwrap(); - assert_eq!(store.get(hash).unwrap(), None); - } - - #[test] - fn test_exists() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(store.exists(hash).unwrap()); - } - - #[test] - fn test_block_does_not_exist() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(!store.exists(other_hash).unwrap()); - } - - #[test] - fn test_delete() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(db.exists(DB_COLUMN, hash).unwrap()); - - store.delete(hash).unwrap(); - assert!(!db.exists(DB_COLUMN, hash).unwrap()); - } + test_crud_for_store!(BeaconBlockStore, DB_COLUMN); #[test] fn test_invalid_block_at_slot() { @@ -197,12 +100,14 @@ mod tests { let store = BeaconBlockStore::new(db.clone()); let ssz = "definitly not a valid block".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); + let hash = &Hash256::from("some hash".as_bytes()); db.put(DB_COLUMN, hash, ssz).unwrap(); assert_eq!( store.block_at_slot(hash, 42), - Err(BeaconBlockAtSlotError::InvalidBeaconBlock) + Err(BeaconBlockAtSlotError::DBError( + "Bad BeaconBlock SSZ.".into() + )) ); } @@ -212,8 +117,8 @@ mod tests { let store = BeaconBlockStore::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); + let hash = &Hash256::from("some hash".as_bytes()); + let other_hash = &Hash256::from("another hash".as_bytes()); db.put(DB_COLUMN, hash, ssz).unwrap(); assert_eq!( @@ -241,7 +146,7 @@ mod tests { for w in 0..wc { let key = (t * w) as u8; let val = 42; - bs.put(&vec![key], &vec![val]).unwrap(); + bs.put(&[key][..].into(), &vec![val]).unwrap(); } }); handles.push(handle); @@ -254,8 +159,8 @@ mod tests { for t in 0..thread_count { for w in 0..write_count { let key = (t * w) as u8; - assert!(bs.exists(&vec![key]).unwrap()); - let val = bs.get(&vec![key]).unwrap().unwrap(); + assert!(bs.exists(&[key][..].into()).unwrap()); + let val = bs.get(&[key][..].into()).unwrap().unwrap(); assert_eq!(vec![42], val); } } @@ -294,13 +199,7 @@ mod tests { block.slot = slots[i]; let ssz = ssz_encode(&block); - db.put(DB_COLUMN, &hashes[i].to_vec(), &ssz).unwrap(); - - // Ensure the slot and parent_root decoding fn works correctly. - let (decoded_slot, decoded_parent_root) = - slot_and_parent_from_block_ssz(&ssz, 0).unwrap(); - assert_eq!(decoded_slot, block.slot); - assert_eq!(decoded_parent_root, block.parent_root); + db.put(DB_COLUMN, &hashes[i], &ssz).unwrap(); blocks.push(block); } @@ -308,14 +207,11 @@ mod tests { // Test that certain slots can be reached from certain hashes. let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; for (hashes_index, slot_index) in test_cases { - let (matched_block_hash, matched_block_ssz) = bs + let (matched_block_hash, _) = bs .block_at_slot(&hashes[hashes_index], slots[slot_index]) .unwrap() .unwrap(); - let (retrieved_slot, _) = - slot_and_parent_from_block_ssz(&matched_block_ssz, 0).unwrap(); - assert_eq!(retrieved_slot, slots[slot_index]); - assert_eq!(matched_block_hash, hashes[slot_index].to_vec()); + assert_eq!(matched_block_hash, hashes[slot_index]); } let ssz = bs.block_at_slot(&hashes[4], 2).unwrap(); diff --git a/lighthouse/db/src/stores/beacon_state_store.rs b/lighthouse/db/src/stores/beacon_state_store.rs index 687fc0f22..122b22509 100644 --- a/lighthouse/db/src/stores/beacon_state_store.rs +++ b/lighthouse/db/src/stores/beacon_state_store.rs @@ -11,29 +11,20 @@ where db: Arc, } +// Implements `put`, `get`, `exists` and `delete` for the store. +impl_crud_for_store!(BeaconStateStore, DB_COLUMN); + impl BeaconStateStore { pub fn new(db: Arc) -> Self { Self { db } } - pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, ssz) - } - - pub fn get(&self, hash: &[u8]) -> Result>, DBError> { - self.db.get(DB_COLUMN, hash) - } - - pub fn exists(&self, hash: &[u8]) -> Result { - self.db.exists(DB_COLUMN, hash) - } - - pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.delete(DB_COLUMN, hash) - } - - /// Retuns a fully de-serialized `BeaconState` (or `None` if hash not known). - pub fn get_deserialized(&self, hash: &[u8]) -> Result, DBError> { + /// Retuns an object implementing `BeaconStateReader`, or `None` (if hash not known). + /// + /// Note: Presently, this function fully deserializes a `BeaconState` and returns that. In the + /// future, it would be ideal to return an object capable of reading directly from serialized + /// SSZ bytes. + pub fn get_reader(&self, hash: &Hash256) -> Result, DBError> { match self.get(&hash)? { None => Ok(None), Some(ssz) => { @@ -45,3 +36,33 @@ impl BeaconStateStore { } } } + +#[cfg(test)] +mod tests { + use super::super::super::MemoryDB; + use super::*; + + use std::sync::Arc; + use ssz::ssz_encode; + use types::Hash256; + use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + + test_crud_for_store!(BeaconStateStore, DB_COLUMN); + + #[test] + fn test_reader() { + let db = Arc::new(MemoryDB::open()); + let store = BeaconStateStore::new(db.clone()); + + let mut rng = XorShiftRng::from_seed([42; 16]); + let state = BeaconState::random_for_test(&mut rng); + let state_root = state.canonical_root(); + + store.put(&state_root, &ssz_encode(&state)).unwrap(); + + let reader = store.get_reader(&state_root).unwrap().unwrap(); + let decoded = reader.into_beacon_state().unwrap(); + + assert_eq!(state, decoded); + } +} diff --git a/lighthouse/db/src/stores/macros.rs b/lighthouse/db/src/stores/macros.rs new file mode 100644 index 000000000..2086dcbac --- /dev/null +++ b/lighthouse/db/src/stores/macros.rs @@ -0,0 +1,102 @@ +macro_rules! impl_crud_for_store { + ($store: ident, $db_column: expr) => { + impl $store { + pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { + self.db.put($db_column, hash, ssz) + } + + pub fn get(&self, hash: &Hash256) -> Result>, DBError> { + self.db.get($db_column, hash) + } + + pub fn exists(&self, hash: &Hash256) -> Result { + self.db.exists($db_column, hash) + } + + pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> { + self.db.delete($db_column, hash) + } + } + }; +} + +macro_rules! test_crud_for_store { + ($store: ident, $db_column: expr) => { + #[test] + fn test_put() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + store.put(hash, ssz).unwrap(); + assert_eq!(db.get(DB_COLUMN, hash).unwrap().unwrap(), ssz); + } + + #[test] + fn test_get() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert_eq!(store.get(hash).unwrap().unwrap(), ssz); + } + + #[test] + fn test_get_unknown() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + let other_hash = &Hash256::from("another hash".as_bytes()); + + db.put(DB_COLUMN, other_hash, ssz).unwrap(); + assert_eq!(store.get(hash).unwrap(), None); + } + + #[test] + fn test_exists() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert!(store.exists(hash).unwrap()); + } + + #[test] + fn test_block_does_not_exist() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + let other_hash = &Hash256::from("another hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert!(!store.exists(other_hash).unwrap()); + } + + #[test] + fn test_delete() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert!(db.exists(DB_COLUMN, hash).unwrap()); + + store.delete(hash).unwrap(); + assert!(!db.exists(DB_COLUMN, hash).unwrap()); + } + }; +} diff --git a/lighthouse/db/src/stores/mod.rs b/lighthouse/db/src/stores/mod.rs index 7fc51abe7..c78d10dbf 100644 --- a/lighthouse/db/src/stores/mod.rs +++ b/lighthouse/db/src/stores/mod.rs @@ -1,5 +1,7 @@ use super::{ClientDB, DBError}; +#[macro_use] +mod macros; mod beacon_block_store; mod beacon_state_store; mod pow_chain_store; @@ -17,4 +19,9 @@ pub const STATES_DB_COLUMN: &str = "states"; pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; pub const VALIDATOR_DB_COLUMN: &str = "validator"; -pub const COLUMNS: [&str; 4] = [BLOCKS_DB_COLUMN, STATES_DB_COLUMN, POW_CHAIN_DB_COLUMN, VALIDATOR_DB_COLUMN]; +pub const COLUMNS: [&str; 4] = [ + BLOCKS_DB_COLUMN, + STATES_DB_COLUMN, + POW_CHAIN_DB_COLUMN, + VALIDATOR_DB_COLUMN, +]; diff --git a/lighthouse/db/src/stores/pow_chain_store.rs b/lighthouse/db/src/stores/pow_chain_store.rs index aa2b267f7..a7c77bab5 100644 --- a/lighthouse/db/src/stores/pow_chain_store.rs +++ b/lighthouse/db/src/stores/pow_chain_store.rs @@ -26,9 +26,9 @@ impl PoWChainStore { #[cfg(test)] mod tests { extern crate types; - - use super::*; + use super::super::super::MemoryDB; + use super::*; use self::types::Hash256;