From 08b1808745a844b30e1bfc0ef58f603f4344b39e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 31 Mar 2019 18:57:48 +1100 Subject: [PATCH 01/44] Modify runtime to allow memory or disk db DiskDB is not working yet, but we'll get there! --- beacon_node/Cargo.toml | 1 + beacon_node/beacon_chain/src/initialise.rs | 43 +++++++++++++++++++++- beacon_node/client/src/client_config.rs | 6 +++ beacon_node/client/src/client_types.rs | 20 ++++++++-- beacon_node/src/main.rs | 9 +++++ beacon_node/src/run.rs | 41 ++++++++++++++++----- 6 files changed, 107 insertions(+), 13 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 37d96a497..da31bfa77 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] types = { path = "../eth2/types" } +db = { path = "./db" } client = { path = "client" } version = { path = "version" } clap = "2.32.0" diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs index 0951e06fb..44cef5fe1 100644 --- a/beacon_node/beacon_chain/src/initialise.rs +++ b/beacon_node/beacon_chain/src/initialise.rs @@ -61,7 +61,7 @@ pub fn initialise_beacon_chain( } /// Initialisation of a test beacon chain, uses an in memory db with fixed genesis time. -pub fn initialise_test_beacon_chain( +pub fn initialise_test_beacon_chain_with_memory_db( spec: &ChainSpec, _db_name: Option<&PathBuf>, ) -> Arc>> { @@ -100,3 +100,44 @@ pub fn initialise_test_beacon_chain( .expect("Terminate if beacon chain generation fails"), ) } + +/// Initialisation of a test beacon chain, uses an in memory db with fixed genesis time. +pub fn initialise_test_beacon_chain_with_disk_db( + spec: &ChainSpec, + db_name: Option<&PathBuf>, +) -> Arc>> { + let db = Arc::new(DiskDB::open(db_name.expect("Must have DB path"), None)); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); + let state_store = Arc::new(BeaconStateStore::new(db.clone())); + + let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); + let (genesis_state, _keypairs) = state_builder.build(); + + let mut genesis_block = BeaconBlock::empty(spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + + // Slot clock + let slot_clock = SystemTimeSlotClock::new( + spec.genesis_slot, + genesis_state.genesis_time, + spec.seconds_per_slot, + ) + .expect("Unable to load SystemTimeSlotClock"); + // Choose the fork choice + let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); + + // Genesis chain + //TODO: Handle error correctly + Arc::new( + BeaconChain::from_genesis( + state_store.clone(), + block_store.clone(), + slot_clock, + genesis_state, + genesis_block, + spec.clone(), + fork_choice, + ) + .expect("Terminate if beacon chain generation fails"), + ) +} diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index cad287f2c..c32379522 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -119,6 +119,12 @@ impl ClientConfig { } } + match args.value_of("db") { + Some("rocks") => config.db_type = DBType::RocksDB, + Some("memory") => config.db_type = DBType::Memory, + _ => unreachable!(), // clap prevents this. + }; + Ok(config) } } diff --git a/beacon_node/client/src/client_types.rs b/beacon_node/client/src/client_types.rs index f5abc77ce..1d2f1d6ec 100644 --- a/beacon_node/client/src/client_types.rs +++ b/beacon_node/client/src/client_types.rs @@ -34,9 +34,9 @@ impl ClientTypes for StandardClientType { } } -pub struct TestingClientType; +pub struct MemoryDBTestingClientType; -impl ClientTypes for TestingClientType { +impl ClientTypes for MemoryDBTestingClientType { type DB = MemoryDB; type SlotClock = SystemTimeSlotClock; type ForkChoice = BitwiseLMDGhost; @@ -44,6 +44,20 @@ impl ClientTypes for TestingClientType { fn initialise_beacon_chain( config: &ClientConfig, ) -> Arc> { - initialise::initialise_test_beacon_chain(&config.spec, None) + initialise::initialise_test_beacon_chain_with_memory_db(&config.spec, None) + } +} + +pub struct DiskDBTestingClientType; + +impl ClientTypes for DiskDBTestingClientType { + type DB = DiskDB; + type SlotClock = SystemTimeSlotClock; + type ForkChoice = BitwiseLMDGhost; + + fn initialise_beacon_chain( + config: &ClientConfig, + ) -> Arc> { + initialise::initialise_test_beacon_chain_with_disk_db(&config.spec, Some(&config.db_name)) } } diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index ea74c7376..8aa6da7d5 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -58,6 +58,15 @@ fn main() { .help("Listen port for RPC endpoint.") .takes_value(true), ) + .arg( + Arg::with_name("db") + .long("db") + .value_name("DB") + .help("Type of database to use.") + .takes_value(true) + .possible_values(&["rocks", "memory"]) + .default_value("memory"), + ) .get_matches(); // invalid arguments, panic diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 1d9156124..1afeb5408 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,15 +1,18 @@ -use client::client_types::TestingClientType; +use client::client_types::{DiskDBTestingClientType, MemoryDBTestingClientType}; use client::error; -use client::{notifier, Client, ClientConfig}; +use client::{notifier, Client, ClientConfig, ClientTypes}; +use db::DBType; use futures::sync::oneshot; use futures::Future; use slog::info; use std::cell::RefCell; use tokio::runtime::Builder; +use tokio::runtime::Runtime; +use tokio::runtime::TaskExecutor; use tokio_timer::clock::Clock; pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Result<()> { - let mut runtime = Builder::new() + let runtime = Builder::new() .name_prefix("main-") .clock(Clock::system()) .build() @@ -20,8 +23,32 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul "data_dir" => &config.data_dir.to_str(), "port" => &config.net_conf.listen_port); + let executor = runtime.executor(); + + match config.db_type { + DBType::RocksDB => { + let client: Client = + Client::new(config, log.clone(), &executor)?; + + run(client, executor, runtime, log) + } + DBType::Memory => { + let client: Client = + Client::new(config, log.clone(), &executor)?; + + run(client, executor, runtime, log) + } + } +} + +pub fn run( + client: Client, + executor: TaskExecutor, + mut runtime: Runtime, + log: &slog::Logger, +) -> error::Result<()> { // run service until ctrl-c - let (ctrlc_send, ctrlc) = oneshot::channel(); + let (ctrlc_send, ctrlc_oneshot) = oneshot::channel(); let ctrlc_send_c = RefCell::new(Some(ctrlc_send)); ctrlc::set_handler(move || { if let Some(ctrlc_send) = ctrlc_send_c.try_borrow_mut().unwrap().take() { @@ -32,14 +59,10 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul let (exit_signal, exit) = exit_future::signal(); - let executor = runtime.executor(); - - // currently testing - using TestingClientType - let client: Client = Client::new(config, log.clone(), &executor)?; notifier::run(&client, executor, exit); runtime - .block_on(ctrlc) + .block_on(ctrlc_oneshot) .map_err(|e| format!("Ctrlc oneshot failed: {:?}", e))?; // perform global shutdown operations. From f4bd46fe6697228a3aa1c58210400261427b0ec9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 31 Mar 2019 19:16:45 +1100 Subject: [PATCH 02/44] Fix rocks db startup issues --- beacon_node/db/src/disk_db.rs | 38 ++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/beacon_node/db/src/disk_db.rs b/beacon_node/db/src/disk_db.rs index 9d8a71bc4..f05320f7f 100644 --- a/beacon_node/db/src/disk_db.rs +++ b/beacon_node/db/src/disk_db.rs @@ -2,6 +2,7 @@ extern crate rocksdb; use super::rocksdb::Error as RocksError; use super::rocksdb::{Options, DB}; +use super::stores::COLUMNS; use super::{ClientDB, DBError, DBValue}; use std::fs; use std::path::Path; @@ -23,31 +24,32 @@ impl DiskDB { /// /// Panics if the database is unable to be created. pub fn open(path: &Path, columns: Option<&[&str]>) -> Self { - /* - * Initialise the options - */ + // Rocks options. let mut options = Options::default(); options.create_if_missing(true); - // TODO: ensure that columns are created (and remove - // the dead_code allow) - - /* - * Initialise the path - */ + // Ensure the path exists. fs::create_dir_all(&path).unwrap_or_else(|_| panic!("Unable to create {:?}", &path)); let db_path = path.join("database"); - /* - * Open the database - */ - let db = match columns { - None => DB::open(&options, db_path), - Some(columns) => DB::open_cf(&options, db_path, columns), - } - .expect("Unable to open local database");; + let columns = columns.unwrap_or(&COLUMNS); - Self { db } + if db_path.exists() { + Self { + db: DB::open_cf(&options, db_path, &COLUMNS) + .expect("Unable to open local database"), + } + } else { + let mut db = Self { + db: DB::open(&options, db_path).expect("Unable to open local database"), + }; + + for cf in columns { + db.create_col(cf).unwrap(); + } + + db + } } /// Create a RocksDB column family. Corresponds to the From b03dfdce593a7d9ab60c5e6cd2094f9ca2107328 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 1 Apr 2019 08:59:59 +1100 Subject: [PATCH 03/44] Fix genesis time issue, add logs --- beacon_node/src/run.rs | 10 ++++++++++ .../src/test_utils/testing_beacon_state_builder.rs | 7 +------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 1afeb5408..52dc3973b 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -27,12 +27,22 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul match config.db_type { DBType::RocksDB => { + info!( + log, + "BeaconNode starting"; + "type" => "DiskDBTestingClientType" + ); let client: Client = Client::new(config, log.clone(), &executor)?; run(client, executor, runtime, log) } DBType::Memory => { + info!( + log, + "BeaconNode starting"; + "type" => "MemoryDBTestingClientType" + ); let client: Client = Client::new(config, log.clone(), &executor)?; diff --git a/eth2/types/src/test_utils/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/testing_beacon_state_builder.rs index def58b0d7..b0168eb78 100644 --- a/eth2/types/src/test_utils/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_state_builder.rs @@ -122,12 +122,7 @@ impl TestingBeaconStateBuilder { }) .collect(); - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - - 30; - let genesis_time = now; // arbitrary + let genesis_time = 1554069200; // arbitrary let mut state = BeaconState::genesis( genesis_time, From ebe47a5b341b1bc710671f4a9dc22ddcf996a017 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 1 Apr 2019 14:56:32 +1100 Subject: [PATCH 04/44] Add `Store` and `db_encode_derive`. Implementation is not complete, but what is here works. --- Cargo.toml | 3 + beacon_node/db/Cargo.toml | 2 - beacon_node/db2/Cargo.toml | 16 + beacon_node/db2/src/disk_db.rs | 199 ++++++++++++ beacon_node/db2/src/lib.rs | 151 +++++++++ beacon_node/db2/src/memory_db.rs | 236 ++++++++++++++ .../db2/src/stores/beacon_block_store.rs | 246 ++++++++++++++ .../db2/src/stores/beacon_state_store.rs | 62 ++++ beacon_node/db2/src/stores/macros.rs | 103 ++++++ beacon_node/db2/src/stores/mod.rs | 25 ++ beacon_node/db2/src/stores/pow_chain_store.rs | 68 ++++ beacon_node/db2/src/stores/validator_store.rs | 215 ++++++++++++ beacon_node/db2/src/traits.rs | 38 +++ beacon_node/db_encode/Cargo.toml | 9 + beacon_node/db_encode/src/lib.rs | 59 ++++ beacon_node/db_encode_derive/Cargo.toml | 13 + beacon_node/db_encode_derive/src/lib.rs | 305 ++++++++++++++++++ 17 files changed, 1748 insertions(+), 2 deletions(-) create mode 100644 beacon_node/db2/Cargo.toml create mode 100644 beacon_node/db2/src/disk_db.rs create mode 100644 beacon_node/db2/src/lib.rs create mode 100644 beacon_node/db2/src/memory_db.rs create mode 100644 beacon_node/db2/src/stores/beacon_block_store.rs create mode 100644 beacon_node/db2/src/stores/beacon_state_store.rs create mode 100644 beacon_node/db2/src/stores/macros.rs create mode 100644 beacon_node/db2/src/stores/mod.rs create mode 100644 beacon_node/db2/src/stores/pow_chain_store.rs create mode 100644 beacon_node/db2/src/stores/validator_store.rs create mode 100644 beacon_node/db2/src/traits.rs create mode 100644 beacon_node/db_encode/Cargo.toml create mode 100644 beacon_node/db_encode/src/lib.rs create mode 100644 beacon_node/db_encode_derive/Cargo.toml create mode 100644 beacon_node/db_encode_derive/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 3ae62248b..008e83bae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,9 @@ members = [ "eth2/utils/test_random_derive", "beacon_node", "beacon_node/db", + "beacon_node/db2", + "beacon_node/db_encode", + "beacon_node/db_encode_derive", "beacon_node/client", "beacon_node/network", "beacon_node/eth2-libp2p", diff --git a/beacon_node/db/Cargo.toml b/beacon_node/db/Cargo.toml index 122aaa34d..ffb3585b9 100644 --- a/beacon_node/db/Cargo.toml +++ b/beacon_node/db/Cargo.toml @@ -9,5 +9,3 @@ blake2-rfc = "0.2.18" bls = { path = "../../eth2/utils/bls" } bytes = "0.4.10" rocksdb = "0.10.1" -ssz = { path = "../../eth2/utils/ssz" } -types = { path = "../../eth2/types" } diff --git a/beacon_node/db2/Cargo.toml b/beacon_node/db2/Cargo.toml new file mode 100644 index 000000000..8a5dbad5e --- /dev/null +++ b/beacon_node/db2/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "db2" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +blake2-rfc = "0.2.18" +bls = { path = "../../eth2/utils/bls" } +bytes = "0.4.10" +db_encode = { path = "../db_encode" } +db_encode_derive = { path = "../db_encode_derive" } +rocksdb = "0.10.1" +ssz = { path = "../../eth2/utils/ssz" } +ssz_derive = { path = "../../eth2/utils/ssz_derive" } +types = { path = "../../eth2/types" } diff --git a/beacon_node/db2/src/disk_db.rs b/beacon_node/db2/src/disk_db.rs new file mode 100644 index 000000000..f05320f7f --- /dev/null +++ b/beacon_node/db2/src/disk_db.rs @@ -0,0 +1,199 @@ +extern crate rocksdb; + +use super::rocksdb::Error as RocksError; +use super::rocksdb::{Options, DB}; +use super::stores::COLUMNS; +use super::{ClientDB, DBError, DBValue}; +use std::fs; +use std::path::Path; + +/// A on-disk database which implements the ClientDB trait. +/// +/// This implementation uses RocksDB with default options. +pub struct DiskDB { + db: DB, +} + +impl DiskDB { + /// Open the RocksDB database, optionally supplying columns if required. + /// + /// The RocksDB database will be contained in a directory titled + /// "database" in the supplied path. + /// + /// # Panics + /// + /// Panics if the database is unable to be created. + pub fn open(path: &Path, columns: Option<&[&str]>) -> Self { + // Rocks options. + let mut options = Options::default(); + options.create_if_missing(true); + + // Ensure the path exists. + fs::create_dir_all(&path).unwrap_or_else(|_| panic!("Unable to create {:?}", &path)); + let db_path = path.join("database"); + + let columns = columns.unwrap_or(&COLUMNS); + + if db_path.exists() { + Self { + db: DB::open_cf(&options, db_path, &COLUMNS) + .expect("Unable to open local database"), + } + } else { + let mut db = Self { + db: DB::open(&options, db_path).expect("Unable to open local database"), + }; + + for cf in columns { + db.create_col(cf).unwrap(); + } + + db + } + } + + /// Create a RocksDB column family. Corresponds to the + /// `create_cf()` function on the RocksDB API. + #[allow(dead_code)] + fn create_col(&mut self, col: &str) -> Result<(), DBError> { + match self.db.create_cf(col, &Options::default()) { + Err(e) => Err(e.into()), + Ok(_) => Ok(()), + } + } +} + +impl From for DBError { + fn from(e: RocksError) -> Self { + Self { + message: e.to_string(), + } + } +} + +impl ClientDB for DiskDB { + /// Get the value for some key on some column. + /// + /// Corresponds to the `get_cf()` method on the RocksDB API. + /// Will attempt to get the `ColumnFamily` and return an Err + /// if it fails. + fn get(&self, col: &str, key: &[u8]) -> Result, DBError> { + match self.db.cf_handle(col) { + None => Err(DBError { + message: "Unknown column".to_string(), + }), + Some(handle) => match self.db.get_cf(handle, key)? { + None => Ok(None), + Some(db_vec) => Ok(Some(DBValue::from(&*db_vec))), + }, + } + } + + /// Set some value for some key on some column. + /// + /// Corresponds to the `cf_handle()` method on the RocksDB API. + /// Will attempt to get the `ColumnFamily` and return an Err + /// if it fails. + fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError> { + match self.db.cf_handle(col) { + None => Err(DBError { + message: "Unknown column".to_string(), + }), + Some(handle) => self.db.put_cf(handle, key, val).map_err(|e| e.into()), + } + } + + /// Return true if some key exists in some column. + fn exists(&self, col: &str, key: &[u8]) -> Result { + /* + * I'm not sure if this is the correct way to read if some + * block exists. Naively I would expect this to unncessarily + * copy some data, but I could be wrong. + */ + match self.db.cf_handle(col) { + None => Err(DBError { + message: "Unknown column".to_string(), + }), + Some(handle) => Ok(self.db.get_cf(handle, key)?.is_some()), + } + } + + /// Delete the value for some key on some column. + /// + /// Corresponds to the `delete_cf()` method on the RocksDB API. + /// Will attempt to get the `ColumnFamily` and return an Err + /// if it fails. + fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError> { + match self.db.cf_handle(col) { + None => Err(DBError { + message: "Unknown column".to_string(), + }), + Some(handle) => { + self.db.delete_cf(handle, key)?; + Ok(()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::super::ClientDB; + use super::*; + use std::sync::Arc; + use std::{env, fs, thread}; + + #[test] + #[ignore] + fn test_rocksdb_can_use_db() { + let pwd = env::current_dir().unwrap(); + let path = pwd.join("testdb_please_remove"); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + + let col_name: &str = "TestColumn"; + let column_families = vec![col_name]; + + let mut db = DiskDB::open(&path, None); + + for cf in column_families { + db.create_col(&cf).unwrap(); + } + + let db = Arc::new(db); + + let thread_count = 10; + let write_count = 10; + + // We're execting the product of these numbers to fit in one byte. + assert!(thread_count * write_count <= 255); + + let mut handles = vec![]; + for t in 0..thread_count { + let wc = write_count; + let db = db.clone(); + let col = col_name.clone(); + let handle = thread::spawn(move || { + for w in 0..wc { + let key = (t * w) as u8; + let val = 42; + db.put(&col, &vec![key], &vec![val]).unwrap(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + for t in 0..thread_count { + for w in 0..write_count { + let key = (t * w) as u8; + let val = db.get(&col_name, &vec![key]).unwrap().unwrap(); + assert_eq!(vec![42], val); + } + } + fs::remove_dir_all(&path).unwrap(); + } +} diff --git a/beacon_node/db2/src/lib.rs b/beacon_node/db2/src/lib.rs new file mode 100644 index 000000000..0704a84f5 --- /dev/null +++ b/beacon_node/db2/src/lib.rs @@ -0,0 +1,151 @@ +extern crate blake2_rfc as blake2; +extern crate bls; +extern crate rocksdb; + +mod disk_db; +mod memory_db; +pub mod stores; +mod traits; + +use self::stores::COLUMNS; +use db_encode::{db_encode, DBDecode, DBEncode}; +use ssz::DecodeError; +use std::sync::Arc; + +pub use self::disk_db::DiskDB; +pub use self::memory_db::MemoryDB; +pub use self::traits::{ClientDB, DBError, DBValue}; +pub use types::*; + +#[derive(Debug, PartialEq)] +pub enum Error { + SszDecodeError(DecodeError), + DBError { message: String }, +} + +impl From for Error { + fn from(e: DecodeError) -> Error { + Error::SszDecodeError(e) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError { message: e.message } + } +} + +/// Currently available database options +#[derive(Debug, Clone)] +pub enum DBType { + Memory, + RocksDB, +} + +pub enum DBColumn { + Block, + State, + BeaconChain, +} + +impl<'a> Into<&'a str> for DBColumn { + /// Returns a `&str` that can be used for keying a key-value data base. + fn into(self) -> &'a str { + match self { + DBColumn::Block => &"blk", + DBColumn::State => &"ste", + DBColumn::BeaconChain => &"bch", + } + } +} + +pub trait DBRecord: DBEncode + DBDecode { + fn db_column() -> DBColumn; +} + +pub struct Store +where + T: ClientDB, +{ + db: Arc, +} + +impl Store { + fn new_in_memory() -> Self { + Self { + db: Arc::new(MemoryDB::open()), + } + } +} + +impl Store +where + T: ClientDB, +{ + /// Put `item` in the store as `key`. + /// + /// The `item` must implement `DBRecord` which defines the db column used. + fn put(&self, key: &Hash256, item: &I) -> Result<(), Error> + where + I: DBRecord, + { + let column = I::db_column().into(); + let key = key.as_bytes(); + let val = db_encode(item); + + self.db.put(column, key, &val).map_err(|e| e.into()) + } + + /// Retrieves an `Ok(Some(item)` from the store if `key` exists, otherwise returns `Ok(None)`. + /// + /// The `item` must implement `DBRecord` which defines the db column used. + fn get(&self, key: &Hash256) -> Result, Error> + where + I: DBRecord, + { + let column = I::db_column().into(); + let key = key.as_bytes(); + + match self.db.get(column, key)? { + Some(bytes) => { + let (item, _index) = I::db_decode(&bytes, 0)?; + Ok(Some(item)) + } + None => Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use db_encode_derive::{DBDecode, DBEncode}; + use ssz::Decodable; + use ssz_derive::{Decode, Encode}; + + #[derive(PartialEq, Debug, Encode, Decode, DBEncode, DBDecode)] + struct StorableThing { + a: u64, + b: u64, + } + + impl DBRecord for StorableThing { + fn db_column() -> DBColumn { + DBColumn::Block + } + } + + #[test] + fn memorydb_can_store() { + let store = Store::new_in_memory(); + + let key = Hash256::random(); + let item = StorableThing { a: 1, b: 42 }; + + store.put(&key, &item).unwrap(); + + let retrieved = store.get(&key).unwrap().unwrap(); + + assert_eq!(item, retrieved); + } +} diff --git a/beacon_node/db2/src/memory_db.rs b/beacon_node/db2/src/memory_db.rs new file mode 100644 index 000000000..008e5912f --- /dev/null +++ b/beacon_node/db2/src/memory_db.rs @@ -0,0 +1,236 @@ +use super::blake2::blake2b::blake2b; +use super::COLUMNS; +use super::{ClientDB, DBError, DBValue}; +use std::collections::{HashMap, HashSet}; +use std::sync::RwLock; + +type DBHashMap = HashMap, Vec>; +type ColumnHashSet = HashSet; + +/// An in-memory database implementing the ClientDB trait. +/// +/// It is not particularily optimized, it exists for ease and speed of testing. It's not expected +/// this DB would be used outside of tests. +pub struct MemoryDB { + db: RwLock, + known_columns: RwLock, +} + +impl MemoryDB { + /// Open the in-memory database. + /// + /// All columns must be supplied initially, you will get an error if you try to access a column + /// that was not declared here. This condition is enforced artificially to simulate RocksDB. + pub fn open() -> Self { + let db: DBHashMap = HashMap::new(); + let mut known_columns: ColumnHashSet = HashSet::new(); + for col in &COLUMNS { + known_columns.insert(col.to_string()); + } + Self { + db: RwLock::new(db), + known_columns: RwLock::new(known_columns), + } + } + + /// Hashes a key and a column name in order to get a unique key for the supplied column. + fn get_key_for_col(col: &str, key: &[u8]) -> Vec { + blake2b(32, col.as_bytes(), key).as_bytes().to_vec() + } +} + +impl ClientDB for MemoryDB { + /// Get the value of some key from the database. Returns `None` if the key does not exist. + fn get(&self, col: &str, key: &[u8]) -> Result, DBError> { + // Panic if the DB locks are poisoned. + let db = self.db.read().unwrap(); + let known_columns = self.known_columns.read().unwrap(); + + if known_columns.contains(&col.to_string()) { + let column_key = MemoryDB::get_key_for_col(col, key); + Ok(db.get(&column_key).and_then(|val| Some(val.clone()))) + } else { + Err(DBError { + message: "Unknown column".to_string(), + }) + } + } + + /// Puts a key in the database. + fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError> { + // Panic if the DB locks are poisoned. + let mut db = self.db.write().unwrap(); + let known_columns = self.known_columns.read().unwrap(); + + if known_columns.contains(&col.to_string()) { + let column_key = MemoryDB::get_key_for_col(col, key); + db.insert(column_key, val.to_vec()); + Ok(()) + } else { + Err(DBError { + message: "Unknown column".to_string(), + }) + } + } + + /// Return true if some key exists in some column. + fn exists(&self, col: &str, key: &[u8]) -> Result { + // Panic if the DB locks are poisoned. + let db = self.db.read().unwrap(); + let known_columns = self.known_columns.read().unwrap(); + + if known_columns.contains(&col.to_string()) { + let column_key = MemoryDB::get_key_for_col(col, key); + Ok(db.contains_key(&column_key)) + } else { + Err(DBError { + message: "Unknown column".to_string(), + }) + } + } + + /// Delete some key from the database. + fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError> { + // Panic if the DB locks are poisoned. + let mut db = self.db.write().unwrap(); + let known_columns = self.known_columns.read().unwrap(); + + if known_columns.contains(&col.to_string()) { + let column_key = MemoryDB::get_key_for_col(col, key); + db.remove(&column_key); + Ok(()) + } else { + Err(DBError { + message: "Unknown column".to_string(), + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::super::stores::{BLOCKS_DB_COLUMN, VALIDATOR_DB_COLUMN}; + use super::super::ClientDB; + use super::*; + use std::sync::Arc; + use std::thread; + + #[test] + fn test_memorydb_can_delete() { + let col_a: &str = BLOCKS_DB_COLUMN; + + let db = MemoryDB::open(); + + db.put(col_a, "dogs".as_bytes(), "lol".as_bytes()).unwrap(); + + assert_eq!( + db.get(col_a, "dogs".as_bytes()).unwrap().unwrap(), + "lol".as_bytes() + ); + + db.delete(col_a, "dogs".as_bytes()).unwrap(); + + assert_eq!(db.get(col_a, "dogs".as_bytes()).unwrap(), None); + } + + #[test] + fn test_memorydb_column_access() { + let col_a: &str = BLOCKS_DB_COLUMN; + let col_b: &str = VALIDATOR_DB_COLUMN; + + let db = MemoryDB::open(); + + /* + * Testing that if we write to the same key in different columns that + * there is not an overlap. + */ + db.put(col_a, "same".as_bytes(), "cat".as_bytes()).unwrap(); + db.put(col_b, "same".as_bytes(), "dog".as_bytes()).unwrap(); + + assert_eq!( + db.get(col_a, "same".as_bytes()).unwrap().unwrap(), + "cat".as_bytes() + ); + assert_eq!( + db.get(col_b, "same".as_bytes()).unwrap().unwrap(), + "dog".as_bytes() + ); + } + + #[test] + fn test_memorydb_unknown_column_access() { + let col_a: &str = BLOCKS_DB_COLUMN; + let col_x: &str = "ColumnX"; + + let db = MemoryDB::open(); + + /* + * Test that we get errors when using undeclared columns + */ + assert!(db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).is_ok()); + assert!(db.put(col_x, "cats".as_bytes(), "lol".as_bytes()).is_err()); + + assert!(db.get(col_a, "cats".as_bytes()).is_ok()); + assert!(db.get(col_x, "cats".as_bytes()).is_err()); + } + + #[test] + fn test_memorydb_exists() { + let col_a: &str = BLOCKS_DB_COLUMN; + let col_b: &str = VALIDATOR_DB_COLUMN; + + let db = MemoryDB::open(); + + /* + * Testing that if we write to the same key in different columns that + * there is not an overlap. + */ + db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).unwrap(); + + assert_eq!(true, db.exists(col_a, "cats".as_bytes()).unwrap()); + assert_eq!(false, db.exists(col_b, "cats".as_bytes()).unwrap()); + + assert_eq!(false, db.exists(col_a, "dogs".as_bytes()).unwrap()); + assert_eq!(false, db.exists(col_b, "dogs".as_bytes()).unwrap()); + } + + #[test] + fn test_memorydb_threading() { + let col_name: &str = BLOCKS_DB_COLUMN; + + let db = Arc::new(MemoryDB::open()); + + let thread_count = 10; + let write_count = 10; + + // We're execting the product of these numbers to fit in one byte. + assert!(thread_count * write_count <= 255); + + let mut handles = vec![]; + for t in 0..thread_count { + let wc = write_count; + let db = db.clone(); + let col = col_name.clone(); + let handle = thread::spawn(move || { + for w in 0..wc { + let key = (t * w) as u8; + let val = 42; + db.put(&col, &vec![key], &vec![val]).unwrap(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + for t in 0..thread_count { + for w in 0..write_count { + let key = (t * w) as u8; + let val = db.get(&col_name, &vec![key]).unwrap().unwrap(); + assert_eq!(vec![42], val); + } + } + } +} diff --git a/beacon_node/db2/src/stores/beacon_block_store.rs b/beacon_node/db2/src/stores/beacon_block_store.rs new file mode 100644 index 000000000..e2e16e60b --- /dev/null +++ b/beacon_node/db2/src/stores/beacon_block_store.rs @@ -0,0 +1,246 @@ +use super::BLOCKS_DB_COLUMN as DB_COLUMN; +use super::{ClientDB, DBError}; +use ssz::Decodable; +use std::sync::Arc; +use types::{BeaconBlock, Hash256, Slot}; + +#[derive(Clone, Debug, PartialEq)] +pub enum BeaconBlockAtSlotError { + UnknownBeaconBlock(Hash256), + InvalidBeaconBlock(Hash256), + DBError(String), +} + +pub struct BeaconBlockStore +where + T: ClientDB, +{ + db: Arc, +} + +// Implements `put`, `get`, `exists` and `delete` for the store. +impl_crud_for_store!(BeaconBlockStore, DB_COLUMN); + +impl BeaconBlockStore { + pub fn new(db: Arc) -> Self { + Self { db } + } + + pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { + match self.get(&hash)? { + None => Ok(None), + Some(ssz) => { + let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { + message: "Bad BeaconBlock SSZ.".to_string(), + })?; + Ok(Some(block)) + } + } + } + + /// Retrieve the block at a slot given a "head_hash" and a slot. + /// + /// A "head_hash" must be a block hash with a slot number greater than or equal to the desired + /// slot. + /// + /// This function will read each block down the chain until it finds a block with the given + /// slot number. If the slot is skipped, the function will return None. + /// + /// If a block is found, a tuple of (block_hash, serialized_block) is returned. + /// + /// Note: this function uses a loop instead of recursion as the compiler is over-strict when it + /// comes to recursion and the `impl Trait` pattern. See: + /// https://stackoverflow.com/questions/54032940/using-impl-trait-in-a-recursive-function + pub fn block_at_slot( + &self, + head_hash: &Hash256, + slot: Slot, + ) -> Result, BeaconBlockAtSlotError> { + let mut current_hash = *head_hash; + + loop { + if let Some(block) = self.get_deserialized(¤t_hash)? { + if block.slot == slot { + break Ok(Some((current_hash, block))); + } else if block.slot < slot { + break Ok(None); + } else { + current_hash = block.previous_block_root; + } + } else { + break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash)); + } + } + } +} + +impl From for BeaconBlockAtSlotError { + fn from(e: DBError) -> Self { + BeaconBlockAtSlotError::DBError(e.message) + } +} + +#[cfg(test)] +mod tests { + use super::super::super::MemoryDB; + use super::*; + + use std::sync::Arc; + use std::thread; + + use ssz::ssz_encode; + use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use types::BeaconBlock; + use types::Hash256; + + test_crud_for_store!(BeaconBlockStore, DB_COLUMN); + + #[test] + fn head_hash_slot_too_low() { + let db = Arc::new(MemoryDB::open()); + let bs = Arc::new(BeaconBlockStore::new(db.clone())); + let mut rng = XorShiftRng::from_seed([42; 16]); + + let mut block = BeaconBlock::random_for_test(&mut rng); + block.slot = Slot::from(10_u64); + + let block_root = block.canonical_root(); + bs.put(&block_root, &ssz_encode(&block)).unwrap(); + + let result = bs.block_at_slot(&block_root, Slot::from(11_u64)).unwrap(); + assert_eq!(result, None); + } + + #[test] + fn test_invalid_block_at_slot() { + let db = Arc::new(MemoryDB::open()); + let store = BeaconBlockStore::new(db.clone()); + + let ssz = "definitly not a valid block".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert_eq!( + store.block_at_slot(hash, Slot::from(42_u64)), + Err(BeaconBlockAtSlotError::DBError( + "Bad BeaconBlock SSZ.".into() + )) + ); + } + + #[test] + fn test_unknown_block_at_slot() { + let db = Arc::new(MemoryDB::open()); + let store = BeaconBlockStore::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + let other_hash = &Hash256::from([0xBB; 32]); + + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert_eq!( + store.block_at_slot(other_hash, Slot::from(42_u64)), + Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash)) + ); + } + + #[test] + fn test_block_store_on_memory_db() { + let db = Arc::new(MemoryDB::open()); + let bs = Arc::new(BeaconBlockStore::new(db.clone())); + + let thread_count = 10; + let write_count = 10; + + let mut handles = vec![]; + for t in 0..thread_count { + let wc = write_count; + let bs = bs.clone(); + let handle = thread::spawn(move || { + for w in 0..wc { + let key = t * w; + let val = 42; + bs.put(&Hash256::from_low_u64_le(key), &vec![val]).unwrap(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + for t in 0..thread_count { + for w in 0..write_count { + let key = t * w; + assert!(bs.exists(&Hash256::from_low_u64_le(key)).unwrap()); + let val = bs.get(&Hash256::from_low_u64_le(key)).unwrap().unwrap(); + assert_eq!(vec![42], val); + } + } + } + + #[test] + #[ignore] + fn test_block_at_slot() { + let db = Arc::new(MemoryDB::open()); + let bs = Arc::new(BeaconBlockStore::new(db.clone())); + let mut rng = XorShiftRng::from_seed([42; 16]); + + // Specify test block parameters. + let hashes = [ + Hash256::from([0; 32]), + Hash256::from([1; 32]), + Hash256::from([2; 32]), + Hash256::from([3; 32]), + Hash256::from([4; 32]), + ]; + let parent_hashes = [ + Hash256::from([255; 32]), // Genesis block. + Hash256::from([0; 32]), + Hash256::from([1; 32]), + Hash256::from([2; 32]), + Hash256::from([3; 32]), + ]; + let unknown_hash = Hash256::from([101; 32]); // different from all above + let slots: Vec = vec![0, 1, 3, 4, 5].iter().map(|x| Slot::new(*x)).collect(); + + // Generate a vec of random blocks and store them in the DB. + let block_count = 5; + let mut blocks: Vec = Vec::with_capacity(5); + for i in 0..block_count { + let mut block = BeaconBlock::random_for_test(&mut rng); + + block.previous_block_root = parent_hashes[i]; + block.slot = slots[i]; + + let ssz = ssz_encode(&block); + db.put(DB_COLUMN, hashes[i].as_bytes(), &ssz).unwrap(); + + blocks.push(block); + } + + // Test that certain slots can be reached from certain hashes. + let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; + for (hashes_index, slot_index) in test_cases { + let (matched_block_hash, block) = bs + .block_at_slot(&hashes[hashes_index], slots[slot_index]) + .unwrap() + .unwrap(); + assert_eq!(matched_block_hash, hashes[slot_index]); + assert_eq!(block.slot, slots[slot_index]); + } + + let ssz = bs.block_at_slot(&hashes[4], Slot::new(2)).unwrap(); + assert_eq!(ssz, None); + + let ssz = bs.block_at_slot(&hashes[4], Slot::new(6)).unwrap(); + assert_eq!(ssz, None); + + let ssz = bs.block_at_slot(&unknown_hash, Slot::new(2)); + assert_eq!( + ssz, + Err(BeaconBlockAtSlotError::UnknownBeaconBlock(unknown_hash)) + ); + } +} diff --git a/beacon_node/db2/src/stores/beacon_state_store.rs b/beacon_node/db2/src/stores/beacon_state_store.rs new file mode 100644 index 000000000..fd6ff569a --- /dev/null +++ b/beacon_node/db2/src/stores/beacon_state_store.rs @@ -0,0 +1,62 @@ +use super::STATES_DB_COLUMN as DB_COLUMN; +use super::{ClientDB, DBError}; +use ssz::Decodable; +use std::sync::Arc; +use types::{BeaconState, Hash256}; + +pub struct BeaconStateStore +where + T: ClientDB, +{ + db: Arc, +} + +// Implements `put`, `get`, `exists` and `delete` for the store. +impl_crud_for_store!(BeaconStateStore, DB_COLUMN); + +impl BeaconStateStore { + pub fn new(db: Arc) -> Self { + Self { db } + } + + pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { + match self.get(&hash)? { + None => Ok(None), + Some(ssz) => { + let (state, _) = BeaconState::ssz_decode(&ssz, 0).map_err(|_| DBError { + message: "Bad State SSZ.".to_string(), + })?; + Ok(Some(state)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::super::super::MemoryDB; + use super::*; + + use ssz::ssz_encode; + use std::sync::Arc; + use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use types::Hash256; + + test_crud_for_store!(BeaconStateStore, DB_COLUMN); + + #[test] + fn test_reader() { + let db = Arc::new(MemoryDB::open()); + let store = BeaconStateStore::new(db.clone()); + + let mut rng = XorShiftRng::from_seed([42; 16]); + let state = BeaconState::random_for_test(&mut rng); + let state_root = state.canonical_root(); + + store.put(&state_root, &ssz_encode(&state)).unwrap(); + + let decoded = store.get_deserialized(&state_root).unwrap().unwrap(); + + assert_eq!(state, decoded); + } +} diff --git a/beacon_node/db2/src/stores/macros.rs b/beacon_node/db2/src/stores/macros.rs new file mode 100644 index 000000000..6c53e40ee --- /dev/null +++ b/beacon_node/db2/src/stores/macros.rs @@ -0,0 +1,103 @@ +macro_rules! impl_crud_for_store { + ($store: ident, $db_column: expr) => { + impl $store { + pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { + self.db.put($db_column, hash.as_bytes(), ssz) + } + + pub fn get(&self, hash: &Hash256) -> Result>, DBError> { + self.db.get($db_column, hash.as_bytes()) + } + + pub fn exists(&self, hash: &Hash256) -> Result { + self.db.exists($db_column, hash.as_bytes()) + } + + pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> { + self.db.delete($db_column, hash.as_bytes()) + } + } + }; +} + +#[cfg(test)] +macro_rules! test_crud_for_store { + ($store: ident, $db_column: expr) => { + #[test] + fn test_put() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + + store.put(hash, ssz).unwrap(); + assert_eq!(db.get(DB_COLUMN, hash.as_bytes()).unwrap().unwrap(), ssz); + } + + #[test] + fn test_get() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert_eq!(store.get(hash).unwrap().unwrap(), ssz); + } + + #[test] + fn test_get_unknown() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + let other_hash = &Hash256::from([0xBB; 32]); + + db.put(DB_COLUMN, other_hash.as_bytes(), ssz).unwrap(); + assert_eq!(store.get(hash).unwrap(), None); + } + + #[test] + fn test_exists() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert!(store.exists(hash).unwrap()); + } + + #[test] + fn test_block_does_not_exist() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + let other_hash = &Hash256::from([0xBB; 32]); + + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert!(!store.exists(other_hash).unwrap()); + } + + #[test] + fn test_delete() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from([0xAA; 32]); + + db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); + assert!(db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); + + store.delete(hash).unwrap(); + assert!(!db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); + } + }; +} diff --git a/beacon_node/db2/src/stores/mod.rs b/beacon_node/db2/src/stores/mod.rs new file mode 100644 index 000000000..44de7eed1 --- /dev/null +++ b/beacon_node/db2/src/stores/mod.rs @@ -0,0 +1,25 @@ +use super::{ClientDB, DBError}; + +#[macro_use] +mod macros; +mod beacon_block_store; +mod beacon_state_store; +mod pow_chain_store; +mod validator_store; + +pub use self::beacon_block_store::{BeaconBlockAtSlotError, BeaconBlockStore}; +pub use self::beacon_state_store::BeaconStateStore; +pub use self::pow_chain_store::PoWChainStore; +pub use self::validator_store::{ValidatorStore, ValidatorStoreError}; + +pub const BLOCKS_DB_COLUMN: &str = "blocks"; +pub const STATES_DB_COLUMN: &str = "states"; +pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; +pub const VALIDATOR_DB_COLUMN: &str = "validator"; + +pub const COLUMNS: [&str; 4] = [ + BLOCKS_DB_COLUMN, + STATES_DB_COLUMN, + POW_CHAIN_DB_COLUMN, + VALIDATOR_DB_COLUMN, +]; diff --git a/beacon_node/db2/src/stores/pow_chain_store.rs b/beacon_node/db2/src/stores/pow_chain_store.rs new file mode 100644 index 000000000..5c8b97907 --- /dev/null +++ b/beacon_node/db2/src/stores/pow_chain_store.rs @@ -0,0 +1,68 @@ +use super::POW_CHAIN_DB_COLUMN as DB_COLUMN; +use super::{ClientDB, DBError}; +use std::sync::Arc; + +pub struct PoWChainStore +where + T: ClientDB, +{ + db: Arc, +} + +impl PoWChainStore { + pub fn new(db: Arc) -> Self { + Self { db } + } + + pub fn put_block_hash(&self, hash: &[u8]) -> Result<(), DBError> { + self.db.put(DB_COLUMN, hash, &[0]) + } + + pub fn block_hash_exists(&self, hash: &[u8]) -> Result { + self.db.exists(DB_COLUMN, hash) + } +} + +#[cfg(test)] +mod tests { + extern crate types; + + use super::super::super::MemoryDB; + use super::*; + + use self::types::Hash256; + + #[test] + fn test_put_block_hash() { + let db = Arc::new(MemoryDB::open()); + let store = PoWChainStore::new(db.clone()); + + let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); + store.put_block_hash(hash).unwrap(); + + assert!(db.exists(DB_COLUMN, hash).unwrap()); + } + + #[test] + fn test_block_hash_exists() { + let db = Arc::new(MemoryDB::open()); + let store = PoWChainStore::new(db.clone()); + + let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); + db.put(DB_COLUMN, hash, &[0]).unwrap(); + + assert!(store.block_hash_exists(hash).unwrap()); + } + + #[test] + fn test_block_hash_does_not_exist() { + let db = Arc::new(MemoryDB::open()); + let store = PoWChainStore::new(db.clone()); + + let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); + let other_hash = &Hash256::from([0xBB; 32]).as_bytes().to_vec(); + db.put(DB_COLUMN, hash, &[0]).unwrap(); + + assert!(!store.block_hash_exists(other_hash).unwrap()); + } +} diff --git a/beacon_node/db2/src/stores/validator_store.rs b/beacon_node/db2/src/stores/validator_store.rs new file mode 100644 index 000000000..02e90dc5c --- /dev/null +++ b/beacon_node/db2/src/stores/validator_store.rs @@ -0,0 +1,215 @@ +extern crate bytes; + +use self::bytes::{BufMut, BytesMut}; +use super::VALIDATOR_DB_COLUMN as DB_COLUMN; +use super::{ClientDB, DBError}; +use bls::PublicKey; +use ssz::{ssz_encode, Decodable}; +use std::sync::Arc; + +#[derive(Debug, PartialEq)] +pub enum ValidatorStoreError { + DBError(String), + DecodeError, +} + +impl From for ValidatorStoreError { + fn from(error: DBError) -> Self { + ValidatorStoreError::DBError(error.message) + } +} + +#[derive(Debug, PartialEq)] +enum KeyPrefixes { + PublicKey, +} + +pub struct ValidatorStore +where + T: ClientDB, +{ + db: Arc, +} + +impl ValidatorStore { + pub fn new(db: Arc) -> Self { + Self { db } + } + + fn prefix_bytes(&self, key_prefix: &KeyPrefixes) -> Vec { + match key_prefix { + KeyPrefixes::PublicKey => b"pubkey".to_vec(), + } + } + + fn get_db_key_for_index(&self, key_prefix: &KeyPrefixes, index: usize) -> Vec { + let mut buf = BytesMut::with_capacity(6 + 8); + buf.put(self.prefix_bytes(key_prefix)); + buf.put_u64_be(index as u64); + buf.take().to_vec() + } + + pub fn put_public_key_by_index( + &self, + index: usize, + public_key: &PublicKey, + ) -> Result<(), ValidatorStoreError> { + let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); + let val = ssz_encode(public_key); + self.db + .put(DB_COLUMN, &key[..], &val[..]) + .map_err(ValidatorStoreError::from) + } + + pub fn get_public_key_by_index( + &self, + index: usize, + ) -> Result, ValidatorStoreError> { + let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); + let val = self.db.get(DB_COLUMN, &key[..])?; + match val { + None => Ok(None), + Some(val) => match PublicKey::ssz_decode(&val, 0) { + Ok((key, _)) => Ok(Some(key)), + Err(_) => Err(ValidatorStoreError::DecodeError), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::super::super::MemoryDB; + use super::*; + use bls::Keypair; + + #[test] + fn test_prefix_bytes() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db.clone()); + + assert_eq!( + store.prefix_bytes(&KeyPrefixes::PublicKey), + b"pubkey".to_vec() + ); + } + + #[test] + fn test_get_db_key_for_index() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db.clone()); + + let mut buf = BytesMut::with_capacity(6 + 8); + buf.put(b"pubkey".to_vec()); + buf.put_u64_be(42); + assert_eq!( + store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42), + buf.take().to_vec() + ) + } + + #[test] + fn test_put_public_key_by_index() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db.clone()); + + let index = 3; + let public_key = Keypair::random().pk; + + store.put_public_key_by_index(index, &public_key).unwrap(); + let public_key_at_index = db + .get( + DB_COLUMN, + &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..], + ) + .unwrap() + .unwrap(); + + assert_eq!(public_key_at_index, ssz_encode(&public_key)); + } + + #[test] + fn test_get_public_key_by_index() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db.clone()); + + let index = 4; + let public_key = Keypair::random().pk; + + db.put( + DB_COLUMN, + &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..], + &ssz_encode(&public_key)[..], + ) + .unwrap(); + + let public_key_at_index = store.get_public_key_by_index(index).unwrap().unwrap(); + assert_eq!(public_key_at_index, public_key); + } + + #[test] + fn test_get_public_key_by_unknown_index() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db.clone()); + + let public_key = Keypair::random().pk; + + db.put( + DB_COLUMN, + &store.get_db_key_for_index(&KeyPrefixes::PublicKey, 3)[..], + &ssz_encode(&public_key)[..], + ) + .unwrap(); + + let public_key_at_index = store.get_public_key_by_index(4).unwrap(); + assert_eq!(public_key_at_index, None); + } + + #[test] + fn test_get_invalid_public_key() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db.clone()); + + let key = store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42); + db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap(); + + assert_eq!( + store.get_public_key_by_index(42), + Err(ValidatorStoreError::DecodeError) + ); + } + + #[test] + fn test_validator_store_put_get() { + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db); + + let keys = vec![ + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + ]; + + for i in 0..keys.len() { + store.put_public_key_by_index(i, &keys[i].pk).unwrap(); + } + + /* + * Check all keys are retrieved correctly. + */ + for i in 0..keys.len() { + let retrieved = store.get_public_key_by_index(i).unwrap().unwrap(); + assert_eq!(retrieved, keys[i].pk); + } + + /* + * Check that an index that wasn't stored returns None. + */ + assert!(store + .get_public_key_by_index(keys.len() + 1) + .unwrap() + .is_none()); + } +} diff --git a/beacon_node/db2/src/traits.rs b/beacon_node/db2/src/traits.rs new file mode 100644 index 000000000..57ebf9353 --- /dev/null +++ b/beacon_node/db2/src/traits.rs @@ -0,0 +1,38 @@ +pub type DBValue = Vec; + +#[derive(Debug)] +pub struct DBError { + pub message: String, +} + +impl DBError { + pub fn new(message: String) -> Self { + Self { message } + } +} + +/// A generic database to be used by the "client' (i.e., +/// the lighthouse blockchain client). +/// +/// The purpose of having this generic trait is to allow the +/// program to use a persistent on-disk database during production, +/// but use a transient database during tests. +pub trait ClientDB: Sync + Send { + fn get(&self, col: &str, key: &[u8]) -> Result, DBError>; + + fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError>; + + fn exists(&self, col: &str, key: &[u8]) -> Result; + + fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError>; +} + +pub enum DBColumn { + Block, + State, + BeaconChain, +} + +pub trait DBStore { + fn db_column(&self) -> DBColumn; +} diff --git a/beacon_node/db_encode/Cargo.toml b/beacon_node/db_encode/Cargo.toml new file mode 100644 index 000000000..b4e919585 --- /dev/null +++ b/beacon_node/db_encode/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "db_encode" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +ethereum-types = "0.5" +ssz = { path = "../../eth2/utils/ssz" } diff --git a/beacon_node/db_encode/src/lib.rs b/beacon_node/db_encode/src/lib.rs new file mode 100644 index 000000000..993ba0e79 --- /dev/null +++ b/beacon_node/db_encode/src/lib.rs @@ -0,0 +1,59 @@ +use ethereum_types::{Address, H256}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; + +/// Convenience function to encode an object. +pub fn db_encode(val: &T) -> Vec +where + T: DBEncode, +{ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(val); + ssz_stream.drain() +} + +/// An encoding scheme based solely upon SSZ. +/// +/// The reason we have a separate encoding scheme is to allows us to store fields in the DB that we +/// don't want to transmit across the wire or hash. +/// +/// For example, the cache fields on `BeaconState` should be stored in the DB, but they should not +/// be hashed or transmitted across the wire. `DBEncode` allows us to define two serialization +/// methods, one that encodes the caches and one that does not. +pub trait DBEncode: Encodable + Sized { + fn db_encode(&self, s: &mut SszStream) { + s.append(&ssz_encode(self)); + } +} + +/// A decoding scheme based solely upon SSZ. +/// +/// See `DBEncode` for reasoning on why this trait exists. +pub trait DBDecode: Decodable { + fn db_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + Self::ssz_decode(bytes, index) + } +} + +// Implement encoding. +impl DBEncode for bool {} +impl DBEncode for u8 {} +impl DBEncode for u16 {} +impl DBEncode for u32 {} +impl DBEncode for u64 {} +impl DBEncode for usize {} +impl DBEncode for Vec where T: Encodable + Sized {} + +impl DBEncode for H256 {} +impl DBEncode for Address {} + +// Implement decoding. +impl DBDecode for bool {} +impl DBDecode for u8 {} +impl DBDecode for u16 {} +impl DBDecode for u32 {} +impl DBDecode for u64 {} +impl DBDecode for usize {} +impl DBDecode for Vec where T: Decodable {} + +impl DBDecode for H256 {} +impl DBDecode for Address {} diff --git a/beacon_node/db_encode_derive/Cargo.toml b/beacon_node/db_encode_derive/Cargo.toml new file mode 100644 index 000000000..b2fba85e3 --- /dev/null +++ b/beacon_node/db_encode_derive/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "db_encode_derive" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" +description = "Procedural derive macros for `db_encode` encoding and decoding." + +[lib] +proc-macro = true + +[dependencies] +syn = "0.15" +quote = "0.6" diff --git a/beacon_node/db_encode_derive/src/lib.rs b/beacon_node/db_encode_derive/src/lib.rs new file mode 100644 index 000000000..1de081419 --- /dev/null +++ b/beacon_node/db_encode_derive/src/lib.rs @@ -0,0 +1,305 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +/// Returns a Vec of `syn::Ident` for each named field in the struct. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. +fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { + struct_data + .fields + .iter() + .map(|f| match &f.ident { + Some(ref ident) => ident, + _ => panic!("db_derive only supports named struct fields."), + }) + .collect() +} + +/// Implements `db_encode::DBEncode` for some `struct`. +/// +/// Fields are encoded in the order they are defined. +#[proc_macro_derive(DBEncode)] +pub fn db_encode_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("db_derive only supports structs."), + }; + + let field_idents = get_named_field_idents(&struct_data); + + let output = quote! { + impl db_encode::DBEncode for #name { + fn db_encode(&self, s: &mut ssz::SszStream) { + #( + s.append(&self.#field_idents); + )* + } + } + }; + output.into() +} + +/// Implements `db_encode::DBEncode` for some `struct`. +/// +/// Fields are encoded in the order they are defined. +#[proc_macro_derive(DBDecode)] +pub fn db_decode_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs."), + }; + + let field_idents = get_named_field_idents(&struct_data); + + // Using a var in an iteration always consumes the var, therefore we must make a `fields_a` and + // a `fields_b` in order to perform two loops. + // + // https://github.com/dtolnay/quote/issues/8 + let field_idents_a = &field_idents; + let field_idents_b = &field_idents; + + let output = quote! { + impl db_encode::DBDecode for #name { + fn db_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), ssz::DecodeError> { + #( + let (#field_idents_a, i) = <_>::ssz_decode(bytes, i)?; + )* + + Ok(( + Self { + #( + #field_idents_b, + )* + }, + i + )) + } + } + }; + output.into() +} + +/* +/// Returns true if some field has an attribute declaring it should not be deserialized. +/// +/// The field attribute is: `#[ssz(skip_deserializing)]` +fn should_skip_deserializing(field: &syn::Field) -> bool { + for attr in &field.attrs { + if attr.tts.to_string() == "( skip_deserializing )" { + return true; + } + } + false +} + +/// Implements `ssz::Decodable` for some `struct`. +/// +/// Fields are decoded in the order they are defined. +#[proc_macro_derive(Decode)] +pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs."), + }; + + let all_idents = get_named_field_idents(&struct_data); + + // Build quotes for fields that should be deserialized and those that should be built from + // `Default`. + let mut quotes = vec![]; + for field in &struct_data.fields { + match &field.ident { + Some(ref ident) => { + if should_skip_deserializing(field) { + quotes.push(quote! { + let #ident = <_>::default(); + }); + } else { + quotes.push(quote! { + let (#ident, i) = <_>::ssz_decode(bytes, i)?; + }); + } + } + _ => panic!("ssz_derive only supports named struct fields."), + }; + } + + let output = quote! { + impl ssz::Decodable for #name { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), ssz::DecodeError> { + #( + #quotes + )* + + Ok(( + Self { + #( + #all_idents, + )* + }, + i + )) + } + } + }; + output.into() +} + +/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields +/// that should not be tree hashed. +/// +/// # Panics +/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. +fn get_tree_hashable_named_field_idents<'a>( + struct_data: &'a syn::DataStruct, +) -> Vec<&'a syn::Ident> { + struct_data + .fields + .iter() + .filter_map(|f| { + if should_skip_tree_hash(&f) { + None + } else { + Some(match &f.ident { + Some(ref ident) => ident, + _ => panic!("ssz_derive only supports named struct fields."), + }) + } + }) + .collect() +} + +/// Returns true if some field has an attribute declaring it should not be tree-hashed. +/// +/// The field attribute is: `#[tree_hash(skip_hashing)]` +fn should_skip_tree_hash(field: &syn::Field) -> bool { + for attr in &field.attrs { + if attr.tts.to_string() == "( skip_hashing )" { + return true; + } + } + false +} + +/// Implements `ssz::TreeHash` for some `struct`. +/// +/// Fields are processed in the order they are defined. +#[proc_macro_derive(TreeHash, attributes(tree_hash))] +pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs."), + }; + + let field_idents = get_tree_hashable_named_field_idents(&struct_data); + + let output = quote! { + impl ssz::TreeHash for #name { + fn hash_tree_root(&self) -> Vec { + let mut list: Vec> = Vec::new(); + #( + list.push(self.#field_idents.hash_tree_root()); + )* + + ssz::merkle_hash(&mut list) + } + } + }; + output.into() +} + +/// Returns `true` if some `Ident` should be considered to be a signature type. +fn type_ident_is_signature(ident: &syn::Ident) -> bool { + match ident.to_string().as_ref() { + "Signature" => true, + "AggregateSignature" => true, + _ => false, + } +} + +/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns +/// the final `Ident` in that path. +/// +/// E.g., for `types::Signature` returns `Signature`. +fn final_type_ident(field: &syn::Field) -> &syn::Ident { + match &field.ty { + syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident, + _ => panic!("ssz_derive only supports Path types."), + } +} + +/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and +/// including a field that is of type "Signature" or "AggregateSignature". +/// +/// See: +/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots +/// +/// This is a rather horrendous macro, it will read the type of the object as a string and decide +/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So, +/// it's important that you use those exact words as your type -- don't alias it to something else. +/// +/// If you can think of a better way to do this, please make an issue! +/// +/// Fields are processed in the order they are defined. +#[proc_macro_derive(SignedRoot)] +pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream { + let item = parse_macro_input!(input as DeriveInput); + + let name = &item.ident; + + let struct_data = match &item.data { + syn::Data::Struct(s) => s, + _ => panic!("ssz_derive only supports structs."), + }; + + let mut field_idents: Vec<&syn::Ident> = vec![]; + + for field in struct_data.fields.iter() { + let final_type_ident = final_type_ident(&field); + + if type_ident_is_signature(final_type_ident) { + break; + } else { + let ident = field + .ident + .as_ref() + .expect("ssz_derive only supports named_struct fields."); + field_idents.push(ident); + } + } + + let output = quote! { + impl ssz::SignedRoot for #name { + fn signed_root(&self) -> Vec { + let mut list: Vec> = Vec::new(); + #( + list.push(self.#field_idents.hash_tree_root()); + )* + + ssz::merkle_hash(&mut list) + } + } + }; + output.into() +} +*/ From d6664cb4ac3353020c3f30b821c0686461da40f7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Apr 2019 13:41:04 +1100 Subject: [PATCH 05/44] Implement all methods on `Store` --- beacon_node/db2/src/lib.rs | 47 +++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/beacon_node/db2/src/lib.rs b/beacon_node/db2/src/lib.rs index 0704a84f5..55f419978 100644 --- a/beacon_node/db2/src/lib.rs +++ b/beacon_node/db2/src/lib.rs @@ -83,8 +83,6 @@ where T: ClientDB, { /// Put `item` in the store as `key`. - /// - /// The `item` must implement `DBRecord` which defines the db column used. fn put(&self, key: &Hash256, item: &I) -> Result<(), Error> where I: DBRecord, @@ -96,9 +94,7 @@ where self.db.put(column, key, &val).map_err(|e| e.into()) } - /// Retrieves an `Ok(Some(item)` from the store if `key` exists, otherwise returns `Ok(None)`. - /// - /// The `item` must implement `DBRecord` which defines the db column used. + /// Retrieves an `Ok(Some(item))` from the store if `key` exists, otherwise returns `Ok(None)`. fn get(&self, key: &Hash256) -> Result, Error> where I: DBRecord, @@ -114,6 +110,28 @@ where None => Ok(None), } } + + /// Returns `Ok(true)` `key` exists in the store. + fn exists(&self, key: &Hash256) -> Result + where + I: DBRecord, + { + let column = I::db_column().into(); + let key = key.as_bytes(); + + self.db.exists(column, key).map_err(|e| e.into()) + } + + /// Returns `Ok(())` if `key` was deleted from the database or did not exist. + fn delete(&self, key: &Hash256) -> Result<(), Error> + where + I: DBRecord, + { + let column = I::db_column().into(); + let key = key.as_bytes(); + + self.db.delete(column, key).map_err(|e| e.into()) + } } #[cfg(test)] @@ -136,7 +154,7 @@ mod tests { } #[test] - fn memorydb_can_store() { + fn memorydb_can_store_and_retrieve() { let store = Store::new_in_memory(); let key = Hash256::random(); @@ -148,4 +166,21 @@ mod tests { assert_eq!(item, retrieved); } + + #[test] + fn exists() { + let store = Store::new_in_memory(); + let key = Hash256::random(); + let item = StorableThing { a: 1, b: 42 }; + + assert_eq!(store.exists::(&key).unwrap(), false); + + store.put(&key, &item).unwrap(); + + assert_eq!(store.exists::(&key).unwrap(), true); + + store.delete::(&key).unwrap(); + + assert_eq!(store.exists::(&key).unwrap(), false); + } } From 3e030c78a870c1402eeb3b08671728280ff58c94 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Wed, 10 Apr 2019 14:42:31 +1000 Subject: [PATCH 06/44] copied builder and setup tests --- .../block_processing_builder.rs | 175 ++++++++++++++++++ .../src/per_block_processing/tests.rs | 23 +++ 2 files changed, 198 insertions(+) create mode 100644 eth2/state_processing/src/per_block_processing/block_processing_builder.rs create mode 100644 eth2/state_processing/src/per_block_processing/tests.rs diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs new file mode 100644 index 000000000..99bbf0944 --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -0,0 +1,175 @@ +use log::info; +use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; +use types::*; + +pub struct BlockProcessingBuilder { + pub state_builder: TestingBeaconStateBuilder, + pub block_builder: TestingBeaconBlockBuilder, + + pub num_validators: usize, + pub num_proposer_slashings: usize, + pub num_attester_slashings: usize, + pub num_indices_per_slashable_vote: usize, + pub num_attestations: usize, + pub num_deposits: usize, + pub num_exits: usize, + pub num_transfers: usize, +} + +impl BlockProcessingBuilder { + pub fn new(num_validators: usize, spec: &ChainSpec) -> Self { + let state_builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec); + let block_builder = TestingBeaconBlockBuilder::new(spec); + + Self { + state_builder, + block_builder, + num_validators: 0, + num_proposer_slashings: 0, + num_attester_slashings: 0, + num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize, + num_attestations: 0, + num_deposits: 0, + num_exits: 0, + num_transfers: 0, + } + } + + pub fn maximize_block_operations(&mut self, spec: &ChainSpec) { + self.num_proposer_slashings = spec.max_proposer_slashings as usize; + self.num_attester_slashings = spec.max_attester_slashings as usize; + self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize; + self.num_attestations = spec.max_attestations as usize; + self.num_deposits = spec.max_deposits as usize; + self.num_exits = spec.max_voluntary_exits as usize; + self.num_transfers = spec.max_transfers as usize; + } + + pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { + self.state_builder.teleport_to_slot(slot, &spec); + } + + pub fn build_caches(&mut self, spec: &ChainSpec) { + // Builds all caches; benches will not contain shuffling/committee building times. + self.state_builder.build_caches(&spec).unwrap(); + } + + pub fn build(mut self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { + let (mut state, keypairs) = self.state_builder.build(); + let builder = &mut self.block_builder; + + builder.set_slot(state.slot); + + let proposer_index = state.get_beacon_proposer_index(state.slot, RelativeEpoch::Current, spec).unwrap(); + let keypair = &keypairs[proposer_index]; + + builder.set_randao_reveal(&keypair.sk, &state.fork, spec); + + // Used as a stream of validator indices for use in slashings, exits, etc. + let mut validators_iter = (0..keypairs.len() as u64).into_iter(); + + // Insert `ProposerSlashing` objects. + for _ in 0..self.num_proposer_slashings { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + builder.insert_proposer_slashing( + validator_index, + &keypairs[validator_index as usize].sk, + &state.fork, + spec, + ); + } + info!( + "Inserted {} proposer slashings.", + builder.block.body.proposer_slashings.len() + ); + + // Insert `AttesterSlashing` objects + for _ in 0..self.num_attester_slashings { + let mut attesters: Vec = vec![]; + let mut secret_keys: Vec<&SecretKey> = vec![]; + + for _ in 0..self.num_indices_per_slashable_vote { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + attesters.push(validator_index); + secret_keys.push(&keypairs[validator_index as usize].sk); + } + + builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec); + } + info!( + "Inserted {} attester slashings.", + builder.block.body.attester_slashings.len() + ); + + // Insert `Attestation` objects. + let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect(); + builder + .insert_attestations( + &state, + &all_secret_keys, + self.num_attestations as usize, + spec, + ) + .unwrap(); + info!( + "Inserted {} attestations.", + builder.block.body.attestations.len() + ); + + // Insert `Deposit` objects. + for i in 0..self.num_deposits { + builder.insert_deposit( + 32_000_000_000, + state.deposit_index + (i as u64), + &state, + spec, + ); + } + info!("Inserted {} deposits.", builder.block.body.deposits.len()); + + // Insert the maximum possible number of `Exit` objects. + for _ in 0..self.num_exits { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + builder.insert_exit( + &state, + validator_index, + &keypairs[validator_index as usize].sk, + spec, + ); + } + info!( + "Inserted {} exits.", + builder.block.body.voluntary_exits.len() + ); + + // Insert the maximum possible number of `Transfer` objects. + for _ in 0..self.num_transfers { + let validator_index = validators_iter.next().expect("Insufficient validators."); + + // Manually set the validator to be withdrawn. + state.validator_registry[validator_index as usize].withdrawable_epoch = + state.previous_epoch(spec); + + builder.insert_transfer( + &state, + validator_index, + validator_index, + 1, + keypairs[validator_index as usize].clone(), + spec, + ); + } + info!("Inserted {} transfers.", builder.block.body.transfers.len()); + + let mut block = self.block_builder.build(&keypair.sk, &state.fork, spec); + + // Set the eth1 data to be different from the state. + block.body.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]); + + (block, state) + } +} diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs new file mode 100644 index 000000000..004f84805 --- /dev/null +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -0,0 +1,23 @@ +#![cfg(test)] +use crate::per_block_processing; +use super::block_processing_builder::BlockProcessingBuilder; +use types::*; + +pub const VALIDATOR_COUNT: usize = 10; + +#[test] +fn runs_without_error() { + let spec = ChainSpec::foundation(); + let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); + + // Set the state and block to be in the last slot of the 4th epoch. + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + builder.set_slot(last_slot_of_epoch, &spec); + + builder.build_caches(&spec); + + let (block, mut state) = builder.build(&spec); + + per_block_processing(&mut state, &block, &spec).unwrap(); +} + From e1c08b1d02dd23f42c1179b72b4b2d10ef0afb09 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Wed, 10 Apr 2019 21:06:25 +1000 Subject: [PATCH 07/44] mapped out invalid states... 59 --- .../src/per_block_processing/tests.rs | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 004f84805..c9318bc63 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -13,7 +13,7 @@ fn runs_without_error() { // Set the state and block to be in the last slot of the 4th epoch. let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); builder.set_slot(last_slot_of_epoch, &spec); - + builder.build_caches(&spec); let (block, mut state) = builder.build(&spec); @@ -21,3 +21,76 @@ fn runs_without_error() { per_block_processing(&mut state, &block, &spec).unwrap(); } +// process_block_header +// Invalid::StateSlotMismatch +// Invalid::ParentBlockRootMismatch + +// verify_block_signature +// Invalid::BadSignature + +// process_randao +// Invalid::BadRandaoSignature + +// process_proposer_slashings +// Invalid::MaxProposerSlashingsExceeded +// verify_proposer_slashing +// Invalid::ProposerUnknown +// Invalid::ProposalSlotMismatch +// Invalid::ProposalsIdentical +// Invalid::ProposerAlreadySlashed +// Invalid::ProposerAlreadyWithdrawn +// Invalid::BadProposal1Signature +// Invalid::BadProposal2Signature + +// process_attester_slashings +// Invalid::MaxAttesterSlashingsExceed +// verify_attester_slashing +// Invalid::AttestationDataIdentical +// Invalid::NotSlashable +// Invalid::SlashableAttestation1Invalid +// Invalid::SlashableAttestation2Invalid + +// process_attestations +// Invalid::MaxAttestationsExceeded +// validate_attestation +// Invalid::PreGenesis +// Invalid::IncludedTooLate +// Invalid::IncludedTooEarly +// Invalid::BadPreviousCrosslink +// Invalid::AggregationBitfieldIsEmpty +// Invalid::CustodyBitfieldHasSetBits +// Invalid::NoCommitteeForShard +// Invalid::BadCustodyBitfieldLength +// Invalid::BadAggregationBitfieldLength +// Invalid::ShardBlockRootNotZero +// verify_justified_epoch_and_root +// Invalid::WrongJustifiedEpoch (current) +// Invalid::WrongJustifiedRoot (current) +// Invalid::WrongJustifiedEpoch (previous) +// Invalid::WrongJustifiedRoot (previous) +// verify_attestation_signature +// Invalid::BadAggregationBitfieldLength +// Invalid::BadCustodyBitfieldLength +// BeaconStateError::UnknownValidator +// Invalid::BadSignature + +// process_deposits +// Invalid::MaxDepositsExceeded +// verify_deposit +// Invalid::BadProofOfPossession +// Invalid::BadMerkleProof +// verify_deposit_index +// Invalid::BadIndex + +// process_exits +// Invalid::MaxExitsExceeded +// verify_exit +// Invalid::ValidatorUnknown +// Invalid::AlreadyExited +// Invalid::AlreadyInitiatedExited +// Invalid::FutureEpoch +// Invalid::TooYoungToLeave +// Invalid::BadSignature + + + From 8366352aaabca7a9f8e0c811353b64fc9d52d018 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Wed, 10 Apr 2019 21:07:56 +1000 Subject: [PATCH 08/44] mapped out invalid states... 59 --- .../src/per_block_processing/tests.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index c9318bc63..28dbc2c55 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -92,5 +92,20 @@ fn runs_without_error() { // Invalid::TooYoungToLeave // Invalid::BadSignature +// process_transfers +// Invalid::MaxTransfersExceed +// verify_transfer +// Invalid::FromValidatorUnknown +// Invalid::FeeOverflow +// Invalid::FromBalanceInsufficient (amount) +// Invalid::FromBalanceInsufficient (fee) +// Invalid::InvalidResultingFromBalance +// Invalid::TransferSlotInPast +// Invalid::StateSlotMismatch +// Invalid::FromValidatorUnknown (???) +// Invalid::FromValidatorIneligableForTransfer +// Invalid::WithdrawalCredentialsMismatch +// Invalid::BadSignature + From f9c1800c00c399594b17323676dded9ac608b843 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Wed, 10 Apr 2019 21:56:31 +1000 Subject: [PATCH 09/44] process_block_header tests --- .../src/per_block_processing/tests.rs | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 28dbc2c55..ac294de69 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -1,6 +1,7 @@ #![cfg(test)] use crate::per_block_processing; use super::block_processing_builder::BlockProcessingBuilder; +use super::errors::*; use types::*; pub const VALIDATOR_COUNT: usize = 10; @@ -8,23 +9,43 @@ pub const VALIDATOR_COUNT: usize = 10; #[test] fn runs_without_error() { let spec = ChainSpec::foundation(); + let (block, mut state) = get_block_state(&spec); + + per_block_processing(&mut state, &block, &spec).unwrap(); +} + +#[test] +fn process_block_header_invalid_state_slot() { + let spec = ChainSpec::foundation(); + let (mut block, mut state) = get_block_state(&spec); + + state.slot = Slot::new(133713); + block.slot = Slot::new(424242); + + let result = per_block_processing(&mut state, &block, &spec); + + assert_eq!(result, Err(BlockProcessingError::Invalid(BlockInvalid::StateSlotMismatch))); +} + +#[test] +#[ignore] +fn process_block_header_invalid_parent_block_root() { + // this will be changed in spec 0.5.1 to use signed root +} + +fn get_block_state(spec: &ChainSpec) -> (BeaconBlock, BeaconState) { let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); // Set the state and block to be in the last slot of the 4th epoch. let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); builder.set_slot(last_slot_of_epoch, &spec); - builder.build_caches(&spec); - - let (block, mut state) = builder.build(&spec); - per_block_processing(&mut state, &block, &spec).unwrap(); + let (block, state) = builder.build(&spec); + + (block, state) } -// process_block_header -// Invalid::StateSlotMismatch -// Invalid::ParentBlockRootMismatch - // verify_block_signature // Invalid::BadSignature From bb0500f11d7f8bbffbe0feec7c4f2ea3c1e07e9a Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Wed, 10 Apr 2019 21:58:27 +1000 Subject: [PATCH 10/44] process_block_header tests --- eth2/state_processing/src/per_block_processing.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 6c52a2676..aac94d55c 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -21,6 +21,8 @@ pub use verify_transfer::{ }; pub mod errors; +pub mod tests; +pub mod block_processing_builder; mod validate_attestation; mod verify_attester_slashing; mod verify_deposit; From d76246e6009a4d21f0a8a0447a6860b3fb6deadf Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Mon, 15 Apr 2019 10:38:13 +1000 Subject: [PATCH 11/44] invalid block signature test --- .../src/per_block_processing.rs | 2 +- .../block_processing_builder.rs | 135 +---------------- .../src/per_block_processing/tests.rs | 142 ++++++------------ eth2/utils/bls/src/lib.rs | 12 -- 4 files changed, 50 insertions(+), 241 deletions(-) diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index aac94d55c..ac874d95e 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -20,9 +20,9 @@ pub use verify_transfer::{ execute_transfer, verify_transfer, verify_transfer_time_independent_only, }; +pub mod block_processing_builder; pub mod errors; pub mod tests; -pub mod block_processing_builder; mod validate_attestation; mod verify_attester_slashing; mod verify_deposit; diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 99bbf0944..489684523 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -1,4 +1,3 @@ -use log::info; use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; use types::*; @@ -7,13 +6,6 @@ pub struct BlockProcessingBuilder { pub block_builder: TestingBeaconBlockBuilder, pub num_validators: usize, - pub num_proposer_slashings: usize, - pub num_attester_slashings: usize, - pub num_indices_per_slashable_vote: usize, - pub num_attestations: usize, - pub num_deposits: usize, - pub num_exits: usize, - pub num_transfers: usize, } impl BlockProcessingBuilder { @@ -26,26 +18,9 @@ impl BlockProcessingBuilder { state_builder, block_builder, num_validators: 0, - num_proposer_slashings: 0, - num_attester_slashings: 0, - num_indices_per_slashable_vote: spec.max_indices_per_slashable_vote as usize, - num_attestations: 0, - num_deposits: 0, - num_exits: 0, - num_transfers: 0, } } - pub fn maximize_block_operations(&mut self, spec: &ChainSpec) { - self.num_proposer_slashings = spec.max_proposer_slashings as usize; - self.num_attester_slashings = spec.max_attester_slashings as usize; - self.num_indices_per_slashable_vote = spec.max_indices_per_slashable_vote as usize; - self.num_attestations = spec.max_attestations as usize; - self.num_deposits = spec.max_deposits as usize; - self.num_exits = spec.max_voluntary_exits as usize; - self.num_transfers = spec.max_transfers as usize; - } - pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { self.state_builder.teleport_to_slot(slot, &spec); } @@ -56,119 +31,19 @@ impl BlockProcessingBuilder { } pub fn build(mut self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { - let (mut state, keypairs) = self.state_builder.build(); + let (state, keypairs) = self.state_builder.build(); let builder = &mut self.block_builder; builder.set_slot(state.slot); - let proposer_index = state.get_beacon_proposer_index(state.slot, RelativeEpoch::Current, spec).unwrap(); + let proposer_index = state + .get_beacon_proposer_index(state.slot, RelativeEpoch::Current, spec) + .unwrap(); let keypair = &keypairs[proposer_index]; builder.set_randao_reveal(&keypair.sk, &state.fork, spec); - // Used as a stream of validator indices for use in slashings, exits, etc. - let mut validators_iter = (0..keypairs.len() as u64).into_iter(); - - // Insert `ProposerSlashing` objects. - for _ in 0..self.num_proposer_slashings { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - builder.insert_proposer_slashing( - validator_index, - &keypairs[validator_index as usize].sk, - &state.fork, - spec, - ); - } - info!( - "Inserted {} proposer slashings.", - builder.block.body.proposer_slashings.len() - ); - - // Insert `AttesterSlashing` objects - for _ in 0..self.num_attester_slashings { - let mut attesters: Vec = vec![]; - let mut secret_keys: Vec<&SecretKey> = vec![]; - - for _ in 0..self.num_indices_per_slashable_vote { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - attesters.push(validator_index); - secret_keys.push(&keypairs[validator_index as usize].sk); - } - - builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec); - } - info!( - "Inserted {} attester slashings.", - builder.block.body.attester_slashings.len() - ); - - // Insert `Attestation` objects. - let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect(); - builder - .insert_attestations( - &state, - &all_secret_keys, - self.num_attestations as usize, - spec, - ) - .unwrap(); - info!( - "Inserted {} attestations.", - builder.block.body.attestations.len() - ); - - // Insert `Deposit` objects. - for i in 0..self.num_deposits { - builder.insert_deposit( - 32_000_000_000, - state.deposit_index + (i as u64), - &state, - spec, - ); - } - info!("Inserted {} deposits.", builder.block.body.deposits.len()); - - // Insert the maximum possible number of `Exit` objects. - for _ in 0..self.num_exits { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - builder.insert_exit( - &state, - validator_index, - &keypairs[validator_index as usize].sk, - spec, - ); - } - info!( - "Inserted {} exits.", - builder.block.body.voluntary_exits.len() - ); - - // Insert the maximum possible number of `Transfer` objects. - for _ in 0..self.num_transfers { - let validator_index = validators_iter.next().expect("Insufficient validators."); - - // Manually set the validator to be withdrawn. - state.validator_registry[validator_index as usize].withdrawable_epoch = - state.previous_epoch(spec); - - builder.insert_transfer( - &state, - validator_index, - validator_index, - 1, - keypairs[validator_index as usize].clone(), - spec, - ); - } - info!("Inserted {} transfers.", builder.block.body.transfers.len()); - - let mut block = self.block_builder.build(&keypair.sk, &state.fork, spec); - - // Set the eth1 data to be different from the state. - block.body.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]); + let block = self.block_builder.build(&keypair.sk, &state.fork, spec); (block, state) } diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index ac294de69..6fe7e40d2 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -1,39 +1,72 @@ #![cfg(test)] -use crate::per_block_processing; use super::block_processing_builder::BlockProcessingBuilder; use super::errors::*; -use types::*; +use crate::per_block_processing; +use ssz::SignedRoot; +use types::{ChainSpec, Domain, Keypair, Signature, Slot}; pub const VALIDATOR_COUNT: usize = 10; #[test] -fn runs_without_error() { +fn valid_block_ok() { let spec = ChainSpec::foundation(); - let (block, mut state) = get_block_state(&spec); + let builder = get_builder(&spec); + let (block, mut state) = builder.build(&spec); - per_block_processing(&mut state, &block, &spec).unwrap(); + let result = per_block_processing(&mut state, &block, &spec); + + assert_eq!(result, Ok(())); } #[test] -fn process_block_header_invalid_state_slot() { +fn invalid_block_header_state_slot() { let spec = ChainSpec::foundation(); - let (mut block, mut state) = get_block_state(&spec); + let builder = get_builder(&spec); + let (mut block, mut state) = builder.build(&spec); state.slot = Slot::new(133713); block.slot = Slot::new(424242); let result = per_block_processing(&mut state, &block, &spec); - assert_eq!(result, Err(BlockProcessingError::Invalid(BlockInvalid::StateSlotMismatch))); + assert_eq!( + result, + Err(BlockProcessingError::Invalid( + BlockInvalid::StateSlotMismatch + )) + ); } #[test] #[ignore] -fn process_block_header_invalid_parent_block_root() { +fn invalid_parent_block_root() { // this will be changed in spec 0.5.1 to use signed root } -fn get_block_state(spec: &ChainSpec) -> (BeaconBlock, BeaconState) { +#[test] +fn invalid_block_signature() { + let spec = ChainSpec::foundation(); + let builder = get_builder(&spec); + let (mut block, mut state) = builder.build(&spec); + + // sign the block with a keypair that is not the expected proposer + let keypair = Keypair::random(); + let message = block.signed_root(); + let epoch = block.slot.epoch(spec.slots_per_epoch); + let domain = spec.get_domain(epoch, Domain::BeaconBlock, &state.fork); + block.signature = Signature::new(&message, domain, &keypair.sk); + + // process block with invalid block signature + let result = per_block_processing(&mut state, &block, &spec); + + // should get a BadSignature error + assert_eq!( + result, + Err(BlockProcessingError::Invalid(BlockInvalid::BadSignature)) + ); +} + +fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); // Set the state and block to be in the last slot of the 4th epoch. @@ -41,92 +74,5 @@ fn get_block_state(spec: &ChainSpec) -> (BeaconBlock, BeaconState) { builder.set_slot(last_slot_of_epoch, &spec); builder.build_caches(&spec); - let (block, state) = builder.build(&spec); - - (block, state) + (builder) } - -// verify_block_signature -// Invalid::BadSignature - -// process_randao -// Invalid::BadRandaoSignature - -// process_proposer_slashings -// Invalid::MaxProposerSlashingsExceeded -// verify_proposer_slashing -// Invalid::ProposerUnknown -// Invalid::ProposalSlotMismatch -// Invalid::ProposalsIdentical -// Invalid::ProposerAlreadySlashed -// Invalid::ProposerAlreadyWithdrawn -// Invalid::BadProposal1Signature -// Invalid::BadProposal2Signature - -// process_attester_slashings -// Invalid::MaxAttesterSlashingsExceed -// verify_attester_slashing -// Invalid::AttestationDataIdentical -// Invalid::NotSlashable -// Invalid::SlashableAttestation1Invalid -// Invalid::SlashableAttestation2Invalid - -// process_attestations -// Invalid::MaxAttestationsExceeded -// validate_attestation -// Invalid::PreGenesis -// Invalid::IncludedTooLate -// Invalid::IncludedTooEarly -// Invalid::BadPreviousCrosslink -// Invalid::AggregationBitfieldIsEmpty -// Invalid::CustodyBitfieldHasSetBits -// Invalid::NoCommitteeForShard -// Invalid::BadCustodyBitfieldLength -// Invalid::BadAggregationBitfieldLength -// Invalid::ShardBlockRootNotZero -// verify_justified_epoch_and_root -// Invalid::WrongJustifiedEpoch (current) -// Invalid::WrongJustifiedRoot (current) -// Invalid::WrongJustifiedEpoch (previous) -// Invalid::WrongJustifiedRoot (previous) -// verify_attestation_signature -// Invalid::BadAggregationBitfieldLength -// Invalid::BadCustodyBitfieldLength -// BeaconStateError::UnknownValidator -// Invalid::BadSignature - -// process_deposits -// Invalid::MaxDepositsExceeded -// verify_deposit -// Invalid::BadProofOfPossession -// Invalid::BadMerkleProof -// verify_deposit_index -// Invalid::BadIndex - -// process_exits -// Invalid::MaxExitsExceeded -// verify_exit -// Invalid::ValidatorUnknown -// Invalid::AlreadyExited -// Invalid::AlreadyInitiatedExited -// Invalid::FutureEpoch -// Invalid::TooYoungToLeave -// Invalid::BadSignature - -// process_transfers -// Invalid::MaxTransfersExceed -// verify_transfer -// Invalid::FromValidatorUnknown -// Invalid::FeeOverflow -// Invalid::FromBalanceInsufficient (amount) -// Invalid::FromBalanceInsufficient (fee) -// Invalid::InvalidResultingFromBalance -// Invalid::TransferSlotInPast -// Invalid::StateSlotMismatch -// Invalid::FromValidatorUnknown (???) -// Invalid::FromValidatorIneligableForTransfer -// Invalid::WithdrawalCredentialsMismatch -// Invalid::BadSignature - - - diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index b9a4d5c1d..caf56ae74 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -6,23 +6,11 @@ mod keypair; mod public_key; mod secret_key; -#[cfg(not(debug_assertions))] mod aggregate_signature; -#[cfg(not(debug_assertions))] mod signature; -#[cfg(not(debug_assertions))] pub use crate::aggregate_signature::AggregateSignature; -#[cfg(not(debug_assertions))] pub use crate::signature::Signature; -#[cfg(debug_assertions)] -mod fake_aggregate_signature; -#[cfg(debug_assertions)] -mod fake_signature; -#[cfg(debug_assertions)] -pub use crate::fake_aggregate_signature::FakeAggregateSignature as AggregateSignature; -#[cfg(debug_assertions)] -pub use crate::fake_signature::FakeSignature as Signature; pub use crate::aggregate_public_key::AggregatePublicKey; pub use crate::keypair::Keypair; From 010d319fdf9cd94d0ebebd0211fe433cfb5a5183 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Thu, 18 Apr 2019 07:00:40 +1000 Subject: [PATCH 12/44] invalid randao signature test implemented --- .../src/per_block_processing.rs | 2 ++ .../block_processing_builder.rs | 8 +++++-- .../src/per_block_processing/tests.rs | 24 ++++++++++++++++--- .../testing_beacon_block_builder.rs | 6 +++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index ac874d95e..331069d44 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -139,6 +139,8 @@ pub fn verify_block_signature( &state.fork, ); + println!("verify {:?}", &block); + verify!( block .signature diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 489684523..fb9dbd153 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -30,7 +30,7 @@ impl BlockProcessingBuilder { self.state_builder.build_caches(&spec).unwrap(); } - pub fn build(mut self, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { + pub fn build(mut self, randao_sk: Option, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { let (state, keypairs) = self.state_builder.build(); let builder = &mut self.block_builder; @@ -41,10 +41,14 @@ impl BlockProcessingBuilder { .unwrap(); let keypair = &keypairs[proposer_index]; - builder.set_randao_reveal(&keypair.sk, &state.fork, spec); + match randao_sk { + Some(sk) => builder.set_randao_reveal(&sk, &state.fork, spec), + None => builder.set_randao_reveal(&keypair.sk, &state.fork, spec), + } let block = self.block_builder.build(&keypair.sk, &state.fork, spec); (block, state) } + } diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 6fe7e40d2..6ae7fd872 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -11,7 +11,7 @@ pub const VALIDATOR_COUNT: usize = 10; fn valid_block_ok() { let spec = ChainSpec::foundation(); let builder = get_builder(&spec); - let (block, mut state) = builder.build(&spec); + let (block, mut state) = builder.build(None, &spec); let result = per_block_processing(&mut state, &block, &spec); @@ -22,7 +22,7 @@ fn valid_block_ok() { fn invalid_block_header_state_slot() { let spec = ChainSpec::foundation(); let builder = get_builder(&spec); - let (mut block, mut state) = builder.build(&spec); + let (mut block, mut state) = builder.build(None, &spec); state.slot = Slot::new(133713); block.slot = Slot::new(424242); @@ -47,7 +47,7 @@ fn invalid_parent_block_root() { fn invalid_block_signature() { let spec = ChainSpec::foundation(); let builder = get_builder(&spec); - let (mut block, mut state) = builder.build(&spec); + let (mut block, mut state) = builder.build(None, &spec); // sign the block with a keypair that is not the expected proposer let keypair = Keypair::random(); @@ -66,6 +66,24 @@ fn invalid_block_signature() { ); } +#[test] +fn invalid_randao_reveal_signature() { + let spec = ChainSpec::foundation(); + let builder = get_builder(&spec); + + // sign randao reveal with random keypair + let keypair = Keypair::random(); + let (block, mut state) = builder.build(Some(keypair.sk), &spec); + + let result = per_block_processing(&mut state, &block, &spec); + + // should get a BadRandaoSignature error + assert_eq!( + result, + Err(BlockProcessingError::Invalid(BlockInvalid::BadRandaoSignature)) + ); +} + fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index c5cd22ed4..c8c32d931 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -33,6 +33,7 @@ impl TestingBeaconBlockBuilder { /// Modifying the block after signing may invalidate the signature. pub fn sign(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { let message = self.block.signed_root(); + println!("block set {:?}", self.block); let epoch = self.block.slot.epoch(spec.slots_per_epoch); let domain = spec.get_domain(epoch, Domain::BeaconBlock, fork); self.block.signature = Signature::new(&message, domain, sk); @@ -48,6 +49,11 @@ impl TestingBeaconBlockBuilder { self.block.body.randao_reveal = Signature::new(&message, domain, sk); } + /// Has the randao reveal been set? + pub fn randao_reveal_not_set(&mut self) -> bool { + self.block.body.randao_reveal.is_empty() + } + /// Inserts a signed, valid `ProposerSlashing` for the validator. pub fn insert_proposer_slashing( &mut self, From 773227d627104ffbc1c0c119c8143725d8aeef15 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Fri, 19 Apr 2019 07:09:20 +1000 Subject: [PATCH 13/44] removed printlns --- eth2/state_processing/src/per_block_processing.rs | 2 -- eth2/types/src/test_utils/testing_beacon_block_builder.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 331069d44..ac874d95e 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -139,8 +139,6 @@ pub fn verify_block_signature( &state.fork, ); - println!("verify {:?}", &block); - verify!( block .signature diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index c8c32d931..c28311002 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -33,7 +33,6 @@ impl TestingBeaconBlockBuilder { /// Modifying the block after signing may invalidate the signature. pub fn sign(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { let message = self.block.signed_root(); - println!("block set {:?}", self.block); let epoch = self.block.slot.epoch(spec.slots_per_epoch); let domain = spec.get_domain(epoch, Domain::BeaconBlock, fork); self.block.signature = Signature::new(&message, domain, sk); From 85266f8db0f41c2790c42e6f6565137b33f1580c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 1 May 2019 11:42:18 +1000 Subject: [PATCH 14/44] Trim db2 down to basic new parts --- beacon_node/db2/Cargo.toml | 1 + beacon_node/db2/src/disk_db.rs | 6 +- beacon_node/db2/src/errors.rs | 30 +++ beacon_node/db2/src/lib.rs | 133 ++++------ beacon_node/db2/src/memory_db.rs | 231 ++-------------- .../db2/src/stores/beacon_block_store.rs | 246 ------------------ .../db2/src/stores/beacon_state_store.rs | 62 ----- beacon_node/db2/src/stores/macros.rs | 103 -------- beacon_node/db2/src/stores/mod.rs | 25 -- beacon_node/db2/src/stores/pow_chain_store.rs | 68 ----- beacon_node/db2/src/stores/validator_store.rs | 215 --------------- beacon_node/db2/src/traits.rs | 38 --- 12 files changed, 113 insertions(+), 1045 deletions(-) create mode 100644 beacon_node/db2/src/errors.rs delete mode 100644 beacon_node/db2/src/stores/beacon_block_store.rs delete mode 100644 beacon_node/db2/src/stores/beacon_state_store.rs delete mode 100644 beacon_node/db2/src/stores/macros.rs delete mode 100644 beacon_node/db2/src/stores/mod.rs delete mode 100644 beacon_node/db2/src/stores/pow_chain_store.rs delete mode 100644 beacon_node/db2/src/stores/validator_store.rs delete mode 100644 beacon_node/db2/src/traits.rs diff --git a/beacon_node/db2/Cargo.toml b/beacon_node/db2/Cargo.toml index 8a5dbad5e..95e87c9ea 100644 --- a/beacon_node/db2/Cargo.toml +++ b/beacon_node/db2/Cargo.toml @@ -10,6 +10,7 @@ bls = { path = "../../eth2/utils/bls" } bytes = "0.4.10" db_encode = { path = "../db_encode" } db_encode_derive = { path = "../db_encode_derive" } +parking_lot = "0.7" rocksdb = "0.10.1" ssz = { path = "../../eth2/utils/ssz" } ssz_derive = { path = "../../eth2/utils/ssz_derive" } diff --git a/beacon_node/db2/src/disk_db.rs b/beacon_node/db2/src/disk_db.rs index f05320f7f..e2162e29a 100644 --- a/beacon_node/db2/src/disk_db.rs +++ b/beacon_node/db2/src/disk_db.rs @@ -1,9 +1,9 @@ extern crate rocksdb; -use super::rocksdb::Error as RocksError; -use super::rocksdb::{Options, DB}; -use super::stores::COLUMNS; +// use super::stores::COLUMNS; use super::{ClientDB, DBError, DBValue}; +use rocksdb::Error as RocksError; +use rocksdb::{Options, DB}; use std::fs; use std::path::Path; diff --git a/beacon_node/db2/src/errors.rs b/beacon_node/db2/src/errors.rs new file mode 100644 index 000000000..815b35a8e --- /dev/null +++ b/beacon_node/db2/src/errors.rs @@ -0,0 +1,30 @@ +use ssz::DecodeError; + +#[derive(Debug, PartialEq)] +pub enum Error { + SszDecodeError(DecodeError), + DBError { message: String }, +} + +impl From for Error { + fn from(e: DecodeError) -> Error { + Error::SszDecodeError(e) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError { message: e.message } + } +} + +#[derive(Debug)] +pub struct DBError { + pub message: String, +} + +impl DBError { + pub fn new(message: String) -> Self { + Self { message } + } +} diff --git a/beacon_node/db2/src/lib.rs b/beacon_node/db2/src/lib.rs index 55f419978..1ed9d5984 100644 --- a/beacon_node/db2/src/lib.rs +++ b/beacon_node/db2/src/lib.rs @@ -1,38 +1,42 @@ -extern crate blake2_rfc as blake2; -extern crate bls; -extern crate rocksdb; - -mod disk_db; +// mod disk_db; +mod errors; mod memory_db; -pub mod stores; -mod traits; -use self::stores::COLUMNS; use db_encode::{db_encode, DBDecode, DBEncode}; -use ssz::DecodeError; -use std::sync::Arc; -pub use self::disk_db::DiskDB; pub use self::memory_db::MemoryDB; -pub use self::traits::{ClientDB, DBError, DBValue}; +pub use errors::Error; pub use types::*; +pub type DBValue = Vec; -#[derive(Debug, PartialEq)] -pub enum Error { - SszDecodeError(DecodeError), - DBError { message: String }, +pub trait StoreDB: Sync + Send + Sized { + fn put(&self, key: &Hash256, item: &impl DBRecord) -> Result<(), Error> { + item.db_put(self, key) + } + + fn get(&self, key: &Hash256) -> Result, Error> { + I::db_get(self, key) + } + + fn exists(&self, key: &Hash256) -> Result { + I::db_exists(self, key) + } + + fn delete(&self, key: &Hash256) -> Result<(), Error> { + I::db_delete(self, key) + } + + fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error>; + + fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error>; + + fn key_exists(&self, col: &str, key: &[u8]) -> Result; + + fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; } -impl From for Error { - fn from(e: DecodeError) -> Error { - Error::SszDecodeError(e) - } -} - -impl From for Error { - fn from(e: DBError) -> Error { - Error::DBError { message: e.message } - } +pub trait DBStore { + fn db_column(&self) -> DBColumn; } /// Currently available database options @@ -61,76 +65,41 @@ impl<'a> Into<&'a str> for DBColumn { pub trait DBRecord: DBEncode + DBDecode { fn db_column() -> DBColumn; -} -pub struct Store -where - T: ClientDB, -{ - db: Arc, -} - -impl Store { - fn new_in_memory() -> Self { - Self { - db: Arc::new(MemoryDB::open()), - } - } -} - -impl Store -where - T: ClientDB, -{ - /// Put `item` in the store as `key`. - fn put(&self, key: &Hash256, item: &I) -> Result<(), Error> - where - I: DBRecord, - { - let column = I::db_column().into(); - let key = key.as_bytes(); - let val = db_encode(item); - - self.db.put(column, key, &val).map_err(|e| e.into()) - } - - /// Retrieves an `Ok(Some(item))` from the store if `key` exists, otherwise returns `Ok(None)`. - fn get(&self, key: &Hash256) -> Result, Error> - where - I: DBRecord, - { - let column = I::db_column().into(); + fn db_put(&self, store: &impl StoreDB, key: &Hash256) -> Result<(), Error> { + let column = Self::db_column().into(); let key = key.as_bytes(); - match self.db.get(column, key)? { + store + .put_bytes(column, key, &db_encode(self)) + .map_err(|e| e.into()) + } + + fn db_get(store: &impl StoreDB, key: &Hash256) -> Result, Error> { + let column = Self::db_column().into(); + let key = key.as_bytes(); + + match store.get_bytes(column, key)? { Some(bytes) => { - let (item, _index) = I::db_decode(&bytes, 0)?; + let (item, _index) = Self::db_decode(&bytes, 0)?; Ok(Some(item)) } None => Ok(None), } } - /// Returns `Ok(true)` `key` exists in the store. - fn exists(&self, key: &Hash256) -> Result - where - I: DBRecord, - { - let column = I::db_column().into(); + fn db_exists(store: &impl StoreDB, key: &Hash256) -> Result { + let column = Self::db_column().into(); let key = key.as_bytes(); - self.db.exists(column, key).map_err(|e| e.into()) + store.key_exists(column, key) } - /// Returns `Ok(())` if `key` was deleted from the database or did not exist. - fn delete(&self, key: &Hash256) -> Result<(), Error> - where - I: DBRecord, - { - let column = I::db_column().into(); + fn db_delete(store: &impl StoreDB, key: &Hash256) -> Result<(), Error> { + let column = Self::db_column().into(); let key = key.as_bytes(); - self.db.delete(column, key).map_err(|e| e.into()) + store.key_delete(column, key) } } @@ -155,7 +124,7 @@ mod tests { #[test] fn memorydb_can_store_and_retrieve() { - let store = Store::new_in_memory(); + let store = MemoryDB::open(); let key = Hash256::random(); let item = StorableThing { a: 1, b: 42 }; @@ -169,7 +138,7 @@ mod tests { #[test] fn exists() { - let store = Store::new_in_memory(); + let store = MemoryDB::open(); let key = Hash256::random(); let item = StorableThing { a: 1, b: 42 }; diff --git a/beacon_node/db2/src/memory_db.rs b/beacon_node/db2/src/memory_db.rs index 008e5912f..bc736e525 100644 --- a/beacon_node/db2/src/memory_db.rs +++ b/beacon_node/db2/src/memory_db.rs @@ -1,236 +1,61 @@ -use super::blake2::blake2b::blake2b; -use super::COLUMNS; -use super::{ClientDB, DBError, DBValue}; -use std::collections::{HashMap, HashSet}; -use std::sync::RwLock; +use super::{DBValue, Error, StoreDB}; +use parking_lot::RwLock; +use std::collections::HashMap; type DBHashMap = HashMap, Vec>; -type ColumnHashSet = HashSet; -/// An in-memory database implementing the ClientDB trait. -/// -/// It is not particularily optimized, it exists for ease and speed of testing. It's not expected -/// this DB would be used outside of tests. pub struct MemoryDB { db: RwLock, - known_columns: RwLock, } impl MemoryDB { - /// Open the in-memory database. - /// - /// All columns must be supplied initially, you will get an error if you try to access a column - /// that was not declared here. This condition is enforced artificially to simulate RocksDB. pub fn open() -> Self { - let db: DBHashMap = HashMap::new(); - let mut known_columns: ColumnHashSet = HashSet::new(); - for col in &COLUMNS { - known_columns.insert(col.to_string()); - } Self { - db: RwLock::new(db), - known_columns: RwLock::new(known_columns), + db: RwLock::new(HashMap::new()), } } - /// Hashes a key and a column name in order to get a unique key for the supplied column. fn get_key_for_col(col: &str, key: &[u8]) -> Vec { - blake2b(32, col.as_bytes(), key).as_bytes().to_vec() + let mut col = col.as_bytes().to_vec(); + col.append(&mut key.to_vec()); + col } } -impl ClientDB for MemoryDB { +impl StoreDB for MemoryDB { /// Get the value of some key from the database. Returns `None` if the key does not exist. - fn get(&self, col: &str, key: &[u8]) -> Result, DBError> { - // Panic if the DB locks are poisoned. - let db = self.db.read().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - Ok(db.get(&column_key).and_then(|val| Some(val.clone()))) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } + Ok(self + .db + .read() + .get(&column_key) + .and_then(|val| Some(val.clone()))) } /// Puts a key in the database. - fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError> { - // Panic if the DB locks are poisoned. - let mut db = self.db.write().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - db.insert(column_key, val.to_vec()); - Ok(()) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } + self.db.write().insert(column_key, val.to_vec()); + + Ok(()) } /// Return true if some key exists in some column. - fn exists(&self, col: &str, key: &[u8]) -> Result { - // Panic if the DB locks are poisoned. - let db = self.db.read().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn key_exists(&self, col: &str, key: &[u8]) -> Result { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - Ok(db.contains_key(&column_key)) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } + Ok(self.db.read().contains_key(&column_key)) } /// Delete some key from the database. - fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError> { - // Panic if the DB locks are poisoned. - let mut db = self.db.write().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - db.remove(&column_key); - Ok(()) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } - } -} - -#[cfg(test)] -mod tests { - use super::super::stores::{BLOCKS_DB_COLUMN, VALIDATOR_DB_COLUMN}; - use super::super::ClientDB; - use super::*; - use std::sync::Arc; - use std::thread; - - #[test] - fn test_memorydb_can_delete() { - let col_a: &str = BLOCKS_DB_COLUMN; - - let db = MemoryDB::open(); - - db.put(col_a, "dogs".as_bytes(), "lol".as_bytes()).unwrap(); - - assert_eq!( - db.get(col_a, "dogs".as_bytes()).unwrap().unwrap(), - "lol".as_bytes() - ); - - db.delete(col_a, "dogs".as_bytes()).unwrap(); - - assert_eq!(db.get(col_a, "dogs".as_bytes()).unwrap(), None); - } - - #[test] - fn test_memorydb_column_access() { - let col_a: &str = BLOCKS_DB_COLUMN; - let col_b: &str = VALIDATOR_DB_COLUMN; - - let db = MemoryDB::open(); - - /* - * Testing that if we write to the same key in different columns that - * there is not an overlap. - */ - db.put(col_a, "same".as_bytes(), "cat".as_bytes()).unwrap(); - db.put(col_b, "same".as_bytes(), "dog".as_bytes()).unwrap(); - - assert_eq!( - db.get(col_a, "same".as_bytes()).unwrap().unwrap(), - "cat".as_bytes() - ); - assert_eq!( - db.get(col_b, "same".as_bytes()).unwrap().unwrap(), - "dog".as_bytes() - ); - } - - #[test] - fn test_memorydb_unknown_column_access() { - let col_a: &str = BLOCKS_DB_COLUMN; - let col_x: &str = "ColumnX"; - - let db = MemoryDB::open(); - - /* - * Test that we get errors when using undeclared columns - */ - assert!(db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).is_ok()); - assert!(db.put(col_x, "cats".as_bytes(), "lol".as_bytes()).is_err()); - - assert!(db.get(col_a, "cats".as_bytes()).is_ok()); - assert!(db.get(col_x, "cats".as_bytes()).is_err()); - } - - #[test] - fn test_memorydb_exists() { - let col_a: &str = BLOCKS_DB_COLUMN; - let col_b: &str = VALIDATOR_DB_COLUMN; - - let db = MemoryDB::open(); - - /* - * Testing that if we write to the same key in different columns that - * there is not an overlap. - */ - db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).unwrap(); - - assert_eq!(true, db.exists(col_a, "cats".as_bytes()).unwrap()); - assert_eq!(false, db.exists(col_b, "cats".as_bytes()).unwrap()); - - assert_eq!(false, db.exists(col_a, "dogs".as_bytes()).unwrap()); - assert_eq!(false, db.exists(col_b, "dogs".as_bytes()).unwrap()); - } - - #[test] - fn test_memorydb_threading() { - let col_name: &str = BLOCKS_DB_COLUMN; - - let db = Arc::new(MemoryDB::open()); - - let thread_count = 10; - let write_count = 10; - - // We're execting the product of these numbers to fit in one byte. - assert!(thread_count * write_count <= 255); - - let mut handles = vec![]; - for t in 0..thread_count { - let wc = write_count; - let db = db.clone(); - let col = col_name.clone(); - let handle = thread::spawn(move || { - for w in 0..wc { - let key = (t * w) as u8; - let val = 42; - db.put(&col, &vec![key], &vec![val]).unwrap(); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - for t in 0..thread_count { - for w in 0..write_count { - let key = (t * w) as u8; - let val = db.get(&col_name, &vec![key]).unwrap().unwrap(); - assert_eq!(vec![42], val); - } - } + self.db.write().remove(&column_key); + + Ok(()) } } diff --git a/beacon_node/db2/src/stores/beacon_block_store.rs b/beacon_node/db2/src/stores/beacon_block_store.rs deleted file mode 100644 index e2e16e60b..000000000 --- a/beacon_node/db2/src/stores/beacon_block_store.rs +++ /dev/null @@ -1,246 +0,0 @@ -use super::BLOCKS_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use ssz::Decodable; -use std::sync::Arc; -use types::{BeaconBlock, Hash256, Slot}; - -#[derive(Clone, Debug, PartialEq)] -pub enum BeaconBlockAtSlotError { - UnknownBeaconBlock(Hash256), - InvalidBeaconBlock(Hash256), - DBError(String), -} - -pub struct BeaconBlockStore -where - T: ClientDB, -{ - db: Arc, -} - -// Implements `put`, `get`, `exists` and `delete` for the store. -impl_crud_for_store!(BeaconBlockStore, DB_COLUMN); - -impl BeaconBlockStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { - match self.get(&hash)? { - None => Ok(None), - Some(ssz) => { - let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { - message: "Bad BeaconBlock SSZ.".to_string(), - })?; - Ok(Some(block)) - } - } - } - - /// Retrieve the block at a slot given a "head_hash" and a slot. - /// - /// A "head_hash" must be a block hash with a slot number greater than or equal to the desired - /// slot. - /// - /// This function will read each block down the chain until it finds a block with the given - /// slot number. If the slot is skipped, the function will return None. - /// - /// If a block is found, a tuple of (block_hash, serialized_block) is returned. - /// - /// Note: this function uses a loop instead of recursion as the compiler is over-strict when it - /// comes to recursion and the `impl Trait` pattern. See: - /// https://stackoverflow.com/questions/54032940/using-impl-trait-in-a-recursive-function - pub fn block_at_slot( - &self, - head_hash: &Hash256, - slot: Slot, - ) -> Result, BeaconBlockAtSlotError> { - let mut current_hash = *head_hash; - - loop { - if let Some(block) = self.get_deserialized(¤t_hash)? { - if block.slot == slot { - break Ok(Some((current_hash, block))); - } else if block.slot < slot { - break Ok(None); - } else { - current_hash = block.previous_block_root; - } - } else { - break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash)); - } - } - } -} - -impl From for BeaconBlockAtSlotError { - fn from(e: DBError) -> Self { - BeaconBlockAtSlotError::DBError(e.message) - } -} - -#[cfg(test)] -mod tests { - use super::super::super::MemoryDB; - use super::*; - - use std::sync::Arc; - use std::thread; - - use ssz::ssz_encode; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use types::BeaconBlock; - use types::Hash256; - - test_crud_for_store!(BeaconBlockStore, DB_COLUMN); - - #[test] - fn head_hash_slot_too_low() { - let db = Arc::new(MemoryDB::open()); - let bs = Arc::new(BeaconBlockStore::new(db.clone())); - let mut rng = XorShiftRng::from_seed([42; 16]); - - let mut block = BeaconBlock::random_for_test(&mut rng); - block.slot = Slot::from(10_u64); - - let block_root = block.canonical_root(); - bs.put(&block_root, &ssz_encode(&block)).unwrap(); - - let result = bs.block_at_slot(&block_root, Slot::from(11_u64)).unwrap(); - assert_eq!(result, None); - } - - #[test] - fn test_invalid_block_at_slot() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "definitly not a valid block".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert_eq!( - store.block_at_slot(hash, Slot::from(42_u64)), - Err(BeaconBlockAtSlotError::DBError( - "Bad BeaconBlock SSZ.".into() - )) - ); - } - - #[test] - fn test_unknown_block_at_slot() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - let other_hash = &Hash256::from([0xBB; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert_eq!( - store.block_at_slot(other_hash, Slot::from(42_u64)), - Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash)) - ); - } - - #[test] - fn test_block_store_on_memory_db() { - let db = Arc::new(MemoryDB::open()); - let bs = Arc::new(BeaconBlockStore::new(db.clone())); - - let thread_count = 10; - let write_count = 10; - - let mut handles = vec![]; - for t in 0..thread_count { - let wc = write_count; - let bs = bs.clone(); - let handle = thread::spawn(move || { - for w in 0..wc { - let key = t * w; - let val = 42; - bs.put(&Hash256::from_low_u64_le(key), &vec![val]).unwrap(); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - for t in 0..thread_count { - for w in 0..write_count { - let key = t * w; - assert!(bs.exists(&Hash256::from_low_u64_le(key)).unwrap()); - let val = bs.get(&Hash256::from_low_u64_le(key)).unwrap().unwrap(); - assert_eq!(vec![42], val); - } - } - } - - #[test] - #[ignore] - fn test_block_at_slot() { - let db = Arc::new(MemoryDB::open()); - let bs = Arc::new(BeaconBlockStore::new(db.clone())); - let mut rng = XorShiftRng::from_seed([42; 16]); - - // Specify test block parameters. - let hashes = [ - Hash256::from([0; 32]), - Hash256::from([1; 32]), - Hash256::from([2; 32]), - Hash256::from([3; 32]), - Hash256::from([4; 32]), - ]; - let parent_hashes = [ - Hash256::from([255; 32]), // Genesis block. - Hash256::from([0; 32]), - Hash256::from([1; 32]), - Hash256::from([2; 32]), - Hash256::from([3; 32]), - ]; - let unknown_hash = Hash256::from([101; 32]); // different from all above - let slots: Vec = vec![0, 1, 3, 4, 5].iter().map(|x| Slot::new(*x)).collect(); - - // Generate a vec of random blocks and store them in the DB. - let block_count = 5; - let mut blocks: Vec = Vec::with_capacity(5); - for i in 0..block_count { - let mut block = BeaconBlock::random_for_test(&mut rng); - - block.previous_block_root = parent_hashes[i]; - block.slot = slots[i]; - - let ssz = ssz_encode(&block); - db.put(DB_COLUMN, hashes[i].as_bytes(), &ssz).unwrap(); - - blocks.push(block); - } - - // Test that certain slots can be reached from certain hashes. - let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; - for (hashes_index, slot_index) in test_cases { - let (matched_block_hash, block) = bs - .block_at_slot(&hashes[hashes_index], slots[slot_index]) - .unwrap() - .unwrap(); - assert_eq!(matched_block_hash, hashes[slot_index]); - assert_eq!(block.slot, slots[slot_index]); - } - - let ssz = bs.block_at_slot(&hashes[4], Slot::new(2)).unwrap(); - assert_eq!(ssz, None); - - let ssz = bs.block_at_slot(&hashes[4], Slot::new(6)).unwrap(); - assert_eq!(ssz, None); - - let ssz = bs.block_at_slot(&unknown_hash, Slot::new(2)); - assert_eq!( - ssz, - Err(BeaconBlockAtSlotError::UnknownBeaconBlock(unknown_hash)) - ); - } -} diff --git a/beacon_node/db2/src/stores/beacon_state_store.rs b/beacon_node/db2/src/stores/beacon_state_store.rs deleted file mode 100644 index fd6ff569a..000000000 --- a/beacon_node/db2/src/stores/beacon_state_store.rs +++ /dev/null @@ -1,62 +0,0 @@ -use super::STATES_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use ssz::Decodable; -use std::sync::Arc; -use types::{BeaconState, Hash256}; - -pub struct BeaconStateStore -where - T: ClientDB, -{ - db: Arc, -} - -// Implements `put`, `get`, `exists` and `delete` for the store. -impl_crud_for_store!(BeaconStateStore, DB_COLUMN); - -impl BeaconStateStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { - match self.get(&hash)? { - None => Ok(None), - Some(ssz) => { - let (state, _) = BeaconState::ssz_decode(&ssz, 0).map_err(|_| DBError { - message: "Bad State SSZ.".to_string(), - })?; - Ok(Some(state)) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::MemoryDB; - use super::*; - - use ssz::ssz_encode; - use std::sync::Arc; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use types::Hash256; - - test_crud_for_store!(BeaconStateStore, DB_COLUMN); - - #[test] - fn test_reader() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconStateStore::new(db.clone()); - - let mut rng = XorShiftRng::from_seed([42; 16]); - let state = BeaconState::random_for_test(&mut rng); - let state_root = state.canonical_root(); - - store.put(&state_root, &ssz_encode(&state)).unwrap(); - - let decoded = store.get_deserialized(&state_root).unwrap().unwrap(); - - assert_eq!(state, decoded); - } -} diff --git a/beacon_node/db2/src/stores/macros.rs b/beacon_node/db2/src/stores/macros.rs deleted file mode 100644 index 6c53e40ee..000000000 --- a/beacon_node/db2/src/stores/macros.rs +++ /dev/null @@ -1,103 +0,0 @@ -macro_rules! impl_crud_for_store { - ($store: ident, $db_column: expr) => { - impl $store { - pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { - self.db.put($db_column, hash.as_bytes(), ssz) - } - - pub fn get(&self, hash: &Hash256) -> Result>, DBError> { - self.db.get($db_column, hash.as_bytes()) - } - - pub fn exists(&self, hash: &Hash256) -> Result { - self.db.exists($db_column, hash.as_bytes()) - } - - pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> { - self.db.delete($db_column, hash.as_bytes()) - } - } - }; -} - -#[cfg(test)] -macro_rules! test_crud_for_store { - ($store: ident, $db_column: expr) => { - #[test] - fn test_put() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - store.put(hash, ssz).unwrap(); - assert_eq!(db.get(DB_COLUMN, hash.as_bytes()).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert_eq!(store.get(hash).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get_unknown() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - let other_hash = &Hash256::from([0xBB; 32]); - - db.put(DB_COLUMN, other_hash.as_bytes(), ssz).unwrap(); - assert_eq!(store.get(hash).unwrap(), None); - } - - #[test] - fn test_exists() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert!(store.exists(hash).unwrap()); - } - - #[test] - fn test_block_does_not_exist() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - let other_hash = &Hash256::from([0xBB; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert!(!store.exists(other_hash).unwrap()); - } - - #[test] - fn test_delete() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert!(db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); - - store.delete(hash).unwrap(); - assert!(!db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); - } - }; -} diff --git a/beacon_node/db2/src/stores/mod.rs b/beacon_node/db2/src/stores/mod.rs deleted file mode 100644 index 44de7eed1..000000000 --- a/beacon_node/db2/src/stores/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -use super::{ClientDB, DBError}; - -#[macro_use] -mod macros; -mod beacon_block_store; -mod beacon_state_store; -mod pow_chain_store; -mod validator_store; - -pub use self::beacon_block_store::{BeaconBlockAtSlotError, BeaconBlockStore}; -pub use self::beacon_state_store::BeaconStateStore; -pub use self::pow_chain_store::PoWChainStore; -pub use self::validator_store::{ValidatorStore, ValidatorStoreError}; - -pub const BLOCKS_DB_COLUMN: &str = "blocks"; -pub const STATES_DB_COLUMN: &str = "states"; -pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; -pub const VALIDATOR_DB_COLUMN: &str = "validator"; - -pub const COLUMNS: [&str; 4] = [ - BLOCKS_DB_COLUMN, - STATES_DB_COLUMN, - POW_CHAIN_DB_COLUMN, - VALIDATOR_DB_COLUMN, -]; diff --git a/beacon_node/db2/src/stores/pow_chain_store.rs b/beacon_node/db2/src/stores/pow_chain_store.rs deleted file mode 100644 index 5c8b97907..000000000 --- a/beacon_node/db2/src/stores/pow_chain_store.rs +++ /dev/null @@ -1,68 +0,0 @@ -use super::POW_CHAIN_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use std::sync::Arc; - -pub struct PoWChainStore -where - T: ClientDB, -{ - db: Arc, -} - -impl PoWChainStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - pub fn put_block_hash(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, &[0]) - } - - pub fn block_hash_exists(&self, hash: &[u8]) -> Result { - self.db.exists(DB_COLUMN, hash) - } -} - -#[cfg(test)] -mod tests { - extern crate types; - - use super::super::super::MemoryDB; - use super::*; - - use self::types::Hash256; - - #[test] - fn test_put_block_hash() { - let db = Arc::new(MemoryDB::open()); - let store = PoWChainStore::new(db.clone()); - - let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); - store.put_block_hash(hash).unwrap(); - - assert!(db.exists(DB_COLUMN, hash).unwrap()); - } - - #[test] - fn test_block_hash_exists() { - let db = Arc::new(MemoryDB::open()); - let store = PoWChainStore::new(db.clone()); - - let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); - db.put(DB_COLUMN, hash, &[0]).unwrap(); - - assert!(store.block_hash_exists(hash).unwrap()); - } - - #[test] - fn test_block_hash_does_not_exist() { - let db = Arc::new(MemoryDB::open()); - let store = PoWChainStore::new(db.clone()); - - let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); - let other_hash = &Hash256::from([0xBB; 32]).as_bytes().to_vec(); - db.put(DB_COLUMN, hash, &[0]).unwrap(); - - assert!(!store.block_hash_exists(other_hash).unwrap()); - } -} diff --git a/beacon_node/db2/src/stores/validator_store.rs b/beacon_node/db2/src/stores/validator_store.rs deleted file mode 100644 index 02e90dc5c..000000000 --- a/beacon_node/db2/src/stores/validator_store.rs +++ /dev/null @@ -1,215 +0,0 @@ -extern crate bytes; - -use self::bytes::{BufMut, BytesMut}; -use super::VALIDATOR_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use bls::PublicKey; -use ssz::{ssz_encode, Decodable}; -use std::sync::Arc; - -#[derive(Debug, PartialEq)] -pub enum ValidatorStoreError { - DBError(String), - DecodeError, -} - -impl From for ValidatorStoreError { - fn from(error: DBError) -> Self { - ValidatorStoreError::DBError(error.message) - } -} - -#[derive(Debug, PartialEq)] -enum KeyPrefixes { - PublicKey, -} - -pub struct ValidatorStore -where - T: ClientDB, -{ - db: Arc, -} - -impl ValidatorStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - fn prefix_bytes(&self, key_prefix: &KeyPrefixes) -> Vec { - match key_prefix { - KeyPrefixes::PublicKey => b"pubkey".to_vec(), - } - } - - fn get_db_key_for_index(&self, key_prefix: &KeyPrefixes, index: usize) -> Vec { - let mut buf = BytesMut::with_capacity(6 + 8); - buf.put(self.prefix_bytes(key_prefix)); - buf.put_u64_be(index as u64); - buf.take().to_vec() - } - - pub fn put_public_key_by_index( - &self, - index: usize, - public_key: &PublicKey, - ) -> Result<(), ValidatorStoreError> { - let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); - let val = ssz_encode(public_key); - self.db - .put(DB_COLUMN, &key[..], &val[..]) - .map_err(ValidatorStoreError::from) - } - - pub fn get_public_key_by_index( - &self, - index: usize, - ) -> Result, ValidatorStoreError> { - let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); - let val = self.db.get(DB_COLUMN, &key[..])?; - match val { - None => Ok(None), - Some(val) => match PublicKey::ssz_decode(&val, 0) { - Ok((key, _)) => Ok(Some(key)), - Err(_) => Err(ValidatorStoreError::DecodeError), - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::MemoryDB; - use super::*; - use bls::Keypair; - - #[test] - fn test_prefix_bytes() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - assert_eq!( - store.prefix_bytes(&KeyPrefixes::PublicKey), - b"pubkey".to_vec() - ); - } - - #[test] - fn test_get_db_key_for_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let mut buf = BytesMut::with_capacity(6 + 8); - buf.put(b"pubkey".to_vec()); - buf.put_u64_be(42); - assert_eq!( - store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42), - buf.take().to_vec() - ) - } - - #[test] - fn test_put_public_key_by_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let index = 3; - let public_key = Keypair::random().pk; - - store.put_public_key_by_index(index, &public_key).unwrap(); - let public_key_at_index = db - .get( - DB_COLUMN, - &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..], - ) - .unwrap() - .unwrap(); - - assert_eq!(public_key_at_index, ssz_encode(&public_key)); - } - - #[test] - fn test_get_public_key_by_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let index = 4; - let public_key = Keypair::random().pk; - - db.put( - DB_COLUMN, - &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..], - &ssz_encode(&public_key)[..], - ) - .unwrap(); - - let public_key_at_index = store.get_public_key_by_index(index).unwrap().unwrap(); - assert_eq!(public_key_at_index, public_key); - } - - #[test] - fn test_get_public_key_by_unknown_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let public_key = Keypair::random().pk; - - db.put( - DB_COLUMN, - &store.get_db_key_for_index(&KeyPrefixes::PublicKey, 3)[..], - &ssz_encode(&public_key)[..], - ) - .unwrap(); - - let public_key_at_index = store.get_public_key_by_index(4).unwrap(); - assert_eq!(public_key_at_index, None); - } - - #[test] - fn test_get_invalid_public_key() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let key = store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42); - db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap(); - - assert_eq!( - store.get_public_key_by_index(42), - Err(ValidatorStoreError::DecodeError) - ); - } - - #[test] - fn test_validator_store_put_get() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db); - - let keys = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - - for i in 0..keys.len() { - store.put_public_key_by_index(i, &keys[i].pk).unwrap(); - } - - /* - * Check all keys are retrieved correctly. - */ - for i in 0..keys.len() { - let retrieved = store.get_public_key_by_index(i).unwrap().unwrap(); - assert_eq!(retrieved, keys[i].pk); - } - - /* - * Check that an index that wasn't stored returns None. - */ - assert!(store - .get_public_key_by_index(keys.len() + 1) - .unwrap() - .is_none()); - } -} diff --git a/beacon_node/db2/src/traits.rs b/beacon_node/db2/src/traits.rs deleted file mode 100644 index 57ebf9353..000000000 --- a/beacon_node/db2/src/traits.rs +++ /dev/null @@ -1,38 +0,0 @@ -pub type DBValue = Vec; - -#[derive(Debug)] -pub struct DBError { - pub message: String, -} - -impl DBError { - pub fn new(message: String) -> Self { - Self { message } - } -} - -/// A generic database to be used by the "client' (i.e., -/// the lighthouse blockchain client). -/// -/// The purpose of having this generic trait is to allow the -/// program to use a persistent on-disk database during production, -/// but use a transient database during tests. -pub trait ClientDB: Sync + Send { - fn get(&self, col: &str, key: &[u8]) -> Result, DBError>; - - fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError>; - - fn exists(&self, col: &str, key: &[u8]) -> Result; - - fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError>; -} - -pub enum DBColumn { - Block, - State, - BeaconChain, -} - -pub trait DBStore { - fn db_column(&self) -> DBColumn; -} From 157d4900aa62ec2619cc6693a3c22ca2e6c54e8d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 1 May 2019 11:59:18 +1000 Subject: [PATCH 15/44] Rename DB traits --- beacon_node/db2/src/lib.rs | 33 +++++++++++--------------------- beacon_node/db2/src/memory_db.rs | 4 ++-- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/beacon_node/db2/src/lib.rs b/beacon_node/db2/src/lib.rs index 1ed9d5984..da00262e6 100644 --- a/beacon_node/db2/src/lib.rs +++ b/beacon_node/db2/src/lib.rs @@ -9,20 +9,20 @@ pub use errors::Error; pub use types::*; pub type DBValue = Vec; -pub trait StoreDB: Sync + Send + Sized { - fn put(&self, key: &Hash256, item: &impl DBRecord) -> Result<(), Error> { +pub trait Store: Sync + Send + Sized { + fn put(&self, key: &Hash256, item: &impl StorableItem) -> Result<(), Error> { item.db_put(self, key) } - fn get(&self, key: &Hash256) -> Result, Error> { + fn get(&self, key: &Hash256) -> Result, Error> { I::db_get(self, key) } - fn exists(&self, key: &Hash256) -> Result { + fn exists(&self, key: &Hash256) -> Result { I::db_exists(self, key) } - fn delete(&self, key: &Hash256) -> Result<(), Error> { + fn delete(&self, key: &Hash256) -> Result<(), Error> { I::db_delete(self, key) } @@ -35,17 +35,6 @@ pub trait StoreDB: Sync + Send + Sized { fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; } -pub trait DBStore { - fn db_column(&self) -> DBColumn; -} - -/// Currently available database options -#[derive(Debug, Clone)] -pub enum DBType { - Memory, - RocksDB, -} - pub enum DBColumn { Block, State, @@ -63,10 +52,10 @@ impl<'a> Into<&'a str> for DBColumn { } } -pub trait DBRecord: DBEncode + DBDecode { +pub trait StorableItem: DBEncode + DBDecode { fn db_column() -> DBColumn; - fn db_put(&self, store: &impl StoreDB, key: &Hash256) -> Result<(), Error> { + fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> { let column = Self::db_column().into(); let key = key.as_bytes(); @@ -75,7 +64,7 @@ pub trait DBRecord: DBEncode + DBDecode { .map_err(|e| e.into()) } - fn db_get(store: &impl StoreDB, key: &Hash256) -> Result, Error> { + fn db_get(store: &impl Store, key: &Hash256) -> Result, Error> { let column = Self::db_column().into(); let key = key.as_bytes(); @@ -88,14 +77,14 @@ pub trait DBRecord: DBEncode + DBDecode { } } - fn db_exists(store: &impl StoreDB, key: &Hash256) -> Result { + fn db_exists(store: &impl Store, key: &Hash256) -> Result { let column = Self::db_column().into(); let key = key.as_bytes(); store.key_exists(column, key) } - fn db_delete(store: &impl StoreDB, key: &Hash256) -> Result<(), Error> { + fn db_delete(store: &impl Store, key: &Hash256) -> Result<(), Error> { let column = Self::db_column().into(); let key = key.as_bytes(); @@ -116,7 +105,7 @@ mod tests { b: u64, } - impl DBRecord for StorableThing { + impl StorableItem for StorableThing { fn db_column() -> DBColumn { DBColumn::Block } diff --git a/beacon_node/db2/src/memory_db.rs b/beacon_node/db2/src/memory_db.rs index bc736e525..83ff77ce1 100644 --- a/beacon_node/db2/src/memory_db.rs +++ b/beacon_node/db2/src/memory_db.rs @@ -1,4 +1,4 @@ -use super::{DBValue, Error, StoreDB}; +use super::{DBValue, Error, Store}; use parking_lot::RwLock; use std::collections::HashMap; @@ -22,7 +22,7 @@ impl MemoryDB { } } -impl StoreDB for MemoryDB { +impl Store for MemoryDB { /// Get the value of some key from the database. Returns `None` if the key does not exist. fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { let column_key = MemoryDB::get_key_for_col(col, key); From cf8a24c2bdb3a32988359b98044c045bb0a73fc6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 1 May 2019 14:29:03 +1000 Subject: [PATCH 16/44] Add enc/decode traits to store --- beacon_node/db2/src/impls.rs | 16 +++++++++++++++ beacon_node/db2/src/lib.rs | 38 +++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 beacon_node/db2/src/impls.rs diff --git a/beacon_node/db2/src/impls.rs b/beacon_node/db2/src/impls.rs new file mode 100644 index 000000000..9e607ddf5 --- /dev/null +++ b/beacon_node/db2/src/impls.rs @@ -0,0 +1,16 @@ +/* +use types::*; + +impl StoreEncode for Hash256 { + fn as_store_bytes(&self) -> Vec { + self.as_bytes().to_vec() + } +} + +impl StoreDecode for Hash256 { + fn from_store_bytes(bytes: &mut [u8]) -> Vec { + Hash256::from_slice() + self.as_bytes().to_vec() + } +} +*/ diff --git a/beacon_node/db2/src/lib.rs b/beacon_node/db2/src/lib.rs index da00262e6..3bff89512 100644 --- a/beacon_node/db2/src/lib.rs +++ b/beacon_node/db2/src/lib.rs @@ -1,9 +1,8 @@ // mod disk_db; mod errors; +mod impls; mod memory_db; -use db_encode::{db_encode, DBDecode, DBEncode}; - pub use self::memory_db::MemoryDB; pub use errors::Error; pub use types::*; @@ -35,6 +34,14 @@ pub trait Store: Sync + Send + Sized { fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; } +pub trait StoreEncode { + fn as_store_bytes(&self) -> Vec; +} + +pub trait StoreDecode: Sized { + fn from_store_bytes(bytes: &mut [u8]) -> Result; +} + pub enum DBColumn { Block, State, @@ -52,7 +59,7 @@ impl<'a> Into<&'a str> for DBColumn { } } -pub trait StorableItem: DBEncode + DBDecode { +pub trait StorableItem: StoreEncode + StoreDecode + Sized { fn db_column() -> DBColumn; fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> { @@ -60,7 +67,7 @@ pub trait StorableItem: DBEncode + DBDecode { let key = key.as_bytes(); store - .put_bytes(column, key, &db_encode(self)) + .put_bytes(column, key, &self.as_store_bytes()) .map_err(|e| e.into()) } @@ -69,10 +76,7 @@ pub trait StorableItem: DBEncode + DBDecode { let key = key.as_bytes(); match store.get_bytes(column, key)? { - Some(bytes) => { - let (item, _index) = Self::db_decode(&bytes, 0)?; - Ok(Some(item)) - } + Some(mut bytes) => Ok(Some(Self::from_store_bytes(&mut bytes[..])?)), None => Ok(None), } } @@ -95,16 +99,28 @@ pub trait StorableItem: DBEncode + DBDecode { #[cfg(test)] mod tests { use super::*; - use db_encode_derive::{DBDecode, DBEncode}; - use ssz::Decodable; + use ssz::{ssz_encode, Decodable}; use ssz_derive::{Decode, Encode}; - #[derive(PartialEq, Debug, Encode, Decode, DBEncode, DBDecode)] + #[derive(PartialEq, Debug, Encode, Decode)] struct StorableThing { a: u64, b: u64, } + impl StoreEncode for StorableThing { + fn as_store_bytes(&self) -> Vec { + ssz_encode(self) + } + } + + impl StoreDecode for StorableThing { + fn from_store_bytes(bytes: &mut [u8]) -> Result { + let (item, _) = Self::ssz_decode(bytes, 0)?; + Ok(item) + } + } + impl StorableItem for StorableThing { fn db_column() -> DBColumn { DBColumn::Block From b9ab3b25236863e8aac3fe4fe706180a7b25a07a Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Thu, 16 May 2019 23:06:41 +1000 Subject: [PATCH 17/44] fake sig back, merge, and fixed up yml checkin issue --- eth2/state_processing/Cargo.toml | 3 ++ .../block_processing_builder.rs | 14 +++++-- .../src/per_block_processing/tests.rs | 42 ++++++++++++------- .../testing_beacon_block_builder.rs | 5 +++ eth2/utils/bls/src/fake_signature.rs | 8 ++++ 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index 4b031022a..0fc7910c8 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -29,3 +29,6 @@ tree_hash = { path = "../utils/tree_hash" } tree_hash_derive = { path = "../utils/tree_hash_derive" } types = { path = "../types" } rayon = "1.0" + +[features] +fake_crypto = ["bls/fake_crypto"] \ No newline at end of file diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index fb9dbd153..307fc4a3d 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -1,14 +1,15 @@ use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; use types::*; +use tree_hash::SignedRoot; -pub struct BlockProcessingBuilder { - pub state_builder: TestingBeaconStateBuilder, +pub struct BlockProcessingBuilder { + pub state_builder: TestingBeaconStateBuilder, pub block_builder: TestingBeaconBlockBuilder, pub num_validators: usize, } -impl BlockProcessingBuilder { +impl BlockProcessingBuilder { pub fn new(num_validators: usize, spec: &ChainSpec) -> Self { let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec); @@ -30,12 +31,17 @@ impl BlockProcessingBuilder { self.state_builder.build_caches(&spec).unwrap(); } - pub fn build(mut self, randao_sk: Option, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { + pub fn build(mut self, randao_sk: Option, previous_block_root: Option, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { let (state, keypairs) = self.state_builder.build(); let builder = &mut self.block_builder; builder.set_slot(state.slot); + match previous_block_root { + Some(root) => builder.set_previous_block_root(root), + None => builder.set_previous_block_root(Hash256::from_slice(&state.latest_block_header.signed_root())), + } + let proposer_index = state .get_beacon_proposer_index(state.slot, RelativeEpoch::Current, spec) .unwrap(); diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 6ae7fd872..715c3e852 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -1,17 +1,17 @@ -#![cfg(test)] +#![cfg(all(test, not(feature = "fake_crypto")))] use super::block_processing_builder::BlockProcessingBuilder; use super::errors::*; use crate::per_block_processing; -use ssz::SignedRoot; -use types::{ChainSpec, Domain, Keypair, Signature, Slot}; +use tree_hash::SignedRoot; +use types::*; pub const VALIDATOR_COUNT: usize = 10; #[test] fn valid_block_ok() { - let spec = ChainSpec::foundation(); + let spec = FoundationEthSpec::spec(); let builder = get_builder(&spec); - let (block, mut state) = builder.build(None, &spec); + let (block, mut state) = builder.build(None, None, &spec); let result = per_block_processing(&mut state, &block, &spec); @@ -20,9 +20,9 @@ fn valid_block_ok() { #[test] fn invalid_block_header_state_slot() { - let spec = ChainSpec::foundation(); + let spec = FoundationEthSpec::spec(); let builder = get_builder(&spec); - let (mut block, mut state) = builder.build(None, &spec); + let (mut block, mut state) = builder.build(None, None, &spec); state.slot = Slot::new(133713); block.slot = Slot::new(424242); @@ -38,16 +38,30 @@ fn invalid_block_header_state_slot() { } #[test] -#[ignore] fn invalid_parent_block_root() { - // this will be changed in spec 0.5.1 to use signed root + let spec = FoundationEthSpec::spec(); + let builder = get_builder(&spec); + let invalid_parent_root = Hash256::from([0xAA; 32]); + let (block, mut state) = builder.build(None, Some(invalid_parent_root), &spec); + + let result = per_block_processing(&mut state, &block, &spec); + + assert_eq!( + result, + Err(BlockProcessingError::Invalid( + BlockInvalid::ParentBlockRootMismatch{ + state: Hash256::from_slice(&state.latest_block_header.signed_root()), + block: block.previous_block_root + } + )) + ); } #[test] fn invalid_block_signature() { - let spec = ChainSpec::foundation(); + let spec = FoundationEthSpec::spec(); let builder = get_builder(&spec); - let (mut block, mut state) = builder.build(None, &spec); + let (mut block, mut state) = builder.build(None, None, &spec); // sign the block with a keypair that is not the expected proposer let keypair = Keypair::random(); @@ -68,12 +82,12 @@ fn invalid_block_signature() { #[test] fn invalid_randao_reveal_signature() { - let spec = ChainSpec::foundation(); + let spec = FoundationEthSpec::spec(); let builder = get_builder(&spec); // sign randao reveal with random keypair let keypair = Keypair::random(); - let (block, mut state) = builder.build(Some(keypair.sk), &spec); + let (block, mut state) = builder.build(Some(keypair.sk), None, &spec); let result = per_block_processing(&mut state, &block, &spec); @@ -84,7 +98,7 @@ fn invalid_randao_reveal_signature() { ); } -fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { +fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); // Set the state and block to be in the last slot of the 4th epoch. diff --git a/eth2/types/src/test_utils/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/testing_beacon_block_builder.rs index 9c75bde0e..9dca6222a 100644 --- a/eth2/types/src/test_utils/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/testing_beacon_block_builder.rs @@ -23,6 +23,11 @@ impl TestingBeaconBlockBuilder { } } + /// Set the previous block root + pub fn set_previous_block_root(&mut self, root: Hash256) { + self.block.previous_block_root = root; + } + /// Set the slot of the block. pub fn set_slot(&mut self, slot: Slot) { self.block.slot = slot; diff --git a/eth2/utils/bls/src/fake_signature.rs b/eth2/utils/bls/src/fake_signature.rs index 0bf3d0a25..de16a05f3 100644 --- a/eth2/utils/bls/src/fake_signature.rs +++ b/eth2/utils/bls/src/fake_signature.rs @@ -14,6 +14,7 @@ use tree_hash::tree_hash_ssz_encoding_as_vector; #[derive(Debug, PartialEq, Clone, Eq)] pub struct FakeSignature { bytes: Vec, + is_empty: bool, } impl FakeSignature { @@ -26,6 +27,7 @@ impl FakeSignature { pub fn zero() -> Self { Self { bytes: vec![0; BLS_SIG_BYTE_SIZE], + is_empty: true, } } @@ -59,6 +61,7 @@ impl FakeSignature { } else { Ok(Self { bytes: bytes.to_vec(), + is_empty: false, }) } } @@ -71,6 +74,11 @@ impl FakeSignature { pub fn empty_signature() -> Self { FakeSignature::zero() } + + // Check for empty Signature + pub fn is_empty(&self) -> bool { + self.is_empty + } } impl_ssz!(FakeSignature, BLS_SIG_BYTE_SIZE, "FakeSignature"); From 39c3526884e41a885552cb2c1b4680608444ee8e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 May 2019 16:47:44 +1000 Subject: [PATCH 18/44] Run `cargo fmt --all` --- .../block_processing_builder.rs | 14 ++++++++++---- .../src/per_block_processing/tests.rs | 10 ++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 307fc4a3d..35e736d5f 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -1,6 +1,6 @@ +use tree_hash::SignedRoot; use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder}; use types::*; -use tree_hash::SignedRoot; pub struct BlockProcessingBuilder { pub state_builder: TestingBeaconStateBuilder, @@ -31,7 +31,12 @@ impl BlockProcessingBuilder { self.state_builder.build_caches(&spec).unwrap(); } - pub fn build(mut self, randao_sk: Option, previous_block_root: Option, spec: &ChainSpec) -> (BeaconBlock, BeaconState) { + pub fn build( + mut self, + randao_sk: Option, + previous_block_root: Option, + spec: &ChainSpec, + ) -> (BeaconBlock, BeaconState) { let (state, keypairs) = self.state_builder.build(); let builder = &mut self.block_builder; @@ -39,7 +44,9 @@ impl BlockProcessingBuilder { match previous_block_root { Some(root) => builder.set_previous_block_root(root), - None => builder.set_previous_block_root(Hash256::from_slice(&state.latest_block_header.signed_root())), + None => builder.set_previous_block_root(Hash256::from_slice( + &state.latest_block_header.signed_root(), + )), } let proposer_index = state @@ -56,5 +63,4 @@ impl BlockProcessingBuilder { (block, state) } - } diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 715c3e852..19418aba1 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -49,8 +49,8 @@ fn invalid_parent_block_root() { assert_eq!( result, Err(BlockProcessingError::Invalid( - BlockInvalid::ParentBlockRootMismatch{ - state: Hash256::from_slice(&state.latest_block_header.signed_root()), + BlockInvalid::ParentBlockRootMismatch { + state: Hash256::from_slice(&state.latest_block_header.signed_root()), block: block.previous_block_root } )) @@ -88,13 +88,15 @@ fn invalid_randao_reveal_signature() { // sign randao reveal with random keypair let keypair = Keypair::random(); let (block, mut state) = builder.build(Some(keypair.sk), None, &spec); - + let result = per_block_processing(&mut state, &block, &spec); // should get a BadRandaoSignature error assert_eq!( result, - Err(BlockProcessingError::Invalid(BlockInvalid::BadRandaoSignature)) + Err(BlockProcessingError::Invalid( + BlockInvalid::BadRandaoSignature + )) ); } From 182135b832299d927e64c447efd3c30fbc9784a1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 May 2019 18:01:51 +1000 Subject: [PATCH 19/44] Remove old DB crates, start fixing fork_choice --- Cargo.toml | 3 - beacon_node/db/Cargo.toml | 6 + beacon_node/db/src/block_at_slot.rs | 46 +++ beacon_node/db/src/disk_db.rs | 8 +- beacon_node/{db2 => db}/src/errors.rs | 0 beacon_node/db/src/impls.rs | 30 ++ beacon_node/db/src/lib.rs | 171 +++++++++- beacon_node/db/src/memory_db.rs | 231 ++----------- .../db/src/stores/beacon_block_store.rs | 246 -------------- .../db/src/stores/beacon_state_store.rs | 65 ---- beacon_node/db/src/stores/macros.rs | 103 ------ beacon_node/db/src/stores/mod.rs | 25 -- beacon_node/db/src/stores/pow_chain_store.rs | 68 ---- beacon_node/db/src/stores/validator_store.rs | 215 ------------ beacon_node/db/src/traits.rs | 28 -- beacon_node/db2/Cargo.toml | 17 - beacon_node/db2/src/disk_db.rs | 199 ------------ beacon_node/db2/src/impls.rs | 16 - beacon_node/db2/src/lib.rs | 160 --------- beacon_node/db2/src/memory_db.rs | 61 ---- beacon_node/db_encode/Cargo.toml | 9 - beacon_node/db_encode/src/lib.rs | 59 ---- beacon_node/db_encode_derive/Cargo.toml | 13 - beacon_node/db_encode_derive/src/lib.rs | 305 ------------------ eth2/fork_choice/src/lib.rs | 17 +- eth2/fork_choice/src/slow_lmd_ghost.rs | 36 +-- 26 files changed, 291 insertions(+), 1846 deletions(-) create mode 100644 beacon_node/db/src/block_at_slot.rs rename beacon_node/{db2 => db}/src/errors.rs (100%) create mode 100644 beacon_node/db/src/impls.rs delete mode 100644 beacon_node/db/src/stores/beacon_block_store.rs delete mode 100644 beacon_node/db/src/stores/beacon_state_store.rs delete mode 100644 beacon_node/db/src/stores/macros.rs delete mode 100644 beacon_node/db/src/stores/mod.rs delete mode 100644 beacon_node/db/src/stores/pow_chain_store.rs delete mode 100644 beacon_node/db/src/stores/validator_store.rs delete mode 100644 beacon_node/db/src/traits.rs delete mode 100644 beacon_node/db2/Cargo.toml delete mode 100644 beacon_node/db2/src/disk_db.rs delete mode 100644 beacon_node/db2/src/impls.rs delete mode 100644 beacon_node/db2/src/lib.rs delete mode 100644 beacon_node/db2/src/memory_db.rs delete mode 100644 beacon_node/db_encode/Cargo.toml delete mode 100644 beacon_node/db_encode/src/lib.rs delete mode 100644 beacon_node/db_encode_derive/Cargo.toml delete mode 100644 beacon_node/db_encode_derive/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 3c657bd4b..893189941 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,6 @@ members = [ "eth2/utils/test_random_derive", "beacon_node", "beacon_node/db", - "beacon_node/db2", - "beacon_node/db_encode", - "beacon_node/db_encode_derive", "beacon_node/client", "beacon_node/network", "beacon_node/eth2-libp2p", diff --git a/beacon_node/db/Cargo.toml b/beacon_node/db/Cargo.toml index ffb3585b9..bb2f659f8 100644 --- a/beacon_node/db/Cargo.toml +++ b/beacon_node/db/Cargo.toml @@ -8,4 +8,10 @@ edition = "2018" blake2-rfc = "0.2.18" bls = { path = "../../eth2/utils/bls" } bytes = "0.4.10" +db_encode = { path = "../db_encode" } +db_encode_derive = { path = "../db_encode_derive" } +parking_lot = "0.7" rocksdb = "0.10.1" +ssz = { path = "../../eth2/utils/ssz" } +ssz_derive = { path = "../../eth2/utils/ssz_derive" } +types = { path = "../../eth2/types" } diff --git a/beacon_node/db/src/block_at_slot.rs b/beacon_node/db/src/block_at_slot.rs new file mode 100644 index 000000000..c18c8998c --- /dev/null +++ b/beacon_node/db/src/block_at_slot.rs @@ -0,0 +1,46 @@ +use super::*; +use ssz::{Decode, DecodeError}; + +fn get_block_bytes(store: &T, root: Hash256) -> Result>, Error> { + store.get_bytes(BeaconBlock::db_column().into(), &root[..]) +} + +fn read_slot_from_block_bytes(bytes: &[u8]) -> Result { + let end = std::cmp::min(Slot::ssz_fixed_len(), bytes.len()); + + Slot::from_ssz_bytes(&bytes[0..end]) +} + +fn read_previous_block_root_from_block_bytes(bytes: &[u8]) -> Result { + let previous_bytes = Slot::ssz_fixed_len(); + let slice = bytes + .get(previous_bytes..previous_bytes + Hash256::ssz_fixed_len()) + .ok_or_else(|| DecodeError::BytesInvalid("Not enough bytes.".to_string()))?; + + Hash256::from_ssz_bytes(slice) +} + +pub fn get_block_at_preceeding_slot( + store: &T, + slot: Slot, + start_root: Hash256, +) -> Result, Error> { + let mut root = start_root; + + loop { + if let Some(bytes) = get_block_bytes(store, root)? { + let this_slot = read_slot_from_block_bytes(&bytes)?; + + if this_slot == slot { + let block = BeaconBlock::from_ssz_bytes(&bytes)?; + break Ok(Some((root, block))); + } else if this_slot < slot { + break Ok(None); + } else { + root = read_previous_block_root_from_block_bytes(&bytes)?; + } + } else { + break Ok(None); + } + } +} diff --git a/beacon_node/db/src/disk_db.rs b/beacon_node/db/src/disk_db.rs index 087941951..e2162e29a 100644 --- a/beacon_node/db/src/disk_db.rs +++ b/beacon_node/db/src/disk_db.rs @@ -1,9 +1,9 @@ extern crate rocksdb; -use super::rocksdb::Error as RocksError; -use super::rocksdb::{Options, DB}; -use super::stores::COLUMNS; +// use super::stores::COLUMNS; use super::{ClientDB, DBError, DBValue}; +use rocksdb::Error as RocksError; +use rocksdb::{Options, DB}; use std::fs; use std::path::Path; @@ -99,7 +99,7 @@ impl ClientDB for DiskDB { None => Err(DBError { message: "Unknown column".to_string(), }), - Some(handle) => self.db.put_cf(handle, key, val).map_err(Into::into), + Some(handle) => self.db.put_cf(handle, key, val).map_err(|e| e.into()), } } diff --git a/beacon_node/db2/src/errors.rs b/beacon_node/db/src/errors.rs similarity index 100% rename from beacon_node/db2/src/errors.rs rename to beacon_node/db/src/errors.rs diff --git a/beacon_node/db/src/impls.rs b/beacon_node/db/src/impls.rs new file mode 100644 index 000000000..91f8d52de --- /dev/null +++ b/beacon_node/db/src/impls.rs @@ -0,0 +1,30 @@ +use crate::*; +use ssz::{Decode, Encode}; + +impl StoreItem for BeaconBlock { + fn db_column() -> DBColumn { + DBColumn::BeaconBlock + } + + fn as_store_bytes(&self) -> Vec { + self.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &mut [u8]) -> Result { + Self::from_ssz_bytes(bytes).map_err(Into::into) + } +} + +impl StoreItem for BeaconState { + fn db_column() -> DBColumn { + DBColumn::BeaconState + } + + fn as_store_bytes(&self) -> Vec { + self.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &mut [u8]) -> Result { + Self::from_ssz_bytes(bytes).map_err(Into::into) + } +} diff --git a/beacon_node/db/src/lib.rs b/beacon_node/db/src/lib.rs index 5e710ae9a..8ac28092a 100644 --- a/beacon_node/db/src/lib.rs +++ b/beacon_node/db/src/lib.rs @@ -1,21 +1,160 @@ -extern crate blake2_rfc as blake2; -extern crate bls; -extern crate rocksdb; - -mod disk_db; +// mod disk_db; +mod block_at_slot; +mod errors; +mod impls; mod memory_db; -pub mod stores; -mod traits; -use self::stores::COLUMNS; - -pub use self::disk_db::DiskDB; pub use self::memory_db::MemoryDB; -pub use self::traits::{ClientDB, DBError, DBValue}; +pub use errors::Error; +pub use types::*; +pub type DBValue = Vec; -/// Currently available database options -#[derive(Debug, Clone)] -pub enum DBType { - Memory, - RocksDB, +pub trait Store: Sync + Send + Sized { + fn put(&self, key: &Hash256, item: &impl StoreItem) -> Result<(), Error> { + item.db_put(self, key) + } + + fn get(&self, key: &Hash256) -> Result, Error> { + I::db_get(self, key) + } + + fn exists(&self, key: &Hash256) -> Result { + I::db_exists(self, key) + } + + fn delete(&self, key: &Hash256) -> Result<(), Error> { + I::db_delete(self, key) + } + + fn get_block_at_preceeding_slot( + &self, + slot: Slot, + start_block_root: Hash256, + ) -> Result, Error> { + block_at_slot::get_block_at_preceeding_slot(self, slot, start_block_root) + } + + fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error>; + + fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error>; + + fn key_exists(&self, col: &str, key: &[u8]) -> Result; + + fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; +} + +pub enum DBColumn { + BeaconBlock, + BeaconState, + BeaconChain, +} + +impl<'a> Into<&'a str> for DBColumn { + /// Returns a `&str` that can be used for keying a key-value data base. + fn into(self) -> &'a str { + match self { + DBColumn::BeaconBlock => &"blk", + DBColumn::BeaconState => &"ste", + DBColumn::BeaconChain => &"bch", + } + } +} + +pub trait StoreItem: Sized { + fn db_column() -> DBColumn; + + fn as_store_bytes(&self) -> Vec; + + fn from_store_bytes(bytes: &mut [u8]) -> Result; + + fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> { + let column = Self::db_column().into(); + let key = key.as_bytes(); + + store + .put_bytes(column, key, &self.as_store_bytes()) + .map_err(|e| e.into()) + } + + fn db_get(store: &impl Store, key: &Hash256) -> Result, Error> { + let column = Self::db_column().into(); + let key = key.as_bytes(); + + match store.get_bytes(column, key)? { + Some(mut bytes) => Ok(Some(Self::from_store_bytes(&mut bytes[..])?)), + None => Ok(None), + } + } + + fn db_exists(store: &impl Store, key: &Hash256) -> Result { + let column = Self::db_column().into(); + let key = key.as_bytes(); + + store.key_exists(column, key) + } + + fn db_delete(store: &impl Store, key: &Hash256) -> Result<(), Error> { + let column = Self::db_column().into(); + let key = key.as_bytes(); + + store.key_delete(column, key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ssz::{Decode, Encode}; + use ssz_derive::{Decode, Encode}; + + #[derive(PartialEq, Debug, Encode, Decode)] + struct StorableThing { + a: u64, + b: u64, + } + + impl StoreItem for StorableThing { + fn db_column() -> DBColumn { + DBColumn::BeaconBlock + } + + fn as_store_bytes(&self) -> Vec { + self.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &mut [u8]) -> Result { + Self::from_ssz_bytes(bytes).map_err(Into::into) + } + } + + #[test] + fn memorydb_can_store_and_retrieve() { + let store = MemoryDB::open(); + + let key = Hash256::random(); + let item = StorableThing { a: 1, b: 42 }; + + store.put(&key, &item).unwrap(); + + let retrieved = store.get(&key).unwrap().unwrap(); + + assert_eq!(item, retrieved); + } + + #[test] + fn exists() { + let store = MemoryDB::open(); + let key = Hash256::random(); + let item = StorableThing { a: 1, b: 42 }; + + assert_eq!(store.exists::(&key).unwrap(), false); + + store.put(&key, &item).unwrap(); + + assert_eq!(store.exists::(&key).unwrap(), true); + + store.delete::(&key).unwrap(); + + assert_eq!(store.exists::(&key).unwrap(), false); + } } diff --git a/beacon_node/db/src/memory_db.rs b/beacon_node/db/src/memory_db.rs index 008e5912f..83ff77ce1 100644 --- a/beacon_node/db/src/memory_db.rs +++ b/beacon_node/db/src/memory_db.rs @@ -1,236 +1,61 @@ -use super::blake2::blake2b::blake2b; -use super::COLUMNS; -use super::{ClientDB, DBError, DBValue}; -use std::collections::{HashMap, HashSet}; -use std::sync::RwLock; +use super::{DBValue, Error, Store}; +use parking_lot::RwLock; +use std::collections::HashMap; type DBHashMap = HashMap, Vec>; -type ColumnHashSet = HashSet; -/// An in-memory database implementing the ClientDB trait. -/// -/// It is not particularily optimized, it exists for ease and speed of testing. It's not expected -/// this DB would be used outside of tests. pub struct MemoryDB { db: RwLock, - known_columns: RwLock, } impl MemoryDB { - /// Open the in-memory database. - /// - /// All columns must be supplied initially, you will get an error if you try to access a column - /// that was not declared here. This condition is enforced artificially to simulate RocksDB. pub fn open() -> Self { - let db: DBHashMap = HashMap::new(); - let mut known_columns: ColumnHashSet = HashSet::new(); - for col in &COLUMNS { - known_columns.insert(col.to_string()); - } Self { - db: RwLock::new(db), - known_columns: RwLock::new(known_columns), + db: RwLock::new(HashMap::new()), } } - /// Hashes a key and a column name in order to get a unique key for the supplied column. fn get_key_for_col(col: &str, key: &[u8]) -> Vec { - blake2b(32, col.as_bytes(), key).as_bytes().to_vec() + let mut col = col.as_bytes().to_vec(); + col.append(&mut key.to_vec()); + col } } -impl ClientDB for MemoryDB { +impl Store for MemoryDB { /// Get the value of some key from the database. Returns `None` if the key does not exist. - fn get(&self, col: &str, key: &[u8]) -> Result, DBError> { - // Panic if the DB locks are poisoned. - let db = self.db.read().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - Ok(db.get(&column_key).and_then(|val| Some(val.clone()))) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } + Ok(self + .db + .read() + .get(&column_key) + .and_then(|val| Some(val.clone()))) } /// Puts a key in the database. - fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError> { - // Panic if the DB locks are poisoned. - let mut db = self.db.write().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - db.insert(column_key, val.to_vec()); - Ok(()) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } + self.db.write().insert(column_key, val.to_vec()); + + Ok(()) } /// Return true if some key exists in some column. - fn exists(&self, col: &str, key: &[u8]) -> Result { - // Panic if the DB locks are poisoned. - let db = self.db.read().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn key_exists(&self, col: &str, key: &[u8]) -> Result { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - Ok(db.contains_key(&column_key)) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } + Ok(self.db.read().contains_key(&column_key)) } /// Delete some key from the database. - fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError> { - // Panic if the DB locks are poisoned. - let mut db = self.db.write().unwrap(); - let known_columns = self.known_columns.read().unwrap(); + fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { + let column_key = MemoryDB::get_key_for_col(col, key); - if known_columns.contains(&col.to_string()) { - let column_key = MemoryDB::get_key_for_col(col, key); - db.remove(&column_key); - Ok(()) - } else { - Err(DBError { - message: "Unknown column".to_string(), - }) - } - } -} - -#[cfg(test)] -mod tests { - use super::super::stores::{BLOCKS_DB_COLUMN, VALIDATOR_DB_COLUMN}; - use super::super::ClientDB; - use super::*; - use std::sync::Arc; - use std::thread; - - #[test] - fn test_memorydb_can_delete() { - let col_a: &str = BLOCKS_DB_COLUMN; - - let db = MemoryDB::open(); - - db.put(col_a, "dogs".as_bytes(), "lol".as_bytes()).unwrap(); - - assert_eq!( - db.get(col_a, "dogs".as_bytes()).unwrap().unwrap(), - "lol".as_bytes() - ); - - db.delete(col_a, "dogs".as_bytes()).unwrap(); - - assert_eq!(db.get(col_a, "dogs".as_bytes()).unwrap(), None); - } - - #[test] - fn test_memorydb_column_access() { - let col_a: &str = BLOCKS_DB_COLUMN; - let col_b: &str = VALIDATOR_DB_COLUMN; - - let db = MemoryDB::open(); - - /* - * Testing that if we write to the same key in different columns that - * there is not an overlap. - */ - db.put(col_a, "same".as_bytes(), "cat".as_bytes()).unwrap(); - db.put(col_b, "same".as_bytes(), "dog".as_bytes()).unwrap(); - - assert_eq!( - db.get(col_a, "same".as_bytes()).unwrap().unwrap(), - "cat".as_bytes() - ); - assert_eq!( - db.get(col_b, "same".as_bytes()).unwrap().unwrap(), - "dog".as_bytes() - ); - } - - #[test] - fn test_memorydb_unknown_column_access() { - let col_a: &str = BLOCKS_DB_COLUMN; - let col_x: &str = "ColumnX"; - - let db = MemoryDB::open(); - - /* - * Test that we get errors when using undeclared columns - */ - assert!(db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).is_ok()); - assert!(db.put(col_x, "cats".as_bytes(), "lol".as_bytes()).is_err()); - - assert!(db.get(col_a, "cats".as_bytes()).is_ok()); - assert!(db.get(col_x, "cats".as_bytes()).is_err()); - } - - #[test] - fn test_memorydb_exists() { - let col_a: &str = BLOCKS_DB_COLUMN; - let col_b: &str = VALIDATOR_DB_COLUMN; - - let db = MemoryDB::open(); - - /* - * Testing that if we write to the same key in different columns that - * there is not an overlap. - */ - db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).unwrap(); - - assert_eq!(true, db.exists(col_a, "cats".as_bytes()).unwrap()); - assert_eq!(false, db.exists(col_b, "cats".as_bytes()).unwrap()); - - assert_eq!(false, db.exists(col_a, "dogs".as_bytes()).unwrap()); - assert_eq!(false, db.exists(col_b, "dogs".as_bytes()).unwrap()); - } - - #[test] - fn test_memorydb_threading() { - let col_name: &str = BLOCKS_DB_COLUMN; - - let db = Arc::new(MemoryDB::open()); - - let thread_count = 10; - let write_count = 10; - - // We're execting the product of these numbers to fit in one byte. - assert!(thread_count * write_count <= 255); - - let mut handles = vec![]; - for t in 0..thread_count { - let wc = write_count; - let db = db.clone(); - let col = col_name.clone(); - let handle = thread::spawn(move || { - for w in 0..wc { - let key = (t * w) as u8; - let val = 42; - db.put(&col, &vec![key], &vec![val]).unwrap(); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - for t in 0..thread_count { - for w in 0..write_count { - let key = (t * w) as u8; - let val = db.get(&col_name, &vec![key]).unwrap().unwrap(); - assert_eq!(vec![42], val); - } - } + self.db.write().remove(&column_key); + + Ok(()) } } diff --git a/beacon_node/db/src/stores/beacon_block_store.rs b/beacon_node/db/src/stores/beacon_block_store.rs deleted file mode 100644 index 868caafe2..000000000 --- a/beacon_node/db/src/stores/beacon_block_store.rs +++ /dev/null @@ -1,246 +0,0 @@ -use super::BLOCKS_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use ssz::Decode; -use std::sync::Arc; -use types::{BeaconBlock, Hash256, Slot}; - -#[derive(Clone, Debug, PartialEq)] -pub enum BeaconBlockAtSlotError { - UnknownBeaconBlock(Hash256), - InvalidBeaconBlock(Hash256), - DBError(String), -} - -pub struct BeaconBlockStore -where - T: ClientDB, -{ - db: Arc, -} - -// Implements `put`, `get`, `exists` and `delete` for the store. -impl_crud_for_store!(BeaconBlockStore, DB_COLUMN); - -impl BeaconBlockStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { - match self.get(&hash)? { - None => Ok(None), - Some(ssz) => { - let block = BeaconBlock::from_ssz_bytes(&ssz).map_err(|_| DBError { - message: "Bad BeaconBlock SSZ.".to_string(), - })?; - Ok(Some(block)) - } - } - } - - /// Retrieve the block at a slot given a "head_hash" and a slot. - /// - /// A "head_hash" must be a block hash with a slot number greater than or equal to the desired - /// slot. - /// - /// This function will read each block down the chain until it finds a block with the given - /// slot number. If the slot is skipped, the function will return None. - /// - /// If a block is found, a tuple of (block_hash, serialized_block) is returned. - /// - /// Note: this function uses a loop instead of recursion as the compiler is over-strict when it - /// comes to recursion and the `impl Trait` pattern. See: - /// https://stackoverflow.com/questions/54032940/using-impl-trait-in-a-recursive-function - pub fn block_at_slot( - &self, - head_hash: &Hash256, - slot: Slot, - ) -> Result, BeaconBlockAtSlotError> { - let mut current_hash = *head_hash; - - loop { - if let Some(block) = self.get_deserialized(¤t_hash)? { - if block.slot == slot { - break Ok(Some((current_hash, block))); - } else if block.slot < slot { - break Ok(None); - } else { - current_hash = block.previous_block_root; - } - } else { - break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash)); - } - } - } -} - -impl From for BeaconBlockAtSlotError { - fn from(e: DBError) -> Self { - BeaconBlockAtSlotError::DBError(e.message) - } -} - -#[cfg(test)] -mod tests { - use super::super::super::MemoryDB; - use super::*; - - use std::sync::Arc; - use std::thread; - - use ssz::ssz_encode; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use types::BeaconBlock; - use types::Hash256; - - test_crud_for_store!(BeaconBlockStore, DB_COLUMN); - - #[test] - fn head_hash_slot_too_low() { - let db = Arc::new(MemoryDB::open()); - let bs = Arc::new(BeaconBlockStore::new(db.clone())); - let mut rng = XorShiftRng::from_seed([42; 16]); - - let mut block = BeaconBlock::random_for_test(&mut rng); - block.slot = Slot::from(10_u64); - - let block_root = block.canonical_root(); - bs.put(&block_root, &ssz_encode(&block)).unwrap(); - - let result = bs.block_at_slot(&block_root, Slot::from(11_u64)).unwrap(); - assert_eq!(result, None); - } - - #[test] - fn test_invalid_block_at_slot() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "definitly not a valid block".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert_eq!( - store.block_at_slot(hash, Slot::from(42_u64)), - Err(BeaconBlockAtSlotError::DBError( - "Bad BeaconBlock SSZ.".into() - )) - ); - } - - #[test] - fn test_unknown_block_at_slot() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - let other_hash = &Hash256::from([0xBB; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert_eq!( - store.block_at_slot(other_hash, Slot::from(42_u64)), - Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash)) - ); - } - - #[test] - fn test_block_store_on_memory_db() { - let db = Arc::new(MemoryDB::open()); - let bs = Arc::new(BeaconBlockStore::new(db.clone())); - - let thread_count = 10; - let write_count = 10; - - let mut handles = vec![]; - for t in 0..thread_count { - let wc = write_count; - let bs = bs.clone(); - let handle = thread::spawn(move || { - for w in 0..wc { - let key = t * w; - let val = 42; - bs.put(&Hash256::from_low_u64_le(key), &vec![val]).unwrap(); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - for t in 0..thread_count { - for w in 0..write_count { - let key = t * w; - assert!(bs.exists(&Hash256::from_low_u64_le(key)).unwrap()); - let val = bs.get(&Hash256::from_low_u64_le(key)).unwrap().unwrap(); - assert_eq!(vec![42], val); - } - } - } - - #[test] - #[ignore] - fn test_block_at_slot() { - let db = Arc::new(MemoryDB::open()); - let bs = Arc::new(BeaconBlockStore::new(db.clone())); - let mut rng = XorShiftRng::from_seed([42; 16]); - - // Specify test block parameters. - let hashes = [ - Hash256::from([0; 32]), - Hash256::from([1; 32]), - Hash256::from([2; 32]), - Hash256::from([3; 32]), - Hash256::from([4; 32]), - ]; - let parent_hashes = [ - Hash256::from([255; 32]), // Genesis block. - Hash256::from([0; 32]), - Hash256::from([1; 32]), - Hash256::from([2; 32]), - Hash256::from([3; 32]), - ]; - let unknown_hash = Hash256::from([101; 32]); // different from all above - let slots: Vec = vec![0, 1, 3, 4, 5].iter().map(|x| Slot::new(*x)).collect(); - - // Generate a vec of random blocks and store them in the DB. - let block_count = 5; - let mut blocks: Vec = Vec::with_capacity(5); - for i in 0..block_count { - let mut block = BeaconBlock::random_for_test(&mut rng); - - block.previous_block_root = parent_hashes[i]; - block.slot = slots[i]; - - let ssz = ssz_encode(&block); - db.put(DB_COLUMN, hashes[i].as_bytes(), &ssz).unwrap(); - - blocks.push(block); - } - - // Test that certain slots can be reached from certain hashes. - let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; - for (hashes_index, slot_index) in test_cases { - let (matched_block_hash, block) = bs - .block_at_slot(&hashes[hashes_index], slots[slot_index]) - .unwrap() - .unwrap(); - assert_eq!(matched_block_hash, hashes[slot_index]); - assert_eq!(block.slot, slots[slot_index]); - } - - let ssz = bs.block_at_slot(&hashes[4], Slot::new(2)).unwrap(); - assert_eq!(ssz, None); - - let ssz = bs.block_at_slot(&hashes[4], Slot::new(6)).unwrap(); - assert_eq!(ssz, None); - - let ssz = bs.block_at_slot(&unknown_hash, Slot::new(2)); - assert_eq!( - ssz, - Err(BeaconBlockAtSlotError::UnknownBeaconBlock(unknown_hash)) - ); - } -} diff --git a/beacon_node/db/src/stores/beacon_state_store.rs b/beacon_node/db/src/stores/beacon_state_store.rs deleted file mode 100644 index 044290592..000000000 --- a/beacon_node/db/src/stores/beacon_state_store.rs +++ /dev/null @@ -1,65 +0,0 @@ -use super::STATES_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use ssz::Decode; -use std::sync::Arc; -use types::{BeaconState, EthSpec, Hash256}; - -pub struct BeaconStateStore -where - T: ClientDB, -{ - db: Arc, -} - -// Implements `put`, `get`, `exists` and `delete` for the store. -impl_crud_for_store!(BeaconStateStore, DB_COLUMN); - -impl BeaconStateStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - pub fn get_deserialized( - &self, - hash: &Hash256, - ) -> Result>, DBError> { - match self.get(&hash)? { - None => Ok(None), - Some(ssz) => { - let state = BeaconState::from_ssz_bytes(&ssz).map_err(|_| DBError { - message: "Bad State SSZ.".to_string(), - })?; - Ok(Some(state)) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::MemoryDB; - use super::*; - - use ssz::ssz_encode; - use std::sync::Arc; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use types::{FoundationBeaconState, Hash256}; - - test_crud_for_store!(BeaconStateStore, DB_COLUMN); - - #[test] - fn test_reader() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconStateStore::new(db.clone()); - - let mut rng = XorShiftRng::from_seed([42; 16]); - let state: FoundationBeaconState = BeaconState::random_for_test(&mut rng); - let state_root = state.canonical_root(); - - store.put(&state_root, &ssz_encode(&state)).unwrap(); - - let decoded = store.get_deserialized(&state_root).unwrap().unwrap(); - - assert_eq!(state, decoded); - } -} diff --git a/beacon_node/db/src/stores/macros.rs b/beacon_node/db/src/stores/macros.rs deleted file mode 100644 index 6c53e40ee..000000000 --- a/beacon_node/db/src/stores/macros.rs +++ /dev/null @@ -1,103 +0,0 @@ -macro_rules! impl_crud_for_store { - ($store: ident, $db_column: expr) => { - impl $store { - pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { - self.db.put($db_column, hash.as_bytes(), ssz) - } - - pub fn get(&self, hash: &Hash256) -> Result>, DBError> { - self.db.get($db_column, hash.as_bytes()) - } - - pub fn exists(&self, hash: &Hash256) -> Result { - self.db.exists($db_column, hash.as_bytes()) - } - - pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> { - self.db.delete($db_column, hash.as_bytes()) - } - } - }; -} - -#[cfg(test)] -macro_rules! test_crud_for_store { - ($store: ident, $db_column: expr) => { - #[test] - fn test_put() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - store.put(hash, ssz).unwrap(); - assert_eq!(db.get(DB_COLUMN, hash.as_bytes()).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert_eq!(store.get(hash).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get_unknown() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - let other_hash = &Hash256::from([0xBB; 32]); - - db.put(DB_COLUMN, other_hash.as_bytes(), ssz).unwrap(); - assert_eq!(store.get(hash).unwrap(), None); - } - - #[test] - fn test_exists() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert!(store.exists(hash).unwrap()); - } - - #[test] - fn test_block_does_not_exist() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - let other_hash = &Hash256::from([0xBB; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert!(!store.exists(other_hash).unwrap()); - } - - #[test] - fn test_delete() { - let db = Arc::new(MemoryDB::open()); - let store = $store::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from([0xAA; 32]); - - db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap(); - assert!(db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); - - store.delete(hash).unwrap(); - assert!(!db.exists(DB_COLUMN, hash.as_bytes()).unwrap()); - } - }; -} diff --git a/beacon_node/db/src/stores/mod.rs b/beacon_node/db/src/stores/mod.rs deleted file mode 100644 index 44de7eed1..000000000 --- a/beacon_node/db/src/stores/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -use super::{ClientDB, DBError}; - -#[macro_use] -mod macros; -mod beacon_block_store; -mod beacon_state_store; -mod pow_chain_store; -mod validator_store; - -pub use self::beacon_block_store::{BeaconBlockAtSlotError, BeaconBlockStore}; -pub use self::beacon_state_store::BeaconStateStore; -pub use self::pow_chain_store::PoWChainStore; -pub use self::validator_store::{ValidatorStore, ValidatorStoreError}; - -pub const BLOCKS_DB_COLUMN: &str = "blocks"; -pub const STATES_DB_COLUMN: &str = "states"; -pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; -pub const VALIDATOR_DB_COLUMN: &str = "validator"; - -pub const COLUMNS: [&str; 4] = [ - BLOCKS_DB_COLUMN, - STATES_DB_COLUMN, - POW_CHAIN_DB_COLUMN, - VALIDATOR_DB_COLUMN, -]; diff --git a/beacon_node/db/src/stores/pow_chain_store.rs b/beacon_node/db/src/stores/pow_chain_store.rs deleted file mode 100644 index 5c8b97907..000000000 --- a/beacon_node/db/src/stores/pow_chain_store.rs +++ /dev/null @@ -1,68 +0,0 @@ -use super::POW_CHAIN_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use std::sync::Arc; - -pub struct PoWChainStore -where - T: ClientDB, -{ - db: Arc, -} - -impl PoWChainStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - pub fn put_block_hash(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, &[0]) - } - - pub fn block_hash_exists(&self, hash: &[u8]) -> Result { - self.db.exists(DB_COLUMN, hash) - } -} - -#[cfg(test)] -mod tests { - extern crate types; - - use super::super::super::MemoryDB; - use super::*; - - use self::types::Hash256; - - #[test] - fn test_put_block_hash() { - let db = Arc::new(MemoryDB::open()); - let store = PoWChainStore::new(db.clone()); - - let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); - store.put_block_hash(hash).unwrap(); - - assert!(db.exists(DB_COLUMN, hash).unwrap()); - } - - #[test] - fn test_block_hash_exists() { - let db = Arc::new(MemoryDB::open()); - let store = PoWChainStore::new(db.clone()); - - let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); - db.put(DB_COLUMN, hash, &[0]).unwrap(); - - assert!(store.block_hash_exists(hash).unwrap()); - } - - #[test] - fn test_block_hash_does_not_exist() { - let db = Arc::new(MemoryDB::open()); - let store = PoWChainStore::new(db.clone()); - - let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec(); - let other_hash = &Hash256::from([0xBB; 32]).as_bytes().to_vec(); - db.put(DB_COLUMN, hash, &[0]).unwrap(); - - assert!(!store.block_hash_exists(other_hash).unwrap()); - } -} diff --git a/beacon_node/db/src/stores/validator_store.rs b/beacon_node/db/src/stores/validator_store.rs deleted file mode 100644 index f653c9f71..000000000 --- a/beacon_node/db/src/stores/validator_store.rs +++ /dev/null @@ -1,215 +0,0 @@ -extern crate bytes; - -use self::bytes::{BufMut, BytesMut}; -use super::VALIDATOR_DB_COLUMN as DB_COLUMN; -use super::{ClientDB, DBError}; -use bls::PublicKey; -use ssz::{Decode, Encode}; -use std::sync::Arc; - -#[derive(Debug, PartialEq)] -pub enum ValidatorStoreError { - DBError(String), - DecodeError, -} - -impl From for ValidatorStoreError { - fn from(error: DBError) -> Self { - ValidatorStoreError::DBError(error.message) - } -} - -#[derive(Debug, PartialEq)] -enum KeyPrefixes { - PublicKey, -} - -pub struct ValidatorStore -where - T: ClientDB, -{ - db: Arc, -} - -impl ValidatorStore { - pub fn new(db: Arc) -> Self { - Self { db } - } - - fn prefix_bytes(&self, key_prefix: &KeyPrefixes) -> Vec { - match key_prefix { - KeyPrefixes::PublicKey => b"pubkey".to_vec(), - } - } - - fn get_db_key_for_index(&self, key_prefix: &KeyPrefixes, index: usize) -> Vec { - let mut buf = BytesMut::with_capacity(6 + 8); - buf.put(self.prefix_bytes(key_prefix)); - buf.put_u64_be(index as u64); - buf.take().to_vec() - } - - pub fn put_public_key_by_index( - &self, - index: usize, - public_key: &PublicKey, - ) -> Result<(), ValidatorStoreError> { - let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); - let val = public_key.as_ssz_bytes(); - self.db - .put(DB_COLUMN, &key[..], &val[..]) - .map_err(ValidatorStoreError::from) - } - - pub fn get_public_key_by_index( - &self, - index: usize, - ) -> Result, ValidatorStoreError> { - let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index); - let val = self.db.get(DB_COLUMN, &key[..])?; - match val { - None => Ok(None), - Some(val) => match PublicKey::from_ssz_bytes(&val) { - Ok(key) => Ok(Some(key)), - Err(_) => Err(ValidatorStoreError::DecodeError), - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::MemoryDB; - use super::*; - use bls::Keypair; - - #[test] - fn test_prefix_bytes() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - assert_eq!( - store.prefix_bytes(&KeyPrefixes::PublicKey), - b"pubkey".to_vec() - ); - } - - #[test] - fn test_get_db_key_for_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let mut buf = BytesMut::with_capacity(6 + 8); - buf.put(b"pubkey".to_vec()); - buf.put_u64_be(42); - assert_eq!( - store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42), - buf.take().to_vec() - ) - } - - #[test] - fn test_put_public_key_by_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let index = 3; - let public_key = Keypair::random().pk; - - store.put_public_key_by_index(index, &public_key).unwrap(); - let public_key_at_index = db - .get( - DB_COLUMN, - &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..], - ) - .unwrap() - .unwrap(); - - assert_eq!(public_key_at_index, public_key.as_ssz_bytes()); - } - - #[test] - fn test_get_public_key_by_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let index = 4; - let public_key = Keypair::random().pk; - - db.put( - DB_COLUMN, - &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..], - &public_key.as_ssz_bytes(), - ) - .unwrap(); - - let public_key_at_index = store.get_public_key_by_index(index).unwrap().unwrap(); - assert_eq!(public_key_at_index, public_key); - } - - #[test] - fn test_get_public_key_by_unknown_index() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let public_key = Keypair::random().pk; - - db.put( - DB_COLUMN, - &store.get_db_key_for_index(&KeyPrefixes::PublicKey, 3)[..], - &public_key.as_ssz_bytes(), - ) - .unwrap(); - - let public_key_at_index = store.get_public_key_by_index(4).unwrap(); - assert_eq!(public_key_at_index, None); - } - - #[test] - fn test_get_invalid_public_key() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db.clone()); - - let key = store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42); - db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap(); - - assert_eq!( - store.get_public_key_by_index(42), - Err(ValidatorStoreError::DecodeError) - ); - } - - #[test] - fn test_validator_store_put_get() { - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db); - - let keys = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - - for i in 0..keys.len() { - store.put_public_key_by_index(i, &keys[i].pk).unwrap(); - } - - /* - * Check all keys are retrieved correctly. - */ - for i in 0..keys.len() { - let retrieved = store.get_public_key_by_index(i).unwrap().unwrap(); - assert_eq!(retrieved, keys[i].pk); - } - - /* - * Check that an index that wasn't stored returns None. - */ - assert!(store - .get_public_key_by_index(keys.len() + 1) - .unwrap() - .is_none()); - } -} diff --git a/beacon_node/db/src/traits.rs b/beacon_node/db/src/traits.rs deleted file mode 100644 index 41be3e23d..000000000 --- a/beacon_node/db/src/traits.rs +++ /dev/null @@ -1,28 +0,0 @@ -pub type DBValue = Vec; - -#[derive(Debug)] -pub struct DBError { - pub message: String, -} - -impl DBError { - pub fn new(message: String) -> Self { - Self { message } - } -} - -/// A generic database to be used by the "client' (i.e., -/// the lighthouse blockchain client). -/// -/// The purpose of having this generic trait is to allow the -/// program to use a persistent on-disk database during production, -/// but use a transient database during tests. -pub trait ClientDB: Sync + Send { - fn get(&self, col: &str, key: &[u8]) -> Result, DBError>; - - fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError>; - - fn exists(&self, col: &str, key: &[u8]) -> Result; - - fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError>; -} diff --git a/beacon_node/db2/Cargo.toml b/beacon_node/db2/Cargo.toml deleted file mode 100644 index 95e87c9ea..000000000 --- a/beacon_node/db2/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "db2" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -blake2-rfc = "0.2.18" -bls = { path = "../../eth2/utils/bls" } -bytes = "0.4.10" -db_encode = { path = "../db_encode" } -db_encode_derive = { path = "../db_encode_derive" } -parking_lot = "0.7" -rocksdb = "0.10.1" -ssz = { path = "../../eth2/utils/ssz" } -ssz_derive = { path = "../../eth2/utils/ssz_derive" } -types = { path = "../../eth2/types" } diff --git a/beacon_node/db2/src/disk_db.rs b/beacon_node/db2/src/disk_db.rs deleted file mode 100644 index e2162e29a..000000000 --- a/beacon_node/db2/src/disk_db.rs +++ /dev/null @@ -1,199 +0,0 @@ -extern crate rocksdb; - -// use super::stores::COLUMNS; -use super::{ClientDB, DBError, DBValue}; -use rocksdb::Error as RocksError; -use rocksdb::{Options, DB}; -use std::fs; -use std::path::Path; - -/// A on-disk database which implements the ClientDB trait. -/// -/// This implementation uses RocksDB with default options. -pub struct DiskDB { - db: DB, -} - -impl DiskDB { - /// Open the RocksDB database, optionally supplying columns if required. - /// - /// The RocksDB database will be contained in a directory titled - /// "database" in the supplied path. - /// - /// # Panics - /// - /// Panics if the database is unable to be created. - pub fn open(path: &Path, columns: Option<&[&str]>) -> Self { - // Rocks options. - let mut options = Options::default(); - options.create_if_missing(true); - - // Ensure the path exists. - fs::create_dir_all(&path).unwrap_or_else(|_| panic!("Unable to create {:?}", &path)); - let db_path = path.join("database"); - - let columns = columns.unwrap_or(&COLUMNS); - - if db_path.exists() { - Self { - db: DB::open_cf(&options, db_path, &COLUMNS) - .expect("Unable to open local database"), - } - } else { - let mut db = Self { - db: DB::open(&options, db_path).expect("Unable to open local database"), - }; - - for cf in columns { - db.create_col(cf).unwrap(); - } - - db - } - } - - /// Create a RocksDB column family. Corresponds to the - /// `create_cf()` function on the RocksDB API. - #[allow(dead_code)] - fn create_col(&mut self, col: &str) -> Result<(), DBError> { - match self.db.create_cf(col, &Options::default()) { - Err(e) => Err(e.into()), - Ok(_) => Ok(()), - } - } -} - -impl From for DBError { - fn from(e: RocksError) -> Self { - Self { - message: e.to_string(), - } - } -} - -impl ClientDB for DiskDB { - /// Get the value for some key on some column. - /// - /// Corresponds to the `get_cf()` method on the RocksDB API. - /// Will attempt to get the `ColumnFamily` and return an Err - /// if it fails. - fn get(&self, col: &str, key: &[u8]) -> Result, DBError> { - match self.db.cf_handle(col) { - None => Err(DBError { - message: "Unknown column".to_string(), - }), - Some(handle) => match self.db.get_cf(handle, key)? { - None => Ok(None), - Some(db_vec) => Ok(Some(DBValue::from(&*db_vec))), - }, - } - } - - /// Set some value for some key on some column. - /// - /// Corresponds to the `cf_handle()` method on the RocksDB API. - /// Will attempt to get the `ColumnFamily` and return an Err - /// if it fails. - fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError> { - match self.db.cf_handle(col) { - None => Err(DBError { - message: "Unknown column".to_string(), - }), - Some(handle) => self.db.put_cf(handle, key, val).map_err(|e| e.into()), - } - } - - /// Return true if some key exists in some column. - fn exists(&self, col: &str, key: &[u8]) -> Result { - /* - * I'm not sure if this is the correct way to read if some - * block exists. Naively I would expect this to unncessarily - * copy some data, but I could be wrong. - */ - match self.db.cf_handle(col) { - None => Err(DBError { - message: "Unknown column".to_string(), - }), - Some(handle) => Ok(self.db.get_cf(handle, key)?.is_some()), - } - } - - /// Delete the value for some key on some column. - /// - /// Corresponds to the `delete_cf()` method on the RocksDB API. - /// Will attempt to get the `ColumnFamily` and return an Err - /// if it fails. - fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError> { - match self.db.cf_handle(col) { - None => Err(DBError { - message: "Unknown column".to_string(), - }), - Some(handle) => { - self.db.delete_cf(handle, key)?; - Ok(()) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::super::ClientDB; - use super::*; - use std::sync::Arc; - use std::{env, fs, thread}; - - #[test] - #[ignore] - fn test_rocksdb_can_use_db() { - let pwd = env::current_dir().unwrap(); - let path = pwd.join("testdb_please_remove"); - let _ = fs::remove_dir_all(&path); - fs::create_dir_all(&path).unwrap(); - - let col_name: &str = "TestColumn"; - let column_families = vec![col_name]; - - let mut db = DiskDB::open(&path, None); - - for cf in column_families { - db.create_col(&cf).unwrap(); - } - - let db = Arc::new(db); - - let thread_count = 10; - let write_count = 10; - - // We're execting the product of these numbers to fit in one byte. - assert!(thread_count * write_count <= 255); - - let mut handles = vec![]; - for t in 0..thread_count { - let wc = write_count; - let db = db.clone(); - let col = col_name.clone(); - let handle = thread::spawn(move || { - for w in 0..wc { - let key = (t * w) as u8; - let val = 42; - db.put(&col, &vec![key], &vec![val]).unwrap(); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - for t in 0..thread_count { - for w in 0..write_count { - let key = (t * w) as u8; - let val = db.get(&col_name, &vec![key]).unwrap().unwrap(); - assert_eq!(vec![42], val); - } - } - fs::remove_dir_all(&path).unwrap(); - } -} diff --git a/beacon_node/db2/src/impls.rs b/beacon_node/db2/src/impls.rs deleted file mode 100644 index 9e607ddf5..000000000 --- a/beacon_node/db2/src/impls.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* -use types::*; - -impl StoreEncode for Hash256 { - fn as_store_bytes(&self) -> Vec { - self.as_bytes().to_vec() - } -} - -impl StoreDecode for Hash256 { - fn from_store_bytes(bytes: &mut [u8]) -> Vec { - Hash256::from_slice() - self.as_bytes().to_vec() - } -} -*/ diff --git a/beacon_node/db2/src/lib.rs b/beacon_node/db2/src/lib.rs deleted file mode 100644 index 3bff89512..000000000 --- a/beacon_node/db2/src/lib.rs +++ /dev/null @@ -1,160 +0,0 @@ -// mod disk_db; -mod errors; -mod impls; -mod memory_db; - -pub use self::memory_db::MemoryDB; -pub use errors::Error; -pub use types::*; -pub type DBValue = Vec; - -pub trait Store: Sync + Send + Sized { - fn put(&self, key: &Hash256, item: &impl StorableItem) -> Result<(), Error> { - item.db_put(self, key) - } - - fn get(&self, key: &Hash256) -> Result, Error> { - I::db_get(self, key) - } - - fn exists(&self, key: &Hash256) -> Result { - I::db_exists(self, key) - } - - fn delete(&self, key: &Hash256) -> Result<(), Error> { - I::db_delete(self, key) - } - - fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error>; - - fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error>; - - fn key_exists(&self, col: &str, key: &[u8]) -> Result; - - fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; -} - -pub trait StoreEncode { - fn as_store_bytes(&self) -> Vec; -} - -pub trait StoreDecode: Sized { - fn from_store_bytes(bytes: &mut [u8]) -> Result; -} - -pub enum DBColumn { - Block, - State, - BeaconChain, -} - -impl<'a> Into<&'a str> for DBColumn { - /// Returns a `&str` that can be used for keying a key-value data base. - fn into(self) -> &'a str { - match self { - DBColumn::Block => &"blk", - DBColumn::State => &"ste", - DBColumn::BeaconChain => &"bch", - } - } -} - -pub trait StorableItem: StoreEncode + StoreDecode + Sized { - fn db_column() -> DBColumn; - - fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> { - let column = Self::db_column().into(); - let key = key.as_bytes(); - - store - .put_bytes(column, key, &self.as_store_bytes()) - .map_err(|e| e.into()) - } - - fn db_get(store: &impl Store, key: &Hash256) -> Result, Error> { - let column = Self::db_column().into(); - let key = key.as_bytes(); - - match store.get_bytes(column, key)? { - Some(mut bytes) => Ok(Some(Self::from_store_bytes(&mut bytes[..])?)), - None => Ok(None), - } - } - - fn db_exists(store: &impl Store, key: &Hash256) -> Result { - let column = Self::db_column().into(); - let key = key.as_bytes(); - - store.key_exists(column, key) - } - - fn db_delete(store: &impl Store, key: &Hash256) -> Result<(), Error> { - let column = Self::db_column().into(); - let key = key.as_bytes(); - - store.key_delete(column, key) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ssz::{ssz_encode, Decodable}; - use ssz_derive::{Decode, Encode}; - - #[derive(PartialEq, Debug, Encode, Decode)] - struct StorableThing { - a: u64, - b: u64, - } - - impl StoreEncode for StorableThing { - fn as_store_bytes(&self) -> Vec { - ssz_encode(self) - } - } - - impl StoreDecode for StorableThing { - fn from_store_bytes(bytes: &mut [u8]) -> Result { - let (item, _) = Self::ssz_decode(bytes, 0)?; - Ok(item) - } - } - - impl StorableItem for StorableThing { - fn db_column() -> DBColumn { - DBColumn::Block - } - } - - #[test] - fn memorydb_can_store_and_retrieve() { - let store = MemoryDB::open(); - - let key = Hash256::random(); - let item = StorableThing { a: 1, b: 42 }; - - store.put(&key, &item).unwrap(); - - let retrieved = store.get(&key).unwrap().unwrap(); - - assert_eq!(item, retrieved); - } - - #[test] - fn exists() { - let store = MemoryDB::open(); - let key = Hash256::random(); - let item = StorableThing { a: 1, b: 42 }; - - assert_eq!(store.exists::(&key).unwrap(), false); - - store.put(&key, &item).unwrap(); - - assert_eq!(store.exists::(&key).unwrap(), true); - - store.delete::(&key).unwrap(); - - assert_eq!(store.exists::(&key).unwrap(), false); - } -} diff --git a/beacon_node/db2/src/memory_db.rs b/beacon_node/db2/src/memory_db.rs deleted file mode 100644 index 83ff77ce1..000000000 --- a/beacon_node/db2/src/memory_db.rs +++ /dev/null @@ -1,61 +0,0 @@ -use super::{DBValue, Error, Store}; -use parking_lot::RwLock; -use std::collections::HashMap; - -type DBHashMap = HashMap, Vec>; - -pub struct MemoryDB { - db: RwLock, -} - -impl MemoryDB { - pub fn open() -> Self { - Self { - db: RwLock::new(HashMap::new()), - } - } - - fn get_key_for_col(col: &str, key: &[u8]) -> Vec { - let mut col = col.as_bytes().to_vec(); - col.append(&mut key.to_vec()); - col - } -} - -impl Store for MemoryDB { - /// Get the value of some key from the database. Returns `None` if the key does not exist. - fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { - let column_key = MemoryDB::get_key_for_col(col, key); - - Ok(self - .db - .read() - .get(&column_key) - .and_then(|val| Some(val.clone()))) - } - - /// Puts a key in the database. - fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { - let column_key = MemoryDB::get_key_for_col(col, key); - - self.db.write().insert(column_key, val.to_vec()); - - Ok(()) - } - - /// Return true if some key exists in some column. - fn key_exists(&self, col: &str, key: &[u8]) -> Result { - let column_key = MemoryDB::get_key_for_col(col, key); - - Ok(self.db.read().contains_key(&column_key)) - } - - /// Delete some key from the database. - fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { - let column_key = MemoryDB::get_key_for_col(col, key); - - self.db.write().remove(&column_key); - - Ok(()) - } -} diff --git a/beacon_node/db_encode/Cargo.toml b/beacon_node/db_encode/Cargo.toml deleted file mode 100644 index b4e919585..000000000 --- a/beacon_node/db_encode/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "db_encode" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -ethereum-types = "0.5" -ssz = { path = "../../eth2/utils/ssz" } diff --git a/beacon_node/db_encode/src/lib.rs b/beacon_node/db_encode/src/lib.rs deleted file mode 100644 index 993ba0e79..000000000 --- a/beacon_node/db_encode/src/lib.rs +++ /dev/null @@ -1,59 +0,0 @@ -use ethereum_types::{Address, H256}; -use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; - -/// Convenience function to encode an object. -pub fn db_encode(val: &T) -> Vec -where - T: DBEncode, -{ - let mut ssz_stream = SszStream::new(); - ssz_stream.append(val); - ssz_stream.drain() -} - -/// An encoding scheme based solely upon SSZ. -/// -/// The reason we have a separate encoding scheme is to allows us to store fields in the DB that we -/// don't want to transmit across the wire or hash. -/// -/// For example, the cache fields on `BeaconState` should be stored in the DB, but they should not -/// be hashed or transmitted across the wire. `DBEncode` allows us to define two serialization -/// methods, one that encodes the caches and one that does not. -pub trait DBEncode: Encodable + Sized { - fn db_encode(&self, s: &mut SszStream) { - s.append(&ssz_encode(self)); - } -} - -/// A decoding scheme based solely upon SSZ. -/// -/// See `DBEncode` for reasoning on why this trait exists. -pub trait DBDecode: Decodable { - fn db_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { - Self::ssz_decode(bytes, index) - } -} - -// Implement encoding. -impl DBEncode for bool {} -impl DBEncode for u8 {} -impl DBEncode for u16 {} -impl DBEncode for u32 {} -impl DBEncode for u64 {} -impl DBEncode for usize {} -impl DBEncode for Vec where T: Encodable + Sized {} - -impl DBEncode for H256 {} -impl DBEncode for Address {} - -// Implement decoding. -impl DBDecode for bool {} -impl DBDecode for u8 {} -impl DBDecode for u16 {} -impl DBDecode for u32 {} -impl DBDecode for u64 {} -impl DBDecode for usize {} -impl DBDecode for Vec where T: Decodable {} - -impl DBDecode for H256 {} -impl DBDecode for Address {} diff --git a/beacon_node/db_encode_derive/Cargo.toml b/beacon_node/db_encode_derive/Cargo.toml deleted file mode 100644 index b2fba85e3..000000000 --- a/beacon_node/db_encode_derive/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "db_encode_derive" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" -description = "Procedural derive macros for `db_encode` encoding and decoding." - -[lib] -proc-macro = true - -[dependencies] -syn = "0.15" -quote = "0.6" diff --git a/beacon_node/db_encode_derive/src/lib.rs b/beacon_node/db_encode_derive/src/lib.rs deleted file mode 100644 index 1de081419..000000000 --- a/beacon_node/db_encode_derive/src/lib.rs +++ /dev/null @@ -1,305 +0,0 @@ -extern crate proc_macro; - -use proc_macro::TokenStream; -use quote::quote; -use syn::{parse_macro_input, DeriveInput}; - -/// Returns a Vec of `syn::Ident` for each named field in the struct. -/// -/// # Panics -/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. -fn get_named_field_idents<'a>(struct_data: &'a syn::DataStruct) -> Vec<&'a syn::Ident> { - struct_data - .fields - .iter() - .map(|f| match &f.ident { - Some(ref ident) => ident, - _ => panic!("db_derive only supports named struct fields."), - }) - .collect() -} - -/// Implements `db_encode::DBEncode` for some `struct`. -/// -/// Fields are encoded in the order they are defined. -#[proc_macro_derive(DBEncode)] -pub fn db_encode_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("db_derive only supports structs."), - }; - - let field_idents = get_named_field_idents(&struct_data); - - let output = quote! { - impl db_encode::DBEncode for #name { - fn db_encode(&self, s: &mut ssz::SszStream) { - #( - s.append(&self.#field_idents); - )* - } - } - }; - output.into() -} - -/// Implements `db_encode::DBEncode` for some `struct`. -/// -/// Fields are encoded in the order they are defined. -#[proc_macro_derive(DBDecode)] -pub fn db_decode_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let field_idents = get_named_field_idents(&struct_data); - - // Using a var in an iteration always consumes the var, therefore we must make a `fields_a` and - // a `fields_b` in order to perform two loops. - // - // https://github.com/dtolnay/quote/issues/8 - let field_idents_a = &field_idents; - let field_idents_b = &field_idents; - - let output = quote! { - impl db_encode::DBDecode for #name { - fn db_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), ssz::DecodeError> { - #( - let (#field_idents_a, i) = <_>::ssz_decode(bytes, i)?; - )* - - Ok(( - Self { - #( - #field_idents_b, - )* - }, - i - )) - } - } - }; - output.into() -} - -/* -/// Returns true if some field has an attribute declaring it should not be deserialized. -/// -/// The field attribute is: `#[ssz(skip_deserializing)]` -fn should_skip_deserializing(field: &syn::Field) -> bool { - for attr in &field.attrs { - if attr.tts.to_string() == "( skip_deserializing )" { - return true; - } - } - false -} - -/// Implements `ssz::Decodable` for some `struct`. -/// -/// Fields are decoded in the order they are defined. -#[proc_macro_derive(Decode)] -pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let all_idents = get_named_field_idents(&struct_data); - - // Build quotes for fields that should be deserialized and those that should be built from - // `Default`. - let mut quotes = vec![]; - for field in &struct_data.fields { - match &field.ident { - Some(ref ident) => { - if should_skip_deserializing(field) { - quotes.push(quote! { - let #ident = <_>::default(); - }); - } else { - quotes.push(quote! { - let (#ident, i) = <_>::ssz_decode(bytes, i)?; - }); - } - } - _ => panic!("ssz_derive only supports named struct fields."), - }; - } - - let output = quote! { - impl ssz::Decodable for #name { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), ssz::DecodeError> { - #( - #quotes - )* - - Ok(( - Self { - #( - #all_idents, - )* - }, - i - )) - } - } - }; - output.into() -} - -/// Returns a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields -/// that should not be tree hashed. -/// -/// # Panics -/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time. -fn get_tree_hashable_named_field_idents<'a>( - struct_data: &'a syn::DataStruct, -) -> Vec<&'a syn::Ident> { - struct_data - .fields - .iter() - .filter_map(|f| { - if should_skip_tree_hash(&f) { - None - } else { - Some(match &f.ident { - Some(ref ident) => ident, - _ => panic!("ssz_derive only supports named struct fields."), - }) - } - }) - .collect() -} - -/// Returns true if some field has an attribute declaring it should not be tree-hashed. -/// -/// The field attribute is: `#[tree_hash(skip_hashing)]` -fn should_skip_tree_hash(field: &syn::Field) -> bool { - for attr in &field.attrs { - if attr.tts.to_string() == "( skip_hashing )" { - return true; - } - } - false -} - -/// Implements `ssz::TreeHash` for some `struct`. -/// -/// Fields are processed in the order they are defined. -#[proc_macro_derive(TreeHash, attributes(tree_hash))] -pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let field_idents = get_tree_hashable_named_field_idents(&struct_data); - - let output = quote! { - impl ssz::TreeHash for #name { - fn hash_tree_root(&self) -> Vec { - let mut list: Vec> = Vec::new(); - #( - list.push(self.#field_idents.hash_tree_root()); - )* - - ssz::merkle_hash(&mut list) - } - } - }; - output.into() -} - -/// Returns `true` if some `Ident` should be considered to be a signature type. -fn type_ident_is_signature(ident: &syn::Ident) -> bool { - match ident.to_string().as_ref() { - "Signature" => true, - "AggregateSignature" => true, - _ => false, - } -} - -/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns -/// the final `Ident` in that path. -/// -/// E.g., for `types::Signature` returns `Signature`. -fn final_type_ident(field: &syn::Field) -> &syn::Ident { - match &field.ty { - syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident, - _ => panic!("ssz_derive only supports Path types."), - } -} - -/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and -/// including a field that is of type "Signature" or "AggregateSignature". -/// -/// See: -/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots -/// -/// This is a rather horrendous macro, it will read the type of the object as a string and decide -/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So, -/// it's important that you use those exact words as your type -- don't alias it to something else. -/// -/// If you can think of a better way to do this, please make an issue! -/// -/// Fields are processed in the order they are defined. -#[proc_macro_derive(SignedRoot)] -pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream { - let item = parse_macro_input!(input as DeriveInput); - - let name = &item.ident; - - let struct_data = match &item.data { - syn::Data::Struct(s) => s, - _ => panic!("ssz_derive only supports structs."), - }; - - let mut field_idents: Vec<&syn::Ident> = vec![]; - - for field in struct_data.fields.iter() { - let final_type_ident = final_type_ident(&field); - - if type_ident_is_signature(final_type_ident) { - break; - } else { - let ident = field - .ident - .as_ref() - .expect("ssz_derive only supports named_struct fields."); - field_idents.push(ident); - } - } - - let output = quote! { - impl ssz::SignedRoot for #name { - fn signed_root(&self) -> Vec { - let mut list: Vec> = Vec::new(); - #( - list.push(self.#field_idents.hash_tree_root()); - )* - - ssz::merkle_hash(&mut list) - } - } - }; - output.into() -} -*/ diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 016cd5dea..50aedb7e2 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -16,22 +16,23 @@ //! [`slow_lmd_ghost`]: struct.SlowLmdGhost.html //! [`bitwise_lmd_ghost`]: struct.OptimisedLmdGhost.html -extern crate db; -extern crate ssz; -extern crate types; - +/* pub mod bitwise_lmd_ghost; pub mod longest_chain; pub mod optimized_lmd_ghost; +*/ pub mod slow_lmd_ghost; -use db::stores::BeaconBlockAtSlotError; -use db::DBError; +// use db::stores::BeaconBlockAtSlotError; +// use db::DBError; +use db::Error as DBError; use types::{BeaconBlock, ChainSpec, Hash256}; +/* pub use bitwise_lmd_ghost::BitwiseLMDGhost; pub use longest_chain::LongestChain; pub use optimized_lmd_ghost::OptimizedLMDGhost; +*/ pub use slow_lmd_ghost::SlowLMDGhost; /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures @@ -77,10 +78,11 @@ pub enum ForkChoiceError { impl From for ForkChoiceError { fn from(e: DBError) -> ForkChoiceError { - ForkChoiceError::StorageError(e.message) + ForkChoiceError::StorageError(format!("{:?}", e)) } } +/* impl From for ForkChoiceError { fn from(e: BeaconBlockAtSlotError) -> ForkChoiceError { match e { @@ -94,6 +96,7 @@ impl From for ForkChoiceError { } } } +*/ /// Fork choice options that are currently implemented. #[derive(Debug, Clone)] diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index c9aaa70d1..7768c2867 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -1,10 +1,7 @@ extern crate db; use crate::{ForkChoice, ForkChoiceError}; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - ClientDB, -}; +use db::{Store, StoreItem}; use log::{debug, trace}; use std::collections::HashMap; use std::marker::PhantomData; @@ -13,32 +10,23 @@ use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot}; //TODO: Pruning and syncing -pub struct SlowLMDGhost { +pub struct SlowLMDGhost { /// The latest attestation targets as a map of validator index to block hash. //TODO: Could this be a fixed size vec latest_attestation_targets: HashMap, /// Stores the children for any given parent. children: HashMap>, - /// Block storage access. - block_store: Arc>, - /// State storage access. - state_store: Arc>, + /// Persistent storage + store: Arc, _phantom: PhantomData, } -impl SlowLMDGhost -where - T: ClientDB + Sized, -{ - pub fn new( - block_store: Arc>, - state_store: Arc>, - ) -> Self { +impl SlowLMDGhost { + pub fn new(store: Arc) -> Self { SlowLMDGhost { latest_attestation_targets: HashMap::new(), children: HashMap::new(), - block_store, - state_store, + store, _phantom: PhantomData, } } @@ -58,8 +46,8 @@ where let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state: BeaconState = self - .state_store - .get_deserialized(&state_root)? + .store + .get(state_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = @@ -90,8 +78,8 @@ where ) -> Result { let mut count = 0; let block_slot = self - .block_store - .get_deserialized(&block_root)? + .store + .get::(&block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))? .slot; @@ -108,7 +96,7 @@ where } } -impl ForkChoice for SlowLMDGhost { +impl ForkChoice for SlowLMDGhost { /// Process when a block is added fn add_block( &mut self, From f8c425d6b4bb56af4873a1ece9f8a63124bd5aff Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 12:58:11 +1000 Subject: [PATCH 20/44] Fix and add tests for db get block at slot --- beacon_node/db/Cargo.toml | 3 +- beacon_node/db/src/block_at_slot.rs | 137 ++++++++++++++++++++++++++++ beacon_node/db/src/lib.rs | 2 +- 3 files changed, 139 insertions(+), 3 deletions(-) diff --git a/beacon_node/db/Cargo.toml b/beacon_node/db/Cargo.toml index bb2f659f8..b6cdafe04 100644 --- a/beacon_node/db/Cargo.toml +++ b/beacon_node/db/Cargo.toml @@ -8,10 +8,9 @@ edition = "2018" blake2-rfc = "0.2.18" bls = { path = "../../eth2/utils/bls" } bytes = "0.4.10" -db_encode = { path = "../db_encode" } -db_encode_derive = { path = "../db_encode_derive" } parking_lot = "0.7" rocksdb = "0.10.1" ssz = { path = "../../eth2/utils/ssz" } ssz_derive = { path = "../../eth2/utils/ssz_derive" } +tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/beacon_node/db/src/block_at_slot.rs b/beacon_node/db/src/block_at_slot.rs index c18c8998c..4fa9635a2 100644 --- a/beacon_node/db/src/block_at_slot.rs +++ b/beacon_node/db/src/block_at_slot.rs @@ -44,3 +44,140 @@ pub fn get_block_at_preceeding_slot( } } } + +#[cfg(test)] +mod tests { + use super::*; + use ssz::Encode; + use tree_hash::TreeHash; + + #[test] + fn read_slot() { + let spec = FewValidatorsEthSpec::spec(); + + let test_slot = |slot: Slot| { + let mut block = BeaconBlock::empty(&spec); + block.slot = slot; + let bytes = block.as_ssz_bytes(); + assert_eq!(read_slot_from_block_bytes(&bytes).unwrap(), slot); + }; + + test_slot(Slot::new(0)); + test_slot(Slot::new(1)); + test_slot(Slot::new(42)); + test_slot(Slot::new(u64::max_value())); + } + + #[test] + fn bad_slot() { + for i in 0..8 { + assert!(read_slot_from_block_bytes(&vec![0; i]).is_err()); + } + } + + #[test] + fn read_previous_block_root() { + let spec = FewValidatorsEthSpec::spec(); + + let test_root = |root: Hash256| { + let mut block = BeaconBlock::empty(&spec); + block.previous_block_root = root; + let bytes = block.as_ssz_bytes(); + assert_eq!( + read_previous_block_root_from_block_bytes(&bytes).unwrap(), + root + ); + }; + + test_root(Hash256::random()); + test_root(Hash256::random()); + test_root(Hash256::random()); + } + + fn build_chain( + store: &impl Store, + slots: &[usize], + spec: &ChainSpec, + ) -> Vec<(Hash256, BeaconBlock)> { + let mut blocks_and_roots: Vec<(Hash256, BeaconBlock)> = vec![]; + + for (i, slot) in slots.iter().enumerate() { + let mut block = BeaconBlock::empty(spec); + block.slot = Slot::from(*slot); + + if i > 0 { + block.previous_block_root = blocks_and_roots[i - 1].0; + } + + let root = Hash256::from_slice(&block.tree_hash_root()); + + store.put(&root, &block).unwrap(); + blocks_and_roots.push((root, block)); + } + + blocks_and_roots + } + + #[test] + fn chain_without_skips() { + let n: usize = 10; + let store = MemoryDB::open(); + let spec = FewValidatorsEthSpec::spec(); + + let slots: Vec = (0..n).collect(); + let blocks_and_roots = build_chain(&store, &slots, &spec); + + for source in 1..n { + for target in 0..=source { + let (source_root, _source_block) = &blocks_and_roots[source]; + let (target_root, target_block) = &blocks_and_roots[target]; + + let (found_root, found_block) = store + .get_block_at_preceeding_slot(*source_root, target_block.slot) + .unwrap() + .unwrap(); + + assert_eq!(found_root, *target_root); + assert_eq!(found_block, *target_block); + } + } + } + + #[test] + fn chain_with_skips() { + let store = MemoryDB::open(); + let spec = FewValidatorsEthSpec::spec(); + + let slots = vec![0, 1, 2, 5]; + + let blocks_and_roots = build_chain(&store, &slots, &spec); + + // Valid slots + for target in 0..3 { + let (source_root, _source_block) = &blocks_and_roots[3]; + let (target_root, target_block) = &blocks_and_roots[target]; + + let (found_root, found_block) = store + .get_block_at_preceeding_slot(*source_root, target_block.slot) + .unwrap() + .unwrap(); + + assert_eq!(found_root, *target_root); + assert_eq!(found_block, *target_block); + } + + // Slot that doesn't exist + let (source_root, _source_block) = &blocks_and_roots[3]; + assert!(store + .get_block_at_preceeding_slot(*source_root, Slot::new(3)) + .unwrap() + .is_none()); + + // Slot too high + let (source_root, _source_block) = &blocks_and_roots[3]; + assert!(store + .get_block_at_preceeding_slot(*source_root, Slot::new(3)) + .unwrap() + .is_none()); + } +} diff --git a/beacon_node/db/src/lib.rs b/beacon_node/db/src/lib.rs index 8ac28092a..21b5b0a75 100644 --- a/beacon_node/db/src/lib.rs +++ b/beacon_node/db/src/lib.rs @@ -28,8 +28,8 @@ pub trait Store: Sync + Send + Sized { fn get_block_at_preceeding_slot( &self, - slot: Slot, start_block_root: Hash256, + slot: Slot, ) -> Result, Error> { block_at_slot::get_block_at_preceeding_slot(self, slot, start_block_root) } From 2128d411bc6544202e159edd72f3d78fdb25d33e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 12:58:51 +1000 Subject: [PATCH 21/44] Migrate fork_choice over to new DB --- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 56 ++++++++------------- eth2/fork_choice/src/lib.rs | 4 -- eth2/fork_choice/src/longest_chain.rs | 28 ++++------- eth2/fork_choice/src/optimized_lmd_ghost.rs | 56 ++++++++------------- eth2/fork_choice/src/slow_lmd_ghost.rs | 20 ++++---- 5 files changed, 65 insertions(+), 99 deletions(-) diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 0bbac6bb6..a76970f01 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -3,10 +3,7 @@ extern crate bit_vec; use crate::{ForkChoice, ForkChoiceError}; use bit_vec::BitVec; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - ClientDB, -}; +use db::Store; use log::{debug, trace}; use std::collections::HashMap; use std::marker::PhantomData; @@ -34,7 +31,7 @@ fn power_of_2_below(x: u64) -> u64 { } /// Stores the necessary data structures to run the optimised bitwise lmd ghost algorithm. -pub struct BitwiseLMDGhost { +pub struct BitwiseLMDGhost { /// A cache of known ancestors at given heights for a specific block. //TODO: Consider FnvHashMap cache: HashMap, Hash256>, @@ -46,30 +43,21 @@ pub struct BitwiseLMDGhost { /// The latest attestation targets as a map of validator index to block hash. //TODO: Could this be a fixed size vec latest_attestation_targets: HashMap, - /// Block storage access. - block_store: Arc>, - /// State storage access. - state_store: Arc>, + /// Block and state storage. + store: Arc, max_known_height: SlotHeight, _phantom: PhantomData, } -impl BitwiseLMDGhost -where - T: ClientDB + Sized, -{ - pub fn new( - block_store: Arc>, - state_store: Arc>, - ) -> Self { +impl BitwiseLMDGhost { + pub fn new(store: Arc) -> Self { BitwiseLMDGhost { cache: HashMap::new(), ancestors: vec![HashMap::new(); 16], latest_attestation_targets: HashMap::new(), children: HashMap::new(), max_known_height: SlotHeight::new(0), - block_store, - state_store, + store, _phantom: PhantomData, } } @@ -89,8 +77,8 @@ where let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state: BeaconState = self - .state_store - .get_deserialized(&state_root)? + .store + .get(&state_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = @@ -121,8 +109,8 @@ where // return None if we can't get the block from the db. let block_height = { let block_slot = self - .block_store - .get_deserialized(&block_hash) + .store + .get::(&block_hash) .ok()? .expect("Should have returned already if None") .slot; @@ -243,7 +231,7 @@ where } } -impl ForkChoice for BitwiseLMDGhost { +impl ForkChoice for BitwiseLMDGhost { fn add_block( &mut self, block: &BeaconBlock, @@ -252,8 +240,8 @@ impl ForkChoice for BitwiseLMDGhost { ) -> Result<(), ForkChoiceError> { // get the height of the parent let parent_height = self - .block_store - .get_deserialized(&block.previous_block_root)? + .store + .get::(&block.previous_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))? .slot .height(spec.genesis_slot); @@ -304,16 +292,16 @@ impl ForkChoice for BitwiseLMDGhost { trace!("Old attestation found: {:?}", attestation_target); // get the height of the target block let block_height = self - .block_store - .get_deserialized(&target_block_root)? + .store + .get::(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot .height(spec.genesis_slot); // get the height of the past target block let past_block_height = self - .block_store - .get_deserialized(&attestation_target)? + .store + .get::(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot .height(spec.genesis_slot); @@ -337,8 +325,8 @@ impl ForkChoice for BitwiseLMDGhost { justified_block_start ); let block = self - .block_store - .get_deserialized(&justified_block_start)? + .store + .get::(&justified_block_start)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; let block_slot = block.slot; @@ -429,8 +417,8 @@ impl ForkChoice for BitwiseLMDGhost { // didn't find head yet, proceed to next iteration // update block height block_height = self - .block_store - .get_deserialized(¤t_head)? + .store + .get::(¤t_head)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? .slot .height(spec.genesis_slot); diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index 50aedb7e2..ed3a1ce08 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -16,11 +16,9 @@ //! [`slow_lmd_ghost`]: struct.SlowLmdGhost.html //! [`bitwise_lmd_ghost`]: struct.OptimisedLmdGhost.html -/* pub mod bitwise_lmd_ghost; pub mod longest_chain; pub mod optimized_lmd_ghost; -*/ pub mod slow_lmd_ghost; // use db::stores::BeaconBlockAtSlotError; @@ -28,11 +26,9 @@ pub mod slow_lmd_ghost; use db::Error as DBError; use types::{BeaconBlock, ChainSpec, Hash256}; -/* pub use bitwise_lmd_ghost::BitwiseLMDGhost; pub use longest_chain::LongestChain; pub use optimized_lmd_ghost::OptimizedLMDGhost; -*/ pub use slow_lmd_ghost::SlowLMDGhost; /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index 423edc567..6aaf56c32 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -1,31 +1,25 @@ use crate::{ForkChoice, ForkChoiceError}; -use db::{stores::BeaconBlockStore, ClientDB}; +use db::Store; use std::sync::Arc; use types::{BeaconBlock, ChainSpec, Hash256, Slot}; -pub struct LongestChain -where - T: ClientDB + Sized, -{ +pub struct LongestChain { /// List of head block hashes head_block_hashes: Vec, - /// Block storage access. - block_store: Arc>, + /// Block storage. + store: Arc, } -impl LongestChain -where - T: ClientDB + Sized, -{ - pub fn new(block_store: Arc>) -> Self { +impl LongestChain { + pub fn new(store: Arc) -> Self { LongestChain { head_block_hashes: Vec::new(), - block_store, + store, } } } -impl ForkChoice for LongestChain { +impl ForkChoice for LongestChain { fn add_block( &mut self, block: &BeaconBlock, @@ -55,9 +49,9 @@ impl ForkChoice for LongestChain { * Load all the head_block hashes from the DB as SszBeaconBlocks. */ for (index, block_hash) in self.head_block_hashes.iter().enumerate() { - let block = self - .block_store - .get_deserialized(&block_hash)? + let block: BeaconBlock = self + .store + .get(&block_hash)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_hash))?; head_blocks.push((index, block)); } diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index 3f585e3c1..aa7d65c85 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -2,10 +2,7 @@ extern crate bit_vec; use crate::{ForkChoice, ForkChoiceError}; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - ClientDB, -}; +use db::Store; use log::{debug, trace}; use std::cmp::Ordering; use std::collections::HashMap; @@ -34,7 +31,7 @@ fn power_of_2_below(x: u64) -> u64 { } /// Stores the necessary data structures to run the optimised lmd ghost algorithm. -pub struct OptimizedLMDGhost { +pub struct OptimizedLMDGhost { /// A cache of known ancestors at given heights for a specific block. //TODO: Consider FnvHashMap cache: HashMap, Hash256>, @@ -46,30 +43,21 @@ pub struct OptimizedLMDGhost { /// The latest attestation targets as a map of validator index to block hash. //TODO: Could this be a fixed size vec latest_attestation_targets: HashMap, - /// Block storage access. - block_store: Arc>, - /// State storage access. - state_store: Arc>, + /// Block and state storage. + store: Arc, max_known_height: SlotHeight, _phantom: PhantomData, } -impl OptimizedLMDGhost -where - T: ClientDB + Sized, -{ - pub fn new( - block_store: Arc>, - state_store: Arc>, - ) -> Self { +impl OptimizedLMDGhost { + pub fn new(store: Arc) -> Self { OptimizedLMDGhost { cache: HashMap::new(), ancestors: vec![HashMap::new(); 16], latest_attestation_targets: HashMap::new(), children: HashMap::new(), max_known_height: SlotHeight::new(0), - block_store, - state_store, + store, _phantom: PhantomData, } } @@ -89,8 +77,8 @@ where let mut latest_votes: HashMap = HashMap::new(); // gets the current weighted votes let current_state: BeaconState = self - .state_store - .get_deserialized(&state_root)? + .store + .get(&state_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = @@ -121,8 +109,8 @@ where // return None if we can't get the block from the db. let block_height = { let block_slot = self - .block_store - .get_deserialized(&block_hash) + .store + .get::(&block_hash) .ok()? .expect("Should have returned already if None") .slot; @@ -214,7 +202,7 @@ where } } -impl ForkChoice for OptimizedLMDGhost { +impl ForkChoice for OptimizedLMDGhost { fn add_block( &mut self, block: &BeaconBlock, @@ -223,8 +211,8 @@ impl ForkChoice for OptimizedLMDGhost { ) -> Result<(), ForkChoiceError> { // get the height of the parent let parent_height = self - .block_store - .get_deserialized(&block.previous_block_root)? + .store + .get::(&block.previous_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))? .slot .height(spec.genesis_slot); @@ -275,16 +263,16 @@ impl ForkChoice for OptimizedLMDGhost { trace!("Old attestation found: {:?}", attestation_target); // get the height of the target block let block_height = self - .block_store - .get_deserialized(&target_block_root)? + .store + .get::(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot .height(spec.genesis_slot); // get the height of the past target block let past_block_height = self - .block_store - .get_deserialized(&attestation_target)? + .store + .get::(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot .height(spec.genesis_slot); @@ -308,8 +296,8 @@ impl ForkChoice for OptimizedLMDGhost { justified_block_start ); let block = self - .block_store - .get_deserialized(&justified_block_start)? + .store + .get::(&justified_block_start)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; let block_slot = block.slot; @@ -400,8 +388,8 @@ impl ForkChoice for OptimizedLMDGhost { // didn't find head yet, proceed to next iteration // update block height block_height = self - .block_store - .get_deserialized(¤t_head)? + .store + .get::(¤t_head)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? .slot .height(spec.genesis_slot); diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 7768c2867..a41eacbb2 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -1,7 +1,7 @@ extern crate db; use crate::{ForkChoice, ForkChoiceError}; -use db::{Store, StoreItem}; +use db::Store; use log::{debug, trace}; use std::collections::HashMap; use std::marker::PhantomData; @@ -16,7 +16,7 @@ pub struct SlowLMDGhost { latest_attestation_targets: HashMap, /// Stores the children for any given parent. children: HashMap>, - /// Persistent storage + /// Block and state storage. store: Arc, _phantom: PhantomData, } @@ -85,8 +85,8 @@ impl SlowLMDGhost { for (vote_hash, votes) in latest_votes.iter() { let (root_at_slot, _) = self - .block_store - .block_at_slot(&vote_hash, block_slot)? + .store + .get_block_at_preceeding_slot(*vote_hash, block_slot)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))?; if root_at_slot == *block_root { count += votes; @@ -138,16 +138,16 @@ impl ForkChoice for SlowLMDGhost { trace!("Old attestation found: {:?}", attestation_target); // get the height of the target block let block_height = self - .block_store - .get_deserialized(&target_block_root)? + .store + .get::(&target_block_root)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? .slot .height(spec.genesis_slot); // get the height of the past target block let past_block_height = self - .block_store - .get_deserialized(&attestation_target)? + .store + .get::(&attestation_target)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? .slot .height(spec.genesis_slot); @@ -168,8 +168,8 @@ impl ForkChoice for SlowLMDGhost { ) -> Result { debug!("Running LMD Ghost Fork-choice rule"); let start = self - .block_store - .get_deserialized(&justified_block_start)? + .store + .get::(&justified_block_start)? .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; let start_state_root = start.state_root; From f1584dada470213d2477832ffc2c400d0e1694b5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 13:36:14 +1000 Subject: [PATCH 22/44] Update BeaconChain struct to use new store --- beacon_node/beacon_chain/src/beacon_chain.rs | 22 +++++++------------ .../testing_beacon_chain_builder.rs | 14 ++++-------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index db5ea1cdb..57b697019 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,9 +1,6 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - ClientDB, DBError, -}; +use db::{Error as DBError, Store}; use fork_choice::{ForkChoice, ForkChoiceError}; use log::{debug, trace}; use operation_pool::DepositInsertStatus; @@ -83,9 +80,8 @@ impl BlockProcessingOutcome { } } -pub struct BeaconChain { - pub block_store: Arc>, - pub state_store: Arc>, +pub struct BeaconChain { + pub store: Arc, pub slot_clock: U, pub op_pool: OperationPool, canonical_head: RwLock>, @@ -97,15 +93,14 @@ pub struct BeaconChain BeaconChain where - T: ClientDB, + T: Store, U: SlotClock, F: ForkChoice, E: EthSpec, { /// Instantiate a new Beacon Chain, from genesis. pub fn from_genesis( - state_store: Arc>, - block_store: Arc>, + store: Arc, slot_clock: U, mut genesis_state: BeaconState, genesis_block: BeaconBlock, @@ -113,10 +108,10 @@ where fork_choice: F, ) -> Result { let state_root = genesis_state.canonical_root(); - state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?; + store.put(&state_root, &genesis_state)?; let block_root = genesis_block.block_header().canonical_root(); - block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; + store.put(&block_root, &genesis_block)?; let finalized_head = RwLock::new(CheckPoint::new( genesis_block.clone(), @@ -134,8 +129,7 @@ where genesis_state.build_all_caches(&spec)?; Ok(Self { - block_store, - state_store, + store, slot_clock, op_pool: OperationPool::new(), state: RwLock::new(genesis_state), diff --git a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs index f7ff3cdae..ce3588674 100644 --- a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs +++ b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs @@ -1,8 +1,5 @@ pub use crate::{BeaconChain, BeaconChainError, CheckPoint}; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - MemoryDB, -}; +use db::MemoryDB; use fork_choice::BitwiseLMDGhost; use slot_clock::TestingSlotClock; use std::sync::Arc; @@ -19,11 +16,9 @@ pub struct TestingBeaconChainBuilder { impl TestingBeaconChainBuilder { pub fn build(self, spec: &ChainSpec) -> TestingBeaconChain { - let db = Arc::new(MemoryDB::open()); - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); + let store = Arc::new(MemoryDB::open()); let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64()); - let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); + let fork_choice = BitwiseLMDGhost::new(store.clone()); let (genesis_state, _keypairs) = self.state_builder.build(); @@ -32,8 +27,7 @@ impl TestingBeaconChainBuilder { // Create the Beacon Chain BeaconChain::from_genesis( - state_store.clone(), - block_store.clone(), + store, slot_clock, genesis_state, genesis_block, From 7d067926dd20beb1f2df314d35e33274fbfc1e97 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 16:29:34 +1000 Subject: [PATCH 23/44] Replace RocksDB with LevelDB --- beacon_node/db/Cargo.toml | 6 +- beacon_node/db/src/leveldb_store.rs | 100 ++++++++++++++++++++++++++++ beacon_node/db/src/lib.rs | 20 ++++++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 beacon_node/db/src/leveldb_store.rs diff --git a/beacon_node/db/Cargo.toml b/beacon_node/db/Cargo.toml index b6cdafe04..808d420a5 100644 --- a/beacon_node/db/Cargo.toml +++ b/beacon_node/db/Cargo.toml @@ -4,12 +4,16 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[dev-dependencies] +tempfile = "3" + [dependencies] blake2-rfc = "0.2.18" bls = { path = "../../eth2/utils/bls" } bytes = "0.4.10" +db-key = "0.0.5" +leveldb = "0.8.4" parking_lot = "0.7" -rocksdb = "0.10.1" ssz = { path = "../../eth2/utils/ssz" } ssz_derive = { path = "../../eth2/utils/ssz_derive" } tree_hash = { path = "../../eth2/utils/tree_hash" } diff --git a/beacon_node/db/src/leveldb_store.rs b/beacon_node/db/src/leveldb_store.rs new file mode 100644 index 000000000..c60e34283 --- /dev/null +++ b/beacon_node/db/src/leveldb_store.rs @@ -0,0 +1,100 @@ +use super::*; +use db_key::Key; +use leveldb::database::kv::KV; +use leveldb::database::Database; +use leveldb::error::Error as LevelDBError; +use leveldb::options::{Options, ReadOptions, WriteOptions}; +use parking_lot::RwLock; +use std::path::Path; + +pub struct LevelDB { + db: RwLock>, +} + +impl LevelDB { + pub fn open(path: &Path) -> Result { + let mut options = Options::new(); + + options.create_if_missing = true; + + let db = Database::open(path, options)?; + + Ok(Self { + db: RwLock::new(db), + }) + } + + fn read_options(&self) -> ReadOptions { + ReadOptions::new() + } + + fn write_options(&self) -> WriteOptions { + WriteOptions::new() + } + + fn get_key_for_col(col: &str, key: &[u8]) -> BytesKey { + let mut col = col.as_bytes().to_vec(); + col.append(&mut key.to_vec()); + BytesKey { key: col } + } +} + +pub struct BytesKey { + key: Vec, +} + +impl Key for BytesKey { + fn from_u8(key: &[u8]) -> Self { + Self { key: key.to_vec() } + } + + fn as_slice T>(&self, f: F) -> T { + f(self.key.as_slice()) + } +} + +impl Store for LevelDB { + fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { + let column_key = Self::get_key_for_col(col, key); + + self.db + .read() + .get(self.read_options(), column_key) + .map_err(Into::into) + } + + fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { + let column_key = Self::get_key_for_col(col, key); + + self.db + .write() + .put(self.write_options(), column_key, val) + .map_err(Into::into) + } + + fn key_exists(&self, col: &str, key: &[u8]) -> Result { + let column_key = Self::get_key_for_col(col, key); + + self.db + .read() + .get(self.read_options(), column_key) + .map_err(Into::into) + .and_then(|val| Ok(val.is_some())) + } + + fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { + let column_key = Self::get_key_for_col(col, key); + self.db + .write() + .delete(self.write_options(), column_key) + .map_err(Into::into) + } +} + +impl From for Error { + fn from(e: LevelDBError) -> Error { + Error::DBError { + message: format!("{:?}", e), + } + } +} diff --git a/beacon_node/db/src/lib.rs b/beacon_node/db/src/lib.rs index 21b5b0a75..d31dc06b0 100644 --- a/beacon_node/db/src/lib.rs +++ b/beacon_node/db/src/lib.rs @@ -2,8 +2,10 @@ mod block_at_slot; mod errors; mod impls; +mod leveldb_store; mod memory_db; +pub use self::leveldb_store::LevelDB; pub use self::memory_db::MemoryDB; pub use errors::Error; pub use types::*; @@ -106,6 +108,7 @@ mod tests { use super::*; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; + use tempfile::tempdir; #[derive(PartialEq, Debug, Encode, Decode)] struct StorableThing { @@ -127,6 +130,23 @@ mod tests { } } + #[test] + fn leveldb_can_store_and_retrieve() { + let dir = tempdir().unwrap(); + let path = dir.path(); + + let store = LevelDB::open(&path).unwrap(); + + let key = Hash256::random(); + let item = StorableThing { a: 1, b: 42 }; + + store.put(&key, &item).unwrap(); + + let retrieved = store.get(&key).unwrap().unwrap(); + + assert_eq!(item, retrieved); + } + #[test] fn memorydb_can_store_and_retrieve() { let store = MemoryDB::open(); From b3a94de086ae3b89d172779b8ab7aa5f7822e3d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 16:36:06 +1000 Subject: [PATCH 24/44] Remove unnecessary RwLock from LevelDB --- beacon_node/db/src/leveldb_store.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/beacon_node/db/src/leveldb_store.rs b/beacon_node/db/src/leveldb_store.rs index c60e34283..a06585b11 100644 --- a/beacon_node/db/src/leveldb_store.rs +++ b/beacon_node/db/src/leveldb_store.rs @@ -4,11 +4,10 @@ use leveldb::database::kv::KV; use leveldb::database::Database; use leveldb::error::Error as LevelDBError; use leveldb::options::{Options, ReadOptions, WriteOptions}; -use parking_lot::RwLock; use std::path::Path; pub struct LevelDB { - db: RwLock>, + db: Database, } impl LevelDB { @@ -19,9 +18,7 @@ impl LevelDB { let db = Database::open(path, options)?; - Ok(Self { - db: RwLock::new(db), - }) + Ok(Self { db }) } fn read_options(&self) -> ReadOptions { @@ -58,7 +55,6 @@ impl Store for LevelDB { let column_key = Self::get_key_for_col(col, key); self.db - .read() .get(self.read_options(), column_key) .map_err(Into::into) } @@ -67,7 +63,6 @@ impl Store for LevelDB { let column_key = Self::get_key_for_col(col, key); self.db - .write() .put(self.write_options(), column_key, val) .map_err(Into::into) } @@ -76,7 +71,6 @@ impl Store for LevelDB { let column_key = Self::get_key_for_col(col, key); self.db - .read() .get(self.read_options(), column_key) .map_err(Into::into) .and_then(|val| Ok(val.is_some())) @@ -85,7 +79,6 @@ impl Store for LevelDB { fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { let column_key = Self::get_key_for_col(col, key); self.db - .write() .delete(self.write_options(), column_key) .map_err(Into::into) } From 54f28df5b18bad81b471ace287f168fa9a8860ef Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 16:37:15 +1000 Subject: [PATCH 25/44] Improve testing for `Store` impls --- beacon_node/db/src/lib.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/beacon_node/db/src/lib.rs b/beacon_node/db/src/lib.rs index d31dc06b0..c0b447c5f 100644 --- a/beacon_node/db/src/lib.rs +++ b/beacon_node/db/src/lib.rs @@ -130,35 +130,40 @@ mod tests { } } - #[test] - fn leveldb_can_store_and_retrieve() { - let dir = tempdir().unwrap(); - let path = dir.path(); - - let store = LevelDB::open(&path).unwrap(); - + fn test_impl(store: impl Store) { let key = Hash256::random(); let item = StorableThing { a: 1, b: 42 }; + assert_eq!(store.exists::(&key), Ok(false)); + store.put(&key, &item).unwrap(); - let retrieved = store.get(&key).unwrap().unwrap(); + assert_eq!(store.exists::(&key), Ok(true)); + let retrieved = store.get(&key).unwrap().unwrap(); assert_eq!(item, retrieved); + + store.delete::(&key).unwrap(); + + assert_eq!(store.exists::(&key), Ok(false)); + + assert_eq!(store.get::(&key), Ok(None)); } #[test] - fn memorydb_can_store_and_retrieve() { + fn leveldb() { + let dir = tempdir().unwrap(); + let path = dir.path(); + let store = LevelDB::open(&path).unwrap(); + + test_impl(store); + } + + #[test] + fn memorydb() { let store = MemoryDB::open(); - let key = Hash256::random(); - let item = StorableThing { a: 1, b: 42 }; - - store.put(&key, &item).unwrap(); - - let retrieved = store.get(&key).unwrap().unwrap(); - - assert_eq!(item, retrieved); + test_impl(store); } #[test] From 78368cc2cdedf3e467ba664fe05993f22f8c33d2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 16:49:56 +1000 Subject: [PATCH 26/44] Make LevelDB key type concrete (not generic) --- beacon_node/db/src/leveldb_store.rs | 10 +++++----- beacon_node/db/src/lib.rs | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beacon_node/db/src/leveldb_store.rs b/beacon_node/db/src/leveldb_store.rs index a06585b11..10643d0cd 100644 --- a/beacon_node/db/src/leveldb_store.rs +++ b/beacon_node/db/src/leveldb_store.rs @@ -6,11 +6,11 @@ use leveldb::error::Error as LevelDBError; use leveldb::options::{Options, ReadOptions, WriteOptions}; use std::path::Path; -pub struct LevelDB { - db: Database, +pub struct LevelDB { + db: Database, } -impl LevelDB { +impl LevelDB { pub fn open(path: &Path) -> Result { let mut options = Options::new(); @@ -21,7 +21,7 @@ impl LevelDB { Ok(Self { db }) } - fn read_options(&self) -> ReadOptions { + fn read_options(&self) -> ReadOptions { ReadOptions::new() } @@ -50,7 +50,7 @@ impl Key for BytesKey { } } -impl Store for LevelDB { +impl Store for LevelDB { fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { let column_key = Self::get_key_for_col(col, key); diff --git a/beacon_node/db/src/lib.rs b/beacon_node/db/src/lib.rs index c0b447c5f..708ac698f 100644 --- a/beacon_node/db/src/lib.rs +++ b/beacon_node/db/src/lib.rs @@ -5,7 +5,7 @@ mod impls; mod leveldb_store; mod memory_db; -pub use self::leveldb_store::LevelDB; +pub use self::leveldb_store::LevelDB as DiskDB; pub use self::memory_db::MemoryDB; pub use errors::Error; pub use types::*; @@ -151,10 +151,10 @@ mod tests { } #[test] - fn leveldb() { + fn diskdb() { let dir = tempdir().unwrap(); let path = dir.path(); - let store = LevelDB::open(&path).unwrap(); + let store = DiskDB::open(&path).unwrap(); test_impl(store); } From 058829b64d9c3d68d2ab12a7f7646b679720d467 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 17:27:06 +1000 Subject: [PATCH 27/44] Update `beacon_chain` to latest DB --- beacon_node/beacon_chain/src/beacon_chain.rs | 48 +++++++++----------- beacon_node/beacon_chain/src/errors.rs | 2 +- beacon_node/beacon_chain/src/initialise.rs | 48 +++++++++----------- 3 files changed, 44 insertions(+), 54 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 57b697019..7c52fed5d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -7,7 +7,6 @@ use operation_pool::DepositInsertStatus; use operation_pool::OperationPool; use parking_lot::{RwLock, RwLockReadGuard}; use slot_clock::SlotClock; -use ssz::ssz_encode; use state_processing::per_block_processing::errors::{ AttestationValidationError, AttesterSlashingValidationError, DepositValidationError, ExitValidationError, ProposerSlashingValidationError, TransferValidationError, @@ -229,7 +228,7 @@ where let new_state_root = state.get_state_root(earliest_historic_slot)?; // Break if the DB is unable to load the state. - state = match self.state_store.get_deserialized(&new_state_root) { + state = match self.store.get(&new_state_root) { Ok(Some(state)) => state, _ => break, } @@ -256,7 +255,7 @@ where /// /// May return a database error. pub fn get_block(&self, block_root: &Hash256) -> Result, Error> { - Ok(self.block_store.get_deserialized(block_root)?) + Ok(self.store.get(block_root)?) } /// Update the canonical head to some new values. @@ -582,7 +581,7 @@ where // Load the blocks parent block from the database, returning invalid if that block is not // found. let parent_block_root = block.previous_block_root; - let parent_block = match self.block_store.get_deserialized(&parent_block_root)? { + let parent_block: BeaconBlock = match self.store.get(&parent_block_root)? { Some(previous_block_root) => previous_block_root, None => { return Ok(BlockProcessingOutcome::InvalidBlock( @@ -595,15 +594,15 @@ where // It is an error because if know the parent block we should also know the parent state. let parent_state_root = parent_block.state_root; let parent_state = self - .state_store - .get_deserialized(&parent_state_root)? + .store + .get(&parent_state_root)? .ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))?; // TODO: check the block proposer signature BEFORE doing a state transition. This will // significantly lower exposure surface to DoS attacks. // Transition the parent state to the block slot. - let mut state = parent_state; + let mut state: BeaconState = parent_state; for _ in state.slot.as_u64()..block.slot.as_u64() { if let Err(e) = per_slot_processing(&mut state, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( @@ -629,8 +628,8 @@ where } // Store the block and state. - self.block_store.put(&block_root, &ssz_encode(&block)[..])?; - self.state_store.put(&state_root, &ssz_encode(&state)[..])?; + self.store.put(&block_root, &block)?; + self.store.put(&state_root, &state)?; // run the fork_choice add_block logic self.fork_choice @@ -723,15 +722,15 @@ where .find_head(&present_head, &self.spec)?; if new_head != present_head { - let block = self - .block_store - .get_deserialized(&new_head)? + let block: BeaconBlock = self + .store + .get(&new_head)? .ok_or_else(|| Error::MissingBeaconBlock(new_head))?; let block_root = block.canonical_root(); - let state = self - .state_store - .get_deserialized(&block.state_root)? + let state: BeaconState = self + .store + .get(&block.state_root)? .ok_or_else(|| Error::MissingBeaconState(block.state_root))?; let state_root = state.canonical_root(); @@ -746,7 +745,7 @@ where /// Returns `true` if the given block root has not been processed. pub fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result { - Ok(!self.block_store.exists(beacon_block_root)?) + Ok(!self.store.exists::(beacon_block_root)?) } /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. @@ -772,19 +771,14 @@ where break; // Genesis has been reached. } - let beacon_block = self - .block_store - .get_deserialized(&beacon_block_root)? - .ok_or_else(|| { + let beacon_block: BeaconBlock = + self.store.get(&beacon_block_root)?.ok_or_else(|| { Error::DBInconsistent(format!("Missing block {}", beacon_block_root)) })?; let beacon_state_root = beacon_block.state_root; - let beacon_state = self - .state_store - .get_deserialized(&beacon_state_root)? - .ok_or_else(|| { - Error::DBInconsistent(format!("Missing state {}", beacon_state_root)) - })?; + let beacon_state = self.store.get(&beacon_state_root)?.ok_or_else(|| { + Error::DBInconsistent(format!("Missing state {}", beacon_state_root)) + })?; let slot = CheckPoint { beacon_block, @@ -805,7 +799,7 @@ where impl From for Error { fn from(e: DBError) -> Error { - Error::DBError(e.message) + Error::DBError(e) } } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index a84e4b10e..dea7f63d9 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -20,7 +20,7 @@ pub enum BeaconChainError { UnableToReadSlot, BeaconStateError(BeaconStateError), DBInconsistent(String), - DBError(String), + DBError(db::Error), ForkChoiceError(ForkChoiceError), MissingBeaconBlock(Hash256), MissingBeaconState(Hash256), diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs index 6bc9c2599..197f7ace0 100644 --- a/beacon_node/beacon_chain/src/initialise.rs +++ b/beacon_node/beacon_chain/src/initialise.rs @@ -3,7 +3,6 @@ // testnet. These are examples. Also. there is code duplication which can/should be cleaned up. use crate::BeaconChain; -use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::{DiskDB, MemoryDB}; use fork_choice::BitwiseLMDGhost; use slot_clock::SystemTimeSlotClock; @@ -26,14 +25,9 @@ pub fn initialise_beacon_chain( FoundationEthSpec, >, > { - // set up the db - let db = Arc::new(DiskDB::open( - db_name.expect("Database directory must be included"), - None, - )); - - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); + let path = db_name.expect("db_name cannot be None."); + let store = DiskDB::open(path).expect("Unable to open DB."); + let store = Arc::new(store); let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); let (genesis_state, _keypairs) = state_builder.build(); @@ -49,14 +43,13 @@ pub fn initialise_beacon_chain( ) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); + let fork_choice = BitwiseLMDGhost::new(store.clone()); // Genesis chain //TODO: Handle error correctly Arc::new( BeaconChain::from_genesis( - state_store.clone(), - block_store.clone(), + store, slot_clock, genesis_state, genesis_block, @@ -79,9 +72,7 @@ pub fn initialise_test_beacon_chain_with_memory_db( FewValidatorsEthSpec, >, > { - let db = Arc::new(MemoryDB::open()); - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); + let store = Arc::new(MemoryDB::open()); let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); let (genesis_state, _keypairs) = state_builder.build(); @@ -97,14 +88,13 @@ pub fn initialise_test_beacon_chain_with_memory_db( ) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); + let fork_choice = BitwiseLMDGhost::new(store.clone()); // Genesis chain //TODO: Handle error correctly Arc::new( BeaconChain::from_genesis( - state_store.clone(), - block_store.clone(), + store, slot_clock, genesis_state, genesis_block, @@ -119,16 +109,23 @@ pub fn initialise_test_beacon_chain_with_memory_db( pub fn initialise_test_beacon_chain_with_disk_db( spec: &ChainSpec, db_name: Option<&PathBuf>, -) -> Arc>> { - let db = Arc::new(DiskDB::open(db_name.expect("Must have DB path"), None)); - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); +) -> Arc< + BeaconChain< + DiskDB, + SystemTimeSlotClock, + BitwiseLMDGhost, + FewValidatorsEthSpec, + >, +> { + let path = db_name.expect("db_name cannot be None."); + let store = DiskDB::open(path).expect("Unable to open DB."); + let store = Arc::new(store); let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); let (genesis_state, _keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root()); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Slot clock let slot_clock = SystemTimeSlotClock::new( @@ -138,14 +135,13 @@ pub fn initialise_test_beacon_chain_with_disk_db( ) .expect("Unable to load SystemTimeSlotClock"); // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); + let fork_choice = BitwiseLMDGhost::new(store.clone()); // Genesis chain //TODO: Handle error correctly Arc::new( BeaconChain::from_genesis( - state_store.clone(), - block_store.clone(), + store, slot_clock, genesis_state, genesis_block, From b62f4477e1ef69bdb1607c5ad00e0f211420ce13 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 17:45:35 +1000 Subject: [PATCH 28/44] More project-wide fixes for new DB --- beacon_node/client/src/client_config.rs | 9 ++++++-- beacon_node/client/src/client_types.rs | 25 ++++++++++++++++----- beacon_node/client/src/lib.rs | 6 ++--- beacon_node/network/src/beacon_chain.rs | 4 ++-- beacon_node/rpc/src/beacon_chain.rs | 4 ++-- beacon_node/src/main.rs | 2 +- beacon_node/src/run.rs | 5 ++--- eth2/fork_choice/tests/tests.rs | 29 ++++++++++--------------- 8 files changed, 48 insertions(+), 36 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 942a907e0..4309b8a64 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -1,5 +1,4 @@ use clap::ArgMatches; -use db::DBType; use fork_choice::ForkChoiceAlgorithm; use network::NetworkConfig; use slog::error; @@ -12,6 +11,12 @@ use types::multiaddr::ToMultiaddr; use types::Multiaddr; use types::{ChainSpec, EthSpec, LighthouseTestnetEthSpec}; +#[derive(Debug, Clone)] +pub enum DBType { + Memory, + Disk, +} + /// Stores the client configuration for this Lighthouse instance. #[derive(Debug, Clone)] pub struct ClientConfig { @@ -132,7 +137,7 @@ impl ClientConfig { } match args.value_of("db") { - Some("rocks") => config.db_type = DBType::RocksDB, + Some("disk") => config.db_type = DBType::Disk, Some("memory") => config.db_type = DBType::Memory, _ => unreachable!(), // clap prevents this. }; diff --git a/beacon_node/client/src/client_types.rs b/beacon_node/client/src/client_types.rs index fdca11caa..c54028d28 100644 --- a/beacon_node/client/src/client_types.rs +++ b/beacon_node/client/src/client_types.rs @@ -1,6 +1,6 @@ use crate::{ArcBeaconChain, ClientConfig}; use beacon_chain::{ - db::{ClientDB, DiskDB, MemoryDB}, + db::{DiskDB, MemoryDB, Store}, fork_choice::BitwiseLMDGhost, initialise, slot_clock::{SlotClock, SystemTimeSlotClock}, @@ -9,7 +9,7 @@ use fork_choice::ForkChoice; use types::{EthSpec, FewValidatorsEthSpec, FoundationEthSpec}; pub trait ClientTypes { - type DB: ClientDB + 'static; + type DB: Store + 'static; type SlotClock: SlotClock + 'static; type ForkChoice: ForkChoice + 'static; type EthSpec: EthSpec + 'static; @@ -24,7 +24,7 @@ pub struct StandardClientType; impl ClientTypes for StandardClientType { type DB = DiskDB; type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; + type ForkChoice = BitwiseLMDGhost; type EthSpec = FoundationEthSpec; fn initialise_beacon_chain( @@ -39,12 +39,27 @@ pub struct MemoryDBTestingClientType; impl ClientTypes for MemoryDBTestingClientType { type DB = MemoryDB; type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; + type ForkChoice = BitwiseLMDGhost; type EthSpec = FewValidatorsEthSpec; fn initialise_beacon_chain( config: &ClientConfig, ) -> ArcBeaconChain { - initialise::initialise_test_beacon_chain(&config.spec, None) + initialise::initialise_test_beacon_chain_with_memory_db(&config.spec, None) + } +} + +pub struct DiskDBTestingClientType; + +impl ClientTypes for DiskDBTestingClientType { + type DB = DiskDB; + type SlotClock = SystemTimeSlotClock; + type ForkChoice = BitwiseLMDGhost; + type EthSpec = FewValidatorsEthSpec; + + fn initialise_beacon_chain( + config: &ClientConfig, + ) -> ArcBeaconChain { + initialise::initialise_test_beacon_chain_with_disk_db(&config.spec, Some(&config.db_name)) } } diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 5d7c221ef..00478b475 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -6,9 +6,9 @@ pub mod error; pub mod notifier; use beacon_chain::BeaconChain; -pub use client_config::ClientConfig; +pub use client_config::{ClientConfig, DBType}; pub use client_types::ClientTypes; -use db::ClientDB; +use db::Store; use exit_future::Signal; use fork_choice::ForkChoice; use futures::{future::Future, Stream}; @@ -146,7 +146,7 @@ impl Client { fn do_state_catchup(chain: &Arc>, log: &slog::Logger) where - T: ClientDB, + T: Store, U: SlotClock, F: ForkChoice, E: EthSpec, diff --git a/beacon_node/network/src/beacon_chain.rs b/beacon_node/network/src/beacon_chain.rs index a98aa73de..f123e4540 100644 --- a/beacon_node/network/src/beacon_chain.rs +++ b/beacon_node/network/src/beacon_chain.rs @@ -1,6 +1,6 @@ use beacon_chain::BeaconChain as RawBeaconChain; use beacon_chain::{ - db::ClientDB, + db::Store, fork_choice::ForkChoice, parking_lot::RwLockReadGuard, slot_clock::SlotClock, @@ -66,7 +66,7 @@ pub trait BeaconChain: Send + Sync { impl BeaconChain for RawBeaconChain where - T: ClientDB + Sized, + T: Store, U: SlotClock, F: ForkChoice, E: EthSpec, diff --git a/beacon_node/rpc/src/beacon_chain.rs b/beacon_node/rpc/src/beacon_chain.rs index 7e75b32ce..9b6b05e9d 100644 --- a/beacon_node/rpc/src/beacon_chain.rs +++ b/beacon_node/rpc/src/beacon_chain.rs @@ -1,6 +1,6 @@ use beacon_chain::BeaconChain as RawBeaconChain; use beacon_chain::{ - db::ClientDB, + db::Store, fork_choice::ForkChoice, parking_lot::{RwLockReadGuard, RwLockWriteGuard}, slot_clock::SlotClock, @@ -36,7 +36,7 @@ pub trait BeaconChain: Send + Sync { impl BeaconChain for RawBeaconChain where - T: ClientDB + Sized, + T: Store, U: SlotClock, F: ForkChoice, E: EthSpec, diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index a1df0b63b..ef2121882 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -74,7 +74,7 @@ fn main() { .value_name("DB") .help("Type of database to use.") .takes_value(true) - .possible_values(&["rocks", "memory"]) + .possible_values(&["disk", "memory"]) .default_value("memory"), ) .get_matches(); diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 52dc3973b..ec421fc6e 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,7 +1,6 @@ use client::client_types::{DiskDBTestingClientType, MemoryDBTestingClientType}; -use client::error; +use client::{error, DBType}; use client::{notifier, Client, ClientConfig, ClientTypes}; -use db::DBType; use futures::sync::oneshot; use futures::Future; use slog::info; @@ -26,7 +25,7 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul let executor = runtime.executor(); match config.db_type { - DBType::RocksDB => { + DBType::Disk => { info!( log, "BeaconNode starting"; diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 067d39da4..b4f4ede2c 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -14,8 +14,8 @@ extern crate yaml_rust; pub use beacon_chain::BeaconChain; use bls::Signature; -use db::stores::{BeaconBlockStore, BeaconStateStore}; use db::MemoryDB; +use db::Store; // use env_logger::{Builder, Env}; use fork_choice::{ BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimizedLMDGhost, SlowLMDGhost, @@ -106,7 +106,7 @@ fn test_yaml_vectors( // process the tests for test_case in test_cases { // setup a fresh test - let (mut fork_choice, block_store, state_root) = + let (mut fork_choice, store, state_root) = setup_inital_state(&fork_choice_algo, emulated_validators); // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) @@ -149,9 +149,7 @@ fn test_yaml_vectors( }; // Store the block. - block_store - .put(&block_hash, &ssz_encode(&beacon_block)[..]) - .unwrap(); + store.put(&block_hash, &beacon_block).unwrap(); // run add block for fork choice if not genesis if parent_id != block_id { @@ -222,29 +220,26 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { fn setup_inital_state( fork_choice_algo: &ForkChoiceAlgorithm, num_validators: usize, -) -> (Box, Arc>, Hash256) { - let db = Arc::new(MemoryDB::open()); - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); +) -> (Box, Arc, Hash256) { + let store = Arc::new(MemoryDB::open()); // the fork choice instantiation let fork_choice: Box = match fork_choice_algo { ForkChoiceAlgorithm::OptimizedLMDGhost => { let f: OptimizedLMDGhost = - OptimizedLMDGhost::new(block_store.clone(), state_store.clone()); + OptimizedLMDGhost::new(store.clone()); Box::new(f) } ForkChoiceAlgorithm::BitwiseLMDGhost => { let f: BitwiseLMDGhost = - BitwiseLMDGhost::new(block_store.clone(), state_store.clone()); + BitwiseLMDGhost::new(store.clone()); Box::new(f) } ForkChoiceAlgorithm::SlowLMDGhost => { - let f: SlowLMDGhost = - SlowLMDGhost::new(block_store.clone(), state_store.clone()); + let f: SlowLMDGhost = SlowLMDGhost::new(store.clone()); Box::new(f) } - ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(block_store.clone())), + ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(store.clone())), }; let spec = FoundationEthSpec::spec(); @@ -255,12 +250,10 @@ fn setup_inital_state( let (state, _keypairs) = state_builder.build(); let state_root = state.canonical_root(); - state_store - .put(&state_root, &ssz_encode(&state)[..]) - .unwrap(); + store.put(&state_root, &state).unwrap(); // return initialised vars - (fork_choice, block_store, state_root) + (fork_choice, store, state_root) } // convert a block_id into a Hash256 -- assume input is hex encoded; From 3bcf5ba7064e92d28dcbf0f7d69d6386bdea4176 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 18:20:23 +1000 Subject: [PATCH 29/44] Rename `db` crate to `store` --- Cargo.toml | 2 +- beacon_node/Cargo.toml | 2 +- beacon_node/beacon_chain/Cargo.toml | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- beacon_node/beacon_chain/src/errors.rs | 2 +- beacon_node/beacon_chain/src/initialise.rs | 20 +++++++------- beacon_node/beacon_chain/src/lib.rs | 2 +- .../testing_beacon_chain_builder.rs | 12 ++++++--- beacon_node/client/Cargo.toml | 2 +- beacon_node/client/src/client_types.rs | 16 ++++++------ beacon_node/client/src/lib.rs | 2 +- beacon_node/network/src/beacon_chain.rs | 2 +- beacon_node/rpc/Cargo.toml | 2 +- beacon_node/rpc/src/beacon_chain.rs | 2 +- beacon_node/src/run.rs | 10 +++---- beacon_node/{db => store}/Cargo.toml | 2 +- .../{db => store}/src/block_at_slot.rs | 4 +-- beacon_node/{db => store}/src/disk_db.rs | 8 +++--- beacon_node/{db => store}/src/errors.rs | 0 beacon_node/{db => store}/src/impls.rs | 0 .../{db => store}/src/leveldb_store.rs | 0 beacon_node/{db => store}/src/lib.rs | 10 +++---- beacon_node/{db => store}/src/memory_db.rs | 14 +++++----- eth2/fork_choice/Cargo.toml | 2 +- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 4 +-- eth2/fork_choice/src/lib.rs | 6 ++--- eth2/fork_choice/src/longest_chain.rs | 2 +- eth2/fork_choice/src/optimized_lmd_ghost.rs | 4 +-- eth2/fork_choice/src/slow_lmd_ghost.rs | 4 +-- eth2/fork_choice/tests/tests.rs | 26 +++++-------------- 30 files changed, 76 insertions(+), 90 deletions(-) rename beacon_node/{db => store}/Cargo.toml (96%) rename beacon_node/{db => store}/src/block_at_slot.rs (98%) rename beacon_node/{db => store}/src/disk_db.rs (97%) rename beacon_node/{db => store}/src/errors.rs (100%) rename beacon_node/{db => store}/src/impls.rs (100%) rename beacon_node/{db => store}/src/leveldb_store.rs (100%) rename beacon_node/{db => store}/src/lib.rs (95%) rename beacon_node/{db => store}/src/memory_db.rs (79%) diff --git a/Cargo.toml b/Cargo.toml index 893189941..00c354309 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ "eth2/utils/fisher_yates_shuffle", "eth2/utils/test_random_derive", "beacon_node", - "beacon_node/db", + "beacon_node/store", "beacon_node/client", "beacon_node/network", "beacon_node/eth2-libp2p", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index da31bfa77..d78a5b596 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] types = { path = "../eth2/types" } -db = { path = "./db" } +store = { path = "./store" } client = { path = "client" } version = { path = "version" } clap = "2.32.0" diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 34b6e11c6..3a84256a7 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] bls = { path = "../../eth2/utils/bls" } boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" } -db = { path = "../db" } +store = { path = "../store" } failure = "0.1" failure_derive = "0.1" hashing = { path = "../../eth2/utils/hashing" } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 7c52fed5d..f2c4b3dbe 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,6 +1,5 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; -use db::{Error as DBError, Store}; use fork_choice::{ForkChoice, ForkChoiceError}; use log::{debug, trace}; use operation_pool::DepositInsertStatus; @@ -16,6 +15,7 @@ use state_processing::{ per_slot_processing, BlockProcessingError, SlotProcessingError, }; use std::sync::Arc; +use store::{Error as DBError, Store}; use types::*; #[derive(Debug, PartialEq)] diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index dea7f63d9..73884916a 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -20,7 +20,7 @@ pub enum BeaconChainError { UnableToReadSlot, BeaconStateError(BeaconStateError), DBInconsistent(String), - DBError(db::Error), + DBError(store::Error), ForkChoiceError(ForkChoiceError), MissingBeaconBlock(Hash256), MissingBeaconState(Hash256), diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs index 197f7ace0..b9d950ed5 100644 --- a/beacon_node/beacon_chain/src/initialise.rs +++ b/beacon_node/beacon_chain/src/initialise.rs @@ -3,11 +3,11 @@ // testnet. These are examples. Also. there is code duplication which can/should be cleaned up. use crate::BeaconChain; -use db::{DiskDB, MemoryDB}; use fork_choice::BitwiseLMDGhost; use slot_clock::SystemTimeSlotClock; use std::path::PathBuf; use std::sync::Arc; +use store::{DiskStore, MemoryStore}; use tree_hash::TreeHash; use types::test_utils::TestingBeaconStateBuilder; use types::{BeaconBlock, ChainSpec, FewValidatorsEthSpec, FoundationEthSpec, Hash256}; @@ -19,14 +19,14 @@ pub fn initialise_beacon_chain( db_name: Option<&PathBuf>, ) -> Arc< BeaconChain< - DiskDB, + DiskStore, SystemTimeSlotClock, - BitwiseLMDGhost, + BitwiseLMDGhost, FoundationEthSpec, >, > { let path = db_name.expect("db_name cannot be None."); - let store = DiskDB::open(path).expect("Unable to open DB."); + let store = DiskStore::open(path).expect("Unable to open DB."); let store = Arc::new(store); let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); @@ -66,13 +66,13 @@ pub fn initialise_test_beacon_chain_with_memory_db( _db_name: Option<&PathBuf>, ) -> Arc< BeaconChain< - MemoryDB, + MemoryStore, SystemTimeSlotClock, - BitwiseLMDGhost, + BitwiseLMDGhost, FewValidatorsEthSpec, >, > { - let store = Arc::new(MemoryDB::open()); + let store = Arc::new(MemoryStore::open()); let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); let (genesis_state, _keypairs) = state_builder.build(); @@ -111,14 +111,14 @@ pub fn initialise_test_beacon_chain_with_disk_db( db_name: Option<&PathBuf>, ) -> Arc< BeaconChain< - DiskDB, + DiskStore, SystemTimeSlotClock, - BitwiseLMDGhost, + BitwiseLMDGhost, FewValidatorsEthSpec, >, > { let path = db_name.expect("db_name cannot be None."); - let store = DiskDB::open(path).expect("Unable to open DB."); + let store = DiskStore::open(path).expect("Unable to open DB."); let store = Arc::new(store); let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index d8d85a8a6..6ac01a5d5 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -7,7 +7,6 @@ pub mod test_utils; pub use self::beacon_chain::{BeaconChain, BlockProcessingOutcome, InvalidBlock, ValidBlock}; pub use self::checkpoint::CheckPoint; pub use self::errors::{BeaconChainError, BlockProductionError}; -pub use db; pub use fork_choice; pub use parking_lot; pub use slot_clock; @@ -15,4 +14,5 @@ pub use state_processing::per_block_processing::errors::{ AttestationValidationError, AttesterSlashingValidationError, DepositValidationError, ExitValidationError, ProposerSlashingValidationError, TransferValidationError, }; +pub use store; pub use types; diff --git a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs index ce3588674..b6b1defcc 100644 --- a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs +++ b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs @@ -1,14 +1,18 @@ pub use crate::{BeaconChain, BeaconChainError, CheckPoint}; -use db::MemoryDB; use fork_choice::BitwiseLMDGhost; use slot_clock::TestingSlotClock; use std::sync::Arc; +use store::MemoryStore; use tree_hash::TreeHash; use types::*; use types::{test_utils::TestingBeaconStateBuilder, EthSpec, FewValidatorsEthSpec}; -type TestingBeaconChain = - BeaconChain, E>; +type TestingBeaconChain = BeaconChain< + MemoryStore, + TestingSlotClock, + BitwiseLMDGhost, + E, +>; pub struct TestingBeaconChainBuilder { state_builder: TestingBeaconStateBuilder, @@ -16,7 +20,7 @@ pub struct TestingBeaconChainBuilder { impl TestingBeaconChainBuilder { pub fn build(self, spec: &ChainSpec) -> TestingBeaconChain { - let store = Arc::new(MemoryDB::open()); + let store = Arc::new(MemoryStore::open()); let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64()); let fork_choice = BitwiseLMDGhost::new(store.clone()); diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 8956dbb07..4a976eec4 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } -db = { path = "../db" } +store = { path = "../store" } rpc = { path = "../rpc" } fork_choice = { path = "../../eth2/fork_choice" } types = { path = "../../eth2/types" } diff --git a/beacon_node/client/src/client_types.rs b/beacon_node/client/src/client_types.rs index c54028d28..4cce42a06 100644 --- a/beacon_node/client/src/client_types.rs +++ b/beacon_node/client/src/client_types.rs @@ -1,9 +1,9 @@ use crate::{ArcBeaconChain, ClientConfig}; use beacon_chain::{ - db::{DiskDB, MemoryDB, Store}, fork_choice::BitwiseLMDGhost, initialise, slot_clock::{SlotClock, SystemTimeSlotClock}, + store::{DiskStore, MemoryStore, Store}, }; use fork_choice::ForkChoice; use types::{EthSpec, FewValidatorsEthSpec, FoundationEthSpec}; @@ -22,7 +22,7 @@ pub trait ClientTypes { pub struct StandardClientType; impl ClientTypes for StandardClientType { - type DB = DiskDB; + type DB = DiskStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = BitwiseLMDGhost; type EthSpec = FoundationEthSpec; @@ -34,10 +34,10 @@ impl ClientTypes for StandardClientType { } } -pub struct MemoryDBTestingClientType; +pub struct MemoryStoreTestingClientType; -impl ClientTypes for MemoryDBTestingClientType { - type DB = MemoryDB; +impl ClientTypes for MemoryStoreTestingClientType { + type DB = MemoryStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = BitwiseLMDGhost; type EthSpec = FewValidatorsEthSpec; @@ -49,10 +49,10 @@ impl ClientTypes for MemoryDBTestingClientType { } } -pub struct DiskDBTestingClientType; +pub struct DiskStoreTestingClientType; -impl ClientTypes for DiskDBTestingClientType { - type DB = DiskDB; +impl ClientTypes for DiskStoreTestingClientType { + type DB = DiskStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = BitwiseLMDGhost; type EthSpec = FewValidatorsEthSpec; diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 00478b475..71d4013d3 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -8,7 +8,6 @@ pub mod notifier; use beacon_chain::BeaconChain; pub use client_config::{ClientConfig, DBType}; pub use client_types::ClientTypes; -use db::Store; use exit_future::Signal; use fork_choice::ForkChoice; use futures::{future::Future, Stream}; @@ -18,6 +17,7 @@ use slot_clock::SlotClock; use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; +use store::Store; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; use types::EthSpec; diff --git a/beacon_node/network/src/beacon_chain.rs b/beacon_node/network/src/beacon_chain.rs index f123e4540..2a42376f7 100644 --- a/beacon_node/network/src/beacon_chain.rs +++ b/beacon_node/network/src/beacon_chain.rs @@ -1,9 +1,9 @@ use beacon_chain::BeaconChain as RawBeaconChain; use beacon_chain::{ - db::Store, fork_choice::ForkChoice, parking_lot::RwLockReadGuard, slot_clock::SlotClock, + store::Store, types::{BeaconState, ChainSpec}, AttestationValidationError, CheckPoint, }; diff --git a/beacon_node/rpc/Cargo.toml b/beacon_node/rpc/Cargo.toml index 3fc52c6b1..a361c94ab 100644 --- a/beacon_node/rpc/Cargo.toml +++ b/beacon_node/rpc/Cargo.toml @@ -17,7 +17,7 @@ protos = { path = "../../protos" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } protobuf = "2.0.2" clap = "2.32.0" -db = { path = "../db" } +store = { path = "../store" } dirs = "1.0.3" futures = "0.1.23" slog = "^2.2.3" diff --git a/beacon_node/rpc/src/beacon_chain.rs b/beacon_node/rpc/src/beacon_chain.rs index 9b6b05e9d..d12baf1d1 100644 --- a/beacon_node/rpc/src/beacon_chain.rs +++ b/beacon_node/rpc/src/beacon_chain.rs @@ -1,9 +1,9 @@ use beacon_chain::BeaconChain as RawBeaconChain; use beacon_chain::{ - db::Store, fork_choice::ForkChoice, parking_lot::{RwLockReadGuard, RwLockWriteGuard}, slot_clock::SlotClock, + store::Store, types::{BeaconState, ChainSpec, Signature}, AttestationValidationError, BlockProductionError, }; diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index ec421fc6e..4cf930060 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,4 +1,4 @@ -use client::client_types::{DiskDBTestingClientType, MemoryDBTestingClientType}; +use client::client_types::{DiskStoreTestingClientType, MemoryStoreTestingClientType}; use client::{error, DBType}; use client::{notifier, Client, ClientConfig, ClientTypes}; use futures::sync::oneshot; @@ -29,9 +29,9 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul info!( log, "BeaconNode starting"; - "type" => "DiskDBTestingClientType" + "type" => "DiskStoreTestingClientType" ); - let client: Client = + let client: Client = Client::new(config, log.clone(), &executor)?; run(client, executor, runtime, log) @@ -40,9 +40,9 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul info!( log, "BeaconNode starting"; - "type" => "MemoryDBTestingClientType" + "type" => "MemoryStoreTestingClientType" ); - let client: Client = + let client: Client = Client::new(config, log.clone(), &executor)?; run(client, executor, runtime, log) diff --git a/beacon_node/db/Cargo.toml b/beacon_node/store/Cargo.toml similarity index 96% rename from beacon_node/db/Cargo.toml rename to beacon_node/store/Cargo.toml index 808d420a5..a95dafa90 100644 --- a/beacon_node/db/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "db" +name = "store" version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" diff --git a/beacon_node/db/src/block_at_slot.rs b/beacon_node/store/src/block_at_slot.rs similarity index 98% rename from beacon_node/db/src/block_at_slot.rs rename to beacon_node/store/src/block_at_slot.rs index 4fa9635a2..4a8abaefd 100644 --- a/beacon_node/db/src/block_at_slot.rs +++ b/beacon_node/store/src/block_at_slot.rs @@ -121,7 +121,7 @@ mod tests { #[test] fn chain_without_skips() { let n: usize = 10; - let store = MemoryDB::open(); + let store = MemoryStore::open(); let spec = FewValidatorsEthSpec::spec(); let slots: Vec = (0..n).collect(); @@ -145,7 +145,7 @@ mod tests { #[test] fn chain_with_skips() { - let store = MemoryDB::open(); + let store = MemoryStore::open(); let spec = FewValidatorsEthSpec::spec(); let slots = vec![0, 1, 2, 5]; diff --git a/beacon_node/db/src/disk_db.rs b/beacon_node/store/src/disk_db.rs similarity index 97% rename from beacon_node/db/src/disk_db.rs rename to beacon_node/store/src/disk_db.rs index e2162e29a..eb2b885c6 100644 --- a/beacon_node/db/src/disk_db.rs +++ b/beacon_node/store/src/disk_db.rs @@ -10,11 +10,11 @@ use std::path::Path; /// A on-disk database which implements the ClientDB trait. /// /// This implementation uses RocksDB with default options. -pub struct DiskDB { +pub struct DiskStore { db: DB, } -impl DiskDB { +impl DiskStore { /// Open the RocksDB database, optionally supplying columns if required. /// /// The RocksDB database will be contained in a directory titled @@ -71,7 +71,7 @@ impl From for DBError { } } -impl ClientDB for DiskDB { +impl ClientDB for DiskStore { /// Get the value for some key on some column. /// /// Corresponds to the `get_cf()` method on the RocksDB API. @@ -154,7 +154,7 @@ mod tests { let col_name: &str = "TestColumn"; let column_families = vec![col_name]; - let mut db = DiskDB::open(&path, None); + let mut db = DiskStore::open(&path, None); for cf in column_families { db.create_col(&cf).unwrap(); diff --git a/beacon_node/db/src/errors.rs b/beacon_node/store/src/errors.rs similarity index 100% rename from beacon_node/db/src/errors.rs rename to beacon_node/store/src/errors.rs diff --git a/beacon_node/db/src/impls.rs b/beacon_node/store/src/impls.rs similarity index 100% rename from beacon_node/db/src/impls.rs rename to beacon_node/store/src/impls.rs diff --git a/beacon_node/db/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs similarity index 100% rename from beacon_node/db/src/leveldb_store.rs rename to beacon_node/store/src/leveldb_store.rs diff --git a/beacon_node/db/src/lib.rs b/beacon_node/store/src/lib.rs similarity index 95% rename from beacon_node/db/src/lib.rs rename to beacon_node/store/src/lib.rs index 708ac698f..096a88184 100644 --- a/beacon_node/db/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -5,8 +5,8 @@ mod impls; mod leveldb_store; mod memory_db; -pub use self::leveldb_store::LevelDB as DiskDB; -pub use self::memory_db::MemoryDB; +pub use self::leveldb_store::LevelDB as DiskStore; +pub use self::memory_db::MemoryStore; pub use errors::Error; pub use types::*; pub type DBValue = Vec; @@ -154,21 +154,21 @@ mod tests { fn diskdb() { let dir = tempdir().unwrap(); let path = dir.path(); - let store = DiskDB::open(&path).unwrap(); + let store = DiskStore::open(&path).unwrap(); test_impl(store); } #[test] fn memorydb() { - let store = MemoryDB::open(); + let store = MemoryStore::open(); test_impl(store); } #[test] fn exists() { - let store = MemoryDB::open(); + let store = MemoryStore::open(); let key = Hash256::random(); let item = StorableThing { a: 1, b: 42 }; diff --git a/beacon_node/db/src/memory_db.rs b/beacon_node/store/src/memory_db.rs similarity index 79% rename from beacon_node/db/src/memory_db.rs rename to beacon_node/store/src/memory_db.rs index 83ff77ce1..38b0c0698 100644 --- a/beacon_node/db/src/memory_db.rs +++ b/beacon_node/store/src/memory_db.rs @@ -4,11 +4,11 @@ use std::collections::HashMap; type DBHashMap = HashMap, Vec>; -pub struct MemoryDB { +pub struct MemoryStore { db: RwLock, } -impl MemoryDB { +impl MemoryStore { pub fn open() -> Self { Self { db: RwLock::new(HashMap::new()), @@ -22,10 +22,10 @@ impl MemoryDB { } } -impl Store for MemoryDB { +impl Store for MemoryStore { /// Get the value of some key from the database. Returns `None` if the key does not exist. fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { - let column_key = MemoryDB::get_key_for_col(col, key); + let column_key = MemoryStore::get_key_for_col(col, key); Ok(self .db @@ -36,7 +36,7 @@ impl Store for MemoryDB { /// Puts a key in the database. fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { - let column_key = MemoryDB::get_key_for_col(col, key); + let column_key = MemoryStore::get_key_for_col(col, key); self.db.write().insert(column_key, val.to_vec()); @@ -45,14 +45,14 @@ impl Store for MemoryDB { /// Return true if some key exists in some column. fn key_exists(&self, col: &str, key: &[u8]) -> Result { - let column_key = MemoryDB::get_key_for_col(col, key); + let column_key = MemoryStore::get_key_for_col(col, key); Ok(self.db.read().contains_key(&column_key)) } /// Delete some key from the database. fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { - let column_key = MemoryDB::get_key_for_col(col, key); + let column_key = MemoryStore::get_key_for_col(col, key); self.db.write().remove(&column_key); diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index 819b84055..f2e6825ed 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Age Manning "] edition = "2018" [dependencies] -db = { path = "../../beacon_node/db" } +store = { path = "../../beacon_node/store" } ssz = { path = "../utils/ssz" } types = { path = "../types" } log = "0.4.6" diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index a76970f01..0e579c0b9 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -1,13 +1,11 @@ //! The optimised bitwise LMD-GHOST fork choice rule. -extern crate bit_vec; - use crate::{ForkChoice, ForkChoiceError}; use bit_vec::BitVec; -use db::Store; use log::{debug, trace}; use std::collections::HashMap; use std::marker::PhantomData; use std::sync::Arc; +use store::Store; use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight}; //TODO: Pruning - Children diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index ed3a1ce08..ffc40e6c6 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -21,9 +21,9 @@ pub mod longest_chain; pub mod optimized_lmd_ghost; pub mod slow_lmd_ghost; -// use db::stores::BeaconBlockAtSlotError; -// use db::DBError; -use db::Error as DBError; +// use store::stores::BeaconBlockAtSlotError; +// use store::DBError; +use store::Error as DBError; use types::{BeaconBlock, ChainSpec, Hash256}; pub use bitwise_lmd_ghost::BitwiseLMDGhost; diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index 6aaf56c32..11453cf49 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -1,6 +1,6 @@ use crate::{ForkChoice, ForkChoiceError}; -use db::Store; use std::sync::Arc; +use store::Store; use types::{BeaconBlock, ChainSpec, Hash256, Slot}; pub struct LongestChain { diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index aa7d65c85..dba6e60da 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -1,13 +1,11 @@ //! The optimised bitwise LMD-GHOST fork choice rule. -extern crate bit_vec; - use crate::{ForkChoice, ForkChoiceError}; -use db::Store; use log::{debug, trace}; use std::cmp::Ordering; use std::collections::HashMap; use std::marker::PhantomData; use std::sync::Arc; +use store::Store; use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight}; //TODO: Pruning - Children diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index a41eacbb2..888356417 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -1,11 +1,9 @@ -extern crate db; - use crate::{ForkChoice, ForkChoiceError}; -use db::Store; use log::{debug, trace}; use std::collections::HashMap; use std::marker::PhantomData; use std::sync::Arc; +use store::Store; use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot}; //TODO: Pruning and syncing diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index b4f4ede2c..0327e8cb3 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -1,26 +1,14 @@ #![cfg(not(debug_assertions))] // Tests the available fork-choice algorithms -extern crate beacon_chain; -extern crate bls; -extern crate db; -// extern crate env_logger; // for debugging -extern crate fork_choice; -extern crate hex; -extern crate log; -extern crate slot_clock; -extern crate types; -extern crate yaml_rust; - pub use beacon_chain::BeaconChain; use bls::Signature; -use db::MemoryDB; -use db::Store; +use store::MemoryStore; +use store::Store; // use env_logger::{Builder, Env}; use fork_choice::{ BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimizedLMDGhost, SlowLMDGhost, }; -use ssz::ssz_encode; use std::collections::HashMap; use std::sync::Arc; use std::{fs::File, io::prelude::*, path::PathBuf}; @@ -220,23 +208,23 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { fn setup_inital_state( fork_choice_algo: &ForkChoiceAlgorithm, num_validators: usize, -) -> (Box, Arc, Hash256) { - let store = Arc::new(MemoryDB::open()); +) -> (Box, Arc, Hash256) { + let store = Arc::new(MemoryStore::open()); // the fork choice instantiation let fork_choice: Box = match fork_choice_algo { ForkChoiceAlgorithm::OptimizedLMDGhost => { - let f: OptimizedLMDGhost = + let f: OptimizedLMDGhost = OptimizedLMDGhost::new(store.clone()); Box::new(f) } ForkChoiceAlgorithm::BitwiseLMDGhost => { - let f: BitwiseLMDGhost = + let f: BitwiseLMDGhost = BitwiseLMDGhost::new(store.clone()); Box::new(f) } ForkChoiceAlgorithm::SlowLMDGhost => { - let f: SlowLMDGhost = SlowLMDGhost::new(store.clone()); + let f: SlowLMDGhost = SlowLMDGhost::new(store.clone()); Box::new(f) } ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(store.clone())), From c840b76cac0609b5816f275fa11858e08cac1e40 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 18:49:24 +1000 Subject: [PATCH 30/44] Tidy `store` crate, add comments --- beacon_node/store/src/leveldb_store.rs | 9 ++- beacon_node/store/src/lib.rs | 56 ++++++++++++++++--- .../src/{memory_db.rs => memory_store.rs} | 6 +- beacon_node/store/src/store.rs | 37 ++++++++++++ 4 files changed, 96 insertions(+), 12 deletions(-) rename beacon_node/store/src/{memory_db.rs => memory_store.rs} (92%) create mode 100644 beacon_node/store/src/store.rs diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs index 10643d0cd..09aec46fa 100644 --- a/beacon_node/store/src/leveldb_store.rs +++ b/beacon_node/store/src/leveldb_store.rs @@ -6,11 +6,13 @@ use leveldb::error::Error as LevelDBError; use leveldb::options::{Options, ReadOptions, WriteOptions}; use std::path::Path; +/// A wrapped leveldb database. pub struct LevelDB { db: Database, } impl LevelDB { + /// Open a database at `path`, creating a new database if one does not already exist. pub fn open(path: &Path) -> Result { let mut options = Options::new(); @@ -36,6 +38,7 @@ impl LevelDB { } } +/// Used for keying leveldb. pub struct BytesKey { key: Vec, } @@ -51,7 +54,8 @@ impl Key for BytesKey { } impl Store for LevelDB { - fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { + /// Retrieve some bytes in `column` with `key`. + fn get_bytes(&self, col: &str, key: &[u8]) -> Result>, Error> { let column_key = Self::get_key_for_col(col, key); self.db @@ -59,6 +63,7 @@ impl Store for LevelDB { .map_err(Into::into) } + /// Store some `value` in `column`, indexed with `key`. fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { let column_key = Self::get_key_for_col(col, key); @@ -67,6 +72,7 @@ impl Store for LevelDB { .map_err(Into::into) } + /// Return `true` if `key` exists in `column`. fn key_exists(&self, col: &str, key: &[u8]) -> Result { let column_key = Self::get_key_for_col(col, key); @@ -76,6 +82,7 @@ impl Store for LevelDB { .and_then(|val| Ok(val.is_some())) } + /// Removes `key` from `column`. fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { let column_key = Self::get_key_for_col(col, key); self.db diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 096a88184..59875601a 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -1,33 +1,55 @@ -// mod disk_db; +//! Storage functionality for Lighthouse. +//! +//! Provides the following stores: +//! +//! - `DiskStore`: an on-disk store backed by leveldb. Used in production. +//! - `MemoryStore`: an in-memory store backed by a hash-map. Used for testing. +//! +//! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See +//! tests for implementation examples. + mod block_at_slot; mod errors; mod impls; mod leveldb_store; -mod memory_db; +mod memory_store; pub use self::leveldb_store::LevelDB as DiskStore; -pub use self::memory_db::MemoryStore; +pub use self::memory_store::MemoryStore; pub use errors::Error; pub use types::*; -pub type DBValue = Vec; +/// An object capable of storing and retrieving objects implementing `StoreItem`. +/// +/// A `Store` is fundamentally backed by a key-value database, however it provides support for +/// columns. A simple column implementation might involve prefixing a key with some bytes unique to +/// each column. pub trait Store: Sync + Send + Sized { + /// Store an item in `Self`. fn put(&self, key: &Hash256, item: &impl StoreItem) -> Result<(), Error> { item.db_put(self, key) } + /// Retrieve an item from `Self`. fn get(&self, key: &Hash256) -> Result, Error> { I::db_get(self, key) } + /// Returns `true` if the given key represents an item in `Self`. fn exists(&self, key: &Hash256) -> Result { I::db_exists(self, key) } + /// Remove an item from `Self`. fn delete(&self, key: &Hash256) -> Result<(), Error> { I::db_delete(self, key) } + /// Given the root of an existing block in the store (`start_block_root`), return a parent + /// block with the specified `slot`. + /// + /// Returns `None` if no parent block exists at that slot, or if `slot` is greater than the + /// slot of `start_block_root`. fn get_block_at_preceeding_slot( &self, start_block_root: Hash256, @@ -36,15 +58,20 @@ pub trait Store: Sync + Send + Sized { block_at_slot::get_block_at_preceeding_slot(self, slot, start_block_root) } - fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error>; + /// Retrieve some bytes in `column` with `key`. + fn get_bytes(&self, column: &str, key: &[u8]) -> Result>, Error>; - fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error>; + /// Store some `value` in `column`, indexed with `key`. + fn put_bytes(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>; - fn key_exists(&self, col: &str, key: &[u8]) -> Result; + /// Return `true` if `key` exists in `column`. + fn key_exists(&self, column: &str, key: &[u8]) -> Result; - fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; + /// Removes `key` from `column`. + fn key_delete(&self, column: &str, key: &[u8]) -> Result<(), Error>; } +/// A unique column identifier. pub enum DBColumn { BeaconBlock, BeaconState, @@ -62,22 +89,31 @@ impl<'a> Into<&'a str> for DBColumn { } } +/// An item that may be stored in a `Store`. +/// +/// Provides default methods that are suitable for most applications, however when overridden they +/// provide full customizability of `Store` operations. pub trait StoreItem: Sized { + /// Identifies which column this item should be placed in. fn db_column() -> DBColumn; + /// Serialize `self` as bytes. fn as_store_bytes(&self) -> Vec; + /// De-serialize `self` from bytes. fn from_store_bytes(bytes: &mut [u8]) -> Result; + /// Store `self`. fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> { let column = Self::db_column().into(); let key = key.as_bytes(); store .put_bytes(column, key, &self.as_store_bytes()) - .map_err(|e| e.into()) + .map_err(Into::into) } + /// Retrieve an instance of `Self`. fn db_get(store: &impl Store, key: &Hash256) -> Result, Error> { let column = Self::db_column().into(); let key = key.as_bytes(); @@ -88,6 +124,7 @@ pub trait StoreItem: Sized { } } + /// Return `true` if an instance of `Self` exists in `Store`. fn db_exists(store: &impl Store, key: &Hash256) -> Result { let column = Self::db_column().into(); let key = key.as_bytes(); @@ -95,6 +132,7 @@ pub trait StoreItem: Sized { store.key_exists(column, key) } + /// Delete `self` from the `Store`. fn db_delete(store: &impl Store, key: &Hash256) -> Result<(), Error> { let column = Self::db_column().into(); let key = key.as_bytes(); diff --git a/beacon_node/store/src/memory_db.rs b/beacon_node/store/src/memory_store.rs similarity index 92% rename from beacon_node/store/src/memory_db.rs rename to beacon_node/store/src/memory_store.rs index 38b0c0698..086a16c26 100644 --- a/beacon_node/store/src/memory_db.rs +++ b/beacon_node/store/src/memory_store.rs @@ -1,14 +1,16 @@ -use super::{DBValue, Error, Store}; +use super::{Error, Store}; use parking_lot::RwLock; use std::collections::HashMap; type DBHashMap = HashMap, Vec>; +/// A thread-safe `HashMap` wrapper. pub struct MemoryStore { db: RwLock, } impl MemoryStore { + /// Create a new, empty database. pub fn open() -> Self { Self { db: RwLock::new(HashMap::new()), @@ -24,7 +26,7 @@ impl MemoryStore { impl Store for MemoryStore { /// Get the value of some key from the database. Returns `None` if the key does not exist. - fn get_bytes(&self, col: &str, key: &[u8]) -> Result, Error> { + fn get_bytes(&self, col: &str, key: &[u8]) -> Result>, Error> { let column_key = MemoryStore::get_key_for_col(col, key); Ok(self diff --git a/beacon_node/store/src/store.rs b/beacon_node/store/src/store.rs new file mode 100644 index 000000000..5d18c7ba5 --- /dev/null +++ b/beacon_node/store/src/store.rs @@ -0,0 +1,37 @@ +use super::*; + +pub type Vec = Vec; + +pub trait Store: Sync + Send + Sized { + fn put(&self, key: &Hash256, item: &impl StoreItem) -> Result<(), Error> { + item.db_put(self, key) + } + + fn get(&self, key: &Hash256) -> Result, Error> { + I::db_get(self, key) + } + + fn exists(&self, key: &Hash256) -> Result { + I::db_exists(self, key) + } + + fn delete(&self, key: &Hash256) -> Result<(), Error> { + I::db_delete(self, key) + } + + fn get_block_at_preceeding_slot( + &self, + start_block_root: Hash256, + slot: Slot, + ) -> Result, Error> { + block_at_slot::get_block_at_preceeding_slot(self, slot, start_block_root) + } + + fn get_bytes(&self, col: &str, key: &[u8]) -> Result>, Error>; + + fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error>; + + fn key_exists(&self, col: &str, key: &[u8]) -> Result; + + fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>; +} From b41f91db1d66b91341a4af95903dd9d154bd6b82 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 May 2019 19:32:07 +1000 Subject: [PATCH 31/44] Rename disk db dir --- beacon_node/client/src/client_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 4309b8a64..8d7176c2c 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -53,7 +53,7 @@ impl Default for ClientConfig { // default to memory db for now db_type: DBType::Memory, // default db name for disk-based dbs - db_name: data_dir.join("chain.db"), + db_name: data_dir.join("chain_db"), rpc_conf: rpc::RPCConfig::default(), } } From 517952fc0f1ea5ec6b9afe87f0f03a5b4423b955 Mon Sep 17 00:00:00 2001 From: Mehdi Zerouali Date: Fri, 24 May 2019 10:07:30 +1000 Subject: [PATCH 32/44] Update instructions to install rustup in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 879f9b8fe..4dd19be7b 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ tests and benchmarks which may be of interest. A few basic steps are needed to get set up: - 1. Install [rustup](https://rustup.rs/). It's a toolchain manager for Rust (Linux | macos | Windows). For installation run the below command in your terminal `$ curl https://sh.rustup.rs -sSf | sh` + 1. Install [rustup](https://rustup.rs/). It's a toolchain manager for Rust (Linux | macOS | Windows). For installation, download the script with `$ curl -f https://sh.rustup.rs > rustup.sh`, review its content (e.g. `$ less ./rustup.sh`) and run the script `$ ./rustup.sh` (you may need to change the permissions to allow execution, i.e. `$ chmod +x rustup.sh`) 2. (Linux & MacOS) To configure your current shell run: `$ source $HOME/.cargo/env` 3. Use the command `rustup show` to get information about the Rust installation. You should see that the active toolchain is the stable version. From 95c218355e9c4bf4cd90f4ff381b1c98587a9c3a Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 23 May 2019 18:27:35 -0700 Subject: [PATCH 33/44] Support multiple bootnodes --- beacon_node/client/src/client_config.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 8d7176c2c..f42336a54 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -95,13 +95,15 @@ impl ClientConfig { } // Custom bootnodes - // TODO: Handle list of addresses if let Some(boot_addresses_str) = args.value_of("boot-nodes") { - if let Ok(boot_address) = boot_addresses_str.parse::() { - config.net_conf.boot_nodes.append(&mut vec![boot_address]); - } else { - error!(log, "Invalid Bootnode multiaddress"; "Multiaddr" => boot_addresses_str); - return Err("Invalid IP Address"); + let mut boot_addresses_split = boot_addresses_str.split(","); + for boot_address in boot_addresses_split { + if let Ok(boot_address) = boot_address.parse::() { + config.net_conf.boot_nodes.append(&mut vec![boot_address]); + } else { + error!(log, "Invalid Bootnode multiaddress"; "Multiaddr" => boot_addresses_str); + return Err("Invalid IP Address"); + } } } From 8dd07dd7d2d8bf16c2f571732bd91e9f2ec8e1c9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 25 May 2019 14:31:13 +1000 Subject: [PATCH 34/44] Add http server to beacon node w/ hello world --- Cargo.toml | 1 + beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/client_config.rs | 4 +- beacon_node/client/src/lib.rs | 13 ++- beacon_node/http_server/Cargo.toml | 31 +++++++ beacon_node/http_server/src/lib.rs | 115 ++++++++++++++++++++++++ 6 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 beacon_node/http_server/Cargo.toml create mode 100644 beacon_node/http_server/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 00c354309..b4d53d420 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "beacon_node", "beacon_node/store", "beacon_node/client", + "beacon_node/http_server", "beacon_node/network", "beacon_node/eth2-libp2p", "beacon_node/rpc", diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 4a976eec4..6634e260d 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } store = { path = "../store" } +http_server = { path = "../http_server" } rpc = { path = "../rpc" } fork_choice = { path = "../../eth2/fork_choice" } types = { path = "../../eth2/types" } diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 8d7176c2c..32722e124 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -1,5 +1,6 @@ use clap::ArgMatches; use fork_choice::ForkChoiceAlgorithm; +use http_server::HttpServerConfig; use network::NetworkConfig; use slog::error; use std::fs; @@ -27,7 +28,7 @@ pub struct ClientConfig { pub db_type: DBType, pub db_name: PathBuf, pub rpc_conf: rpc::RPCConfig, - //pub ipc_conf: + pub http_conf: HttpServerConfig, //pub ipc_conf: } impl Default for ClientConfig { @@ -55,6 +56,7 @@ impl Default for ClientConfig { // default db name for disk-based dbs db_name: data_dir.join("chain_db"), rpc_conf: rpc::RPCConfig::default(), + http_conf: HttpServerConfig::default(), } } } diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 71d4013d3..6433b94e2 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -98,7 +98,7 @@ impl Client { Some(rpc::start_server( &config.rpc_conf, executor, - network_send, + network_send.clone(), beacon_chain.clone(), &log, )) @@ -106,6 +106,17 @@ impl Client { None }; + // Start the `http_server` service. + // + // Note: presently we are ignoring the config and _always_ starting a HTTP server. + http_server::start_service( + &config.http_conf, + executor, + network_send, + beacon_chain.clone(), + &log, + ); + let (slot_timer_exit_signal, exit) = exit_future::signal(); if let Ok(Some(duration_to_next_slot)) = beacon_chain.slot_clock.duration_to_next_slot() { // set up the validator work interval - start at next slot and proceed every slot diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml new file mode 100644 index 000000000..5d5d7e492 --- /dev/null +++ b/beacon_node/http_server/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "http_server" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +bls = { path = "../../eth2/utils/bls" } +beacon_chain = { path = "../beacon_chain" } +iron = "^0.6" +router = "^0.6" +network = { path = "../network" } +eth2-libp2p = { path = "../eth2-libp2p" } +version = { path = "../version" } +types = { path = "../../eth2/types" } +ssz = { path = "../../eth2/utils/ssz" } +slot_clock = { path = "../../eth2/utils/slot_clock" } +protos = { path = "../../protos" } +fork_choice = { path = "../../eth2/fork_choice" } +grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } +protobuf = "2.0.2" +clap = "2.32.0" +store = { path = "../store" } +dirs = "1.0.3" +futures = "0.1.23" +slog = "^2.2.3" +slog-term = "^2.4.0" +slog-async = "^2.3.0" +tokio = "0.1.17" +exit-future = "0.1.4" +crossbeam-channel = "0.3.8" diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs new file mode 100644 index 000000000..676928ce6 --- /dev/null +++ b/beacon_node/http_server/src/lib.rs @@ -0,0 +1,115 @@ +use beacon_chain::BeaconChain; +use futures::Future; +use grpcio::{Environment, ServerBuilder}; +use network::NetworkMessage; +use protos::services_grpc::{ + create_attestation_service, create_beacon_block_service, create_beacon_node_service, + create_validator_service, +}; +use slog::{info, o, warn}; +use std::net::Ipv4Addr; +use std::sync::Arc; +use tokio::runtime::TaskExecutor; +use types::EthSpec; + +use iron::prelude::*; +use iron::{status::Status, Handler, IronResult, Request, Response}; +use router::Router; + +#[derive(PartialEq, Clone, Debug)] +pub struct HttpServerConfig { + pub enabled: bool, + pub listen_address: String, + /* + pub listen_address: Ipv4Addr, + pub port: u16, + */ +} + +impl Default for HttpServerConfig { + fn default() -> Self { + Self { + enabled: false, + listen_address: "127.0.0.1:5051".to_string(), + /* + listen_address: Ipv4Addr::new(127, 0, 0, 1), + port: 5051, + */ + } + } +} + +pub struct IndexHandler { + message: String, +} + +impl Handler for IndexHandler { + fn handle(&self, _: &mut Request) -> IronResult { + Ok(Response::with((Status::Ok, self.message.clone()))) + } +} + +pub fn create_iron_http_server() -> Iron { + let index_handler = IndexHandler { + message: "Hello world".to_string(), + }; + + let mut router = Router::new(); + router.get("/", index_handler, "index"); + Iron::new(router) +} + +pub fn start_service( + config: &HttpServerConfig, + executor: &TaskExecutor, + network_chan: crossbeam_channel::Sender, + beacon_chain: Arc>, + log: &slog::Logger, +) -> exit_future::Signal +where + T: store::Store, + U: slot_clock::SlotClock, + F: fork_choice::ForkChoice, + E: EthSpec, +{ + let log = log.new(o!("Service"=>"RPC")); + let env = Arc::new(Environment::new(1)); + + // Create: + // - `shutdown_trigger` a one-shot to shut down this service. + // - `wait_for_shutdown` a future that will wait until someone calls shutdown. + let (shutdown_trigger, wait_for_shutdown) = exit_future::signal(); + + let iron = create_iron_http_server(); + + let spawn_rpc = { + let result = iron.http(config.listen_address.clone()); + + if result.is_ok() { + info!(log, "HTTP server running on {}", config.listen_address); + } else { + warn!( + log, + "HTTP server failed to start on {}", config.listen_address + ); + } + + wait_for_shutdown.and_then(move |_| { + info!(log, "HTTP server shutting down"); + + // TODO: shutdown server. + /* + server + .shutdown() + .wait() + .map(|_| ()) + .map_err(|e| warn!(log, "RPC server failed to shutdown: {:?}", e))?; + Ok(()) + */ + info!(log, "HTTP server exited"); + Ok(()) + }) + }; + executor.spawn(spawn_rpc); + shutdown_trigger +} From 596ff5178bbf052c10fdfd999d7748ccd57c422d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 25 May 2019 16:17:48 +1000 Subject: [PATCH 35/44] Add http server shutdown, tidy --- beacon_node/http_server/src/lib.rs | 43 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 676928ce6..b230f924f 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -1,17 +1,10 @@ use beacon_chain::BeaconChain; use futures::Future; -use grpcio::{Environment, ServerBuilder}; use network::NetworkMessage; -use protos::services_grpc::{ - create_attestation_service, create_beacon_block_service, create_beacon_node_service, - create_validator_service, -}; use slog::{info, o, warn}; -use std::net::Ipv4Addr; use std::sync::Arc; use tokio::runtime::TaskExecutor; use types::EthSpec; - use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; use router::Router; @@ -62,8 +55,8 @@ pub fn create_iron_http_server() -> Iron { pub fn start_service( config: &HttpServerConfig, executor: &TaskExecutor, - network_chan: crossbeam_channel::Sender, - beacon_chain: Arc>, + _network_chan: crossbeam_channel::Sender, + _beacon_chain: Arc>, log: &slog::Logger, ) -> exit_future::Signal where @@ -73,19 +66,20 @@ where E: EthSpec, { let log = log.new(o!("Service"=>"RPC")); - let env = Arc::new(Environment::new(1)); // Create: // - `shutdown_trigger` a one-shot to shut down this service. // - `wait_for_shutdown` a future that will wait until someone calls shutdown. let (shutdown_trigger, wait_for_shutdown) = exit_future::signal(); + // Create an `iron` http, without starting it yet. let iron = create_iron_http_server(); let spawn_rpc = { - let result = iron.http(config.listen_address.clone()); + // Start the HTTP server + let server_start_result = iron.http(config.listen_address.clone()); - if result.is_ok() { + if server_start_result.is_ok() { info!(log, "HTTP server running on {}", config.listen_address); } else { warn!( @@ -94,19 +88,24 @@ where ); } + // Build a future that will shutdown the HTTP server when the `shutdown_trigger` is + // triggered. wait_for_shutdown.and_then(move |_| { info!(log, "HTTP server shutting down"); - // TODO: shutdown server. - /* - server - .shutdown() - .wait() - .map(|_| ()) - .map_err(|e| warn!(log, "RPC server failed to shutdown: {:?}", e))?; - Ok(()) - */ - info!(log, "HTTP server exited"); + if let Ok(mut server) = server_start_result { + // According to the documentation, this function "doesn't work" and the server + // keeps listening. + // + // It is being called anyway, because it seems like the right thing to do. If you + // know this has negative side-effects, please create an issue to discuss. + // + // See: https://docs.rs/iron/0.6.0/iron/struct.Listening.html#impl + match server.close() { + _=> () + }; + } + info!(log, "HTTP server shutdown complete."); Ok(()) }) }; From 85211ebccd884b09336d42464d69bb4b22484aae Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 25 May 2019 17:25:21 +1000 Subject: [PATCH 36/44] Add basic prometheus endpoint --- beacon_node/client/src/lib.rs | 7 ++- beacon_node/http_server/Cargo.toml | 1 + beacon_node/http_server/src/lib.rs | 51 ++++++++++++++----- .../http_server/src/prometheus_handler.rs | 43 ++++++++++++++++ 4 files changed, 86 insertions(+), 16 deletions(-) create mode 100644 beacon_node/http_server/src/prometheus_handler.rs diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 6433b94e2..9445799d5 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -35,6 +35,8 @@ pub struct Client { pub network: Arc>, /// Signal to terminate the RPC server. pub rpc_exit_signal: Option, + /// Signal to terminate the HTTP server. + pub http_exit_signal: Option, /// Signal to terminate the slot timer. pub slot_timer_exit_signal: Option, /// The clients logger. @@ -109,13 +111,13 @@ impl Client { // Start the `http_server` service. // // Note: presently we are ignoring the config and _always_ starting a HTTP server. - http_server::start_service( + let http_exit_signal = Some(http_server::start_service( &config.http_conf, executor, network_send, beacon_chain.clone(), &log, - ); + )); let (slot_timer_exit_signal, exit) = exit_future::signal(); if let Ok(Some(duration_to_next_slot)) = beacon_chain.slot_clock.duration_to_next_slot() { @@ -146,6 +148,7 @@ impl Client { Ok(Client { _config: config, _beacon_chain: beacon_chain, + http_exit_signal, rpc_exit_signal, slot_timer_exit_signal: Some(slot_timer_exit_signal), log, diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml index 5d5d7e492..6f4579d17 100644 --- a/beacon_node/http_server/Cargo.toml +++ b/beacon_node/http_server/Cargo.toml @@ -19,6 +19,7 @@ protos = { path = "../../protos" } fork_choice = { path = "../../eth2/fork_choice" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } protobuf = "2.0.2" +prometheus = "^0.6" clap = "2.32.0" store = { path = "../store" } dirs = "1.0.3" diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index b230f924f..b2c3a86fc 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -1,13 +1,16 @@ +mod prometheus_handler; + use beacon_chain::BeaconChain; use futures::Future; +use iron::prelude::*; +use iron::{status::Status, Handler, IronResult, Request, Response}; use network::NetworkMessage; +use prometheus_handler::PrometheusHandler; +use router::Router; use slog::{info, o, warn}; use std::sync::Arc; use tokio::runtime::TaskExecutor; use types::EthSpec; -use iron::prelude::*; -use iron::{status::Status, Handler, IronResult, Request, Response}; -use router::Router; #[derive(PartialEq, Clone, Debug)] pub struct HttpServerConfig { @@ -42,13 +45,25 @@ impl Handler for IndexHandler { } } -pub fn create_iron_http_server() -> Iron { +pub fn create_iron_http_server( + beacon_chain: Arc>, +) -> Iron +where + T: store::Store + 'static, + U: slot_clock::SlotClock + 'static, + F: fork_choice::ForkChoice + 'static, + E: EthSpec + 'static, +{ let index_handler = IndexHandler { message: "Hello world".to_string(), }; + let prom_handler = PrometheusHandler { + beacon_chain: beacon_chain, + }; let mut router = Router::new(); router.get("/", index_handler, "index"); + router.get("/prometheus/", prom_handler, "prometheus"); Iron::new(router) } @@ -56,16 +71,16 @@ pub fn start_service( config: &HttpServerConfig, executor: &TaskExecutor, _network_chan: crossbeam_channel::Sender, - _beacon_chain: Arc>, + beacon_chain: Arc>, log: &slog::Logger, ) -> exit_future::Signal where - T: store::Store, - U: slot_clock::SlotClock, - F: fork_choice::ForkChoice, - E: EthSpec, + T: store::Store + 'static, + U: slot_clock::SlotClock + 'static, + F: fork_choice::ForkChoice + 'static, + E: EthSpec + 'static, { - let log = log.new(o!("Service"=>"RPC")); + let log = log.new(o!("Service"=>"HTTP")); // Create: // - `shutdown_trigger` a one-shot to shut down this service. @@ -73,9 +88,14 @@ where let (shutdown_trigger, wait_for_shutdown) = exit_future::signal(); // Create an `iron` http, without starting it yet. - let iron = create_iron_http_server(); + let iron = create_iron_http_server(beacon_chain); - let spawn_rpc = { + // Create a HTTP server future. + // + // 1. Start the HTTP server + // 2. Build an exit future that will shutdown the server when requested. + // 3. Return the exit future, so the caller may shutdown the service when desired. + let http_service = { // Start the HTTP server let server_start_result = iron.http(config.listen_address.clone()); @@ -102,13 +122,16 @@ where // // See: https://docs.rs/iron/0.6.0/iron/struct.Listening.html#impl match server.close() { - _=> () + _ => (), }; } info!(log, "HTTP server shutdown complete."); Ok(()) }) }; - executor.spawn(spawn_rpc); + + // Attach the HTTP server to the executor. + executor.spawn(http_service); + shutdown_trigger } diff --git a/beacon_node/http_server/src/prometheus_handler.rs b/beacon_node/http_server/src/prometheus_handler.rs new file mode 100644 index 000000000..cf577a9eb --- /dev/null +++ b/beacon_node/http_server/src/prometheus_handler.rs @@ -0,0 +1,43 @@ +use beacon_chain::BeaconChain; +use iron::{status::Status, Handler, IronResult, Request, Response}; +use prometheus::{IntCounter, Encoder, Opts, Registry, TextEncoder}; +use std::sync::Arc; +use types::EthSpec; + +pub struct PrometheusHandler { + pub beacon_chain: Arc>, +} + +impl PrometheusHandler where E: EthSpec {} + +impl Handler for PrometheusHandler +where + E: EthSpec + 'static, + U: slot_clock::SlotClock + Send + Sync + 'static, + T: Send + Sync + 'static, + F: Send + Sync + 'static, +{ + fn handle(&self, _: &mut Request) -> IronResult { + // Create a Counter. + let counter_opts = Opts::new("present_slot", "direct_slot_clock_reading"); + let counter = IntCounter::with_opts(counter_opts).unwrap(); + + // Create a Registry and register Counter. + let r = Registry::new(); + r.register(Box::new(counter.clone())).unwrap(); + + if let Ok(Some(slot)) = self.beacon_chain.slot_clock.present_slot() { + counter.inc_by(slot.as_u64() as i64); + } + + // Gather the metrics. + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + let metric_families = r.gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + let prom_string = String::from_utf8(buffer).unwrap(); + + Ok(Response::with((Status::Ok, prom_string))) + } +} From 45e3a1759cb75fb732a300486038ad26882e738c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 25 May 2019 17:56:53 +1000 Subject: [PATCH 37/44] Add slot to prometheus endpoint --- .../http_server/src/prometheus_handler.rs | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/beacon_node/http_server/src/prometheus_handler.rs b/beacon_node/http_server/src/prometheus_handler.rs index cf577a9eb..ae84d4871 100644 --- a/beacon_node/http_server/src/prometheus_handler.rs +++ b/beacon_node/http_server/src/prometheus_handler.rs @@ -1,8 +1,8 @@ use beacon_chain::BeaconChain; use iron::{status::Status, Handler, IronResult, Request, Response}; -use prometheus::{IntCounter, Encoder, Opts, Registry, TextEncoder}; +use prometheus::{Encoder, IntCounter, Opts, Registry, TextEncoder}; use std::sync::Arc; -use types::EthSpec; +use types::{EthSpec, Slot}; pub struct PrometheusHandler { pub beacon_chain: Arc>, @@ -18,17 +18,19 @@ where F: Send + Sync + 'static, { fn handle(&self, _: &mut Request) -> IronResult { - // Create a Counter. - let counter_opts = Opts::new("present_slot", "direct_slot_clock_reading"); - let counter = IntCounter::with_opts(counter_opts).unwrap(); - - // Create a Registry and register Counter. let r = Registry::new(); - r.register(Box::new(counter.clone())).unwrap(); - if let Ok(Some(slot)) = self.beacon_chain.slot_clock.present_slot() { - counter.inc_by(slot.as_u64() as i64); - } + let present_slot = if let Ok(Some(slot)) = self.beacon_chain.slot_clock.present_slot() { + slot + } else { + Slot::new(0) + }; + register_and_set_slot( + &r, + "present_slot", + "direct_slock_clock_reading", + present_slot, + ); // Gather the metrics. let mut buffer = vec![]; @@ -41,3 +43,10 @@ where Ok(Response::with((Status::Ok, prom_string))) } } + +fn register_and_set_slot(registry: &Registry, name: &str, help: &str, slot: Slot) { + let counter_opts = Opts::new(name, help); + let counter = IntCounter::with_opts(counter_opts).unwrap(); + registry.register(Box::new(counter.clone())).unwrap(); + counter.inc_by(slot.as_u64() as i64); +} From ee8d13573fd90386a50b7a4a953c3ae810dfa0aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 25 May 2019 20:51:15 +1000 Subject: [PATCH 38/44] Create `BeaconChainTypes`, thread through runtime --- beacon_node/beacon_chain/src/beacon_chain.rs | 61 +++---- beacon_node/beacon_chain/src/initialise.rs | 153 ------------------ beacon_node/beacon_chain/src/lib.rs | 6 +- .../beacon_chain/src/test_utils/mod.rs | 3 - .../testing_beacon_chain_builder.rs | 49 ------ beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/beacon_chain_types.rs | 109 +++++++++++++ beacon_node/client/src/client_types.rs | 65 -------- beacon_node/client/src/lib.rs | 33 ++-- beacon_node/client/src/notifier.rs | 8 +- beacon_node/http_server/src/lib.rs | 27 +--- .../http_server/src/prometheus_handler.rs | 19 +-- beacon_node/network/src/beacon_chain.rs | 27 ++-- beacon_node/network/src/message_handler.rs | 13 +- beacon_node/network/src/service.rs | 12 +- beacon_node/network/src/sync/import_queue.rs | 12 +- beacon_node/network/src/sync/simple_sync.rs | 18 +-- beacon_node/rpc/src/attestation.rs | 10 +- beacon_node/rpc/src/beacon_block.rs | 10 +- beacon_node/rpc/src/beacon_chain.rs | 27 ++-- beacon_node/rpc/src/beacon_node.rs | 9 +- beacon_node/rpc/src/lib.rs | 7 +- beacon_node/rpc/src/validator.rs | 10 +- beacon_node/src/run.rs | 17 +- 24 files changed, 254 insertions(+), 452 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/initialise.rs delete mode 100644 beacon_node/beacon_chain/src/test_utils/mod.rs delete mode 100644 beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs create mode 100644 beacon_node/client/src/beacon_chain_types.rs delete mode 100644 beacon_node/client/src/client_types.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f2c4b3dbe..9f08b6f64 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -79,32 +79,33 @@ impl BlockProcessingOutcome { } } -pub struct BeaconChain { - pub store: Arc, - pub slot_clock: U, - pub op_pool: OperationPool, - canonical_head: RwLock>, - finalized_head: RwLock>, - pub state: RwLock>, - pub spec: ChainSpec, - pub fork_choice: RwLock, +pub trait BeaconChainTypes { + type Store: store::Store; + type SlotClock: slot_clock::SlotClock; + type ForkChoice: fork_choice::ForkChoice; + type EthSpec: types::EthSpec; } -impl BeaconChain -where - T: Store, - U: SlotClock, - F: ForkChoice, - E: EthSpec, -{ +pub struct BeaconChain { + pub store: Arc, + pub slot_clock: T::SlotClock, + pub op_pool: OperationPool, + canonical_head: RwLock>, + finalized_head: RwLock>, + pub state: RwLock>, + pub spec: ChainSpec, + pub fork_choice: RwLock, +} + +impl BeaconChain { /// Instantiate a new Beacon Chain, from genesis. pub fn from_genesis( - store: Arc, - slot_clock: U, - mut genesis_state: BeaconState, + store: Arc, + slot_clock: T::SlotClock, + mut genesis_state: BeaconState, genesis_block: BeaconBlock, spec: ChainSpec, - fork_choice: F, + fork_choice: T::ForkChoice, ) -> Result { let state_root = genesis_state.canonical_root(); store.put(&state_root, &genesis_state)?; @@ -223,7 +224,7 @@ where Err(BeaconStateError::SlotOutOfBounds) => { // Read the earliest historic state in the current slot. let earliest_historic_slot = - state.slot - Slot::from(E::SlotsPerHistoricalRoot::to_usize()); + state.slot - Slot::from(T::EthSpec::slots_per_historical_root()); // Load the earlier state from disk. let new_state_root = state.get_state_root(earliest_historic_slot)?; @@ -263,7 +264,7 @@ where &self, new_beacon_block: BeaconBlock, new_beacon_block_root: Hash256, - new_beacon_state: BeaconState, + new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { debug!( @@ -285,7 +286,7 @@ where /// It is important to note that the `beacon_state` returned may not match the present slot. It /// is the state as it was when the head block was received, which could be some slots prior to /// now. - pub fn head(&self) -> RwLockReadGuard> { + pub fn head(&self) -> RwLockReadGuard> { self.canonical_head.read() } @@ -295,7 +296,7 @@ where /// state and calling `catchup_state` as it will not result in an old state being installed and /// then having it iteratively updated -- in such a case it's possible for another thread to /// find the state at an old slot. - pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { + pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, _ => return Err(Error::UnableToReadSlot), @@ -350,7 +351,7 @@ where &self, new_beacon_block: BeaconBlock, new_beacon_block_root: Hash256, - new_beacon_state: BeaconState, + new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { let mut finalized_head = self.finalized_head.write(); @@ -364,7 +365,7 @@ where /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen, /// indirectly, by the fork-choice rule). - pub fn finalized_head(&self) -> RwLockReadGuard> { + pub fn finalized_head(&self) -> RwLockReadGuard> { self.finalized_head.read() } @@ -602,7 +603,7 @@ where // significantly lower exposure surface to DoS attacks. // Transition the parent state to the block slot. - let mut state: BeaconState = parent_state; + let mut state: BeaconState = parent_state; for _ in state.slot.as_u64()..block.slot.as_u64() { if let Err(e) = per_slot_processing(&mut state, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( @@ -657,7 +658,7 @@ where pub fn produce_block( &self, randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { + ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { debug!("Producing block at slot {}...", self.state.read().slot); let mut state = self.state.read().clone(); @@ -728,7 +729,7 @@ where .ok_or_else(|| Error::MissingBeaconBlock(new_head))?; let block_root = block.canonical_root(); - let state: BeaconState = self + let state: BeaconState = self .store .get(&block.state_root)? .ok_or_else(|| Error::MissingBeaconState(block.state_root))?; @@ -752,7 +753,7 @@ where /// /// This could be a very expensive operation and should only be done in testing/analysis /// activities. - pub fn chain_dump(&self) -> Result>, Error> { + pub fn chain_dump(&self) -> Result>, Error> { let mut dump = vec![]; let mut last_slot = CheckPoint { diff --git a/beacon_node/beacon_chain/src/initialise.rs b/beacon_node/beacon_chain/src/initialise.rs deleted file mode 100644 index b9d950ed5..000000000 --- a/beacon_node/beacon_chain/src/initialise.rs +++ /dev/null @@ -1,153 +0,0 @@ -// Initialisation functions to generate a new BeaconChain. -// Note: A new version of ClientTypes may need to be implemented for the lighthouse -// testnet. These are examples. Also. there is code duplication which can/should be cleaned up. - -use crate::BeaconChain; -use fork_choice::BitwiseLMDGhost; -use slot_clock::SystemTimeSlotClock; -use std::path::PathBuf; -use std::sync::Arc; -use store::{DiskStore, MemoryStore}; -use tree_hash::TreeHash; -use types::test_utils::TestingBeaconStateBuilder; -use types::{BeaconBlock, ChainSpec, FewValidatorsEthSpec, FoundationEthSpec, Hash256}; - -//TODO: Correct this for prod -//TODO: Account for historical db -pub fn initialise_beacon_chain( - spec: &ChainSpec, - db_name: Option<&PathBuf>, -) -> Arc< - BeaconChain< - DiskStore, - SystemTimeSlotClock, - BitwiseLMDGhost, - FoundationEthSpec, - >, -> { - let path = db_name.expect("db_name cannot be None."); - let store = DiskStore::open(path).expect("Unable to open DB."); - let store = Arc::new(store); - - let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); - let (genesis_state, _keypairs) = state_builder.build(); - - let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); - - // Slot clock - let slot_clock = SystemTimeSlotClock::new( - spec.genesis_slot, - genesis_state.genesis_time, - spec.seconds_per_slot, - ) - .expect("Unable to load SystemTimeSlotClock"); - // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(store.clone()); - - // Genesis chain - //TODO: Handle error correctly - Arc::new( - BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec.clone(), - fork_choice, - ) - .expect("Terminate if beacon chain generation fails"), - ) -} - -/// Initialisation of a test beacon chain, uses an in memory db with fixed genesis time. -pub fn initialise_test_beacon_chain_with_memory_db( - spec: &ChainSpec, - _db_name: Option<&PathBuf>, -) -> Arc< - BeaconChain< - MemoryStore, - SystemTimeSlotClock, - BitwiseLMDGhost, - FewValidatorsEthSpec, - >, -> { - let store = Arc::new(MemoryStore::open()); - - let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); - let (genesis_state, _keypairs) = state_builder.build(); - - let mut genesis_block = BeaconBlock::empty(spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); - - // Slot clock - let slot_clock = SystemTimeSlotClock::new( - spec.genesis_slot, - genesis_state.genesis_time, - spec.seconds_per_slot, - ) - .expect("Unable to load SystemTimeSlotClock"); - // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(store.clone()); - - // Genesis chain - //TODO: Handle error correctly - Arc::new( - BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec.clone(), - fork_choice, - ) - .expect("Terminate if beacon chain generation fails"), - ) -} - -/// Initialisation of a test beacon chain, uses an in memory db with fixed genesis time. -pub fn initialise_test_beacon_chain_with_disk_db( - spec: &ChainSpec, - db_name: Option<&PathBuf>, -) -> Arc< - BeaconChain< - DiskStore, - SystemTimeSlotClock, - BitwiseLMDGhost, - FewValidatorsEthSpec, - >, -> { - let path = db_name.expect("db_name cannot be None."); - let store = DiskStore::open(path).expect("Unable to open DB."); - let store = Arc::new(store); - - let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec); - let (genesis_state, _keypairs) = state_builder.build(); - - let mut genesis_block = BeaconBlock::empty(spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); - - // Slot clock - let slot_clock = SystemTimeSlotClock::new( - spec.genesis_slot, - genesis_state.genesis_time, - spec.seconds_per_slot, - ) - .expect("Unable to load SystemTimeSlotClock"); - // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(store.clone()); - - // Genesis chain - //TODO: Handle error correctly - Arc::new( - BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec.clone(), - fork_choice, - ) - .expect("Terminate if beacon chain generation fails"), - ) -} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 6ac01a5d5..9f3058d0b 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,10 +1,10 @@ mod beacon_chain; mod checkpoint; mod errors; -pub mod initialise; -pub mod test_utils; -pub use self::beacon_chain::{BeaconChain, BlockProcessingOutcome, InvalidBlock, ValidBlock}; +pub use self::beacon_chain::{ + BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock, ValidBlock, +}; pub use self::checkpoint::CheckPoint; pub use self::errors::{BeaconChainError, BlockProductionError}; pub use fork_choice; diff --git a/beacon_node/beacon_chain/src/test_utils/mod.rs b/beacon_node/beacon_chain/src/test_utils/mod.rs deleted file mode 100644 index ad251a3c9..000000000 --- a/beacon_node/beacon_chain/src/test_utils/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod testing_beacon_chain_builder; - -pub use testing_beacon_chain_builder::TestingBeaconChainBuilder; diff --git a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs b/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs deleted file mode 100644 index b6b1defcc..000000000 --- a/beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs +++ /dev/null @@ -1,49 +0,0 @@ -pub use crate::{BeaconChain, BeaconChainError, CheckPoint}; -use fork_choice::BitwiseLMDGhost; -use slot_clock::TestingSlotClock; -use std::sync::Arc; -use store::MemoryStore; -use tree_hash::TreeHash; -use types::*; -use types::{test_utils::TestingBeaconStateBuilder, EthSpec, FewValidatorsEthSpec}; - -type TestingBeaconChain = BeaconChain< - MemoryStore, - TestingSlotClock, - BitwiseLMDGhost, - E, ->; - -pub struct TestingBeaconChainBuilder { - state_builder: TestingBeaconStateBuilder, -} - -impl TestingBeaconChainBuilder { - pub fn build(self, spec: &ChainSpec) -> TestingBeaconChain { - let store = Arc::new(MemoryStore::open()); - let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64()); - let fork_choice = BitwiseLMDGhost::new(store.clone()); - - let (genesis_state, _keypairs) = self.state_builder.build(); - - let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); - - // Create the Beacon Chain - BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec.clone(), - fork_choice, - ) - .unwrap() - } -} - -impl From> for TestingBeaconChainBuilder { - fn from(state_builder: TestingBeaconStateBuilder) -> TestingBeaconChainBuilder { - TestingBeaconChainBuilder { state_builder } - } -} diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 6634e260d..387bf1675 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -12,6 +12,7 @@ http_server = { path = "../http_server" } rpc = { path = "../rpc" } fork_choice = { path = "../../eth2/fork_choice" } types = { path = "../../eth2/types" } +tree_hash = { path = "../../eth2/utils/tree_hash" } slot_clock = { path = "../../eth2/utils/slot_clock" } error-chain = "0.12.0" slog = "^2.2.3" diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs new file mode 100644 index 000000000..b8236c679 --- /dev/null +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -0,0 +1,109 @@ +use crate::ClientConfig; +use beacon_chain::{ + fork_choice::BitwiseLMDGhost, + slot_clock::SystemTimeSlotClock, + store::{DiskStore, MemoryStore, Store}, + BeaconChain, BeaconChainTypes, +}; +use std::sync::Arc; +use tree_hash::TreeHash; +use types::{ + test_utils::TestingBeaconStateBuilder, BeaconBlock, EthSpec, FewValidatorsEthSpec, Hash256, +}; + +/// Provides a new, initialized `BeaconChain` +pub trait InitialiseBeaconChain { + fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain; +} + +/// A testnet-suitable BeaconChainType, using `MemoryStore`. +#[derive(Clone)] +pub struct TestnetMemoryBeaconChainTypes; + +impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { + type Store = MemoryStore; + type SlotClock = SystemTimeSlotClock; + type ForkChoice = BitwiseLMDGhost; + type EthSpec = FewValidatorsEthSpec; +} + +impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes +where + T: BeaconChainTypes< + Store = MemoryStore, + SlotClock = SystemTimeSlotClock, + ForkChoice = BitwiseLMDGhost, + >, +{ + fn initialise_beacon_chain(_config: &ClientConfig) -> BeaconChain { + initialize_chain(MemoryStore::open()) + } +} + +/// A testnet-suitable BeaconChainType, using `DiskStore`. +#[derive(Clone)] +pub struct TestnetDiskBeaconChainTypes; + +impl BeaconChainTypes for TestnetDiskBeaconChainTypes { + type Store = DiskStore; + type SlotClock = SystemTimeSlotClock; + type ForkChoice = BitwiseLMDGhost; + type EthSpec = FewValidatorsEthSpec; +} + +impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes +where + T: BeaconChainTypes< + Store = DiskStore, + SlotClock = SystemTimeSlotClock, + ForkChoice = BitwiseLMDGhost, + >, +{ + fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain { + let store = DiskStore::open(&config.db_name).expect("Unable to open DB."); + + initialize_chain(store) + } +} + +/// Produces a `BeaconChain` given some pre-initialized `Store`. +fn initialize_chain(store: U) -> BeaconChain +where + T: BeaconChainTypes< + Store = U, + SlotClock = SystemTimeSlotClock, + ForkChoice = BitwiseLMDGhost, + >, +{ + let spec = T::EthSpec::spec(); + + let store = Arc::new(store); + + let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); + let (genesis_state, _keypairs) = state_builder.build(); + + let mut genesis_block = BeaconBlock::empty(&spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); + + // Slot clock + let slot_clock = SystemTimeSlotClock::new( + spec.genesis_slot, + genesis_state.genesis_time, + spec.seconds_per_slot, + ) + .expect("Unable to load SystemTimeSlotClock"); + // Choose the fork choice + let fork_choice = BitwiseLMDGhost::new(store.clone()); + + // Genesis chain + //TODO: Handle error correctly + BeaconChain::from_genesis( + store, + slot_clock, + genesis_state, + genesis_block, + spec.clone(), + fork_choice, + ) + .expect("Terminate if beacon chain generation fails") +} diff --git a/beacon_node/client/src/client_types.rs b/beacon_node/client/src/client_types.rs deleted file mode 100644 index 4cce42a06..000000000 --- a/beacon_node/client/src/client_types.rs +++ /dev/null @@ -1,65 +0,0 @@ -use crate::{ArcBeaconChain, ClientConfig}; -use beacon_chain::{ - fork_choice::BitwiseLMDGhost, - initialise, - slot_clock::{SlotClock, SystemTimeSlotClock}, - store::{DiskStore, MemoryStore, Store}, -}; -use fork_choice::ForkChoice; -use types::{EthSpec, FewValidatorsEthSpec, FoundationEthSpec}; - -pub trait ClientTypes { - type DB: Store + 'static; - type SlotClock: SlotClock + 'static; - type ForkChoice: ForkChoice + 'static; - type EthSpec: EthSpec + 'static; - - fn initialise_beacon_chain( - config: &ClientConfig, - ) -> ArcBeaconChain; -} - -pub struct StandardClientType; - -impl ClientTypes for StandardClientType { - type DB = DiskStore; - type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; - type EthSpec = FoundationEthSpec; - - fn initialise_beacon_chain( - config: &ClientConfig, - ) -> ArcBeaconChain { - initialise::initialise_beacon_chain(&config.spec, Some(&config.db_name)) - } -} - -pub struct MemoryStoreTestingClientType; - -impl ClientTypes for MemoryStoreTestingClientType { - type DB = MemoryStore; - type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; - type EthSpec = FewValidatorsEthSpec; - - fn initialise_beacon_chain( - config: &ClientConfig, - ) -> ArcBeaconChain { - initialise::initialise_test_beacon_chain_with_memory_db(&config.spec, None) - } -} - -pub struct DiskStoreTestingClientType; - -impl ClientTypes for DiskStoreTestingClientType { - type DB = DiskStore; - type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; - type EthSpec = FewValidatorsEthSpec; - - fn initialise_beacon_chain( - config: &ClientConfig, - ) -> ArcBeaconChain { - initialise::initialise_test_beacon_chain_with_disk_db(&config.spec, Some(&config.db_name)) - } -} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 9445799d5..40be9b7b8 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -1,15 +1,13 @@ extern crate slog; +mod beacon_chain_types; mod client_config; -pub mod client_types; pub mod error; pub mod notifier; use beacon_chain::BeaconChain; -pub use client_config::{ClientConfig, DBType}; -pub use client_types::ClientTypes; +use beacon_chain_types::InitialiseBeaconChain; use exit_future::Signal; -use fork_choice::ForkChoice; use futures::{future::Future, Stream}; use network::Service as NetworkService; use slog::{error, info, o}; @@ -17,22 +15,22 @@ use slot_clock::SlotClock; use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; -use store::Store; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; -use types::EthSpec; -type ArcBeaconChain = Arc>; +pub use beacon_chain::BeaconChainTypes; +pub use beacon_chain_types::{TestnetDiskBeaconChainTypes, TestnetMemoryBeaconChainTypes}; +pub use client_config::{ClientConfig, DBType}; /// Main beacon node client service. This provides the connection and initialisation of the clients /// sub-services in multiple threads. -pub struct Client { +pub struct Client { /// Configuration for the lighthouse client. _config: ClientConfig, /// The beacon chain for the running client. - _beacon_chain: ArcBeaconChain, + _beacon_chain: Arc>, /// Reference to the network service. - pub network: Arc>, + pub network: Arc>, /// Signal to terminate the RPC server. pub rpc_exit_signal: Option, /// Signal to terminate the HTTP server. @@ -45,7 +43,10 @@ pub struct Client { phantom: PhantomData, } -impl Client { +impl Client +where + T: BeaconChainTypes + InitialiseBeaconChain + Clone + 'static, +{ /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( config: ClientConfig, @@ -53,7 +54,7 @@ impl Client { executor: &TaskExecutor, ) -> error::Result { // generate a beacon chain - let beacon_chain = TClientType::initialise_beacon_chain(&config); + let beacon_chain = Arc::new(T::initialise_beacon_chain(&config)); if beacon_chain.read_slot_clock().is_none() { panic!("Cannot start client before genesis!") @@ -158,13 +159,7 @@ impl Client { } } -fn do_state_catchup(chain: &Arc>, log: &slog::Logger) -where - T: Store, - U: SlotClock, - F: ForkChoice, - E: EthSpec, -{ +fn do_state_catchup(chain: &Arc>, log: &slog::Logger) { if let Some(genesis_height) = chain.slots_since_genesis() { let result = chain.catchup_state(); diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index aa1e43c3c..977342b1a 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -1,5 +1,5 @@ use crate::Client; -use crate::ClientTypes; +use beacon_chain::BeaconChainTypes; use exit_future::Exit; use futures::{Future, Stream}; use slog::{debug, o}; @@ -10,7 +10,11 @@ use tokio::timer::Interval; /// Thread that monitors the client and reports useful statistics to the user. -pub fn run(client: &Client, executor: TaskExecutor, exit: Exit) { +pub fn run( + client: &Client, + executor: TaskExecutor, + exit: Exit, +) { // notification heartbeat let interval = Interval::new(Instant::now(), Duration::from_secs(5)); diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index b2c3a86fc..13754980d 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -1,6 +1,6 @@ mod prometheus_handler; -use beacon_chain::BeaconChain; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; @@ -10,7 +10,6 @@ use router::Router; use slog::{info, o, warn}; use std::sync::Arc; use tokio::runtime::TaskExecutor; -use types::EthSpec; #[derive(PartialEq, Clone, Debug)] pub struct HttpServerConfig { @@ -45,15 +44,9 @@ impl Handler for IndexHandler { } } -pub fn create_iron_http_server( - beacon_chain: Arc>, -) -> Iron -where - T: store::Store + 'static, - U: slot_clock::SlotClock + 'static, - F: fork_choice::ForkChoice + 'static, - E: EthSpec + 'static, -{ +pub fn create_iron_http_server( + beacon_chain: Arc>, +) -> Iron { let index_handler = IndexHandler { message: "Hello world".to_string(), }; @@ -67,19 +60,13 @@ where Iron::new(router) } -pub fn start_service( +pub fn start_service( config: &HttpServerConfig, executor: &TaskExecutor, _network_chan: crossbeam_channel::Sender, - beacon_chain: Arc>, + beacon_chain: Arc>, log: &slog::Logger, -) -> exit_future::Signal -where - T: store::Store + 'static, - U: slot_clock::SlotClock + 'static, - F: fork_choice::ForkChoice + 'static, - E: EthSpec + 'static, -{ +) -> exit_future::Signal { let log = log.new(o!("Service"=>"HTTP")); // Create: diff --git a/beacon_node/http_server/src/prometheus_handler.rs b/beacon_node/http_server/src/prometheus_handler.rs index ae84d4871..60f56084c 100644 --- a/beacon_node/http_server/src/prometheus_handler.rs +++ b/beacon_node/http_server/src/prometheus_handler.rs @@ -1,22 +1,17 @@ -use beacon_chain::BeaconChain; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::{status::Status, Handler, IronResult, Request, Response}; use prometheus::{Encoder, IntCounter, Opts, Registry, TextEncoder}; +use slot_clock::SlotClock; use std::sync::Arc; -use types::{EthSpec, Slot}; +use types::Slot; -pub struct PrometheusHandler { - pub beacon_chain: Arc>, +pub struct PrometheusHandler { + pub beacon_chain: Arc>, } -impl PrometheusHandler where E: EthSpec {} +impl PrometheusHandler {} -impl Handler for PrometheusHandler -where - E: EthSpec + 'static, - U: slot_clock::SlotClock + Send + Sync + 'static, - T: Send + Sync + 'static, - F: Send + Sync + 'static, -{ +impl Handler for PrometheusHandler { fn handle(&self, _: &mut Request) -> IronResult { let r = Registry::new(); diff --git a/beacon_node/network/src/beacon_chain.rs b/beacon_node/network/src/beacon_chain.rs index 2a42376f7..6324e3a94 100644 --- a/beacon_node/network/src/beacon_chain.rs +++ b/beacon_node/network/src/beacon_chain.rs @@ -1,9 +1,6 @@ use beacon_chain::BeaconChain as RawBeaconChain; use beacon_chain::{ - fork_choice::ForkChoice, parking_lot::RwLockReadGuard, - slot_clock::SlotClock, - store::Store, types::{BeaconState, ChainSpec}, AttestationValidationError, CheckPoint, }; @@ -12,17 +9,17 @@ use types::{ Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot, }; -pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome, InvalidBlock}; +pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; /// The network's API to the beacon chain. -pub trait BeaconChain: Send + Sync { +pub trait BeaconChain: Send + Sync { fn get_spec(&self) -> &ChainSpec; - fn get_state(&self) -> RwLockReadGuard>; + fn get_state(&self) -> RwLockReadGuard>; fn slot(&self) -> Slot; - fn head(&self) -> RwLockReadGuard>; + fn head(&self) -> RwLockReadGuard>; fn get_block(&self, block_root: &Hash256) -> Result, BeaconChainError>; @@ -30,7 +27,7 @@ pub trait BeaconChain: Send + Sync { fn best_block_root(&self) -> Hash256; - fn finalized_head(&self) -> RwLockReadGuard>; + fn finalized_head(&self) -> RwLockReadGuard>; fn finalized_epoch(&self) -> Epoch; @@ -64,18 +61,12 @@ pub trait BeaconChain: Send + Sync { fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result; } -impl BeaconChain for RawBeaconChain -where - T: Store, - U: SlotClock, - F: ForkChoice, - E: EthSpec, -{ +impl BeaconChain for RawBeaconChain { fn get_spec(&self) -> &ChainSpec { &self.spec } - fn get_state(&self) -> RwLockReadGuard> { + fn get_state(&self) -> RwLockReadGuard> { self.state.read() } @@ -83,7 +74,7 @@ where self.get_state().slot } - fn head(&self) -> RwLockReadGuard> { + fn head(&self) -> RwLockReadGuard> { self.head() } @@ -95,7 +86,7 @@ where self.get_state().finalized_epoch } - fn finalized_head(&self) -> RwLockReadGuard> { + fn finalized_head(&self) -> RwLockReadGuard> { self.finalized_head() } diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index a7d0ff2a1..f6a27ad60 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use crate::error; use crate::service::{NetworkMessage, OutgoingMessage}; use crate::sync::SimpleSync; @@ -13,7 +13,6 @@ use slog::{debug, warn}; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; -use types::EthSpec; /// Timeout for RPC requests. // const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); @@ -21,11 +20,11 @@ use types::EthSpec; // const HELLO_TIMEOUT: Duration = Duration::from_secs(30); /// Handles messages received from the network and client and organises syncing. -pub struct MessageHandler { +pub struct MessageHandler { /// Currently loaded and initialised beacon chain. - _chain: Arc>, + _chain: Arc>, /// The syncing framework. - sync: SimpleSync, + sync: SimpleSync, /// The context required to send messages to, and process messages from peers. network_context: NetworkContext, /// The `MessageHandler` logger. @@ -45,10 +44,10 @@ pub enum HandlerMessage { PubsubMessage(PeerId, Box), } -impl MessageHandler { +impl MessageHandler { /// Initializes and runs the MessageHandler. pub fn spawn( - beacon_chain: Arc>, + beacon_chain: Arc>, network_send: crossbeam_channel::Sender, executor: &tokio::runtime::TaskExecutor, log: slog::Logger, diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 50454a875..d87b9e5a9 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use crate::error; use crate::message_handler::{HandlerMessage, MessageHandler}; use crate::NetworkConfig; @@ -13,20 +13,20 @@ use slog::{debug, info, o, trace}; use std::marker::PhantomData; use std::sync::Arc; use tokio::runtime::TaskExecutor; -use types::{EthSpec, Topic}; +use types::Topic; /// Service that handles communication between internal services and the eth2_libp2p network service. -pub struct Service { +pub struct Service { //libp2p_service: Arc>, _libp2p_exit: oneshot::Sender<()>, network_send: crossbeam_channel::Sender, - _phantom: PhantomData, //message_handler: MessageHandler, + _phantom: PhantomData, //message_handler: MessageHandler, //message_handler_send: Sender } -impl Service { +impl Service { pub fn new( - beacon_chain: Arc>, + beacon_chain: Arc>, config: &NetworkConfig, executor: &TaskExecutor, log: slog::Logger, diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 6c2fc33ee..793f4c395 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -1,11 +1,11 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::PeerId; use slog::{debug, error}; use std::sync::Arc; use std::time::{Duration, Instant}; use tree_hash::TreeHash; -use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, EthSpec, Hash256, Slot}; +use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot}; /// Provides a queue for fully and partially built `BeaconBlock`s. /// @@ -19,8 +19,8 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, EthSpec, Hash256, S /// `BeaconBlockBody` as the key. /// - It is possible for multiple distinct blocks to have identical `BeaconBlockBodies`. Therefore /// we cannot use a `HashMap` keyed by the root of `BeaconBlockBody`. -pub struct ImportQueue { - pub chain: Arc>, +pub struct ImportQueue { + pub chain: Arc>, /// Partially imported blocks, keyed by the root of `BeaconBlockBody`. pub partials: Vec, /// Time before a queue entry is considered state. @@ -29,9 +29,9 @@ pub struct ImportQueue { log: slog::Logger, } -impl ImportQueue { +impl ImportQueue { /// Return a new, empty queue. - pub fn new(chain: Arc>, stale_time: Duration, log: slog::Logger) -> Self { + pub fn new(chain: Arc>, stale_time: Duration, log: slog::Logger) -> Self { Self { chain, partials: vec![], diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index d44ffd4b7..6ab8ea7d9 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,5 +1,5 @@ use super::import_queue::ImportQueue; -use crate::beacon_chain::{BeaconChain, BlockProcessingOutcome, InvalidBlock}; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; use crate::message_handler::NetworkContext; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; @@ -9,7 +9,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tree_hash::TreeHash; -use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot}; +use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot}; /// The number of slots that we can import blocks ahead of us, before going into full Sync mode. const SLOT_IMPORT_TOLERANCE: u64 = 100; @@ -88,8 +88,8 @@ impl From for PeerSyncInfo { } } -impl From<&Arc>> for PeerSyncInfo { - fn from(chain: &Arc>) -> PeerSyncInfo { +impl From<&Arc>> for PeerSyncInfo { + fn from(chain: &Arc>) -> PeerSyncInfo { Self::from(chain.hello_message()) } } @@ -103,22 +103,22 @@ pub enum SyncState { } /// Simple Syncing protocol. -pub struct SimpleSync { +pub struct SimpleSync { /// A reference to the underlying beacon chain. - chain: Arc>, + chain: Arc>, /// A mapping of Peers to their respective PeerSyncInfo. known_peers: HashMap, /// A queue to allow importing of blocks - import_queue: ImportQueue, + import_queue: ImportQueue, /// The current state of the syncing protocol. state: SyncState, /// Sync logger. log: slog::Logger, } -impl SimpleSync { +impl SimpleSync { /// Instantiate a `SimpleSync` instance, with no peers and an empty queue. - pub fn new(beacon_chain: Arc>, log: &slog::Logger) -> Self { + pub fn new(beacon_chain: Arc>, log: &slog::Logger) -> Self { let sync_logger = log.new(o!("Service"=> "Sync")); let queue_item_stale_time = Duration::from_secs(QUEUE_STALE_SECS); diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index e22715b55..6048e42b1 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use protos::services::{ @@ -9,15 +9,15 @@ use protos::services_grpc::AttestationService; use slog::{error, info, trace, warn}; use ssz::{ssz_encode, Decode}; use std::sync::Arc; -use types::{Attestation, EthSpec}; +use types::Attestation; #[derive(Clone)] -pub struct AttestationServiceInstance { - pub chain: Arc>, +pub struct AttestationServiceInstance { + pub chain: Arc>, pub log: slog::Logger, } -impl AttestationService for AttestationServiceInstance { +impl AttestationService for AttestationServiceInstance { /// Produce the `AttestationData` for signing by a validator. fn produce_attestation_data( &mut self, diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index bbe6a8ee2..e553b79e7 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use crossbeam_channel; use eth2_libp2p::PubsubMessage; use futures::Future; @@ -13,16 +13,16 @@ use slog::Logger; use slog::{error, info, trace, warn}; use ssz::{ssz_encode, Decode}; use std::sync::Arc; -use types::{BeaconBlock, EthSpec, Signature, Slot}; +use types::{BeaconBlock, Signature, Slot}; #[derive(Clone)] -pub struct BeaconBlockServiceInstance { - pub chain: Arc>, +pub struct BeaconBlockServiceInstance { + pub chain: Arc>, pub network_chan: crossbeam_channel::Sender, pub log: Logger, } -impl BeaconBlockService for BeaconBlockServiceInstance { +impl BeaconBlockService for BeaconBlockServiceInstance { /// Produce a `BeaconBlock` for signing by a validator. fn produce_beacon_block( &mut self, diff --git a/beacon_node/rpc/src/beacon_chain.rs b/beacon_node/rpc/src/beacon_chain.rs index d12baf1d1..b0a490137 100644 --- a/beacon_node/rpc/src/beacon_chain.rs +++ b/beacon_node/rpc/src/beacon_chain.rs @@ -1,22 +1,19 @@ use beacon_chain::BeaconChain as RawBeaconChain; use beacon_chain::{ - fork_choice::ForkChoice, parking_lot::{RwLockReadGuard, RwLockWriteGuard}, - slot_clock::SlotClock, - store::Store, types::{BeaconState, ChainSpec, Signature}, AttestationValidationError, BlockProductionError, }; -pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome}; +pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome}; use types::{Attestation, AttestationData, BeaconBlock, EthSpec}; /// The RPC's API to the beacon chain. -pub trait BeaconChain: Send + Sync { +pub trait BeaconChain: Send + Sync { fn get_spec(&self) -> &ChainSpec; - fn get_state(&self) -> RwLockReadGuard>; + fn get_state(&self) -> RwLockReadGuard>; - fn get_mut_state(&self) -> RwLockWriteGuard>; + fn get_mut_state(&self) -> RwLockWriteGuard>; fn process_block(&self, block: BeaconBlock) -> Result; @@ -24,7 +21,7 @@ pub trait BeaconChain: Send + Sync { fn produce_block( &self, randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), BlockProductionError>; + ) -> Result<(BeaconBlock, BeaconState), BlockProductionError>; fn produce_attestation_data(&self, shard: u64) -> Result; @@ -34,22 +31,16 @@ pub trait BeaconChain: Send + Sync { ) -> Result<(), AttestationValidationError>; } -impl BeaconChain for RawBeaconChain -where - T: Store, - U: SlotClock, - F: ForkChoice, - E: EthSpec, -{ +impl BeaconChain for RawBeaconChain { fn get_spec(&self) -> &ChainSpec { &self.spec } - fn get_state(&self) -> RwLockReadGuard> { + fn get_state(&self) -> RwLockReadGuard> { self.state.read() } - fn get_mut_state(&self) -> RwLockWriteGuard> { + fn get_mut_state(&self) -> RwLockWriteGuard> { self.state.write() } @@ -63,7 +54,7 @@ where fn produce_block( &self, randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { + ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { self.produce_block(randao_reveal) } diff --git a/beacon_node/rpc/src/beacon_node.rs b/beacon_node/rpc/src/beacon_node.rs index 2ca39ae51..a923bbb35 100644 --- a/beacon_node/rpc/src/beacon_node.rs +++ b/beacon_node/rpc/src/beacon_node.rs @@ -1,19 +1,18 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use grpcio::{RpcContext, UnarySink}; use protos::services::{Empty, Fork, NodeInfoResponse}; use protos::services_grpc::BeaconNodeService; use slog::{trace, warn}; use std::sync::Arc; -use types::EthSpec; #[derive(Clone)] -pub struct BeaconNodeServiceInstance { - pub chain: Arc>, +pub struct BeaconNodeServiceInstance { + pub chain: Arc>, pub log: slog::Logger, } -impl BeaconNodeService for BeaconNodeServiceInstance { +impl BeaconNodeService for BeaconNodeServiceInstance { /// Provides basic node information. fn info(&mut self, ctx: RpcContext, _req: Empty, sink: UnarySink) { trace!(self.log, "Node info requested via RPC"); diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index f1d5e9c88..9646135b6 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -7,7 +7,7 @@ mod validator; use self::attestation::AttestationServiceInstance; use self::beacon_block::BeaconBlockServiceInstance; -use self::beacon_chain::BeaconChain; +use self::beacon_chain::{BeaconChain, BeaconChainTypes}; use self::beacon_node::BeaconNodeServiceInstance; use self::validator::ValidatorServiceInstance; pub use config::Config as RPCConfig; @@ -21,13 +21,12 @@ use protos::services_grpc::{ use slog::{info, o, warn}; use std::sync::Arc; use tokio::runtime::TaskExecutor; -use types::EthSpec; -pub fn start_server( +pub fn start_server( config: &RPCConfig, executor: &TaskExecutor, network_chan: crossbeam_channel::Sender, - beacon_chain: Arc>, + beacon_chain: Arc>, log: &slog::Logger, ) -> exit_future::Signal { let log = log.new(o!("Service"=>"RPC")); diff --git a/beacon_node/rpc/src/validator.rs b/beacon_node/rpc/src/validator.rs index 34fbba5c4..e58c202d6 100644 --- a/beacon_node/rpc/src/validator.rs +++ b/beacon_node/rpc/src/validator.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::BeaconChain; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; @@ -7,16 +7,16 @@ use protos::services_grpc::ValidatorService; use slog::{trace, warn}; use ssz::Decode; use std::sync::Arc; -use types::{Epoch, EthSpec, RelativeEpoch}; +use types::{Epoch, RelativeEpoch}; #[derive(Clone)] -pub struct ValidatorServiceInstance { - pub chain: Arc>, +pub struct ValidatorServiceInstance { + pub chain: Arc>, pub log: slog::Logger, } //TODO: Refactor Errors -impl ValidatorService for ValidatorServiceInstance { +impl ValidatorService for ValidatorServiceInstance { /// For a list of validator public keys, this function returns the slot at which each /// validator must propose a block, attest to a shard, their shard committee and the shard they /// need to attest to. diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 4cf930060..6ec65a92d 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,6 +1,7 @@ -use client::client_types::{DiskStoreTestingClientType, MemoryStoreTestingClientType}; -use client::{error, DBType}; -use client::{notifier, Client, ClientConfig, ClientTypes}; +use client::{ + error, notifier, BeaconChainTypes, Client, ClientConfig, DBType, TestnetDiskBeaconChainTypes, + TestnetMemoryBeaconChainTypes, +}; use futures::sync::oneshot; use futures::Future; use slog::info; @@ -29,9 +30,9 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul info!( log, "BeaconNode starting"; - "type" => "DiskStoreTestingClientType" + "type" => "TestnetDiskBeaconChainTypes" ); - let client: Client = + let client: Client = Client::new(config, log.clone(), &executor)?; run(client, executor, runtime, log) @@ -40,9 +41,9 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul info!( log, "BeaconNode starting"; - "type" => "MemoryStoreTestingClientType" + "type" => "TestnetMemoryBeaconChainTypes" ); - let client: Client = + let client: Client = Client::new(config, log.clone(), &executor)?; run(client, executor, runtime, log) @@ -50,7 +51,7 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul } } -pub fn run( +pub fn run( client: Client, executor: TaskExecutor, mut runtime: Runtime, From 855222fa28b291c9b0e3335e33c05f9b2dfd5f58 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 26 May 2019 15:59:52 +1000 Subject: [PATCH 39/44] Rename prom HTTP endpoint --- beacon_node/http_server/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 13754980d..fe7b9b1b3 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -56,7 +56,7 @@ pub fn create_iron_http_server( let mut router = Router::new(); router.get("/", index_handler, "index"); - router.get("/prometheus/", prom_handler, "prometheus"); + router.get("/metrics", prom_handler, "metrics"); Iron::new(router) } From 705edf0e455b2d43c1151a4046a6ac7607893f8e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 09:01:50 +1000 Subject: [PATCH 40/44] Remove commented-out code --- beacon_node/http_server/src/lib.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index fe7b9b1b3..6bc69e72d 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -15,10 +15,6 @@ use tokio::runtime::TaskExecutor; pub struct HttpServerConfig { pub enabled: bool, pub listen_address: String, - /* - pub listen_address: Ipv4Addr, - pub port: u16, - */ } impl Default for HttpServerConfig { @@ -26,10 +22,6 @@ impl Default for HttpServerConfig { Self { enabled: false, listen_address: "127.0.0.1:5051".to_string(), - /* - listen_address: Ipv4Addr::new(127, 0, 0, 1), - port: 5051, - */ } } } From eb83fd48a8c811bf84c8cde63d074d7eb742f6b0 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 27 May 2019 10:24:44 +1000 Subject: [PATCH 41/44] Update readme with donation ens --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4dd19be7b..2151a0db8 100644 --- a/README.md +++ b/README.md @@ -176,4 +176,4 @@ Ping @paulhauner or @AgeManning to get the quickest response. If you support the cause, we could certainly use donations to help fund development: -`0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b` +`0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b` (donation.sigmaprime.eth) From 255590ef3bca45d8d2306b707a22c58d01858c63 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 11:34:22 +1000 Subject: [PATCH 42/44] Add `node/fork` endpoint to HTTP API, tidy --- beacon_node/http_server/Cargo.toml | 4 ++ beacon_node/http_server/src/api.rs | 69 +++++++++++++++++++ beacon_node/http_server/src/key.rs | 12 ++++ beacon_node/http_server/src/lib.rs | 38 +++++----- beacon_node/http_server/src/metrics.rs | 57 +++++++++++++++ .../http_server/src/prometheus_handler.rs | 47 ------------- 6 files changed, 158 insertions(+), 69 deletions(-) create mode 100644 beacon_node/http_server/src/api.rs create mode 100644 beacon_node/http_server/src/key.rs create mode 100644 beacon_node/http_server/src/metrics.rs delete mode 100644 beacon_node/http_server/src/prometheus_handler.rs diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml index 6f4579d17..fb8bf9f4b 100644 --- a/beacon_node/http_server/Cargo.toml +++ b/beacon_node/http_server/Cargo.toml @@ -18,12 +18,16 @@ slot_clock = { path = "../../eth2/utils/slot_clock" } protos = { path = "../../protos" } fork_choice = { path = "../../eth2/fork_choice" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } +persistent = "^0.4" protobuf = "2.0.2" prometheus = "^0.6" clap = "2.32.0" store = { path = "../store" } dirs = "1.0.3" futures = "0.1.23" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" slog = "^2.2.3" slog-term = "^2.4.0" slog-async = "^2.3.0" diff --git a/beacon_node/http_server/src/api.rs b/beacon_node/http_server/src/api.rs new file mode 100644 index 000000000..c89cacd9a --- /dev/null +++ b/beacon_node/http_server/src/api.rs @@ -0,0 +1,69 @@ +use crate::key::BeaconChainKey; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use iron::prelude::*; +use iron::{ + headers::{CacheControl, CacheDirective, ContentType}, + status::Status, + AfterMiddleware, Handler, IronResult, Request, Response, +}; +use persistent::Read; +use router::Router; +use serde_json::json; +use std::sync::Arc; + +pub fn build_handler( + beacon_chain: Arc>, +) -> impl Handler { + let mut router = Router::new(); + + router.get("/node/fork", handle_fork::, "fork"); + + let mut chain = Chain::new(router); + + // Insert `BeaconChain` so it may be accessed in a request. + chain.link(Read::>::both(beacon_chain.clone())); + // Set the content-type headers. + chain.link_after(SetJsonContentType); + // Set the cache headers. + chain.link_after(SetCacheDirectives); + + chain +} + +/// Sets the `cache-control` headers on _all_ responses, unless they are already set. +struct SetCacheDirectives; +impl AfterMiddleware for SetCacheDirectives { + fn after(&self, _req: &mut Request, mut resp: Response) -> IronResult { + // This is run for every requests, AFTER all handlers have been executed + if resp.headers.get::() == None { + resp.headers.set(CacheControl(vec![ + CacheDirective::NoCache, + CacheDirective::NoStore, + ])); + } + Ok(resp) + } +} + +/// Sets the `content-type` headers on _all_ responses, unless they are already set. +struct SetJsonContentType; +impl AfterMiddleware for SetJsonContentType { + fn after(&self, _req: &mut Request, mut resp: Response) -> IronResult { + if resp.headers.get::() == None { + resp.headers.set(ContentType::json()); + } + Ok(resp) + } +} + +fn handle_fork(req: &mut Request) -> IronResult { + // TODO: investigate unwrap - I'm _guessing_ we'll never hit it but we should check to be sure. + let beacon_chain = req.get::>>().unwrap(); + + let response = json!({ + "fork": beacon_chain.head().beacon_state.fork, + "chain_id": beacon_chain.spec.chain_id + }); + + Ok(Response::with((Status::Ok, response.to_string()))) +} diff --git a/beacon_node/http_server/src/key.rs b/beacon_node/http_server/src/key.rs new file mode 100644 index 000000000..2d27ce9f0 --- /dev/null +++ b/beacon_node/http_server/src/key.rs @@ -0,0 +1,12 @@ +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use iron::typemap::Key; +use std::marker::PhantomData; +use std::sync::Arc; + +pub struct BeaconChainKey { + _phantom: PhantomData, +} + +impl Key for BeaconChainKey { + type Value = Arc>; +} diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 6bc69e72d..8f6378e9a 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -1,11 +1,11 @@ -mod prometheus_handler; +mod api; +mod key; +mod metrics; use beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use iron::prelude::*; -use iron::{status::Status, Handler, IronResult, Request, Response}; use network::NetworkMessage; -use prometheus_handler::PrometheusHandler; use router::Router; use slog::{info, o, warn}; use std::sync::Arc; @@ -26,32 +26,26 @@ impl Default for HttpServerConfig { } } -pub struct IndexHandler { - message: String, -} - -impl Handler for IndexHandler { - fn handle(&self, _: &mut Request) -> IronResult { - Ok(Response::with((Status::Ok, self.message.clone()))) - } -} - +/// Build the `iron` HTTP server, defining the core routes. pub fn create_iron_http_server( beacon_chain: Arc>, ) -> Iron { - let index_handler = IndexHandler { - message: "Hello world".to_string(), - }; - let prom_handler = PrometheusHandler { - beacon_chain: beacon_chain, - }; - let mut router = Router::new(); - router.get("/", index_handler, "index"); - router.get("/metrics", prom_handler, "metrics"); + + // A `GET` request to `/metrics` is handled by the `metrics` module. + router.get( + "/metrics", + metrics::build_handler(beacon_chain.clone()), + "metrics", + ); + + // Any request to all other endpoints is handled by the `api` module. + router.any("/*", api::build_handler(beacon_chain.clone()), "api"); + Iron::new(router) } +/// Start the HTTP service on the tokio `TaskExecutor`. pub fn start_service( config: &HttpServerConfig, executor: &TaskExecutor, diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs new file mode 100644 index 000000000..57fa70623 --- /dev/null +++ b/beacon_node/http_server/src/metrics.rs @@ -0,0 +1,57 @@ +use crate::key::BeaconChainKey; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use iron::prelude::*; +use iron::{status::Status, Handler, IronResult, Request, Response}; +use persistent::Read; +use prometheus::{Encoder, IntCounter, Opts, Registry, TextEncoder}; +use slot_clock::SlotClock; +use std::sync::Arc; +use types::Slot; + +pub fn build_handler( + beacon_chain: Arc>, +) -> impl Handler { + let mut chain = Chain::new(handle_metrics::); + + chain.link(Read::>::both(beacon_chain)); + + chain +} + +/// Handle a request for Prometheus metrics. +/// +/// Returns a text string containing all metrics. +fn handle_metrics(req: &mut Request) -> IronResult { + let beacon_chain = req.get::>>().unwrap(); + + let r = Registry::new(); + + let present_slot = if let Ok(Some(slot)) = beacon_chain.slot_clock.present_slot() { + slot + } else { + Slot::new(0) + }; + register_and_set_slot( + &r, + "present_slot", + "direct_slock_clock_reading", + present_slot, + ); + + // Gather the metrics. + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + let metric_families = r.gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + let prom_string = String::from_utf8(buffer).unwrap(); + + Ok(Response::with((Status::Ok, prom_string))) +} + +fn register_and_set_slot(registry: &Registry, name: &str, help: &str, slot: Slot) { + let counter_opts = Opts::new(name, help); + let counter = IntCounter::with_opts(counter_opts).unwrap(); + registry.register(Box::new(counter.clone())).unwrap(); + counter.inc_by(slot.as_u64() as i64); +} diff --git a/beacon_node/http_server/src/prometheus_handler.rs b/beacon_node/http_server/src/prometheus_handler.rs deleted file mode 100644 index 60f56084c..000000000 --- a/beacon_node/http_server/src/prometheus_handler.rs +++ /dev/null @@ -1,47 +0,0 @@ -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use iron::{status::Status, Handler, IronResult, Request, Response}; -use prometheus::{Encoder, IntCounter, Opts, Registry, TextEncoder}; -use slot_clock::SlotClock; -use std::sync::Arc; -use types::Slot; - -pub struct PrometheusHandler { - pub beacon_chain: Arc>, -} - -impl PrometheusHandler {} - -impl Handler for PrometheusHandler { - fn handle(&self, _: &mut Request) -> IronResult { - let r = Registry::new(); - - let present_slot = if let Ok(Some(slot)) = self.beacon_chain.slot_clock.present_slot() { - slot - } else { - Slot::new(0) - }; - register_and_set_slot( - &r, - "present_slot", - "direct_slock_clock_reading", - present_slot, - ); - - // Gather the metrics. - let mut buffer = vec![]; - let encoder = TextEncoder::new(); - let metric_families = r.gather(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let prom_string = String::from_utf8(buffer).unwrap(); - - Ok(Response::with((Status::Ok, prom_string))) - } -} - -fn register_and_set_slot(registry: &Registry, name: &str, help: &str, slot: Slot) { - let counter_opts = Opts::new(name, help); - let counter = IntCounter::with_opts(counter_opts).unwrap(); - registry.register(Box::new(counter.clone())).unwrap(); - counter.inc_by(slot.as_u64() as i64); -} From 3a65f84b129100d405d7e441a29a1d4659517704 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 12:56:09 +1000 Subject: [PATCH 43/44] Improve comments for `http_server` --- beacon_node/http_server/src/api.rs | 1 + beacon_node/http_server/src/lib.rs | 2 +- beacon_node/http_server/src/metrics.rs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_node/http_server/src/api.rs b/beacon_node/http_server/src/api.rs index c89cacd9a..11afdb9df 100644 --- a/beacon_node/http_server/src/api.rs +++ b/beacon_node/http_server/src/api.rs @@ -11,6 +11,7 @@ use router::Router; use serde_json::json; use std::sync::Arc; +/// Yields a handler for the HTTP API. pub fn build_handler( beacon_chain: Arc>, ) -> impl Handler { diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 8f6378e9a..77665db8d 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -87,7 +87,7 @@ pub fn start_service( info!(log, "HTTP server shutting down"); if let Ok(mut server) = server_start_result { - // According to the documentation, this function "doesn't work" and the server + // According to the documentation, `server.close()` "doesn't work" and the server // keeps listening. // // It is being called anyway, because it seems like the right thing to do. If you diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 57fa70623..30acb8853 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -8,6 +8,7 @@ use slot_clock::SlotClock; use std::sync::Arc; use types::Slot; +/// Yields a handler for the metrics endpoint. pub fn build_handler( beacon_chain: Arc>, ) -> impl Handler { From ed4d7aa44a03d32cceaabc26c1e4017a6da4987e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 17:09:16 +1000 Subject: [PATCH 44/44] Replace `http_server` unwrap with 500 error --- beacon_node/http_server/src/api.rs | 7 ++++--- beacon_node/http_server/src/lib.rs | 8 ++++++++ beacon_node/http_server/src/metrics.rs | 6 ++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/beacon_node/http_server/src/api.rs b/beacon_node/http_server/src/api.rs index 11afdb9df..a91080899 100644 --- a/beacon_node/http_server/src/api.rs +++ b/beacon_node/http_server/src/api.rs @@ -1,4 +1,4 @@ -use crate::key::BeaconChainKey; +use crate::{key::BeaconChainKey, map_persistent_err_to_500}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::prelude::*; use iron::{ @@ -58,8 +58,9 @@ impl AfterMiddleware for SetJsonContentType { } fn handle_fork(req: &mut Request) -> IronResult { - // TODO: investigate unwrap - I'm _guessing_ we'll never hit it but we should check to be sure. - let beacon_chain = req.get::>>().unwrap(); + let beacon_chain = req + .get::>>() + .map_err(map_persistent_err_to_500)?; let response = json!({ "fork": beacon_chain.head().beacon_state.fork, diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 77665db8d..486badaff 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -108,3 +108,11 @@ pub fn start_service( shutdown_trigger } + +/// Helper function for mapping a failure to read state to a 500 server error. +fn map_persistent_err_to_500(e: persistent::PersistentError) -> iron::error::IronError { + iron::error::IronError { + error: Box::new(e), + response: iron::Response::with(iron::status::Status::InternalServerError), + } +} diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 30acb8853..eb7816d0e 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -1,4 +1,4 @@ -use crate::key::BeaconChainKey; +use crate::{key::BeaconChainKey, map_persistent_err_to_500}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; @@ -23,7 +23,9 @@ pub fn build_handler( /// /// Returns a text string containing all metrics. fn handle_metrics(req: &mut Request) -> IronResult { - let beacon_chain = req.get::>>().unwrap(); + let beacon_chain = req + .get::>>() + .map_err(map_persistent_err_to_500)?; let r = Registry::new();