2019-05-21 08:49:24 +00:00
|
|
|
//! Storage functionality for Lighthouse.
|
|
|
|
//!
|
|
|
|
//! Provides the following stores:
|
|
|
|
//!
|
|
|
|
//! - `DiskStore`: an on-disk store backed by leveldb. Used in production.
|
|
|
|
//! - `MemoryStore`: an in-memory store backed by a hash-map. Used for testing.
|
|
|
|
//!
|
|
|
|
//! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See
|
|
|
|
//! tests for implementation examples.
|
2019-08-19 11:02:34 +00:00
|
|
|
#[macro_use]
|
|
|
|
extern crate lazy_static;
|
2019-05-21 08:49:24 +00:00
|
|
|
|
2019-12-06 07:52:11 +00:00
|
|
|
pub mod chunked_iter;
|
2019-11-26 23:54:46 +00:00
|
|
|
pub mod chunked_vector;
|
2019-12-06 03:29:06 +00:00
|
|
|
pub mod config;
|
2019-05-20 08:01:51 +00:00
|
|
|
mod errors;
|
2019-12-06 07:52:11 +00:00
|
|
|
mod forwards_iter;
|
2020-04-20 09:59:56 +00:00
|
|
|
pub mod hot_cold_store;
|
2019-05-20 08:01:51 +00:00
|
|
|
mod impls;
|
2019-05-21 06:29:34 +00:00
|
|
|
mod leveldb_store;
|
2019-05-21 08:49:24 +00:00
|
|
|
mod memory_store;
|
2019-08-19 11:02:34 +00:00
|
|
|
mod metrics;
|
2019-11-26 23:54:46 +00:00
|
|
|
mod partial_beacon_state;
|
2020-02-10 00:30:21 +00:00
|
|
|
mod state_batch;
|
2019-02-14 01:09:18 +00:00
|
|
|
|
2019-06-15 13:56:41 +00:00
|
|
|
pub mod iter;
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
use std::sync::Arc;
|
|
|
|
|
2019-12-06 03:29:06 +00:00
|
|
|
pub use self::config::StoreConfig;
|
2020-02-10 00:30:21 +00:00
|
|
|
pub use self::hot_cold_store::{HotColdDB as DiskStore, HotStateSummary};
|
2019-11-26 23:54:46 +00:00
|
|
|
pub use self::leveldb_store::LevelDB as SimpleDiskStore;
|
2019-05-21 08:49:24 +00:00
|
|
|
pub use self::memory_store::MemoryStore;
|
2019-11-26 23:54:46 +00:00
|
|
|
pub use self::partial_beacon_state::PartialBeaconState;
|
2019-05-20 08:01:51 +00:00
|
|
|
pub use errors::Error;
|
2019-12-06 05:44:03 +00:00
|
|
|
pub use impls::beacon_state::StorageContainer as BeaconStateStorageContainer;
|
2019-08-19 11:02:34 +00:00
|
|
|
pub use metrics::scrape_for_metrics;
|
2020-02-10 00:30:21 +00:00
|
|
|
pub use state_batch::StateBatch;
|
2019-05-20 08:01:51 +00:00
|
|
|
pub use types::*;
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// An object capable of storing and retrieving objects implementing `StoreItem`.
|
|
|
|
///
|
|
|
|
/// A `Store` is fundamentally backed by a key-value database, however it provides support for
|
|
|
|
/// columns. A simple column implementation might involve prefixing a key with some bytes unique to
|
|
|
|
/// each column.
|
2019-12-06 07:52:11 +00:00
|
|
|
pub trait Store<E: EthSpec>: Sync + Send + Sized + 'static {
|
|
|
|
type ForwardsBlockRootsIterator: Iterator<Item = (Hash256, Slot)>;
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
/// Retrieve some bytes in `column` with `key`.
|
|
|
|
fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
|
|
|
|
|
|
|
|
/// Store some `value` in `column`, indexed with `key`.
|
|
|
|
fn put_bytes(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>;
|
|
|
|
|
|
|
|
/// Return `true` if `key` exists in `column`.
|
|
|
|
fn key_exists(&self, column: &str, key: &[u8]) -> Result<bool, Error>;
|
|
|
|
|
|
|
|
/// Removes `key` from `column`.
|
|
|
|
fn key_delete(&self, column: &str, key: &[u8]) -> Result<(), Error>;
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// Store an item in `Self`.
|
2020-05-25 00:26:54 +00:00
|
|
|
fn put<I: SimpleStoreItem>(&self, key: &Hash256, item: &I) -> Result<(), Error> {
|
|
|
|
let column = I::db_column().into();
|
|
|
|
let key = key.as_bytes();
|
|
|
|
|
|
|
|
self.put_bytes(column, key, &item.as_store_bytes())
|
|
|
|
.map_err(Into::into)
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// Retrieve an item from `Self`.
|
2020-05-25 00:26:54 +00:00
|
|
|
fn get<I: SimpleStoreItem>(&self, key: &Hash256) -> Result<Option<I>, Error> {
|
|
|
|
let column = I::db_column().into();
|
|
|
|
let key = key.as_bytes();
|
|
|
|
|
|
|
|
match self.get_bytes(column, key)? {
|
|
|
|
Some(bytes) => Ok(Some(I::from_store_bytes(&bytes[..])?)),
|
|
|
|
None => Ok(None),
|
|
|
|
}
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// Returns `true` if the given key represents an item in `Self`.
|
2020-05-25 00:26:54 +00:00
|
|
|
fn exists<I: SimpleStoreItem>(&self, key: &Hash256) -> Result<bool, Error> {
|
|
|
|
let column = I::db_column().into();
|
|
|
|
let key = key.as_bytes();
|
|
|
|
|
|
|
|
self.key_exists(column, key)
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// Remove an item from `Self`.
|
2020-05-25 00:26:54 +00:00
|
|
|
fn delete<I: SimpleStoreItem>(&self, key: &Hash256) -> Result<(), Error> {
|
|
|
|
let column = I::db_column().into();
|
|
|
|
let key = key.as_bytes();
|
|
|
|
|
|
|
|
self.key_delete(column, key)
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
2020-02-10 00:30:21 +00:00
|
|
|
/// Store a block in the store.
|
2020-02-10 23:19:36 +00:00
|
|
|
fn put_block(&self, block_root: &Hash256, block: SignedBeaconBlock<E>) -> Result<(), Error> {
|
2020-02-10 00:30:21 +00:00
|
|
|
self.put(block_root, &block)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Fetch a block from the store.
|
2020-02-10 23:19:36 +00:00
|
|
|
fn get_block(&self, block_root: &Hash256) -> Result<Option<SignedBeaconBlock<E>>, Error> {
|
2020-02-10 00:30:21 +00:00
|
|
|
self.get(block_root)
|
|
|
|
}
|
|
|
|
|
2020-03-04 05:48:35 +00:00
|
|
|
/// Delete a block from the store.
|
|
|
|
fn delete_block(&self, block_root: &Hash256) -> Result<(), Error> {
|
2020-05-16 03:23:32 +00:00
|
|
|
self.key_delete(DBColumn::BeaconBlock.into(), block_root.as_bytes())
|
2020-03-04 05:48:35 +00:00
|
|
|
}
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
/// Store a state in the store.
|
2020-04-06 00:53:33 +00:00
|
|
|
fn put_state(&self, state_root: &Hash256, state: &BeaconState<E>) -> Result<(), Error>;
|
2020-02-10 00:30:21 +00:00
|
|
|
|
2020-05-16 03:23:32 +00:00
|
|
|
/// Execute either all of the operations in `batch` or none at all, returning an error.
|
|
|
|
fn do_atomically(&self, batch: &[StoreOp]) -> Result<(), Error>;
|
|
|
|
|
2020-02-10 00:30:21 +00:00
|
|
|
/// Store a state summary in the store.
|
|
|
|
// NOTE: this is a hack for the HotColdDb, we could consider splitting this
|
|
|
|
// trait and removing the generic `S: Store` types everywhere?
|
|
|
|
fn put_state_summary(
|
|
|
|
&self,
|
|
|
|
state_root: &Hash256,
|
|
|
|
summary: HotStateSummary,
|
|
|
|
) -> Result<(), Error> {
|
2020-05-25 00:26:54 +00:00
|
|
|
self.put(state_root, &summary).map_err(Into::into)
|
2020-02-10 00:30:21 +00:00
|
|
|
}
|
2019-11-26 23:54:46 +00:00
|
|
|
|
|
|
|
/// Fetch a state from the store.
|
2019-12-06 07:52:11 +00:00
|
|
|
fn get_state(
|
2019-11-26 23:54:46 +00:00
|
|
|
&self,
|
|
|
|
state_root: &Hash256,
|
|
|
|
slot: Option<Slot>,
|
|
|
|
) -> Result<Option<BeaconState<E>>, Error>;
|
|
|
|
|
2020-02-10 00:30:21 +00:00
|
|
|
/// Fetch a state from the store, controlling which cache fields are cloned.
|
|
|
|
fn get_state_with(
|
|
|
|
&self,
|
|
|
|
state_root: &Hash256,
|
|
|
|
slot: Option<Slot>,
|
|
|
|
) -> Result<Option<BeaconState<E>>, Error> {
|
|
|
|
// Default impl ignores config. Overriden in `HotColdDb`.
|
|
|
|
self.get_state(state_root, slot)
|
|
|
|
}
|
|
|
|
|
2020-03-04 05:48:35 +00:00
|
|
|
/// Delete a state from the store.
|
|
|
|
fn delete_state(&self, state_root: &Hash256, _slot: Slot) -> Result<(), Error> {
|
|
|
|
self.key_delete(DBColumn::BeaconState.into(), state_root.as_bytes())
|
|
|
|
}
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
/// (Optionally) Move all data before the frozen slot to the freezer database.
|
2020-04-20 09:59:56 +00:00
|
|
|
fn process_finalization(
|
2019-11-26 23:54:46 +00:00
|
|
|
_store: Arc<Self>,
|
|
|
|
_frozen_head_root: Hash256,
|
|
|
|
_frozen_head: &BeaconState<E>,
|
|
|
|
) -> Result<(), Error> {
|
|
|
|
Ok(())
|
|
|
|
}
|
2019-12-06 07:52:11 +00:00
|
|
|
|
|
|
|
/// Get a forwards (slot-ascending) iterator over the beacon block roots since `start_slot`.
|
|
|
|
///
|
|
|
|
/// Will be efficient for frozen portions of the database if using `DiskStore`.
|
|
|
|
///
|
|
|
|
/// The `end_state` and `end_block_root` are required for backtracking in the post-finalization
|
|
|
|
/// part of the chain, and should be usually be set to the current head. Importantly, the
|
|
|
|
/// `end_state` must be a state that has had a block applied to it, and the hash of that
|
|
|
|
/// block must be `end_block_root`.
|
|
|
|
// NOTE: could maybe optimise by getting the `BeaconState` and end block root from a closure, as
|
|
|
|
// it's not always required.
|
|
|
|
fn forwards_block_roots_iterator(
|
|
|
|
store: Arc<Self>,
|
|
|
|
start_slot: Slot,
|
|
|
|
end_state: BeaconState<E>,
|
|
|
|
end_block_root: Hash256,
|
|
|
|
spec: &ChainSpec,
|
|
|
|
) -> Self::ForwardsBlockRootsIterator;
|
2020-01-08 02:58:01 +00:00
|
|
|
|
|
|
|
/// Load the most recent ancestor state of `state_root` which lies on an epoch boundary.
|
|
|
|
///
|
|
|
|
/// If `state_root` corresponds to an epoch boundary state, then that state itself should be
|
|
|
|
/// returned.
|
|
|
|
fn load_epoch_boundary_state(
|
|
|
|
&self,
|
|
|
|
state_root: &Hash256,
|
|
|
|
) -> Result<Option<BeaconState<E>>, Error> {
|
|
|
|
// The default implementation is not very efficient, but isn't used in prod.
|
|
|
|
// See `HotColdDB` for the optimized implementation.
|
|
|
|
if let Some(state) = self.get_state(state_root, None)? {
|
|
|
|
let epoch_boundary_slot = state.slot / E::slots_per_epoch() * E::slots_per_epoch();
|
|
|
|
if state.slot == epoch_boundary_slot {
|
|
|
|
Ok(Some(state))
|
|
|
|
} else {
|
|
|
|
let epoch_boundary_state_root = state.get_state_root(epoch_boundary_slot)?;
|
|
|
|
self.get_state(epoch_boundary_state_root, Some(epoch_boundary_slot))
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
Ok(None)
|
|
|
|
}
|
|
|
|
}
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
2020-05-16 03:23:32 +00:00
|
|
|
/// Reified key-value storage operation. Helps in modifying the storage atomically.
|
|
|
|
/// See also https://github.com/sigp/lighthouse/issues/692
|
|
|
|
pub enum StoreOp {
|
|
|
|
DeleteBlock(SignedBeaconBlockHash),
|
|
|
|
DeleteState(BeaconStateHash, Slot),
|
|
|
|
}
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// A unique column identifier.
|
2019-11-26 23:54:46 +00:00
|
|
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
2019-05-20 08:01:51 +00:00
|
|
|
pub enum DBColumn {
|
2019-11-26 23:54:46 +00:00
|
|
|
/// For data related to the database itself.
|
|
|
|
BeaconMeta,
|
2019-05-20 08:01:51 +00:00
|
|
|
BeaconBlock,
|
|
|
|
BeaconState,
|
2020-03-06 05:09:41 +00:00
|
|
|
/// For persisting in-memory state to the database.
|
2019-05-20 08:01:51 +00:00
|
|
|
BeaconChain,
|
2020-03-06 05:09:41 +00:00
|
|
|
OpPool,
|
|
|
|
Eth1Cache,
|
|
|
|
ForkChoice,
|
2019-12-06 03:29:06 +00:00
|
|
|
/// For the table mapping restore point numbers to state roots.
|
|
|
|
BeaconRestorePoint,
|
2020-01-08 02:58:01 +00:00
|
|
|
/// For the mapping from state roots to their slots or summaries.
|
|
|
|
BeaconStateSummary,
|
2019-11-26 23:54:46 +00:00
|
|
|
BeaconBlockRoots,
|
|
|
|
BeaconStateRoots,
|
|
|
|
BeaconHistoricalRoots,
|
|
|
|
BeaconRandaoMixes,
|
2020-01-23 07:16:11 +00:00
|
|
|
DhtEnrs,
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
impl Into<&'static str> for DBColumn {
|
2019-05-20 08:01:51 +00:00
|
|
|
/// Returns a `&str` that can be used for keying a key-value data base.
|
2019-11-26 23:54:46 +00:00
|
|
|
fn into(self) -> &'static str {
|
2019-05-20 08:01:51 +00:00
|
|
|
match self {
|
2019-11-26 23:54:46 +00:00
|
|
|
DBColumn::BeaconMeta => "bma",
|
|
|
|
DBColumn::BeaconBlock => "blk",
|
|
|
|
DBColumn::BeaconState => "ste",
|
|
|
|
DBColumn::BeaconChain => "bch",
|
2020-03-06 05:09:41 +00:00
|
|
|
DBColumn::OpPool => "opo",
|
|
|
|
DBColumn::Eth1Cache => "etc",
|
|
|
|
DBColumn::ForkChoice => "frk",
|
2019-12-06 03:29:06 +00:00
|
|
|
DBColumn::BeaconRestorePoint => "brp",
|
2020-01-08 02:58:01 +00:00
|
|
|
DBColumn::BeaconStateSummary => "bss",
|
2019-11-26 23:54:46 +00:00
|
|
|
DBColumn::BeaconBlockRoots => "bbr",
|
|
|
|
DBColumn::BeaconStateRoots => "bsr",
|
|
|
|
DBColumn::BeaconHistoricalRoots => "bhr",
|
|
|
|
DBColumn::BeaconRandaoMixes => "brm",
|
2020-01-23 07:16:11 +00:00
|
|
|
DBColumn::DhtEnrs => "dht",
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
/// An item that may stored in a `Store` by serializing and deserializing from bytes.
|
|
|
|
pub trait SimpleStoreItem: Sized {
|
2019-05-21 08:49:24 +00:00
|
|
|
/// Identifies which column this item should be placed in.
|
2019-05-20 08:01:51 +00:00
|
|
|
fn db_column() -> DBColumn;
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// Serialize `self` as bytes.
|
2019-05-20 08:01:51 +00:00
|
|
|
fn as_store_bytes(&self) -> Vec<u8>;
|
|
|
|
|
2019-05-21 08:49:24 +00:00
|
|
|
/// De-serialize `self` from bytes.
|
2019-11-26 23:54:46 +00:00
|
|
|
///
|
|
|
|
/// Return an instance of the type and the number of bytes that were read.
|
|
|
|
fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error>;
|
|
|
|
}
|
|
|
|
|
2019-05-20 08:01:51 +00:00
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
|
|
|
use super::*;
|
|
|
|
use ssz::{Decode, Encode};
|
|
|
|
use ssz_derive::{Decode, Encode};
|
2019-05-21 06:29:34 +00:00
|
|
|
use tempfile::tempdir;
|
2019-05-20 08:01:51 +00:00
|
|
|
|
|
|
|
#[derive(PartialEq, Debug, Encode, Decode)]
|
|
|
|
struct StorableThing {
|
|
|
|
a: u64,
|
|
|
|
b: u64,
|
|
|
|
}
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
impl SimpleStoreItem for StorableThing {
|
2019-05-20 08:01:51 +00:00
|
|
|
fn db_column() -> DBColumn {
|
|
|
|
DBColumn::BeaconBlock
|
|
|
|
}
|
|
|
|
|
|
|
|
fn as_store_bytes(&self) -> Vec<u8> {
|
|
|
|
self.as_ssz_bytes()
|
|
|
|
}
|
|
|
|
|
2019-11-26 23:54:46 +00:00
|
|
|
fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
|
2019-05-20 08:01:51 +00:00
|
|
|
Self::from_ssz_bytes(bytes).map_err(Into::into)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-06 07:52:11 +00:00
|
|
|
fn test_impl(store: impl Store<MinimalEthSpec>) {
|
2019-05-21 06:29:34 +00:00
|
|
|
let key = Hash256::random();
|
|
|
|
let item = StorableThing { a: 1, b: 42 };
|
|
|
|
|
2020-05-21 00:21:44 +00:00
|
|
|
assert_eq!(store.exists::<StorableThing>(&key).unwrap(), false);
|
2019-05-21 06:37:15 +00:00
|
|
|
|
2019-05-21 06:29:34 +00:00
|
|
|
store.put(&key, &item).unwrap();
|
|
|
|
|
2020-05-21 00:21:44 +00:00
|
|
|
assert_eq!(store.exists::<StorableThing>(&key).unwrap(), true);
|
2019-05-21 06:29:34 +00:00
|
|
|
|
2019-05-21 06:37:15 +00:00
|
|
|
let retrieved = store.get(&key).unwrap().unwrap();
|
2019-05-21 06:29:34 +00:00
|
|
|
assert_eq!(item, retrieved);
|
2019-05-21 06:37:15 +00:00
|
|
|
|
|
|
|
store.delete::<StorableThing>(&key).unwrap();
|
|
|
|
|
2020-05-21 00:21:44 +00:00
|
|
|
assert_eq!(store.exists::<StorableThing>(&key).unwrap(), false);
|
2019-05-21 06:37:15 +00:00
|
|
|
|
2020-05-21 00:21:44 +00:00
|
|
|
assert_eq!(store.get::<StorableThing>(&key).unwrap(), None);
|
2019-05-21 06:29:34 +00:00
|
|
|
}
|
|
|
|
|
2019-05-20 08:01:51 +00:00
|
|
|
#[test]
|
2019-05-21 06:49:56 +00:00
|
|
|
fn diskdb() {
|
2019-11-26 23:54:46 +00:00
|
|
|
use sloggers::{null::NullLoggerBuilder, Build};
|
|
|
|
|
|
|
|
let hot_dir = tempdir().unwrap();
|
|
|
|
let cold_dir = tempdir().unwrap();
|
|
|
|
let spec = MinimalEthSpec::default_spec();
|
|
|
|
let log = NullLoggerBuilder.build().unwrap();
|
2019-12-06 07:52:11 +00:00
|
|
|
let store = DiskStore::open(
|
2019-12-06 03:29:06 +00:00
|
|
|
&hot_dir.path(),
|
|
|
|
&cold_dir.path(),
|
2020-02-10 00:30:21 +00:00
|
|
|
StoreConfig::default(),
|
2019-12-06 03:29:06 +00:00
|
|
|
spec,
|
|
|
|
log,
|
|
|
|
)
|
|
|
|
.unwrap();
|
2019-11-26 23:54:46 +00:00
|
|
|
|
|
|
|
test_impl(store);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn simplediskdb() {
|
2019-05-21 06:37:15 +00:00
|
|
|
let dir = tempdir().unwrap();
|
|
|
|
let path = dir.path();
|
2019-11-26 23:54:46 +00:00
|
|
|
let store = SimpleDiskStore::open(&path).unwrap();
|
2019-05-20 08:01:51 +00:00
|
|
|
|
2019-05-21 06:37:15 +00:00
|
|
|
test_impl(store);
|
|
|
|
}
|
2019-05-20 08:01:51 +00:00
|
|
|
|
2019-05-21 06:37:15 +00:00
|
|
|
#[test]
|
|
|
|
fn memorydb() {
|
2019-05-21 08:20:23 +00:00
|
|
|
let store = MemoryStore::open();
|
2019-05-20 08:01:51 +00:00
|
|
|
|
2019-05-21 06:37:15 +00:00
|
|
|
test_impl(store);
|
2019-05-20 08:01:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn exists() {
|
2019-12-06 07:52:11 +00:00
|
|
|
let store = MemoryStore::<MinimalEthSpec>::open();
|
2019-05-20 08:01:51 +00:00
|
|
|
let key = Hash256::random();
|
|
|
|
let item = StorableThing { a: 1, b: 42 };
|
|
|
|
|
|
|
|
assert_eq!(store.exists::<StorableThing>(&key).unwrap(), false);
|
|
|
|
|
|
|
|
store.put(&key, &item).unwrap();
|
|
|
|
|
|
|
|
assert_eq!(store.exists::<StorableThing>(&key).unwrap(), true);
|
|
|
|
|
|
|
|
store.delete::<StorableThing>(&key).unwrap();
|
2019-02-27 23:24:27 +00:00
|
|
|
|
2019-05-20 08:01:51 +00:00
|
|
|
assert_eq!(store.exists::<StorableThing>(&key).unwrap(), false);
|
|
|
|
}
|
2019-02-27 23:24:27 +00:00
|
|
|
}
|