Merge branch 'sync'

boolean-bitfield/Cargo.toml

@@ -4,3 +4,4 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 
 [dependencies]
+ssz = { path = "../ssz" }

boolean-bitfield/src/lib.rs

@@ -8,66 +8,78 @@
  */
 use std::cmp::max;
 
-#[derive(Eq)]
+#[derive(Eq,Clone)]
 pub struct BooleanBitfield{
     len: usize,
     vec: Vec<u8>
 }
 
 impl BooleanBitfield {
+    /// Create a new bitfield with a length of zero.
     pub fn new() -> Self {
         Self {
             len: 0,
             vec: vec![]
         }
     }
-    
+
+    /// Create a new bitfield of a certain capacity
     pub fn with_capacity(capacity: usize) -> Self {
         Self {
             len: 0,
-            vec: Vec::with_capacity(capacity)
+            vec: Vec::with_capacity(capacity / 8 + 1)
         }
     }
 
-    // Output the bitfield as a big-endian vec of u8.
+    /// Read the value of a bit.
-    pub fn to_be_vec(&self) -> Vec<u8> {
+    ///
-        let mut o = self.vec.clone();
+    /// Will return `true` if the bit has been set to `true`
-        o.reverse();
+    /// without then being set to `False`.
-        o
-    }
-
     pub fn get_bit(&self, i: &usize) -> bool {
-        self.get_bit_on_byte(*i % 8, *i / 8)
+        let bit = |i: &usize| *i % 8;
-    }
+        let byte = |i: &usize| *i / 8;
 
-    fn get_bit_on_byte(&self, bit: usize, byte: usize) -> bool {
+        if byte(i) >= self.vec.len() {
-        assert!(bit < 8);
-        if byte >= self.vec.len() {
             false
         } else {
-             self.vec[byte] & (1 << (bit as u8)) != 0
+             self.vec[byte(i)] & (1 << (bit(i) as u8)) != 0
         }
     }
 
-    pub fn set_bit(&mut self, bit: &usize, to: &bool) {
+    /// Set the value of a bit.
-        self.len = max(self.len, *bit + 1);
+    ///
-        self.set_bit_on_byte(*bit % 8, *bit / 8, to);
+    /// If this bit is larger than the length of the underlying byte
-    }
+    /// array it will be extended.
+    pub fn set_bit(&mut self, i: &usize, to: &bool) {
+        let bit = |i: &usize| *i % 8;
+        let byte = |i: &usize| *i / 8;
 
-    fn set_bit_on_byte(&mut self, bit: usize, byte: usize, val: &bool) {
+        self.len = max(self.len, i + 1);
-        assert!(bit < 8);
+
-        if byte >= self.vec.len() {
+        if byte(i) >= self.vec.len() {
-            self.vec.resize(byte + 1, 0);
+            self.vec.resize(byte(i) + 1, 0);
         }
-        match val {
+        match to {
-            true =>  self.vec[byte] = self.vec[byte] |  (1 << (bit as u8)),
+            true =>  {
-            false => self.vec[byte] = self.vec[byte] & !(1 << (bit as u8))
+                self.vec[byte(i)] =
+                    self.vec[byte(i)] |  (1 << (bit(i) as u8))
+            }
+            false => {
+                self.vec[byte(i)] =
+                    self.vec[byte(i)] & !(1 << (bit(i) as u8))
+            }
         }
     }
 
+    /// Return the "length" of this bitfield. Length is defined as
+    /// the highest bit that has been set.
+    ///
+    /// Note: this is distinct from the length of the underlying
+    /// vector.
     pub fn len(&self) -> usize { self.len }
 
-    // Return the total number of bits set to true.
+    /// Iterate through the underlying vector and count the number of
+    /// true bits.
     pub fn num_true_bits(&self) -> u64 {
         let mut count: u64 = 0;
         for byte in &self.vec {
@@ -79,6 +91,13 @@ impl BooleanBitfield {
         }
         count
     }
+
+    /// Clone and return the underlying byte array (`Vec<u8>`).
+    pub fn to_be_vec(&self) -> Vec<u8> {
+        let mut o = self.vec.clone();
+        o.reverse();
+        o
+    }
 }
 
 impl PartialEq for BooleanBitfield {
@@ -88,34 +107,24 @@ impl PartialEq for BooleanBitfield {
     }
 }
 
-impl Clone for BooleanBitfield {
-    fn clone(&self) -> Self {
-        Self {
-            vec: self.vec.to_vec(),
-            ..*self
-        }
-    }
-}
-
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use super::rlp;
 
     #[test]
     fn test_bitfield_set() {
         let mut b = BooleanBitfield::new();
         b.set_bit(&0, &false);
         assert_eq!(b.to_be_vec(), [0]);
-        
+
         b = BooleanBitfield::new();
         b.set_bit(&7, &true);
         assert_eq!(b.to_be_vec(), [128]);
         b.set_bit(&7, &false);
         assert_eq!(b.to_be_vec(), [0]);
         assert_eq!(b.len(), 8);
-        
+
         b = BooleanBitfield::new();
         b.set_bit(&7, &true);
         b.set_bit(&0, &true);
@@ -123,21 +132,22 @@ mod tests {
         b.set_bit(&7, &false);
         assert_eq!(b.to_be_vec(), [1]);
         assert_eq!(b.len(), 8);
-        
+
         b = BooleanBitfield::new();
         b.set_bit(&8, &true);
         assert_eq!(b.to_be_vec(), [1, 0]);
+        assert_eq!(b.len(), 9);
         b.set_bit(&8, &false);
         assert_eq!(b.to_be_vec(), [0, 0]);
         assert_eq!(b.len(), 9);
-        
+
         b = BooleanBitfield::new();
         b.set_bit(&15, &true);
         assert_eq!(b.to_be_vec(), [128, 0]);
         b.set_bit(&15, &false);
         assert_eq!(b.to_be_vec(), [0, 0]);
         assert_eq!(b.len(), 16);
-        
+
         b = BooleanBitfield::new();
         b.set_bit(&8, &true);
         b.set_bit(&15, &true);
@@ -156,9 +166,9 @@ mod tests {
             b.set_bit(&i, &true);
             assert_eq!(b.get_bit(&i), true);
             b.set_bit(&i, &true);
-        } 
+        }
     }
-    
+
     #[test]
     fn test_bitfield_num_true_bits() {
         let mut b = BooleanBitfield::new();

lighthouse/client/mod.rs

@@ -8,7 +8,7 @@ use super::futures::sync::mpsc::{
 use super::network_libp2p::service::listen as network_listen;
 use super::network_libp2p::state::NetworkState;
 use super::slog::Logger;
-use super::sync::start_sync;
+use super::sync::run_sync_future;
 
 /// Represents the co-ordination of the
 /// networking, syncing and RPC (not-yet-implemented) threads.
@@ -59,7 +59,7 @@ impl Client {
             let sync_log = log.new(o!());
             let sync_db = Arc::clone(&db);
             let thread = thread::spawn(move || {
-                start_sync(
+                run_sync_future(
                     sync_db,
                     network_tx.clone(),
                     network_rx,

lighthouse/state/attestation_record.rs

@@ -1,5 +1,6 @@
 use super::utils::types::{ Hash256, Bitfield };
 use super::utils::bls::{ AggregateSignature };
+use super::ssz::{ Encodable, SszStream };
 
 
 pub struct AttestationRecord {
@@ -11,6 +12,17 @@ pub struct AttestationRecord {
     pub aggregate_sig: Option<AggregateSignature>,
 }
 
+impl Encodable for AttestationRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.slot);
+        s.append(&self.shard_id);
+        s.append_vec(&self.oblique_parent_hashes);
+        s.append(&self.shard_block_hash);
+        s.append_vec(&self.attester_bitfield.to_be_vec());
+        // TODO: add aggregate signature
+    }
+}
+
 impl AttestationRecord {
     pub fn zero() -> Self {
         Self {

lighthouse/state/block.rs

@@ -1,6 +1,6 @@
 use super::utils::types::Hash256;
 use super::attestation_record::AttestationRecord;
-use super::ssz;
+use super::ssz::{ Encodable, SszStream };
 
 const SSZ_BLOCK_LENGTH: usize = 192;
 
@@ -12,7 +12,7 @@ pub struct Block {
     pub pow_chain_ref: Hash256,
     pub active_state_root: Hash256,
     pub crystallized_state_root: Hash256,
-} 
+}
 
 impl Block {
     pub fn zero() -> Self {
@@ -27,12 +27,12 @@ impl Block {
         }
     }
 
-    // Not sure if this will be useful, leaving it here for the
+    /// Return the bytes that should be signed in order to
-    // time being.
+    /// attest for this block.
-    pub fn ssz_encode_without_attestations(&self) 
+    pub fn encode_for_signing(&self)
         -> [u8; SSZ_BLOCK_LENGTH]
     {
-        let mut s = ssz::SszStream::new();
+        let mut s = SszStream::new();
         s.append(&self.parent_hash);
         s.append(&self.slot_number);
         s.append(&self.randao_reveal);
@@ -45,6 +45,18 @@ impl Block {
     }
 }
 
+impl Encodable for Block {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.parent_hash);
+        s.append(&self.slot_number);
+        s.append(&self.randao_reveal);
+        s.append_vec(&self.attestations);
+        s.append(&self.pow_chain_ref);
+        s.append(&self.active_state_root);
+        s.append(&self.crystallized_state_root);
+    }
+}
+
 
 #[cfg(test)]
 mod tests {

lighthouse/sync/messages.rs

@@ -0,0 +1,12 @@
+pub enum SyncEventType {
+    Invalid,
+    PeerConnect,
+    PeerDrop,
+    ReceiveBlocks,
+    ReceiveAttestationRecords,
+}
+
+pub struct SyncEvent {
+    event: SyncEventType,
+    data: Option<Vec<u8>>
+}

lighthouse/sync/mod.rs

@@ -1,50 +1,13 @@
 extern crate futures;
 extern crate slog;
 extern crate tokio;
+extern crate network_libp2p;
 
-use self::futures::sync::mpsc::{
+pub mod messages;
-    UnboundedReceiver,
+pub mod network;
-    UnboundedSender,
+pub mod sync_future;
-};
+pub mod wire_protocol;
-use self::tokio::prelude::*;
-use std::sync::{ RwLock, Arc };
-use super::network_libp2p::message::{
-    NetworkEvent,
-    OutgoingMessage,
-};
-use super::db::DB;
-use slog::Logger;
 
-type NetworkSender = UnboundedSender<OutgoingMessage>;
+pub use self::sync_future::run_sync_future;
-type NetworkReceiver = UnboundedReceiver<NetworkEvent>;
 
-type SyncSender = UnboundedSender<Vec<u8>>;
+use super::db;
-type SyncReceiver = UnboundedReceiver<Vec<u8>>;
-
-/// Start a syncing tokio future.
-///
-/// This is effectively a stub function being
-/// used to test network functionality.
-///
-/// Expect a full re-write.
-pub fn start_sync(
-    _db: Arc<RwLock<DB>>,
-    _network_tx: NetworkSender,
-    network_rx: NetworkReceiver,
-    _sync_tx: SyncSender,
-    _sync_rx: SyncReceiver,
-    log: Logger) {
-    let rx_future = network_rx
-        .for_each(move |event| {
-            debug!(&log, "Sync receive";
-                   "msg" => format!("{:?}", event));
-            Ok(())
-        })
-        .map_err(|_| panic!("rx failed"));
-
-    /*
-     * This is an unfinished stub function.
-     */
-
-    tokio::run(rx_future);
-}

lighthouse/sync/network.rs

@@ -0,0 +1,66 @@
+use std::sync::{ RwLock, Arc };
+use super::db::DB;
+use slog::Logger;
+
+use super::network_libp2p::message::{
+    NetworkEvent,
+    OutgoingMessage,
+    NetworkEventType,
+};
+
+use super::wire_protocol::{ WireMessageType, message_type };
+
+use super::futures::sync::mpsc::{
+    UnboundedSender,
+};
+
+/// Accept a network event and perform all required processing.
+///
+/// This function should be called whenever an underlying network
+/// (e.g., libp2p) has an event to push up to the sync process.
+pub fn handle_network_event(
+    event: NetworkEvent,
+    db: Arc<RwLock<DB>>,
+    network_tx: UnboundedSender<OutgoingMessage>,
+    log: Logger)
+    -> Result<(), ()>
+{
+        debug!(&log, "";
+               "network_event" => format!("{:?}", &event));
+        match event.event {
+            NetworkEventType::PeerConnect => Ok(()),
+            NetworkEventType::PeerDrop => Ok(()),
+            NetworkEventType::Message => {
+                if let Some(data) = event.data {
+                    handle_network_message(
+                        data,
+                        db,
+                        network_tx,
+                        log)
+                } else {
+                    Ok(())
+                }
+            }
+        }
+}
+
+/// Accept a message from the network and perform all required
+/// processing.
+///
+/// This function should be called whenever a peer from a network
+/// (e.g., libp2p) has sent a message to us.
+fn handle_network_message(
+    message: Vec<u8>,
+    _db: Arc<RwLock<DB>>,
+    _network_tx: UnboundedSender<OutgoingMessage>,
+    _log: Logger)
+    -> Result<(), ()>
+{
+    match message_type(&message) {
+        Some(WireMessageType::Blocks) => {
+            // Do something with inbound blocks.
+            Ok(())
+        }
+        _ => Ok(())
+    }
+}

lighthouse/sync/sync_future.rs

@@ -0,0 +1,48 @@
+use super::tokio;
+use super::futures::{ Future, Stream };
+use super::futures::sync::mpsc::{
+    UnboundedReceiver,
+    UnboundedSender,
+};
+use super::network_libp2p::message::{
+    NetworkEvent,
+    OutgoingMessage,
+};
+use super::network::handle_network_event;
+use std::sync::{ RwLock, Arc };
+use super::db::DB;
+use slog::Logger;
+
+type NetworkSender = UnboundedSender<OutgoingMessage>;
+type NetworkReceiver = UnboundedReceiver<NetworkEvent>;
+
+type SyncSender = UnboundedSender<Vec<u8>>;
+type SyncReceiver = UnboundedReceiver<Vec<u8>>;
+
+/// Start a syncing tokio future.
+///
+/// Uses green-threading to process messages
+/// from the network and the RPC and update
+/// the state.
+pub fn run_sync_future(
+    db: Arc<RwLock<DB>>,
+    network_tx: NetworkSender,
+    network_rx: NetworkReceiver,
+    _sync_tx: SyncSender,
+    _sync_rx: SyncReceiver,
+    log: Logger)
+{
+    let network_future = {
+        network_rx
+            .for_each(move |event| {
+                handle_network_event(
+                    event,
+                    db.clone(),
+                    network_tx.clone(),
+                    log.clone())
+            })
+            .map_err(|_| panic!("rx failed"))
+    };
+
+    tokio::run(network_future);
+}

lighthouse/sync/wire_protocol.rs

@@ -0,0 +1,24 @@
+pub enum WireMessageType {
+    Status,
+    NewBlockHashes,
+    GetBlockHashes,
+    BlockHashes,
+    GetBlocks,
+    Blocks,
+    NewBlock,
+}
+
+
+/// Determines the message type of some given
+/// message.
+///
+/// Does not check the validity of the message data,
+/// it just reads the first byte.
+pub fn message_type(message: &Vec<u8>)
+    -> Option<WireMessageType>
+{
+    match message.get(0) {
+        Some(0x06) => Some(WireMessageType::Blocks),
+        _ => None
+    }
+}

network-libp2p/src/service.rs

@@ -18,7 +18,7 @@ use super::state::NetworkState;
 use super::message::{ NetworkEvent, NetworkEventType, OutgoingMessage };
 use self::bigint::U512;
 use self::futures::{ Future, Stream, Poll };
-use self::futures::sync::mpsc::{ 
+use self::futures::sync::mpsc::{
     UnboundedSender, UnboundedReceiver
 };
 use self::libp2p_core::{ AddrComponent, Endpoint, Multiaddr,
@@ -39,14 +39,14 @@ pub use self::libp2p_floodsub::Message;
 pub fn listen(state: NetworkState,
           events_to_app: UnboundedSender<NetworkEvent>,
           raw_rx: UnboundedReceiver<OutgoingMessage>,
-          log: Logger) 
+          log: Logger)
 {
     let peer_store = state.peer_store;
     let peer_id = state.peer_id;
     let listen_multiaddr = state.listen_multiaddr;
     let listened_addrs = Arc::new(RwLock::new(vec![]));
     let rx = ApplicationReciever{ inner: raw_rx };
-    
+
     // Build a tokio core
     let mut core =  tokio_core::reactor::Core::new().expect("tokio failure.");
     // Build a base TCP libp2p transport
@@ -65,10 +65,10 @@ pub fn listen(state: NetworkState,
             // is stored not the internal addr.
             .map(move |out, _, _| {
                 if let(Some(ref observed), ref listen_multiaddr) =
-                    (out.observed_addr, listen_multiaddr) 
+                    (out.observed_addr, listen_multiaddr)
                 {
-                    if let Some(viewed_from_outside) = 
+                    if let Some(viewed_from_outside) =
-                        transport.nat_traversal(listen_multiaddr, observed) 
+                        transport.nat_traversal(listen_multiaddr, observed)
                     {
                         listened_addrs.write().unwrap()
                             .push(viewed_from_outside);
@@ -79,7 +79,7 @@ pub fn listen(state: NetworkState,
     };
 
     // Configure and build a Kademlia upgrade to be applied
-    // to the base TCP transport. 
+    // to the base TCP transport.
     let kad_config = libp2p_kad::KademliaConfig {
         parallelism: 3,
         record_store: (),
@@ -91,10 +91,10 @@ pub fn listen(state: NetworkState,
         KademliaControllerPrototype::new(kad_config);
     let kad_upgrade = libp2p_kad::
         KademliaUpgrade::from_prototype(&kad_ctl_proto);
-   
+
     // Build a floodsub upgrade to allow pushing topic'ed
     // messages across the network.
-    let (floodsub_upgrade, floodsub_rx) = 
+    let (floodsub_upgrade, floodsub_rx) =
         FloodSubUpgrade::new(peer_id.clone());
 
     // Combine the Kademlia and Identify upgrades into a single
@@ -104,7 +104,7 @@ pub fn listen(state: NetworkState,
         floodsub: floodsub_upgrade.clone(),
         identify: libp2p_identify::IdentifyProtocolConfig,
     };
-  
+
     // Build a Swarm to manage upgrading connections to peers.
     let swarm_listened_addrs = listened_addrs.clone();
     let swarm_peer_id = peer_id.clone();
@@ -166,7 +166,7 @@ pub fn listen(state: NetworkState,
                 for peer in peers {
                     let peer_hash = U512::from(peer.hash());
                     let distance = 512 - (local_hash ^ peer_hash).leading_zeros();
-                    info!(kad_poll_log, "Discovered peer"; 
+                    info!(kad_poll_log, "Discovered peer";
                           "distance" => distance,
                           "peer_id" => peer.to_base58());
                     let peer_addr = AddrComponent::P2P(peer.into_bytes()).into();
@@ -240,7 +240,7 @@ struct ConnectionUpgrader<P, R> {
 }
 
 impl<C, P, R, Pc> ConnectionUpgrade<C> for ConnectionUpgrader<P, R>
-where 
+where
     C: AsyncRead + AsyncWrite + 'static,
     P: Deref<Target = Pc> + Clone + 'static,
     for<'r> &'r Pc: libp2p_peerstore::Peerstore,
@@ -251,7 +251,7 @@ where
     type Output = FinalUpgrade<C>;
     type Future = Box<Future<Item = FinalUpgrade<C>, Error = IoError>>;
 
-    #[inline] 
+    #[inline]
     fn protocol_names(&self) -> Self::NamesIter {
         vec![
             (Bytes::from("/ipfs/kad/1.0.0"), 0),

ssz/src/decode.rs

@@ -0,0 +1,135 @@
+use super::{
+    LENGTH_BYTES,
+};
+
+#[derive(Debug, PartialEq)]
+pub enum DecodeError {
+    OutOfBounds,
+    TooShort,
+    TooLong,
+}
+
+pub trait Decodable: Sized {
+    fn ssz_decode(bytes: &[u8]) -> Result<Self, DecodeError>;
+}
+
+/// Decode the nth element of some ssz list.
+///
+/// A single ssz encoded value can be considered a list of
+/// one element, so this function will work on it too.
+pub fn decode_ssz_list_element<T>(ssz_bytes: &[u8], n: usize)
+    -> Result<T, DecodeError>
+    where T: Decodable
+{
+    T::ssz_decode(nth_value(ssz_bytes, n)?)
+}
+
+/// Return the nth value in some ssz encoded list.
+///
+/// The four-byte length prefix is not included in the return.
+///
+/// A single ssz encoded value can be considered a list of
+/// one element, so this function will work on it too.
+fn nth_value(ssz_bytes: &[u8], n: usize)
+    -> Result<&[u8], DecodeError>
+{
+    let mut c: usize = 0;
+    for i in 0..(n + 1) {
+        let length = decode_length(&ssz_bytes[c..], LENGTH_BYTES)?;
+        let next = c + LENGTH_BYTES + length;
+
+        if i == n {
+            return Ok(&ssz_bytes[c + LENGTH_BYTES..next]);
+        } else {
+            if next >= ssz_bytes.len() {
+                return Err(DecodeError::OutOfBounds);
+            } else {
+                c = next;
+            }
+        }
+    }
+    Err(DecodeError::OutOfBounds)
+}
+
+/// Given some number of bytes, interpret the first four
+/// bytes as a 32-bit big-endian integer and return the
+/// result.
+fn decode_length(bytes: &[u8], length_bytes: usize)
+    -> Result<usize, DecodeError>
+{
+    if bytes.len() < length_bytes {
+        return Err(DecodeError::TooShort);
+    };
+    let mut len: usize = 0;
+    for i in 0..length_bytes {
+        let offset = (length_bytes - i - 1) * 8;
+        len = ((bytes[i] as usize) << offset) | len;
+    };
+    Ok(len)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use super::super::encode::encode_length;
+
+    #[test]
+    fn test_ssz_decode_length() {
+        let decoded = decode_length(
+            &vec![0, 0, 1],
+            LENGTH_BYTES);
+        assert_eq!(decoded.unwrap(), 1);
+
+        let decoded = decode_length(
+            &vec![0, 1, 0],
+            LENGTH_BYTES);
+        assert_eq!(decoded.unwrap(), 256);
+
+        let decoded = decode_length(
+            &vec![0, 1, 255],
+            LENGTH_BYTES);
+        assert_eq!(decoded.unwrap(), 511);
+
+        let decoded = decode_length(
+            &vec![255, 255, 255],
+            LENGTH_BYTES);
+        assert_eq!(decoded.unwrap(), 16777215);
+    }
+
+    #[test]
+    fn test_encode_decode_length() {
+        let params: Vec<usize> = vec![
+            0, 1, 2, 3, 7, 8, 16,
+            2^8,  2^8  + 1,
+            2^16, 2^16 + 1,
+            2^24, 2^24 + 1,
+            2^32,
+        ];
+        for i in params {
+            let decoded = decode_length(
+                &encode_length(i, LENGTH_BYTES),
+                LENGTH_BYTES).unwrap();
+            assert_eq!(i, decoded);
+        }
+    }
+
+    #[test]
+    fn test_ssz_nth_value() {
+        let ssz = vec![0, 0, 1, 0];
+        let result = nth_value(&ssz, 0).unwrap();
+        assert_eq!(result, vec![0].as_slice());
+
+        let ssz = vec![0, 0, 4, 1, 2, 3, 4];
+        let result = nth_value(&ssz, 0).unwrap();
+        assert_eq!(result, vec![1, 2, 3, 4].as_slice());
+
+        let ssz = vec![0, 0, 1, 0, 0, 0, 1, 1];
+        let result = nth_value(&ssz, 1).unwrap();
+        assert_eq!(result, vec![1].as_slice());
+
+        let mut ssz = vec![0, 1, 255];
+        ssz.append(&mut vec![42; 511]);
+        let result = nth_value(&ssz, 0).unwrap();
+        assert_eq!(result, vec![42; 511].as_slice());
+    }
+}

ssz/src/encode.rs

@@ -0,0 +1,166 @@
+use super::LENGTH_BYTES;
+
+pub trait Encodable {
+    fn ssz_append(&self, s: &mut SszStream);
+}
+
+/// Provides a buffer for appending ssz-encodable values.
+///
+/// Use the `append()` fn to add a value to a list, then use
+/// the `drain()` method to consume the struct and return the
+/// ssz encoded bytes.
+pub struct SszStream {
+    buffer: Vec<u8>
+}
+
+impl SszStream {
+    /// Create a new, empty stream for writing ssz values.
+    pub fn new() -> Self {
+        SszStream {
+            buffer: Vec::new()
+        }
+    }
+
+    /// Append some ssz encodable value to the stream.
+    pub fn append<E>(&mut self, value: &E) -> &mut Self
+        where E: Encodable
+    {
+        value.ssz_append(self);
+        self
+    }
+
+    /// Append some ssz encoded bytes to the stream.
+    ///
+    /// The length of the supplied bytes will be concatenated
+    /// to the stream before the supplied bytes.
+    pub fn append_encoded_val(&mut self, vec: &Vec<u8>) {
+        self.buffer.extend_from_slice(
+            &encode_length(vec.len(),
+            LENGTH_BYTES));
+        self.buffer.extend_from_slice(&vec);
+    }
+
+    /// Append some vector (list) of encodable values to the stream.
+    ///
+    /// The length of the list will be concatenated to the stream, then
+    /// each item in the vector will be encoded and concatenated.
+    pub fn append_vec<E>(&mut self, vec: &Vec<E>)
+        where E: Encodable
+    {
+        self.buffer.extend_from_slice(&encode_length(vec.len(), LENGTH_BYTES));
+        for v in vec {
+            v.ssz_append(self);
+        }
+    }
+
+    /// Consume the stream and return the underlying bytes.
+    pub fn drain(self) -> Vec<u8> {
+        self.buffer
+    }
+}
+
+/// Encode some length into a ssz size prefix.
+///
+/// The ssz size prefix is 4 bytes, which is treated as a continuious
+/// 32bit big-endian integer.
+pub fn encode_length(len: usize, length_bytes: usize) -> Vec<u8> {
+    assert!(length_bytes > 0);  // For sanity
+    assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8));
+    let mut header: Vec<u8> = vec![0; length_bytes];
+    for i in 0..length_bytes {
+        let offset = (length_bytes - i - 1) * 8;
+        header[i] = ((len >> offset) & 0xff) as u8;
+    };
+    header
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    fn test_encode_length_0_bytes_panic() {
+        encode_length(0, 0);
+    }
+
+    #[test]
+    fn test_encode_length_4_bytes() {
+        assert_eq!(
+            encode_length(0, LENGTH_BYTES),
+            vec![0; 3]
+        );
+        assert_eq!(
+            encode_length(1, LENGTH_BYTES),
+            vec![0, 0, 1]
+        );
+        assert_eq!(
+            encode_length(255, LENGTH_BYTES),
+            vec![0, 0, 255]
+        );
+        assert_eq!(
+            encode_length(256, LENGTH_BYTES),
+            vec![0, 1, 0]
+        );
+        assert_eq!(
+            encode_length(16777215, LENGTH_BYTES),  // 2^(3*8) - 1
+            vec![255, 255, 255]
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_encode_length_4_bytes_panic() {
+        encode_length(16777216, LENGTH_BYTES);  // 2^(3*8)
+    }
+
+    /*
+    #[test]
+    fn test_encode_struct() {
+        pub struct TestStruct {
+            pub one: u32,
+            pub two: H256,
+            pub three: u64,
+            pub four: U256,
+        }
+
+        impl Encodable for TestStruct {
+            fn ssz_append(&self, s: &mut SszStream) {
+                s.append(&self.one);
+                s.append(&self.two);
+                s.append(&self.three);
+                s.append(&self.four);
+            }
+        }
+
+        let t = TestStruct {
+            one: 1,
+            two: H256::zero(),
+            three: 100,
+            four: U256::zero(),
+        };
+
+        let mut s = SszStream::new();
+        s.append(&t);
+        let e = s.drain();
+
+        let expected_len = {
+            3 + 4 +
+            3 + 32 +
+            3 + 8 +
+            3 + 32
+        };
+
+        assert_eq!(e[0..4], [0, 0, 0, 4]);
+        assert_eq!(e[4..8], [0, 0, 0, 1]);
+        assert_eq!(e[8..12], [0, 0, 0, 32]);
+        assert_eq!(e[12..44], [0; 32]);
+        assert_eq!(e[44..48], [0, 0, 0, 8]);
+        assert_eq!(e[48..56], [0, 0, 0, 0, 0, 0, 0, 100]);
+        assert_eq!(e[56..60], [0, 0, 0, 32]);
+        assert_eq!(e[60..92], [0; 32]);
+        assert_eq!(e.len(), expected_len);
+    }
+    */
+}

ssz/src/impl_decode.rs

@@ -0,0 +1,114 @@
+use super::{
+    DecodeError,
+    Decodable,
+};
+macro_rules! impl_decodable_for_uint {
+    ($type: ident, $bit_size: expr) => {
+        impl Decodable for $type {
+            fn ssz_decode(bytes: &[u8])
+                -> Result<Self, DecodeError>
+            {
+                assert!((0 < $bit_size) &
+                        ($bit_size <= 64) &
+                        ($bit_size % 8 == 0));
+                let max_bytes = $bit_size / 8;
+                if bytes.len() <= max_bytes {
+                    let mut result: $type = 0;
+                    for i in 0..bytes.len() {
+                        let offset = (bytes.len() - i - 1) * 8;
+                        result = ((bytes[i] as $type) << offset) | result;
+                    };
+                    Ok(result)
+                } else {
+                    Err(DecodeError::TooLong)
+                }
+            }
+        }
+    }
+}
+
+impl_decodable_for_uint!(u16, 16);
+impl_decodable_for_uint!(u32, 32);
+impl_decodable_for_uint!(u64, 64);
+impl_decodable_for_uint!(usize, 64);
+
+
+#[cfg(test)]
+mod tests {
+    use super::super::{
+        DecodeError,
+        decode_ssz_list_element,
+    };
+
+    #[test]
+    fn test_ssz_decode_u16() {
+        let ssz = vec![0, 0, 1, 0];
+        let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 0);
+
+        let ssz = vec![0, 0, 1, 16];
+        let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 16);
+
+        let ssz = vec![0, 0, 2, 1, 0];
+        let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 256);
+
+        let ssz = vec![0, 0, 2, 255, 255];
+        let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 65535);
+
+        let ssz = vec![0, 0, 3, 0, 0, 1];
+        let result: Result<u16, DecodeError> =
+            decode_ssz_list_element(&ssz, 0);
+        assert_eq!(result, Err(DecodeError::TooLong));
+    }
+
+    #[test]
+    fn test_ssz_decode_u32() {
+        let ssz = vec![0, 0, 1, 0];
+        let result: u32 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 0);
+
+        let ssz = vec![0, 0, 4, 255, 255, 255, 255];
+        let result: u32 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 4294967295);
+
+        let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1];
+        let result: Result<u32, DecodeError> =
+            decode_ssz_list_element(&ssz, 0);
+        assert_eq!(result, Err(DecodeError::TooLong));
+    }
+
+    #[test]
+    fn test_ssz_decode_u64() {
+        let ssz = vec![0, 0, 1, 0];
+        let result: u64 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 0);
+
+        let ssz = vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255];
+        let result: u64 = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 18446744073709551615);
+
+        let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1];
+        let result: Result<u64, DecodeError> =
+            decode_ssz_list_element(&ssz, 0);
+        assert_eq!(result, Err(DecodeError::TooLong));
+    }
+
+    #[test]
+    fn test_ssz_decode_usize() {
+        let ssz = vec![0, 0, 1, 0];
+        let result: usize = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 0);
+
+        let ssz = vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255];
+        let result: usize = decode_ssz_list_element(&ssz, 0).unwrap();
+        assert_eq!(result, 18446744073709551615);
+
+        let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1];
+        let result: Result<usize, DecodeError> =
+            decode_ssz_list_element(&ssz, 0);
+        assert_eq!(result, Err(DecodeError::TooLong));
+    }
+}

ssz/src/impl_encode.rs

@@ -0,0 +1,186 @@
+use super::{
+    Encodable,
+    SszStream
+};
+use super::ethereum_types::{ H256, U256 };
+
+/*
+ * Note: there is a "to_bytes" function for integers
+ * in Rust nightly. When it is in stable, we should
+ * use it instead.
+ */
+macro_rules! impl_encodable_for_uint {
+    ($type: ident) => {
+        impl Encodable for $type {
+            fn ssz_append(&self, s: &mut SszStream)
+            {
+                // Number of bits required to represent this integer.
+                // This could be optimised at the expense of complexity.
+                let num_bits = {
+                    let mut n = *self;
+                    let mut r: usize = 0;
+                    while n > 0 {
+                        n >>= 1;
+                        r += 1;
+                    }
+                    if r == 0 { 1 } else { r }
+                };
+                // Number of bytes required to represent this bit
+                let num_bytes = (num_bits + 8 - 1) / 8;
+                let mut ssz_val: Vec<u8> = Vec::with_capacity(num_bytes);
+                ssz_val.resize(num_bytes, 0);
+                for i in (0..num_bytes).rev() {
+                    let offset = (num_bytes - i - 1) * 8;
+                    ssz_val[i] = 0_u8 | (self >> offset) as u8
+                }
+                s.append_encoded_val(&ssz_val);
+            }
+        }
+    }
+}
+
+impl_encodable_for_uint!(u8);
+impl_encodable_for_uint!(u16);
+impl_encodable_for_uint!(u32);
+impl_encodable_for_uint!(u64);
+impl_encodable_for_uint!(usize);
+
+impl Encodable for H256 {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_encoded_val(&self.to_vec());
+    }
+}
+
+impl Encodable for U256 {
+    fn ssz_append(&self, s: &mut SszStream) {
+        let mut a = [0; 32];
+        self.to_big_endian(&mut a);
+        s.append_encoded_val(&a.to_vec());
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ssz_encode_u8() {
+        let x: u16 = 0;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 0]);
+
+        let x: u16 = 1;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
+
+        let x: u16 = 100;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
+
+        let x: u16 = 255;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 255]);
+    }
+
+    #[test]
+    fn test_ssz_encode_u16() {
+        let x: u16 = 1;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
+
+        let x: u16 = 100;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
+
+        let x: u16 = 1 << 8;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 2, 1, 0]);
+
+        let x: u16 = 65535;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 2, 255, 255]);
+    }
+
+    #[test]
+    fn test_ssz_encode_u32() {
+        let x: u32 = 1;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
+
+        let x: u32 = 100;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
+
+        let x: u32 = 1 << 16;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 3, 1, 0, 0]);
+
+        let x: u32 = 1 << 24;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 4, 1, 0, 0, 0]);
+
+        let x: u32 = !0;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 4, 255, 255, 255, 255]);
+    }
+
+    #[test]
+    fn test_ssz_encode_u64() {
+        let x: u64 = 1;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
+
+        let x: u64 = 100;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
+
+        let x: u64 = 1 << 32;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 5, 1, 0, 0, 0, 0]);
+
+        let x: u64 = !0;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]);
+    }
+
+    #[test]
+    fn test_ssz_encode_usize() {
+        let x: usize = 1;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
+
+        let x: usize = 100;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
+
+        let x: usize = 1 << 32;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 5, 1, 0, 0, 0, 0]);
+
+        let x: usize = !0;
+        let mut ssz = SszStream::new();
+        ssz.append(&x);
+        assert_eq!(ssz.drain(), vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]);
+    }
+}

ssz/src/lib.rs

@@ -1,8 +1,8 @@
 /*
- * This is a WIP of implementing an alternative 
+ * This is a WIP of implementing an alternative
  * serialization strategy. It attempts to follow Vitalik's
- * "ssz" format here: 
+ * "simpleserialize" format here:
- * https://github.com/ethereum/research/tree/master/py_ssz
+ * https://github.com/ethereum/beacon_chain/blob/master/beacon_chain/utils/simpleserialize.py
  *
  * This implementation is not final and would almost certainly
  * have issues.
@@ -10,172 +10,19 @@
 extern crate bytes;
 extern crate ethereum_types;
 
-use self::bytes::{ BytesMut, BufMut };
+mod encode;
-use self::ethereum_types::{ H256, U256 };
+mod decode;
+mod impl_encode;
+mod impl_decode;
 
-pub const LENGTH_BYTES: usize = 4;
+pub use decode::{
+    Decodable,
+    DecodeError,
+    decode_ssz_list_element,
+};
+pub use encode::{
+    Encodable,
+    SszStream,
+};
 
-pub trait Encodable {
+pub const LENGTH_BYTES: usize = 3;
-    fn ssz_append(&self, s: &mut SszStream);
-}
-
-pub struct SszStream {
-    buffer: Vec<u8>
-}
-
-impl SszStream {
-    pub fn new() -> Self {
-        SszStream {
-            buffer: Vec::new()
-        }
-    }
-
-    pub fn append<E>(&mut self, value: &E) -> &mut Self
-        where E: Encodable
-    {
-        value.ssz_append(self);
-        self
-    }
-
-    fn append_encoded_vec(&mut self, v: &mut Vec<u8>) {
-        self.buffer.append(&mut encode_length(v.len(), LENGTH_BYTES));
-        self.buffer.append(v) ;
-    }
-    
-    fn append_encoded_array(&mut self, a: &mut [u8]) {
-        let len = a.len();
-        self.buffer.append(&mut encode_length(len, LENGTH_BYTES));
-        self.buffer.extend_from_slice(&a[0..len]);
-    }
-
-    pub fn drain(self) -> Vec<u8> {
-        self.buffer
-    }
-}
-
-pub fn encode<E>(value: &E) -> Vec<u8>
-    where E: Encodable
-{
-    let mut stream = SszStream::new();
-    stream.append(value);
-    stream.drain()
-}
-
-fn encode_length(len: usize, length_bytes: usize) -> Vec<u8> {
-    assert!(length_bytes > 0);  // For sanity
-    assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8));
-    let mut header: Vec<u8> = vec![0; length_bytes];
-    for i in 0..length_bytes {
-        let offset = (length_bytes - i - 1) * 8;
-        header[i] = ((len >> offset) & 0xff) as u8;
-    };
-    header
-}
-
-/*
- * Implementations for various types
- */
-
-impl Encodable for u32 {
-    fn ssz_append(&self, s: &mut SszStream) {
-        let mut buf = BytesMut::with_capacity(32/8);
-        buf.put_u32_be(*self);
-        s.append_encoded_vec(&mut buf.to_vec());
-    }
-}
-
-impl Encodable for u64 {
-    fn ssz_append(&self, s: &mut SszStream) {
-        let mut buf = BytesMut::with_capacity(64/8);
-        buf.put_u64_be(*self);
-        s.append_encoded_vec(&mut buf.to_vec());
-    }
-}
-
-impl Encodable for H256 {
-    fn ssz_append(&self, s: &mut SszStream) {
-        s.append_encoded_vec(&mut self.to_vec());
-    }
-}
-
-impl Encodable for U256 {
-    fn ssz_append(&self, s: &mut SszStream) {
-        let mut a = [0; 32];
-        self.to_big_endian(&mut a);
-        s.append_encoded_array(&mut a);
-    }
-}
-
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    
-    #[test]
-    #[should_panic]
-    fn test_encode_length_0_bytes_panic() {
-        encode_length(0, 0);
-    }
-
-    #[test]
-    fn test_encode_length_4_bytes() {
-        assert_eq!(
-            encode_length(0, 4),
-            vec![0; 4]
-        );
-        assert_eq!(
-            encode_length(1, 4),
-            vec![0, 0, 0, 1]
-        );
-        assert_eq!(
-            encode_length(255, 4),
-            vec![0, 0, 0, 255]
-        );
-        assert_eq!(
-            encode_length(256, 4),
-            vec![0, 0, 1, 0]
-        );
-        assert_eq!(
-            encode_length(4294967295, 4),  // 2^(4*8) - 1
-            vec![255, 255, 255, 255]
-        );
-    }
-    
-    #[test]
-    #[should_panic]
-    fn test_encode_length_4_bytes_panic() {
-        encode_length(4294967296, 4);  // 2^(4*8)
-    }
-
-    #[test]
-    fn test_serialization() {
-        pub struct TestStruct {
-            pub one: u32,
-            pub two: H256,
-            pub three: u64,        
-        }
-
-        impl Encodable for TestStruct {
-            fn ssz_append(&self, s: &mut SszStream) {
-                s.append(&self.one);
-                s.append(&self.two);
-                s.append(&self.three);
-            }
-        }
-
-        let t = TestStruct {
-            one: 1,
-            two: H256::zero(),
-            three: 100
-        };
-
-        let e = encode(&t);
-        assert_eq!(e[0..4], [0, 0, 0, 4]);
-        assert_eq!(e[4..8], [0, 0, 0, 1]);
-        assert_eq!(e[8..12], [0, 0, 0, 32]);
-        assert_eq!(e[12..44], [0; 32]);
-        assert_eq!(e[44..48], [0, 0, 0, 8]);
-        assert_eq!(e[48..56], [0, 0, 0, 0, 0, 0, 0, 100]);
-        assert_eq!(e.len(), 56);
-    }
-}