Add optimized SSZ decoding for fixed-len items (#865)

* Add custom SSZ decode for Validator

* Move efficient decode into macro

* Don't allocate SSZ offset to heap

* Use smallvec in SszDecoder

* Fix test compile error

Cargo.lock

@@ -1131,7 +1131,7 @@ dependencies = [
  "slog-async 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "slog-stdlog 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "slog-term 2.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io-timeout 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1182,6 +1182,7 @@ version = "0.1.2"
 dependencies = [
  "eth2_ssz_derive 0.1.0",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -2059,7 +2060,7 @@ dependencies = [
  "protobuf 2.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2655,7 +2656,7 @@ dependencies = [
  "rlp 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "sloggers 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "store 0.1.0",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3877,7 +3878,7 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -4608,7 +4609,7 @@ name = "unicode-normalization"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -5452,7 +5453,7 @@ dependencies = [
 "checksum slog-term 2.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "54b50e85b73c2bd42ceb97b6ded235576d405bd1e974242ccfe634fa269f6da7"
 "checksum sloggers 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d41aa58f9a02e205e21117ffa08e94c37f06e1f1009be2639b621f351a75796d"
 "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
-"checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4"
+"checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc"
 "checksum snow 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "afb767eee7d257ba202f0b9b08673bc13b22281632ef45267b19f13100accd2f"
 "checksum soketto 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bceb1a3a15232d013d9a3b7cac9e5ce8e2313f348f01d4bc1097e5e53aa07095"
 "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"

eth2/utils/ssz/Cargo.toml

@@ -14,3 +14,4 @@ eth2_ssz_derive = "0.1.0"
 
 [dependencies]
 ethereum-types = "0.8.0"
+smallvec = "1.2.0"

eth2/utils/ssz/src/decode.rs

@@ -1,4 +1,7 @@
 use super::*;
+use smallvec::{smallvec, SmallVec};
+
+type SmallVec8<T> = SmallVec<[T; 8]>;
 
 pub mod impls;
 
@@ -62,8 +65,8 @@ pub struct Offset {
 /// See [`SszDecoder`](struct.SszDecoder.html) for usage examples.
 pub struct SszDecoderBuilder<'a> {
     bytes: &'a [u8],
-    items: Vec<&'a [u8]>,
+    items: SmallVec8<&'a [u8]>,
-    offsets: Vec<Offset>,
+    offsets: SmallVec8<Offset>,
     items_index: usize,
 }
 
@@ -73,8 +76,8 @@ impl<'a> SszDecoderBuilder<'a> {
     pub fn new(bytes: &'a [u8]) -> Self {
         Self {
             bytes,
-            items: vec![],
+            items: smallvec![],
-            offsets: vec![],
+            offsets: smallvec![],
             items_index: 0,
         }
     }
@@ -204,7 +207,7 @@ impl<'a> SszDecoderBuilder<'a> {
 ///
 /// ```
 pub struct SszDecoder<'a> {
-    items: Vec<&'a [u8]>,
+    items: SmallVec8<&'a [u8]>,
 }
 
 impl<'a> SszDecoder<'a> {

eth2/utils/ssz/src/encode.rs

@@ -115,7 +115,7 @@ impl<'a> SszEncoder<'a> {
             item.ssz_append(&mut self.buf);
         } else {
             self.buf
-                .append(&mut encode_length(self.offset + self.variable_bytes.len()));
+                .extend_from_slice(&encode_length(self.offset + self.variable_bytes.len()));
 
             item.ssz_append(&mut self.variable_bytes);
         }
@@ -132,17 +132,17 @@ impl<'a> SszEncoder<'a> {
     }
 }
 
-/// Encode `index` as a little-endian byte vec of `BYTES_PER_LENGTH_OFFSET` length.
+/// Encode `index` as a little-endian byte array of `BYTES_PER_LENGTH_OFFSET` length.
 ///
 /// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised.
-pub fn encode_union_index(index: usize) -> Vec<u8> {
+pub fn encode_union_index(index: usize) -> [u8; BYTES_PER_LENGTH_OFFSET] {
     encode_length(index)
 }
 
-/// Encode `len` as a little-endian byte vec of `BYTES_PER_LENGTH_OFFSET` length.
+/// Encode `len` as a little-endian byte array of `BYTES_PER_LENGTH_OFFSET` length.
 ///
 /// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised.
-pub fn encode_length(len: usize) -> Vec<u8> {
+pub fn encode_length(len: usize) -> [u8; BYTES_PER_LENGTH_OFFSET] {
     // Note: it is possible for `len` to be larger than what can be encoded in
     // `BYTES_PER_LENGTH_OFFSET` bytes, triggering this debug assertion.
     //
@@ -166,7 +166,9 @@ pub fn encode_length(len: usize) -> Vec<u8> {
     // If you have a different opinion, feel free to start an issue and tag @paulhauner.
     debug_assert!(len <= MAX_LENGTH_VALUE);
 
-    len.to_le_bytes()[0..BYTES_PER_LENGTH_OFFSET].to_vec()
+    let mut bytes = [0; BYTES_PER_LENGTH_OFFSET];
+    bytes.copy_from_slice(&len.to_le_bytes()[0..BYTES_PER_LENGTH_OFFSET]);
+    bytes
 }
 
 #[cfg(test)]
@@ -175,13 +177,13 @@ mod tests {
 
     #[test]
     fn test_encode_length() {
-        assert_eq!(encode_length(0), vec![0; 4]);
+        assert_eq!(encode_length(0), [0; 4]);
 
-        assert_eq!(encode_length(1), vec![1, 0, 0, 0]);
+        assert_eq!(encode_length(1), [1, 0, 0, 0]);
 
         assert_eq!(
             encode_length(MAX_LENGTH_VALUE),
-            vec![255; BYTES_PER_LENGTH_OFFSET]
+            [255; BYTES_PER_LENGTH_OFFSET]
         );
     }
 
@@ -195,6 +197,6 @@ mod tests {
     #[test]
     #[cfg(not(debug_assertions))]
     fn test_encode_length_above_max_not_debug_does_not_panic() {
-        assert_eq!(encode_length(MAX_LENGTH_VALUE + 1), vec![0; 4]);
+        assert_eq!(&encode_length(MAX_LENGTH_VALUE + 1)[..], &[0; 4]);
     }
 }

eth2/utils/ssz/src/encode/impls.rs

@@ -221,9 +221,9 @@ impl<T: Encode> Encode for Option<T> {
 
     fn ssz_append(&self, buf: &mut Vec<u8>) {
         match self {
-            None => buf.append(&mut encode_union_index(0)),
+            None => buf.extend_from_slice(&encode_union_index(0)),
             Some(t) => {
-                buf.append(&mut encode_union_index(1));
+                buf.extend_from_slice(&encode_union_index(1));
                 t.ssz_append(buf);
             }
         }

eth2/utils/ssz_derive/src/lib.rs

@@ -1,4 +1,4 @@
-#![recursion_limit = "128"]
+#![recursion_limit = "256"]
 //! Provides procedural derive macros for the `Encode` and `Decode` traits of the `eth2_ssz` crate.
 //!
 //! Supports field attributes, see each derive macro for more information.
@@ -173,6 +173,7 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
     };
 
     let mut register_types = vec![];
+    let mut fixed_decodes = vec![];
     let mut decodes = vec![];
     let mut is_fixed_lens = vec![];
     let mut fixed_lens = vec![];
@@ -187,6 +188,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
                     decodes.push(quote! {
                         #ident: <_>::default()
                     });
+
+                    fixed_decodes.push(quote! {
+                        #ident: <_>::default()
+                    });
                 } else {
                     let ty = &field.ty;
 
@@ -198,6 +203,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
                         #ident: decoder.decode_next()?
                     });
 
+                    fixed_decodes.push(quote! {
+                        #ident: decode_field!(#ty)
+                    });
+
                     is_fixed_lens.push(quote! {
                         <#ty as ssz::Decode>::is_ssz_fixed_len()
                     });
@@ -232,19 +241,50 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
             }
 
             fn from_ssz_bytes(bytes: &[u8]) -> std::result::Result<Self, ssz::DecodeError> {
-                let mut builder = ssz::SszDecoderBuilder::new(bytes);
+                if <Self as ssz::Decode>::is_ssz_fixed_len() {
+                    if bytes.len() != <Self as ssz::Decode>::ssz_fixed_len() {
+                        return Err(ssz::DecodeError::InvalidByteLength {
+                            len: bytes.len(),
+                            expected: <Self as ssz::Decode>::ssz_fixed_len(),
+                        });
+                    }
 
-                #(
+                    let mut start = 0;
-                    #register_types
+                    let mut end = start;
-                )*
 
-                let mut decoder = builder.build()?;
+                    macro_rules! decode_field {
+                        ($type: ty) => {{
+                            start = end;
+                            end += <$type as ssz::Decode>::ssz_fixed_len();
+                            let slice = bytes.get(start..end)
+                                .ok_or_else(|| ssz::DecodeError::InvalidByteLength {
+                                    len: bytes.len(),
+                                    expected: end
+                                })?;
+                            <$type as ssz::Decode>::from_ssz_bytes(slice)?
+                        }};
+                    }
+
+                    Ok(Self {
+                        #(
+                            #fixed_decodes,
+                        )*
+                    })
+                } else {
+                    let mut builder = ssz::SszDecoderBuilder::new(bytes);
 
-                Ok(Self {
                     #(
-                        #decodes,
+                        #register_types
                     )*
-                })
+
+                    let mut decoder = builder.build()?;
+
+                    Ok(Self {
+                        #(
+                            #decodes,
+                        )*
+                    })
+                }
             }
         }
     };