cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Add optimized SSZ decoding for fixed-len items (#865)

Browse Source 
* Add custom SSZ decode for Validator

* Move efficient decode into macro

* Don't allocate SSZ offset to heap

* Use smallvec in SszDecoder

* Fix test compile error
This commit is contained in:
Paul Hauner 2020-03-04 11:45:01 +11:00 committed by GitHub
parent 58fb144276
commit 871163aecc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 79 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@ -1131,7 +1131,7 @@ dependencies = [
"slog-async 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-stdlog 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-term 2.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io-timeout 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1182,6 +1182,7 @@ version = "0.1.2"
dependencies = [
"eth2_ssz_derive 0.1.0",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -2059,7 +2060,7 @@ dependencies = [
"protobuf 2.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
"sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2655,7 +2656,7 @@ dependencies = [
"rlp 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"sloggers 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"store 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3877,7 +3878,7 @@ dependencies = [
[[package]]
name = "smallvec"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -4608,7 +4609,7 @@ name = "unicode-normalization"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -5452,7 +5453,7 @@ dependencies = [
"checksum slog-term 2.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "54b50e85b73c2bd42ceb97b6ded235576d405bd1e974242ccfe634fa269f6da7"
"checksum sloggers 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d41aa58f9a02e205e21117ffa08e94c37f06e1f1009be2639b621f351a75796d"
"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
"checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4"
"checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc"
"checksum snow 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "afb767eee7d257ba202f0b9b08673bc13b22281632ef45267b19f13100accd2f"
"checksum soketto 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bceb1a3a15232d013d9a3b7cac9e5ce8e2313f348f01d4bc1097e5e53aa07095"
"checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"

1
eth2/utils/ssz/Cargo.toml
View File

@ -14,3 +14,4 @@ eth2_ssz_derive = "0.1.0"
[dependencies]
ethereum-types = "0.8.0"
smallvec = "1.2.0"

13
eth2/utils/ssz/src/decode.rs
View File

@ -1,4 +1,7 @@
use super::*;
use smallvec::{smallvec, SmallVec};
type SmallVec8<T> = SmallVec<[T; 8]>;
pub mod impls;
@ -62,8 +65,8 @@ pub struct Offset {
/// See [`SszDecoder`](struct.SszDecoder.html) for usage examples.
pub struct SszDecoderBuilder<'a> {
bytes: &'a [u8],
items: Vec<&'a [u8]>,
offsets: Vec<Offset>,
items: SmallVec8<&'a [u8]>,
offsets: SmallVec8<Offset>,
items_index: usize,
}
@ -73,8 +76,8 @@ impl<'a> SszDecoderBuilder<'a> {
pub fn new(bytes: &'a [u8]) -> Self {
Self {
bytes,
items: vec![],
offsets: vec![],
items: smallvec![],
offsets: smallvec![],
items_index: 0,
}
}
@ -204,7 +207,7 @@ impl<'a> SszDecoderBuilder<'a> {
///
/// ```
pub struct SszDecoder<'a> {
items: Vec<&'a [u8]>,
items: SmallVec8<&'a [u8]>,
}
impl<'a> SszDecoder<'a> {

22
eth2/utils/ssz/src/encode.rs
View File

@ -115,7 +115,7 @@ impl<'a> SszEncoder<'a> {
item.ssz_append(&mut self.buf);
} else {
self.buf
.append(&mut encode_length(self.offset + self.variable_bytes.len()));
.extend_from_slice(&encode_length(self.offset + self.variable_bytes.len()));
item.ssz_append(&mut self.variable_bytes);
}
@ -132,17 +132,17 @@ impl<'a> SszEncoder<'a> {
}
}
/// Encode `index` as a little-endian byte vec of `BYTES_PER_LENGTH_OFFSET` length.
/// Encode `index` as a little-endian byte array of `BYTES_PER_LENGTH_OFFSET` length.
///
/// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised.
pub fn encode_union_index(index: usize) -> Vec<u8> {
pub fn encode_union_index(index: usize) -> [u8; BYTES_PER_LENGTH_OFFSET] {
encode_length(index)
}
/// Encode `len` as a little-endian byte vec of `BYTES_PER_LENGTH_OFFSET` length.
/// Encode `len` as a little-endian byte array of `BYTES_PER_LENGTH_OFFSET` length.
///
/// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised.
pub fn encode_length(len: usize) -> Vec<u8> {
pub fn encode_length(len: usize) -> [u8; BYTES_PER_LENGTH_OFFSET] {
// Note: it is possible for `len` to be larger than what can be encoded in
// `BYTES_PER_LENGTH_OFFSET` bytes, triggering this debug assertion.
//
@ -166,7 +166,9 @@ pub fn encode_length(len: usize) -> Vec<u8> {
// If you have a different opinion, feel free to start an issue and tag @paulhauner.
debug_assert!(len <= MAX_LENGTH_VALUE);
len.to_le_bytes()[0..BYTES_PER_LENGTH_OFFSET].to_vec()
let mut bytes = [0; BYTES_PER_LENGTH_OFFSET];
bytes.copy_from_slice(&len.to_le_bytes()[0..BYTES_PER_LENGTH_OFFSET]);
bytes
}
#[cfg(test)]
@ -175,13 +177,13 @@ mod tests {
#[test]
fn test_encode_length() {
assert_eq!(encode_length(0), vec![0; 4]);
assert_eq!(encode_length(0), [0; 4]);
assert_eq!(encode_length(1), vec![1, 0, 0, 0]);
assert_eq!(encode_length(1), [1, 0, 0, 0]);
assert_eq!(
encode_length(MAX_LENGTH_VALUE),
vec![255; BYTES_PER_LENGTH_OFFSET]
[255; BYTES_PER_LENGTH_OFFSET]
);
}
@ -195,6 +197,6 @@ mod tests {
#[test]
#[cfg(not(debug_assertions))]
fn test_encode_length_above_max_not_debug_does_not_panic() {
assert_eq!(encode_length(MAX_LENGTH_VALUE + 1), vec![0; 4]);
assert_eq!(&encode_length(MAX_LENGTH_VALUE + 1)[..], &[0; 4]);
}
}

4
eth2/utils/ssz/src/encode/impls.rs
View File

@ -221,9 +221,9 @@ impl<T: Encode> Encode for Option<T> {
fn ssz_append(&self, buf: &mut Vec<u8>) {
match self {
None => buf.append(&mut encode_union_index(0)),
None => buf.extend_from_slice(&encode_union_index(0)),
Some(t) => {
buf.append(&mut encode_union_index(1));
buf.extend_from_slice(&encode_union_index(1));
t.ssz_append(buf);
}
}

58
eth2/utils/ssz_derive/src/lib.rs
View File

@ -1,4 +1,4 @@
#![recursion_limit = "128"]
#![recursion_limit = "256"]
//! Provides procedural derive macros for the `Encode` and `Decode` traits of the `eth2_ssz` crate.
//!
//! Supports field attributes, see each derive macro for more information.
@ -173,6 +173,7 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
};
let mut register_types = vec![];
let mut fixed_decodes = vec![];
let mut decodes = vec![];
let mut is_fixed_lens = vec![];
let mut fixed_lens = vec![];
@ -187,6 +188,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
decodes.push(quote! {
#ident: <_>::default()
});
fixed_decodes.push(quote! {
#ident: <_>::default()
});
} else {
let ty = &field.ty;
@ -198,6 +203,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
#ident: decoder.decode_next()?
});
fixed_decodes.push(quote! {
#ident: decode_field!(#ty)
});
is_fixed_lens.push(quote! {
<#ty as ssz::Decode>::is_ssz_fixed_len()
});
@ -232,19 +241,50 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
}
fn from_ssz_bytes(bytes: &[u8]) -> std::result::Result<Self, ssz::DecodeError> {
let mut builder = ssz::SszDecoderBuilder::new(bytes);
if <Self as ssz::Decode>::is_ssz_fixed_len() {
if bytes.len() != <Self as ssz::Decode>::ssz_fixed_len() {
return Err(ssz::DecodeError::InvalidByteLength {
len: bytes.len(),
expected: <Self as ssz::Decode>::ssz_fixed_len(),
});
}
#(
#register_types
)*
let mut start = 0;
let mut end = start;
let mut decoder = builder.build()?;
macro_rules! decode_field {
($type: ty) => {{
start = end;
end += <$type as ssz::Decode>::ssz_fixed_len();
let slice = bytes.get(start..end)
.ok_or_else(|| ssz::DecodeError::InvalidByteLength {
len: bytes.len(),
expected: end
})?;
<$type as ssz::Decode>::from_ssz_bytes(slice)?
}};
}
Ok(Self {
#(
#fixed_decodes,
)*
})
} else {
let mut builder = ssz::SszDecoderBuilder::new(bytes);
Ok(Self {
#(
#decodes,
#register_types
)*
})
let mut decoder = builder.build()?;
Ok(Self {
#(
#decodes,
)*
})
}
}
}
};