From daf6912d18ec8af6f40013aeea156aaceaff0567 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 6 May 2019 08:47:49 +1000 Subject: [PATCH] Begin refactor for less allocation --- eth2/utils/ssz/examples/large_list.rs | 4 +- eth2/utils/ssz/examples/struct_definition.rs | 23 ++++-- eth2/utils/ssz/src/encode.rs | 40 ++++++++++- eth2/utils/ssz/src/encode/impls.rs | 73 +++++--------------- eth2/utils/ssz/src/lib.rs | 8 ++- eth2/utils/ssz/tests/tests.rs | 2 + 6 files changed, 82 insertions(+), 68 deletions(-) diff --git a/eth2/utils/ssz/examples/large_list.rs b/eth2/utils/ssz/examples/large_list.rs index b235771fc..d6cba8240 100644 --- a/eth2/utils/ssz/examples/large_list.rs +++ b/eth2/utils/ssz/examples/large_list.rs @@ -1,4 +1,4 @@ -//! Encode and decode a list 10,000 times. +//! Encode and decode a list many times. //! //! Useful for `cargo flamegraph`. @@ -7,7 +7,7 @@ use ssz::{Decodable, Encodable}; fn main() { let vec: Vec = vec![4242; 8196]; - let output: Vec> = (0..10_000) + let output: Vec> = (0..40_000) .into_iter() .map(|_| Vec::from_ssz_bytes(&vec.as_ssz_bytes()).unwrap()) .collect(); diff --git a/eth2/utils/ssz/examples/struct_definition.rs b/eth2/utils/ssz/examples/struct_definition.rs index d80093fc1..f363030f5 100644 --- a/eth2/utils/ssz/examples/struct_definition.rs +++ b/eth2/utils/ssz/examples/struct_definition.rs @@ -1,4 +1,4 @@ -use ssz::{Decodable, DecodeError, Encodable, SszDecoderBuilder, SszStream}; +use ssz::{encode_length, Decodable, DecodeError, Encodable, SszDecoderBuilder, SszStream}; #[derive(Debug, PartialEq)] pub struct Foo { @@ -12,14 +12,23 @@ impl Encodable for Foo { ::is_ssz_fixed_len() && as Encodable>::is_ssz_fixed_len() } - fn as_ssz_bytes(&self) -> Vec { - let mut stream = SszStream::new(); + fn ssz_append(&self, buf: &mut Vec) { + let offset = ::ssz_fixed_len() + + as Encodable>::ssz_fixed_len() + + ::ssz_fixed_len(); - stream.append(&self.a); - stream.append(&self.b); - stream.append(&self.c); + let mut fixed = Vec::with_capacity(offset); + let mut variable = vec![]; - stream.drain() + if ::is_ssz_fixed_len() { + self.a.ssz_append(&mut fixed); + } else { + fixed.append(encode_length()) + } + + if as Encodable>::is_ssz_fixed_len() { + self.a.ssz_append(&mut fixed); + } } } diff --git a/eth2/utils/ssz/src/encode.rs b/eth2/utils/ssz/src/encode.rs index 6650f8e90..dccc652a2 100644 --- a/eth2/utils/ssz/src/encode.rs +++ b/eth2/utils/ssz/src/encode.rs @@ -3,10 +3,10 @@ use super::*; mod impls; pub trait Encodable { - fn as_ssz_bytes(&self) -> Vec; - fn is_ssz_fixed_len() -> bool; + fn ssz_append(&self, buf: &mut Vec); + /// The number of bytes this object occupies in the fixed-length portion of the SSZ bytes. /// /// By default, this is set to `BYTES_PER_LENGTH_OFFSET` which is suitable for variable length @@ -15,6 +15,14 @@ pub trait Encodable { fn ssz_fixed_len() -> usize { BYTES_PER_LENGTH_OFFSET } + + fn as_ssz_bytes(&self) -> Vec { + let mut buf = vec![]; + + self.ssz_append(&mut buf); + + buf + } } pub struct VariableLengths { @@ -40,11 +48,13 @@ impl SszStream { } } + /* /// Append some item to the stream. pub fn append(&mut self, item: &T) { let mut bytes = item.as_ssz_bytes(); if T::is_ssz_fixed_len() { + self.app self.fixed_bytes.append(&mut bytes); } else { self.variable_lengths.push(VariableLengths { @@ -57,6 +67,32 @@ impl SszStream { self.variable_bytes.append(&mut bytes); } } + */ + pub fn reserve(&mut self, additional: usize) { + if T::is_ssz_fixed_len() { + self.fixed_bytes.reserve(additional * T::ssz_fixed_len()); + } else { + self.fixed_bytes + .reserve(additional * BYTES_PER_LENGTH_OFFSET); + self.variable_lengths.reserve(additional); + } + } + + pub fn append_fixed_bytes(&mut self, bytes: &[u8]) { + self.fixed_bytes.extend_from_slice(bytes) + } + + pub fn append_variable_bytes(&mut self, bytes: &[u8]) { + self.variable_lengths.push(VariableLengths { + fixed_bytes_position: self.fixed_bytes.len(), + variable_bytes_length: bytes.len(), + }); + + self.fixed_bytes + .append(&mut vec![0; BYTES_PER_LENGTH_OFFSET]); + + self.variable_bytes.extend_from_slice(bytes); + } /// Update the offsets (if any) in the fixed-length bytes to correctly point to the values in /// the variable length part. diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index 24665a049..017b0f29f 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -12,8 +12,8 @@ macro_rules! impl_encodable_for_uint { $bit_size / 8 } - fn as_ssz_bytes(&self) -> Vec { - self.to_le_bytes().to_vec() + fn ssz_append(&self, buf: &mut Vec) { + buf.extend_from_slice(&self.to_le_bytes()); } } }; @@ -30,16 +30,24 @@ impl Encodable for Vec { false } - fn as_ssz_bytes(&self) -> Vec { + fn ssz_append(&self, buf: &mut Vec) { + if T::is_ssz_fixed_len() { - let mut bytes = Vec::with_capacity(T::ssz_fixed_len() * self.len()); + buf.reserve(T::ssz_fixed_len() * self.len()); for item in self { - bytes.append(&mut item.as_ssz_bytes()); + item.ssz_append(buf); } - - bytes } else { + /* + for item in self { + let mut substream = SszStream::new(); + + item.ssz_append(&mut substream); + + s.append_variable_bytes(&substream.drain()); + } + */ let mut offset = self.len() * BYTES_PER_LENGTH_OFFSET; let mut fixed = Vec::with_capacity(offset); let mut variable = vec![]; @@ -51,9 +59,8 @@ impl Encodable for Vec { variable.append(&mut bytes); } - fixed.append(&mut variable); - - fixed + buf.append(&mut fixed); + buf.append(&mut variable); } } } @@ -64,8 +71,8 @@ impl Encodable for bool { Some(8) } - fn as_ssz_bytes(&self) -> Vec { - (*self as u8).to_le_bytes().to_vec() + fn ssz_append(&self, s: &mut SszStream) { + s.append_fixed_bytes(&(self as u8).to_le_bytes()); } } @@ -94,48 +101,6 @@ macro_rules! impl_encodable_for_u8_array { } impl_encodable_for_u8_array!(4); - -macro_rules! impl_encodable_for_u8_array { - ($len: expr) => { - impl Encodable for [u8; $len] { - - fn ssz_append(&self, s: &mut SszStream) { - let bytes: Vec = self.iter().cloned().collect(); - s.append_encoded_raw(&bytes); - } - } - }; -} - -impl_encodable_for_u8_array!(4); - -impl Encodable for bool { - fn ssz_append(&self, s: &mut SszStream) { - let byte = if *self { 0b0000_0001 } else { 0b0000_0000 }; - s.append_encoded_raw(&[byte]); - } -} - -impl Encodable for H256 { - fn ssz_append(&self, s: &mut SszStream) { - s.append_encoded_raw(self.as_bytes()); - } -} - -impl Encodable for Address { - fn ssz_append(&self, s: &mut SszStream) { - s.append_encoded_raw(self.as_bytes()); - } -} - -impl Encodable for Vec -where - T: Encodable, -{ - fn ssz_append(&self, s: &mut SszStream) { - s.append_vec(&self); - } -} */ #[cfg(test)] diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index b38c01e80..809daf49f 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -27,9 +27,11 @@ pub fn ssz_encode(val: &T) -> Vec where T: Encodable, { - let mut ssz_stream = SszStream::new(); - ssz_stream.append(val); - ssz_stream.drain() + let mut buf = vec![]; + + val.ssz_append(&mut buf); + + buf } /* diff --git a/eth2/utils/ssz/tests/tests.rs b/eth2/utils/ssz/tests/tests.rs index 876cb4289..c3db0c757 100644 --- a/eth2/utils/ssz/tests/tests.rs +++ b/eth2/utils/ssz/tests/tests.rs @@ -1,6 +1,7 @@ use ssz::{Decodable, Encodable}; use ssz_derive::{Decode, Encode}; +/* fn round_trip(items: Vec) { for item in items { let encoded = &item.as_ssz_bytes(); @@ -157,3 +158,4 @@ fn vec_of_variable_len_struct() { round_trip(items); } +*/