From c59bab58d269cadc8f57ad1fdb5f35d4c8bb57b0 Mon Sep 17 00:00:00 2001 From: Darren Langley Date: Fri, 15 Feb 2019 20:46:06 +1000 Subject: [PATCH] big to little endian, some more tests --- eth2/utils/ssz/README.md | 16 ++++---- eth2/utils/ssz/src/decode.rs | 51 ++++++++++++++++--------- eth2/utils/ssz/src/encode.rs | 62 +++++++++++++++++++++++++++---- eth2/utils/ssz/src/impl_decode.rs | 18 ++++----- eth2/utils/ssz/src/impl_encode.rs | 53 ++++++++++++++++++-------- 5 files changed, 141 insertions(+), 59 deletions(-) diff --git a/eth2/utils/ssz/README.md b/eth2/utils/ssz/README.md index 7355ca4cc..30d8ded72 100644 --- a/eth2/utils/ssz/README.md +++ b/eth2/utils/ssz/README.md @@ -69,7 +69,7 @@ Syntax: | Shorthand | Meaning | |:-------------|:----------------------------------------------------| -| `big` | ``big endian`` | +| `little` | ``little endian`` | | `to_bytes` | convert to bytes. Params: ``(size, byte order)`` | | `from_bytes` | convert from bytes. Params: ``(bytes, byte order)`` | | `value` | the value to serialize | @@ -82,7 +82,7 @@ Syntax: Convert directly to bytes the size of the int. (e.g. ``int16 = 2 bytes``) -All integers are serialized as **big endian**. +All integers are serialized as **little endian**. | Check to perform | Code | |:-----------------------|:------------------------| @@ -92,7 +92,7 @@ All integers are serialized as **big endian**. ```python buffer_size = int_size / 8 -return value.to_bytes(buffer_size, 'big') +return value.to_bytes(buffer_size, 'little') ``` #### Address @@ -131,7 +131,7 @@ For general `byte` type: value_bytes ]`` ```python -byte_length = (len(value)).to_bytes(4, 'big') +byte_length = (len(value)).to_bytes(4, 'little') return byte_length + value ``` @@ -175,12 +175,12 @@ At each step, the following checks should be made: Convert directly from bytes into integer utilising the number of bytes the same size as the integer length. (e.g. ``int16 == 2 bytes``) -All integers are interpreted as **big endian**. +All integers are interpreted as **little endian**. ```python byte_length = int_size / 8 new_index = current_index + int_size -return int.from_bytes(rawbytes[current_index:current_index+int_size], 'big'), new_index +return int.from_bytes(rawbytes[current_index:current_index+int_size], 'little'), new_index ``` #### Address @@ -206,7 +206,7 @@ return rawbytes[current_index:current_index+32], new_index Get the length of the bytes, return the bytes. ```python -bytes_length = int.from_bytes(rawbytes[current_index:current_index+4], 'big') +bytes_length = int.from_bytes(rawbytes[current_index:current_index+4], 'little') new_index = current_index + 4 + bytes_lenth return rawbytes[current_index+4:current_index+4+bytes_length], new_index ``` @@ -224,7 +224,7 @@ entire length of the list. | rawbytes has enough left for length | ``len(rawbytes) > current_index + 4`` | ```python -total_length = int.from_bytes(rawbytes[current_index:current_index+4], 'big') +total_length = int.from_bytes(rawbytes[current_index:current_index+4], 'little') new_index = current_index + 4 + total_length item_index = current_index + 4 deserialized_list = [] diff --git a/eth2/utils/ssz/src/decode.rs b/eth2/utils/ssz/src/decode.rs index 426baeace..6053ee940 100644 --- a/eth2/utils/ssz/src/decode.rs +++ b/eth2/utils/ssz/src/decode.rs @@ -65,7 +65,7 @@ where } /// Given some number of bytes, interpret the first four -/// bytes as a 32-bit big-endian integer and return the +/// bytes as a 32-bit little-endian integer and return the /// result. pub fn decode_length( bytes: &[u8], @@ -82,7 +82,7 @@ pub fn decode_length( .take(index + length_bytes) .skip(index) { - let offset = (index + length_bytes - i - 1) * 8; + let offset = (length_bytes - (length_bytes - (i - index))) * 8; len |= (*byte as usize) << offset; } Ok(len) @@ -90,18 +90,18 @@ pub fn decode_length( #[cfg(test)] mod tests { - use super::super::encode::encode_length; + use super::super::encode::*; use super::*; #[test] fn test_ssz_decode_length() { - let decoded = decode_length(&vec![0, 0, 0, 1], 0, LENGTH_BYTES); + let decoded = decode_length(&vec![1, 0, 0, 0], 0, LENGTH_BYTES); assert_eq!(decoded.unwrap(), 1); - let decoded = decode_length(&vec![0, 0, 1, 0], 0, LENGTH_BYTES); + let decoded = decode_length(&vec![0, 1, 0, 0], 0, LENGTH_BYTES); assert_eq!(decoded.unwrap(), 256); - let decoded = decode_length(&vec![0, 0, 1, 255], 0, LENGTH_BYTES); + let decoded = decode_length(&vec![255, 1, 0, 0], 0, LENGTH_BYTES); assert_eq!(decoded.unwrap(), 511); let decoded = decode_length(&vec![255, 255, 255, 255], 0, LENGTH_BYTES); @@ -132,21 +132,35 @@ mod tests { } } + #[test] + fn test_encode_decode_ssz_list() { + let test_vec: Vec = vec![256; 12]; + let mut stream = SszStream::new(); + stream.append_vec(&test_vec); + let ssz = stream.drain(); + + // u16 + let decoded: (Vec, usize) = decode_ssz_list(&ssz, 0).unwrap(); + + assert_eq!(decoded.0, test_vec); + assert_eq!(decoded.1, LENGTH_BYTES + (12 * 2)); + } + #[test] fn test_decode_ssz_list() { // u16 let v: Vec = vec![10, 10, 10, 10]; let decoded: (Vec, usize) = - decode_ssz_list(&vec![0, 0, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10], 0).unwrap(); + decode_ssz_list(&vec![8, 0, 0, 0, 10, 0, 10, 0, 10, 0, 10, 0], 0).unwrap(); assert_eq!(decoded.0, v); - assert_eq!(decoded.1, 12); + assert_eq!(decoded.1, LENGTH_BYTES + (4 * 2)); // u32 let v: Vec = vec![10, 10, 10, 10]; let decoded: (Vec, usize) = decode_ssz_list( &vec![ - 0, 0, 0, 16, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10, + 16, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 00, ], 0, ) @@ -158,36 +172,37 @@ mod tests { let v: Vec = vec![10, 10, 10, 10]; let decoded: (Vec, usize) = decode_ssz_list( &vec![ - 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, - 10, 0, 0, 0, 0, 0, 0, 0, 10, + 32, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, + 0, 10, 0, 0, 0, 0, 0, 0, 0, ], 0, ) .unwrap(); assert_eq!(decoded.0, v); - assert_eq!(decoded.1, 36); + assert_eq!(decoded.1, LENGTH_BYTES + (8 * 4)); // Check that it can accept index let v: Vec = vec![15, 15, 15, 15]; + let offset = 10; let decoded: (Vec, usize) = decode_ssz_list( &vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, - 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 32, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, + 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, ], - 10, + offset, ) .unwrap(); assert_eq!(decoded.0, v); - assert_eq!(decoded.1, 46); + assert_eq!(decoded.1, offset + LENGTH_BYTES + (8 * 4)); // Check that length > bytes throws error let decoded: Result<(Vec, usize), DecodeError> = - decode_ssz_list(&vec![0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 15], 0); + decode_ssz_list(&vec![32, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0], 0); assert_eq!(decoded, Err(DecodeError::TooShort)); // Check that incorrect index throws error let decoded: Result<(Vec, usize), DecodeError> = - decode_ssz_list(&vec![0, 0, 0, 0, 0, 0, 0, 15], 16); + decode_ssz_list(&vec![15, 0, 0, 0, 0, 0, 0, 0], 16); assert_eq!(decoded, Err(DecodeError::TooShort)); } } diff --git a/eth2/utils/ssz/src/encode.rs b/eth2/utils/ssz/src/encode.rs index dfb969a8d..f310c8d1b 100644 --- a/eth2/utils/ssz/src/encode.rs +++ b/eth2/utils/ssz/src/encode.rs @@ -70,13 +70,13 @@ impl SszStream { /// Encode some length into a ssz size prefix. /// /// The ssz size prefix is 4 bytes, which is treated as a continuious -/// 32bit big-endian integer. +/// 32bit little-endian integer. pub fn encode_length(len: usize, length_bytes: usize) -> Vec { assert!(length_bytes > 0); // For sanity assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8)); let mut header: Vec = vec![0; length_bytes]; for (i, header_byte) in header.iter_mut().enumerate() { - let offset = (length_bytes - i - 1) * 8; + let offset = (length_bytes - (length_bytes - i)) * 8; *header_byte = ((len >> offset) & 0xff) as u8; } header @@ -95,15 +95,27 @@ mod tests { #[test] fn test_encode_length_4_bytes() { assert_eq!(encode_length(0, LENGTH_BYTES), vec![0; 4]); - assert_eq!(encode_length(1, LENGTH_BYTES), vec![0, 0, 0, 1]); - assert_eq!(encode_length(255, LENGTH_BYTES), vec![0, 0, 0, 255]); - assert_eq!(encode_length(256, LENGTH_BYTES), vec![0, 0, 1, 0]); + assert_eq!(encode_length(1, LENGTH_BYTES), vec![1, 0, 0, 0]); + assert_eq!(encode_length(255, LENGTH_BYTES), vec![255, 0, 0, 0]); + assert_eq!(encode_length(256, LENGTH_BYTES), vec![0, 1, 0, 0]); assert_eq!( encode_length(4294967295, LENGTH_BYTES), // 2^(3*8) - 1 vec![255, 255, 255, 255] ); } + #[test] + fn test_encode_lower_length() { + assert_eq!(encode_length(0, LENGTH_BYTES - 2), vec![0; 2]); + assert_eq!(encode_length(1, LENGTH_BYTES - 2), vec![1, 0]); + } + + #[test] + fn test_encode_higher_length() { + assert_eq!(encode_length(0, LENGTH_BYTES + 2), vec![0; 6]); + assert_eq!(encode_length(1, LENGTH_BYTES + 2), vec![1, 0, 0, 0, 0, 0]); + } + #[test] #[should_panic] fn test_encode_length_4_bytes_panic() { @@ -117,8 +129,42 @@ mod tests { stream.append_vec(&test_vec); let ssz = stream.drain(); - assert_eq!(ssz.len(), 4 + (12 * 2)); - assert_eq!(ssz[0..4], *vec![0, 0, 0, 24]); - assert_eq!(ssz[4..6], *vec![1, 0]); + assert_eq!(ssz.len(), LENGTH_BYTES + (12 * 2)); + assert_eq!(ssz[0..4], *vec![24, 0, 0, 0]); + assert_eq!(ssz[4..6], *vec![0, 1]); + } + + #[test] + fn test_encode_mixed_prefixed() { + let test_vec: Vec = vec![100, 200]; + let test_value: u8 = 5; + + let mut stream = SszStream::new(); + stream.append_vec(&test_vec); + stream.append(&test_value); + let ssz = stream.drain(); + + assert_eq!(ssz.len(), LENGTH_BYTES + (2 * 2) + 1); + assert_eq!(ssz[0..4], *vec![4, 0, 0, 0]); + assert_eq!(ssz[4..6], *vec![100, 0]); + assert_eq!(ssz[6..8], *vec![200, 0]); + assert_eq!(ssz[8], 5); + } + + #[test] + fn test_encode_mixed_postfixed() { + let test_value: u8 = 5; + let test_vec: Vec = vec![100, 200]; + + let mut stream = SszStream::new(); + stream.append(&test_value); + stream.append_vec(&test_vec); + let ssz = stream.drain(); + + assert_eq!(ssz.len(), 1 + LENGTH_BYTES + (2 * 2)); + assert_eq!(ssz[0], 5); + assert_eq!(ssz[1..5], *vec![4, 0, 0, 0]); + assert_eq!(ssz[5..7], *vec![100, 0]); + assert_eq!(ssz[7..9], *vec![200, 0]); } } diff --git a/eth2/utils/ssz/src/impl_decode.rs b/eth2/utils/ssz/src/impl_decode.rs index 134e438e1..13ad24c49 100644 --- a/eth2/utils/ssz/src/impl_decode.rs +++ b/eth2/utils/ssz/src/impl_decode.rs @@ -12,7 +12,7 @@ macro_rules! impl_decodable_for_uint { let end_bytes = index + max_bytes; let mut result: $type = 0; for (i, byte) in bytes.iter().enumerate().take(end_bytes).skip(index) { - let offset = (end_bytes - i - 1) * 8; + let offset = (end_bytes - (end_bytes - (i - index))) * 8; result |= ($type::from(*byte)) << offset; } Ok((result, end_bytes)) @@ -108,12 +108,12 @@ mod tests { assert_eq!(result, 0); assert_eq!(index, 2); - let ssz = vec![0, 16]; + let ssz = vec![16, 0]; let (result, index): (u16, usize) = decode_ssz(&ssz, 0).unwrap(); assert_eq!(result, 16); assert_eq!(index, 2); - let ssz = vec![1, 0]; + let ssz = vec![0, 1]; let (result, index): (u16, usize) = decode_ssz(&ssz, 0).unwrap(); assert_eq!(result, 256); assert_eq!(index, 2); @@ -135,17 +135,17 @@ mod tests { assert_eq!(result, 0); assert_eq!(index, 4); - let ssz = vec![0, 0, 1, 0]; + let ssz = vec![0, 1, 0, 0]; let (result, index): (u32, usize) = decode_ssz(&ssz, 0).unwrap(); assert_eq!(index, 4); assert_eq!(result, 256); - let ssz = vec![255, 255, 255, 0, 0, 1, 0]; + let ssz = vec![255, 255, 255, 0, 1, 0, 0]; let (result, index): (u32, usize) = decode_ssz(&ssz, 3).unwrap(); assert_eq!(index, 7); assert_eq!(result, 256); - let ssz = vec![0, 200, 1, 0]; + let ssz = vec![0, 1, 200, 0]; let (result, index): (u32, usize) = decode_ssz(&ssz, 0).unwrap(); assert_eq!(index, 4); assert_eq!(result, 13107456); @@ -155,7 +155,7 @@ mod tests { assert_eq!(index, 4); assert_eq!(result, 4294967295); - let ssz = vec![0, 0, 1]; + let ssz = vec![1, 0, 0]; let result: Result<(u32, usize), DecodeError> = decode_ssz(&ssz, 0); assert_eq!(result, Err(DecodeError::TooShort)); } @@ -172,7 +172,7 @@ mod tests { assert_eq!(index, 8); assert_eq!(result, 18446744073709551615); - let ssz = vec![0, 0, 8, 255, 0, 0, 0, 0, 0, 0, 0]; + let ssz = vec![0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 255]; let (result, index): (u64, usize) = decode_ssz(&ssz, 3).unwrap(); assert_eq!(index, 11); assert_eq!(result, 18374686479671623680); @@ -212,7 +212,7 @@ mod tests { let err: Result<(u16, usize), DecodeError> = decode_ssz(&vec![0, 0, 0, 0], 3); assert_eq!(err, Err(DecodeError::TooShort)); - let result: u16 = decode_ssz(&vec![0, 0, 0, 0, 1], 3).unwrap().0; + let result: u16 = decode_ssz(&vec![0, 0, 0, 1, 0], 3).unwrap().0; assert_eq!(result, 1); } } diff --git a/eth2/utils/ssz/src/impl_encode.rs b/eth2/utils/ssz/src/impl_encode.rs index 8714cf75f..c41799d92 100644 --- a/eth2/utils/ssz/src/impl_encode.rs +++ b/eth2/utils/ssz/src/impl_encode.rs @@ -27,9 +27,9 @@ macro_rules! impl_encodable_for_uint { // Match bit size with encoding match $bit_size { 8 => buf.put_u8(*self as u8), - 16 => buf.put_u16_be(*self as u16), - 32 => buf.put_u32_be(*self as u32), - 64 => buf.put_u64_be(*self as u64), + 16 => buf.put_u16_le(*self as u16), + 32 => buf.put_u32_le(*self as u32), + 64 => buf.put_u64_le(*self as u64), _ => {} } @@ -115,17 +115,17 @@ mod tests { let x: u16 = 1; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 1]); + assert_eq!(ssz.drain(), vec![1, 0]); let x: u16 = 100; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 100]); + assert_eq!(ssz.drain(), vec![100, 0]); let x: u16 = 1 << 8; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![1, 0]); + assert_eq!(ssz.drain(), vec![0, 1]); let x: u16 = 65535; let mut ssz = SszStream::new(); @@ -138,22 +138,22 @@ mod tests { let x: u32 = 1; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 1]); + assert_eq!(ssz.drain(), vec![1, 0, 0, 0]); let x: u32 = 100; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 100]); + assert_eq!(ssz.drain(), vec![100, 0, 0, 0]); let x: u32 = 1 << 16; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 1, 0, 0]); + assert_eq!(ssz.drain(), vec![0, 0, 1, 0]); let x: u32 = 1 << 24; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![1, 0, 0, 0]); + assert_eq!(ssz.drain(), vec![0, 0, 0, 1]); let x: u32 = !0; let mut ssz = SszStream::new(); @@ -166,17 +166,17 @@ mod tests { let x: u64 = 1; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 0, 0, 0, 0, 1]); + assert_eq!(ssz.drain(), vec![1, 0, 0, 0, 0, 0, 0, 0]); let x: u64 = 100; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 0, 0, 0, 0, 100]); + assert_eq!(ssz.drain(), vec![100, 0, 0, 0, 0, 0, 0, 0]); let x: u64 = 1 << 32; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 1, 0, 0, 0, 0]); + assert_eq!(ssz.drain(), vec![0, 0, 0, 0, 1, 0, 0, 0]); let x: u64 = !0; let mut ssz = SszStream::new(); @@ -189,21 +189,42 @@ mod tests { let x: usize = 1; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 0, 0, 0, 0, 1]); + assert_eq!(ssz.drain(), vec![1, 0, 0, 0, 0, 0, 0, 0]); let x: usize = 100; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 0, 0, 0, 0, 100]); + assert_eq!(ssz.drain(), vec![100, 0, 0, 0, 0, 0, 0, 0]); let x: usize = 1 << 32; let mut ssz = SszStream::new(); ssz.append(&x); - assert_eq!(ssz.drain(), vec![0, 0, 0, 1, 0, 0, 0, 0]); + assert_eq!(ssz.drain(), vec![0, 0, 0, 0, 1, 0, 0, 0]); let x: usize = !0; let mut ssz = SszStream::new(); ssz.append(&x); assert_eq!(ssz.drain(), vec![255, 255, 255, 255, 255, 255, 255, 255]); } + + #[test] + fn test_ssz_mixed() { + let mut stream = SszStream::new(); + + let h = Address::zero(); + let a: u8 = 100; + let b: u16 = 65535; + let c: u32 = 1 << 24; + + stream.append(&h); + stream.append(&a); + stream.append(&b); + stream.append(&c); + + let ssz = stream.drain(); + assert_eq!(ssz[0..20], *vec![0; 20]); + assert_eq!(ssz[20], 100); + assert_eq!(ssz[21..23], *vec![255, 255]); + assert_eq!(ssz[23..27], *vec![0, 0, 0, 1]); + } }