Merge branch 'shuffle_underflow' into genesis

beacon_chain/utils/ssz_helpers/src/ssz_beacon_block.rs

@@ -14,7 +14,16 @@ pub enum SszBeaconBlockError {
     TooLong,
 }
 
-const LENGTH_BYTES: usize = 4;
+/*
+ * Constants used for navigating the SSZ bytes.
+ */
+const LENGTH_PREFIX_BYTES: usize = 4;
+const SLOT_BYTES: usize = 8;
+const HASH_SIZE: usize = 32;
+const RANDAO_REVEAL_BYTES: usize = HASH_SIZE;
+const POW_CHAIN_REF_BYTES: usize = HASH_SIZE;
+const ACTIVE_STATE_BYTES: usize = HASH_SIZE;
+const CRYSTALLIZED_STATE_BYTES: usize = HASH_SIZE;
 
 /// Allows for reading of block values directly from serialized ssz bytes.
 ///
@@ -74,41 +83,35 @@ impl<'a> SszBeaconBlock<'a> {
          * Determine how many bytes are used to store ancestor hashes.
          */
         let ancestors_position =
-            8 +     // slot
+            SLOT_BYTES +
-            32 +    // randao_reveal
+            RANDAO_REVEAL_BYTES +
-            32;     // pow_chain_reference
+            POW_CHAIN_REF_BYTES;
-        let ancestors_len = decode_length(untrimmed_ssz, ancestors_position, LENGTH_BYTES)
+        let ancestors_len = decode_length(untrimmed_ssz, ancestors_position, LENGTH_PREFIX_BYTES)
             .map_err(|_| SszBeaconBlockError::TooShort)?;
 
         /*
          * Determine how many bytes are used to store attestation records.
          */
         let attestations_position =
-            ancestors_position + LENGTH_BYTES + ancestors_len +     // end of ancestor bytes
+            ancestors_position + LENGTH_PREFIX_BYTES + ancestors_len +     // end of ancestor bytes
-            32 +                                                    // active_state_root
+            ACTIVE_STATE_BYTES +
-            32;                                                     // crystallized_state_root
+            CRYSTALLIZED_STATE_BYTES;
-        let attestations_len = decode_length(untrimmed_ssz, attestations_position, LENGTH_BYTES)
+        let attestations_len = decode_length(untrimmed_ssz, attestations_position, LENGTH_PREFIX_BYTES)
             .map_err(|_| SszBeaconBlockError::TooShort)?;
 
         /*
          * Determine how many bytes are used to store specials.
          */
-        let specials_position =
+        let specials_position = attestations_position + LENGTH_PREFIX_BYTES + attestations_len;
-            attestations_position +  // location of attestations
+        let specials_len = decode_length(untrimmed_ssz, specials_position, LENGTH_PREFIX_BYTES)
-            LENGTH_BYTES +          // attestations length prefix
-            attestations_len;        // # of attestation bytes
-        let specials_len = decode_length(untrimmed_ssz, specials_position, LENGTH_BYTES)
             .map_err(|_| SszBeaconBlockError::TooShort)?;
 
         /*
          * Now that all variable field lengths are known (ancestors, attestations, specials) we can
          * know the exact length of the block and reject it if the slice is too short.
          */
-        let block_ssz_len = {
+        let block_ssz_len = MIN_SSZ_BLOCK_LENGTH + ancestors_len + attestations_len + specials_len;
-            MIN_SSZ_BLOCK_LENGTH + ancestors_len + attestations_len + specials_len
-        };
         if vec.len() < block_ssz_len {
-            println!("block_ssz_len: {:?}, len: {:?}", block_ssz_len, vec.len());
             return Err(SszBeaconBlockError::TooShort);
         }
 
@@ -135,12 +138,8 @@ impl<'a> SszBeaconBlock<'a> {
     /// The first hash in `ancestor_hashes` is the parent of the block.
     pub fn parent_hash(&self) -> Option<&[u8]> {
         let ancestor_ssz = self.ancestor_hashes();
-        let start = LENGTH_BYTES;
+        let start = LENGTH_PREFIX_BYTES;
-        if ancestor_ssz.len() >= 32 {
+        ancestor_ssz.get(start..start + HASH_SIZE)
-            Some(&ancestor_ssz[start..start + 32])
-        } else {
-            None
-        }
     }
 
     /// Return the `slot` field.
@@ -162,54 +161,54 @@ impl<'a> SszBeaconBlock<'a> {
 
     /// Return the `randao_reveal` field.
     pub fn randao_reveal(&self) -> &[u8] {
-        let start = 8;  // slot is 8 bytes
+        let start = SLOT_BYTES;
-        &self.ssz[start..start + 32]
+        &self.ssz[start..start + RANDAO_REVEAL_BYTES]
     }
 
     /// Return the `pow_chain_reference` field.
     pub fn pow_chain_reference(&self) -> &[u8] {
         let start =
-            8 +     // slot
+            SLOT_BYTES +
-            32;     // randao_reveal
+            RANDAO_REVEAL_BYTES;
-        &self.ssz[start..start + 32]
+        &self.ssz[start..start + POW_CHAIN_REF_BYTES]
     }
 
     /// Return the serialized `ancestor_hashes` bytes, including length prefix.
     pub fn ancestor_hashes(&self) -> &[u8] {
         let start = self.ancestors_position;
-        &self.ssz[start..(start + self.ancestors_len + LENGTH_BYTES)]
+        &self.ssz[start..(start + self.ancestors_len + LENGTH_PREFIX_BYTES)]
     }
 
     /// Return the `active_state_root` field.
     pub fn act_state_root(&self) -> &[u8] {
-        let start = self.ancestors_position + LENGTH_BYTES + self.ancestors_len;
+        let start = self.ancestors_position + LENGTH_PREFIX_BYTES + self.ancestors_len;
         &self.ssz[start..(start + 32)]
     }
 
     /// Return the `active_state_root` field.
     pub fn cry_state_root(&self) -> &[u8] {
         let start =
-            self.ancestors_position + LENGTH_BYTES + self.ancestors_len +   // ancestors
+            self.ancestors_position + LENGTH_PREFIX_BYTES + self.ancestors_len +
-            32;                                                             // active_state_root;
+            ACTIVE_STATE_BYTES;
         &self.ssz[start..(start + 32)]
     }
 
     /// Return the serialized `attestations` bytes, including length prefix.
     pub fn attestations(&self) -> &[u8] {
         let start = self.attestations_position;
-        &self.ssz[start..(start + self.attestations_len + LENGTH_BYTES)]
+        &self.ssz[start..(start + self.attestations_len + LENGTH_PREFIX_BYTES)]
     }
 
     /// Return the serialized `attestations` bytes _without_ the length prefix.
     pub fn attestations_without_length(&self) -> &[u8] {
-        let start = self.attestations_position + LENGTH_BYTES;
+        let start = self.attestations_position + LENGTH_PREFIX_BYTES;
         &self.ssz[start..start + self.attestations_len]
     }
 
     /// Return the serialized `specials` bytes, including length prefix.
     pub fn specials(&self) -> &[u8] {
         let start = self.specials_position;
-        &self.ssz[start..(start + self.specials_len + LENGTH_BYTES)]
+        &self.ssz[start..(start + self.specials_len + LENGTH_PREFIX_BYTES)]
     }
 }
 
@@ -349,7 +348,7 @@ mod tests {
         let serialized = get_block_ssz(&block);
         let ssz_block = SszBeaconBlock::from_slice(&serialized).unwrap();
 
-        let mut expected = encode_length(32, LENGTH_BYTES);
+        let mut expected = encode_length(32, LENGTH_PREFIX_BYTES);
         expected.append(&mut h.to_vec());
 
         assert_eq!(ssz_block.ancestor_hashes(), &expected[..]);
@@ -383,7 +382,7 @@ mod tests {
         let ssz_block = SszBeaconBlock::from_slice(&serialized).unwrap();
         let sr_ssz = get_special_record_ssz(&s);
 
-        let mut expected = encode_length(sr_ssz.len(), LENGTH_BYTES);
+        let mut expected = encode_length(sr_ssz.len(), LENGTH_PREFIX_BYTES);
         expected.append(&mut sr_ssz.to_vec());
 
         assert_eq!(ssz_block.specials(), &expected[..]);
@@ -399,7 +398,7 @@ mod tests {
         let ssz_block = SszBeaconBlock::from_slice(&serialized).unwrap();
         let sr_ssz = get_special_record_ssz(&s);
 
-        let mut expected = encode_length(sr_ssz.len(), LENGTH_BYTES);
+        let mut expected = encode_length(sr_ssz.len(), LENGTH_PREFIX_BYTES);
         expected.append(&mut sr_ssz.to_vec());
 
         assert_eq!(ssz_block.specials(), &expected[..]);
@@ -417,7 +416,7 @@ mod tests {
         let ssz_block = SszBeaconBlock::from_slice(&serialized).unwrap();
         let ssz_ar = get_attestation_record_ssz(&AttestationRecord::zero());
 
-        let mut expected = encode_length(ssz_ar.len(), LENGTH_BYTES);
+        let mut expected = encode_length(ssz_ar.len(), LENGTH_PREFIX_BYTES);
         expected.append(&mut ssz_ar.to_vec());
 
         assert_eq!(ssz_block.attestations(), &expected[..]);
@@ -434,7 +433,7 @@ mod tests {
         let mut ssz_ar = get_attestation_record_ssz(&AttestationRecord::zero());
         ssz_ar.append(&mut get_attestation_record_ssz(&AttestationRecord::zero()));
 
-        let mut expected = encode_length(ssz_ar.len(), LENGTH_BYTES);
+        let mut expected = encode_length(ssz_ar.len(), LENGTH_PREFIX_BYTES);
         expected.append(&mut ssz_ar.to_vec());
 
         assert_eq!(ssz_block.attestations(), &expected[..]);

beacon_chain/utils/vec_shuffle/Cargo.toml

@@ -5,3 +5,6 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 
 [dependencies]
 hashing = { path = "../hashing" }
+
+[dev-dependencies]
+yaml-rust = "0.4.2"

beacon_chain/utils/vec_shuffle/src/lib.rs

@@ -25,10 +25,16 @@ pub fn shuffle<T>(
     -> Result<Vec<T>, ShuffleErr>
 {
     let mut rng = ShuffleRng::new(seed);
+
     if list.len() > rng.rand_max as usize {
         return Err(ShuffleErr::ExceedsListLength);
     }
-    for i in 0..(list.len() - 1) {
+
+    if list.is_empty() {
+        return Ok(list);
+    }
+
+    for i in 0..(list.len().saturating_sub(1)) {
         let n = list.len() - i;
         let j = rng.rand_range(n as u32) as usize + i;
         list.swap(i, j);
@@ -39,17 +45,40 @@ pub fn shuffle<T>(
 
 #[cfg(test)]
 mod tests {
+    extern crate yaml_rust;
+
     use super::*;
     use super::hashing::canonical_hash;
+    use std::fs::File;
+    use std::io::prelude::*;
+    use self::yaml_rust::yaml;
 
     #[test]
     fn test_shuffling() {
-        let seed = canonical_hash(b"4kn4driuctg8");
+        let mut file = File::open("./src/specs/shuffle_test_vectors.yaml").unwrap();
-        let list: Vec<usize> = (0..12).collect();
+        let mut yaml_str = String::new();
-        let s = shuffle(&seed, list).unwrap();
+
-        assert_eq!(
+        file.read_to_string(&mut yaml_str).unwrap();
-            s,
+
-            vec![7, 3, 2, 5, 11, 9, 1, 0, 4, 6, 10, 8],
+        let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap();
-        )
+        let doc = &docs[0];
+        let test_cases = doc["test_cases"].as_vec();
+
+        for test_case in test_cases.unwrap() {
+            let input = test_case["input"].clone().into_vec().unwrap();
+            let output = test_case["output"].clone().into_vec().unwrap();
+            let seed_bytes = test_case["seed"].as_str().unwrap().as_bytes();
+            let mut seed;
+
+            if seed_bytes.len() > 0 {
+                seed = canonical_hash(seed_bytes);
+            } else {
+                seed = vec![];
+            }
+
+            let mut s = shuffle(&seed, input).unwrap();
+
+            assert_eq!(s, output);
+        }
     }
 }

beacon_chain/utils/vec_shuffle/src/rng.rs

@@ -2,7 +2,7 @@ use super::hashing::canonical_hash;
 
 const SEED_SIZE_BYTES: usize = 32;
 const RAND_BYTES: usize = 3;      // 24 / 8
-const RAND_MAX: u32 = 16_777_216;   // 2**24
+const RAND_MAX: u32 = 16_777_215;   // 2 ** (rand_bytes * 8) - 1
 
 /// A pseudo-random number generator which given a seed
 /// uses successive blake2s hashing to generate "entropy".

beacon_chain/utils/vec_shuffle/src/specs/shuffle_test_vectors.yaml

@@ -0,0 +1,133 @@
+# This file was generated with sigp/shuffling_sandbox
+# python3 sandbox.py test_vectors
+title: Shuffling Algorithm Tests
+summary: Test vectors for shuffling a list based upon a seed.
+test_suite: Shuffling
+
+test_cases:
+- input: []
+  output: []
+  seed: ''
+- input: [0]
+  output: [0]
+  seed: ''
+- input: [255]
+  output: [255]
+  seed: ''
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [1, 6, 4, 1, 6, 6, 2, 2, 4, 5]
+  seed: ''
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [4, 7, 10, 13, 3, 1, 2, 9, 12, 6, 11, 8, 5]
+  seed: ''
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [1, 6, 65, 1, 6, 6, 2, 2, 4, 5]
+  seed: ''
+- input: []
+  output: []
+  seed: 4kn4driuctg8
+- input: [0]
+  output: [0]
+  seed: 4kn4driuctg8
+- input: [255]
+  output: [255]
+  seed: 4kn4driuctg8
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 4, 2, 5, 4, 2, 6, 6, 1, 1]
+  seed: 4kn4driuctg8
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [13, 1, 9, 8, 3, 10, 6, 2, 5, 12, 11, 4, 7]
+  seed: 4kn4driuctg8
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 65, 2, 5, 4, 2, 6, 6, 1, 1]
+  seed: 4kn4driuctg8
+- input: []
+  output: []
+  seed: ytre1p
+- input: [0]
+  output: [0]
+  seed: ytre1p
+- input: [255]
+  output: [255]
+  seed: ytre1p
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 2, 5, 1, 6, 4, 1, 2, 4, 6]
+  seed: ytre1p
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [3, 8, 10, 4, 7, 11, 6, 1, 2, 5, 13, 9, 12]
+  seed: ytre1p
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 2, 5, 1, 6, 4, 1, 2, 65, 6]
+  seed: ytre1p
+- input: []
+  output: []
+  seed: mytobcffnkvj
+- input: [0]
+  output: [0]
+  seed: mytobcffnkvj
+- input: [255]
+  output: [255]
+  seed: mytobcffnkvj
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [5, 6, 2, 1, 6, 4, 6, 4, 1, 2]
+  seed: mytobcffnkvj
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [12, 4, 11, 6, 13, 10, 9, 2, 3, 7, 8, 1, 5]
+  seed: mytobcffnkvj
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [5, 6, 2, 1, 6, 65, 6, 4, 1, 2]
+  seed: mytobcffnkvj
+- input: []
+  output: []
+  seed: myzu3g7evxp5nkvj
+- input: [0]
+  output: [0]
+  seed: myzu3g7evxp5nkvj
+- input: [255]
+  output: [255]
+  seed: myzu3g7evxp5nkvj
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 2, 6, 5, 4, 4, 1, 6, 2, 1]
+  seed: myzu3g7evxp5nkvj
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [10, 12, 13, 3, 7, 11, 2, 4, 9, 8, 6, 5, 1]
+  seed: myzu3g7evxp5nkvj
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 2, 6, 5, 65, 4, 1, 6, 2, 1]
+  seed: myzu3g7evxp5nkvj
+- input: []
+  output: []
+  seed: xdpli1jsx5xb
+- input: [0]
+  output: [0]
+  seed: xdpli1jsx5xb
+- input: [255]
+  output: [255]
+  seed: xdpli1jsx5xb
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 2, 4, 1, 2, 6, 5, 1, 6, 4]
+  seed: xdpli1jsx5xb
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [11, 8, 12, 9, 2, 1, 10, 4, 13, 5, 7, 3, 6]
+  seed: xdpli1jsx5xb
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [6, 2, 65, 1, 2, 6, 5, 1, 6, 4]
+  seed: xdpli1jsx5xb
+- input: []
+  output: []
+  seed: oab3mbb3xe8qsx5xb
+- input: [0]
+  output: [0]
+  seed: oab3mbb3xe8qsx5xb
+- input: [255]
+  output: [255]
+  seed: oab3mbb3xe8qsx5xb
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [2, 5, 1, 6, 1, 2, 6, 6, 4, 4]
+  seed: oab3mbb3xe8qsx5xb
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [5, 13, 9, 7, 11, 10, 12, 2, 6, 8, 3, 1, 4]
+  seed: oab3mbb3xe8qsx5xb
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [2, 5, 1, 6, 1, 2, 6, 6, 65, 4]
+  seed: oab3mbb3xe8qsx5xb