add quoted serialization util for FixedVector and VariableList (#1794)

## Issue Addressed

This comment: https://github.com/sigp/lighthouse/issues/1776#issuecomment-712349841

## Proposed Changes

- Add quoted serde utils for `FixedVector` and `VariableList`
- Had to remove the dependency that `ssz_types` has on `serde_utils` to avoid a circular dependency.

## Additional Info


Co-authored-by: realbigsean <seananderson33@gmail.com>

Cargo.lock

@@ -1797,6 +1797,7 @@ dependencies = [
  "eth2_ssz",
  "serde",
  "serde_derive",
+ "serde_json",
  "serde_utils",
  "tree_hash",
  "tree_hash_derive",

consensus/ssz_types/Cargo.toml

@@ -17,4 +17,5 @@ typenum = "1.12.0"
 arbitrary = { version = "0.4.6", features = ["derive"], optional = true }
 
 [dev-dependencies]
+serde_json = "1.0.58"
 tree_hash_derive = "0.2.0"

consensus/ssz_types/src/lib.rs

@@ -40,6 +40,7 @@
 #[macro_use]
 mod bitfield;
 mod fixed_vector;
+pub mod serde_utils;
 mod tree_hash;
 mod variable_list;
 

consensus/ssz_types/src/serde_utils/mod.rs

@@ -0,0 +1,2 @@
+pub mod quoted_u64_fixed_vec;
+pub mod quoted_u64_var_list;

consensus/ssz_types/src/serde_utils/quoted_u64_fixed_vec.rs

@@ -0,0 +1,113 @@
+//! Formats `FixedVector<u64,N>` using quotes.
+//!
+//! E.g., `FixedVector::from(vec![0, 1, 2])` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding. If `N` does not equal the length deserialization will fail.
+
+use crate::serde_utils::quoted_u64_var_list::deserialize_max;
+use crate::FixedVector;
+use serde::ser::SerializeSeq;
+use serde::{Deserializer, Serializer};
+use serde_utils::quoted_u64_vec::QuotedIntWrapper;
+use std::marker::PhantomData;
+use typenum::Unsigned;
+
+pub struct QuotedIntFixedVecVisitor<N> {
+    _phantom: PhantomData<N>,
+}
+
+impl<'a, N> serde::de::Visitor<'a> for QuotedIntFixedVecVisitor<N>
+where
+    N: Unsigned,
+{
+    type Value = FixedVector<u64, N>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "a list of quoted or unquoted integers")
+    }
+
+    fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'a>,
+    {
+        let vec = deserialize_max(seq, N::to_usize())?;
+        let fix: FixedVector<u64, N> = FixedVector::new(vec)
+            .map_err(|e| serde::de::Error::custom(format!("FixedVector: {:?}", e)))?;
+        Ok(fix)
+    }
+}
+
+pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut seq = serializer.serialize_seq(Some(value.len()))?;
+    for &int in value {
+        seq.serialize_element(&QuotedIntWrapper { int })?;
+    }
+    seq.end()
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<FixedVector<u64, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    deserializer.deserialize_any(QuotedIntFixedVecVisitor {
+        _phantom: PhantomData,
+    })
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use serde_derive::{Deserialize, Serialize};
+    use typenum::U4;
+
+    #[derive(Debug, Serialize, Deserialize)]
+    struct Obj {
+        #[serde(with = "crate::serde_utils::quoted_u64_fixed_vec")]
+        values: FixedVector<u64, U4>,
+    }
+
+    #[test]
+    fn quoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn unquoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn mixed_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn empty_list_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": [] }"#).unwrap_err();
+    }
+
+    #[test]
+    fn short_list_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": [1, 2] }"#).unwrap_err();
+    }
+
+    #[test]
+    fn long_list_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": [1, 2, 3, 4, 5] }"#).unwrap_err();
+    }
+
+    #[test]
+    fn whole_list_quoted_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
+    }
+}

consensus/ssz_types/src/serde_utils/quoted_u64_var_list.rs

@@ -0,0 +1,139 @@
+//! Formats `VariableList<u64,N>` using quotes.
+//!
+//! E.g., `VariableList::from(vec![0, 1, 2])` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding. If the length of the `Vec` is greater than `N`, deserialization fails.
+
+use crate::VariableList;
+use serde::ser::SerializeSeq;
+use serde::{Deserializer, Serializer};
+use serde_utils::quoted_u64_vec::QuotedIntWrapper;
+use std::marker::PhantomData;
+use typenum::Unsigned;
+
+pub struct QuotedIntVarListVisitor<N> {
+    _phantom: PhantomData<N>,
+}
+
+impl<'a, N> serde::de::Visitor<'a> for QuotedIntVarListVisitor<N>
+where
+    N: Unsigned,
+{
+    type Value = VariableList<u64, N>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "a list of quoted or unquoted integers")
+    }
+
+    fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'a>,
+    {
+        let vec = deserialize_max(seq, N::to_usize())?;
+        let list: VariableList<u64, N> = VariableList::new(vec)
+            .map_err(|e| serde::de::Error::custom(format!("VariableList: {:?}", e)))?;
+        Ok(list)
+    }
+}
+
+pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut seq = serializer.serialize_seq(Some(value.len()))?;
+    for &int in value {
+        seq.serialize_element(&QuotedIntWrapper { int })?;
+    }
+    seq.end()
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u64, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    deserializer.deserialize_any(QuotedIntVarListVisitor {
+        _phantom: PhantomData,
+    })
+}
+
+/// Returns a `Vec` of no more than `max_items` length.
+pub(crate) fn deserialize_max<'a, A>(mut seq: A, max_items: usize) -> Result<Vec<u64>, A::Error>
+where
+    A: serde::de::SeqAccess<'a>,
+{
+    let mut vec = vec![];
+    let mut counter = 0;
+
+    while let Some(val) = seq.next_element()? {
+        let val: QuotedIntWrapper = val;
+        counter += 1;
+        if counter > max_items {
+            return Err(serde::de::Error::custom(format!(
+                "Deserialization failed. Length cannot be greater than {}.",
+                max_items
+            )));
+        }
+
+        vec.push(val.int);
+    }
+
+    Ok(vec)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use serde_derive::{Deserialize, Serialize};
+    use typenum::U4;
+
+    #[derive(Debug, Serialize, Deserialize)]
+    struct Obj {
+        #[serde(with = "crate::serde_utils::quoted_u64_var_list")]
+        values: VariableList<u64, U4>,
+    }
+
+    #[test]
+    fn quoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn unquoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn mixed_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn empty_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [] }"#).unwrap();
+        assert!(obj.values.is_empty());
+    }
+
+    #[test]
+    fn short_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn long_list_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": [1, 2, 3, 4, 5] }"#).unwrap_err();
+    }
+
+    #[test]
+    fn whole_list_quoted_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
+    }
+}

consensus/ssz_types/src/variable_list.rs

@@ -320,7 +320,7 @@ mod test {
 
         let vec = vec![];
         let fixed: VariableList<u64, U4> = VariableList::from(vec);
-        assert_eq!(&fixed[..], &vec![][..]);
+        assert_eq!(&fixed[..], &[] as &[u64]);
     }
 
     #[test]

consensus/types/src/beacon_state.rs

@@ -181,12 +181,14 @@ where
     #[compare_fields(as_slice)]
     pub validators: VariableList<Validator, T::ValidatorRegistryLimit>,
     #[compare_fields(as_slice)]
+    #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")]
     pub balances: VariableList<u64, T::ValidatorRegistryLimit>,
 
     // Randomness
     pub randao_mixes: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
 
     // Slashings
+    #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")]
     pub slashings: FixedVector<u64, T::EpochsPerSlashingsVector>,
 
     // Attestations