Marge fixes to test_harness, add serdehex crate

Cargo.toml

@@ -11,6 +11,7 @@ members = [
 	"eth2/utils/honey-badger-split",
 	"eth2/utils/merkle_proof",
 	"eth2/utils/int_to_bytes",
+	"eth2/utils/serde_hex",
 	"eth2/utils/slot_clock",
 	"eth2/utils/ssz",
 	"eth2/utils/ssz_derive",

eth2/block_proposer/src/test_utils/epoch_map.rs

@@ -28,8 +28,8 @@ impl DutiesReader for EpochMap {
 
     fn fork(&self) -> Result<Fork, DutiesReaderError> {
         Ok(Fork {
-            previous_version: 0,
+            previous_version: [0; 4],
-            current_version: 0,
+            current_version: [0; 4],
             epoch: Epoch::new(0),
         })
     }

eth2/state_processing/Cargo.toml

@@ -11,6 +11,9 @@ harness = false
 [dev-dependencies]
 criterion = "0.2"
 env_logger = "0.6.0"
+serde = "1.0"
+serde_derive = "1.0"
+serde_yaml = "0.8"
 
 [dependencies]
 bls = { path = "../utils/bls" }

eth2/types/src/attestation.rs

@@ -1,7 +1,7 @@
 use super::{AggregateSignature, AttestationData, Bitfield};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -9,7 +9,18 @@ use test_random_derive::TestRandom;
 /// Details an attestation that can be slashable.
 ///
 /// Spec v0.4.0
-#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct Attestation {
     pub aggregation_bitfield: Bitfield,
     pub data: AttestationData,

eth2/types/src/attestation_data.rs

@@ -1,7 +1,7 @@
 use crate::test_utils::TestRandom;
 use crate::{Crosslink, Epoch, Hash256, Slot};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -15,6 +15,7 @@ use test_random_derive::TestRandom;
     PartialEq,
     Default,
     Serialize,
+    Deserialize,
     Hash,
     Encode,
     Decode,

eth2/types/src/attester_slashing.rs

@@ -1,13 +1,13 @@
 use crate::{test_utils::TestRandom, SlashableAttestation};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Two conflicting attestations.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct AttesterSlashing {
     pub slashable_attestation_1: SlashableAttestation,
     pub slashable_attestation_2: SlashableAttestation,

eth2/types/src/beacon_block.rs

@@ -2,7 +2,7 @@ use crate::test_utils::TestRandom;
 use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, Proposal, Slot};
 use bls::Signature;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::{SignedRoot, TreeHash};
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// A block of the `BeaconChain`.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct BeaconBlock {
     pub slot: Slot,
     pub parent_root: Hash256,

eth2/types/src/beacon_block_body.rs

@@ -1,14 +1,16 @@
 use super::{Attestation, AttesterSlashing, Deposit, ProposerSlashing, Transfer, VoluntaryExit};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// The body of a `BeaconChain` block, containing operations.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct BeaconBlockBody {
     pub proposer_slashings: Vec<ProposerSlashing>,
     pub attester_slashings: Vec<AttesterSlashing>,

eth2/types/src/beacon_state.rs

@@ -7,7 +7,7 @@ use int_to_bytes::int_to_bytes32;
 use log::{debug, error, trace};
 use pubkey_cache::PubkeyCache;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::{hash, Decodable, DecodeError, Encodable, SignedRoot, SszStream, TreeHash};
 use std::collections::HashMap;
 use swap_or_not_shuffle::shuffle_list;
@@ -72,7 +72,7 @@ macro_rules! safe_sub_assign {
     };
 }
 
-#[derive(Debug, PartialEq, Clone, Default, Serialize)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)]
 pub struct BeaconState {
     // Misc
     pub slot: Slot,
@@ -114,7 +114,9 @@ pub struct BeaconState {
 
     // Caching (not in the spec)
     pub cache_index_offset: usize,
+    #[serde(default)]
     pub caches: Vec<EpochCache>,
+    #[serde(default)]
     pub pubkey_cache: PubkeyCache,
 }
 
@@ -137,11 +139,7 @@ impl BeaconState {
              */
             slot: spec.genesis_slot,
             genesis_time,
-            fork: Fork {
+            fork: Fork::genesis(spec),
-                previous_version: spec.genesis_fork_version,
-                current_version: spec.genesis_fork_version,
-                epoch: spec.genesis_epoch,
-            },
 
             /*
              * Validator registry
@@ -193,8 +191,8 @@ impl BeaconState {
              * Caching (not in spec)
              */
             cache_index_offset: 0,
-            caches: vec![EpochCache::empty(); CACHED_EPOCHS],
+            caches: vec![EpochCache::default(); CACHED_EPOCHS],
-            pubkey_cache: PubkeyCache::empty(),
+            pubkey_cache: PubkeyCache::default(),
         }
     }
 
@@ -276,7 +274,7 @@ impl BeaconState {
     /// Removes the specified cache and sets it to uninitialized.
     pub fn drop_cache(&mut self, relative_epoch: RelativeEpoch) {
         let previous_cache_index = self.cache_index(relative_epoch);
-        self.caches[previous_cache_index] = EpochCache::empty();
+        self.caches[previous_cache_index] = EpochCache::default();
     }
 
     /// Returns the index of `self.caches` for some `RelativeEpoch`.
@@ -324,7 +322,7 @@ impl BeaconState {
 
     /// Completely drops the `pubkey_cache`, replacing it with a new, empty cache.
     pub fn drop_pubkey_cache(&mut self) {
-        self.pubkey_cache = PubkeyCache::empty()
+        self.pubkey_cache = PubkeyCache::default()
     }
 
     /// If a validator pubkey exists in the validator registry, returns `Some(i)`, otherwise
@@ -1227,8 +1225,8 @@ impl Decodable for BeaconState {
                 eth1_data_votes,
                 deposit_index,
                 cache_index_offset: 0,
-                caches: vec![EpochCache::empty(); CACHED_EPOCHS],
+                caches: vec![EpochCache::default(); CACHED_EPOCHS],
-                pubkey_cache: PubkeyCache::empty(),
+                pubkey_cache: PubkeyCache::default(),
             },
             i,
         ))
@@ -1298,8 +1296,8 @@ impl<T: RngCore> TestRandom<T> for BeaconState {
             eth1_data_votes: <_>::random_for_test(rng),
             deposit_index: <_>::random_for_test(rng),
             cache_index_offset: 0,
-            caches: vec![EpochCache::empty(); CACHED_EPOCHS],
+            caches: vec![EpochCache::default(); CACHED_EPOCHS],
-            pubkey_cache: PubkeyCache::empty(),
+            pubkey_cache: PubkeyCache::default(),
         }
     }
 }

eth2/types/src/beacon_state/epoch_cache.rs

@@ -1,8 +1,8 @@
 use super::{AttestationDuty, BeaconState, CrosslinkCommittees, Error};
 use crate::{ChainSpec, Epoch};
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 
-#[derive(Debug, PartialEq, Clone, Serialize)]
+#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
 pub struct EpochCache {
     /// True if this cache has been initialized.
     pub initialized: bool,
@@ -15,16 +15,6 @@ pub struct EpochCache {
 }
 
 impl EpochCache {
-    /// Return a new, completely empty cache.
-    pub fn empty() -> EpochCache {
-        EpochCache {
-            initialized: false,
-            committees: vec![],
-            attestation_duties: vec![],
-            shard_committee_indices: vec![],
-        }
-    }
-
     /// Return a new, fully initialized cache.
     pub fn initialized(
         state: &BeaconState,

eth2/types/src/beacon_state/pubkey_cache.rs

@@ -1,22 +1,15 @@
 use crate::*;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 type ValidatorIndex = usize;
 
-#[derive(Debug, PartialEq, Clone, Default, Serialize)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)]
 pub struct PubkeyCache {
     map: HashMap<PublicKey, ValidatorIndex>,
 }
 
 impl PubkeyCache {
-    /// Instantiates a new, empty cache.
-    pub fn empty() -> Self {
-        Self {
-            map: HashMap::new(),
-        }
-    }
-
     /// Returns the number of validator indices already in the map.
     pub fn len(&self) -> ValidatorIndex {
         self.map.len()

eth2/types/src/chain_spec.rs

@@ -1,5 +1,7 @@
 use crate::{Address, Epoch, Fork, Hash256, Slot};
 use bls::Signature;
+use int_to_bytes::int_to_bytes4;
+use serde_derive::Deserialize;
 
 const GWEI: u64 = 1_000_000_000;
 
@@ -15,7 +17,8 @@ pub enum Domain {
 /// Holds all the "constants" for a BeaconChain.
 ///
 /// Spec v0.4.0
-#[derive(PartialEq, Debug, Clone)]
+#[derive(PartialEq, Debug, Clone, Deserialize)]
+#[serde(default)]
 pub struct ChainSpec {
     /*
      * Misc
@@ -45,7 +48,7 @@ pub struct ChainSpec {
     /*
      * Initial Values
      */
-    pub genesis_fork_version: u64,
+    pub genesis_fork_version: u32,
     pub genesis_slot: Slot,
     pub genesis_epoch: Epoch,
     pub genesis_start_shard: u64,
@@ -100,12 +103,12 @@ pub struct ChainSpec {
      *
      * Use `ChainSpec::get_domain(..)` to access these values.
      */
-    domain_deposit: u64,
+    domain_deposit: u32,
-    domain_attestation: u64,
+    domain_attestation: u32,
-    domain_proposal: u64,
+    domain_proposal: u32,
-    domain_exit: u64,
+    domain_exit: u32,
-    domain_randao: u64,
+    domain_randao: u32,
-    domain_transfer: u64,
+    domain_transfer: u32,
 }
 
 impl ChainSpec {
@@ -135,8 +138,11 @@ impl ChainSpec {
             Domain::Transfer => self.domain_transfer,
         };
 
-        let fork_version = fork.get_fork_version(epoch);
+        let mut fork_and_domain = [0; 8];
-        fork_version * u64::pow(2, 32) + domain_constant
+        fork_and_domain.copy_from_slice(&fork.get_fork_version(epoch));
+        fork_and_domain.copy_from_slice(&int_to_bytes4(domain_constant));
+
+        u64::from_le_bytes(fork_and_domain)
     }
 
     /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification.
@@ -254,6 +260,12 @@ impl ChainSpec {
     }
 }
 
+impl Default for ChainSpec {
+    fn default() -> Self {
+        Self::foundation()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

eth2/types/src/crosslink.rs

@@ -1,7 +1,7 @@
 use crate::test_utils::TestRandom;
 use crate::{Epoch, Hash256};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
@@ -9,7 +9,17 @@ use test_random_derive::TestRandom;
 ///
 /// Spec v0.4.0
 #[derive(
-    Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash, TestRandom,
+    Debug,
+    Clone,
+    PartialEq,
+    Default,
+    Serialize,
+    Deserialize,
+    Hash,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
 )]
 pub struct Crosslink {
     pub epoch: Epoch,

eth2/types/src/eth1_data.rs

@@ -1,14 +1,16 @@
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Contains data obtained from the Eth1 chain.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Eth1Data {
     pub deposit_root: Hash256,
     pub block_hash: Hash256,

eth2/types/src/eth1_data_vote.rs

@@ -1,14 +1,16 @@
 use super::Eth1Data;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// A summation of votes for some `Eth1Data`.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Eth1DataVote {
     pub eth1_data: Eth1Data,
     pub vote_count: u64,

eth2/types/src/fork.rs

@@ -1,24 +1,41 @@
-use crate::{test_utils::TestRandom, Epoch};
+use crate::{test_utils::TestRandom, ChainSpec, Epoch};
+use int_to_bytes::int_to_bytes4;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Specifies a fork of the `BeaconChain`, to prevent replay attacks.
 ///
-/// Spec v0.4.0
+/// Spec v0.5.0
-#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Fork {
-    pub previous_version: u64,
+    pub previous_version: [u8; 4],
-    pub current_version: u64,
+    pub current_version: [u8; 4],
     pub epoch: Epoch,
 }
 
 impl Fork {
+    /// Initialize the `Fork` from the genesis parameters in the `spec`.
+    ///
+    /// Spec v0.5.0
+    pub fn genesis(spec: &ChainSpec) -> Self {
+        let mut current_version: [u8; 4] = [0; 4];
+        current_version.copy_from_slice(&int_to_bytes4(spec.genesis_fork_version));
+
+        Self {
+            previous_version: current_version,
+            current_version,
+            epoch: spec.genesis_epoch,
+        }
+    }
+
     /// Return the fork version of the given ``epoch``.
     ///
-    /// Spec v0.4.0
+    /// Spec v0.5.0
-    pub fn get_fork_version(&self, epoch: Epoch) -> u64 {
+    pub fn get_fork_version(&self, epoch: Epoch) -> [u8; 4] {
         if epoch < self.epoch {
             return self.previous_version;
         }

eth2/types/src/pending_attestation.rs

@@ -1,14 +1,14 @@
 use crate::test_utils::TestRandom;
 use crate::{AttestationData, Bitfield, Slot};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// An attestation that has been included in the state but not yet fully processed.
 ///
 /// Spec v0.4.0
-#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct PendingAttestation {
     pub aggregation_bitfield: Bitfield,
     pub data: AttestationData,

eth2/types/src/proposal.rs

@@ -2,7 +2,7 @@ use crate::test_utils::TestRandom;
 use crate::{Hash256, Slot};
 use bls::Signature;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// A proposal for some shard or beacon block.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct Proposal {
     pub slot: Slot,
     /// Shard number (spec.beacon_chain_shard_number for beacon chain)

eth2/types/src/proposer_slashing.rs

@@ -1,14 +1,14 @@
 use super::Proposal;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Two conflicting proposals from the same proposer (validator).
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct ProposerSlashing {
     pub proposer_index: u64,
     pub proposal_1: Proposal,

eth2/types/src/slashable_attestation.rs

@@ -1,6 +1,6 @@
 use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// To be included in an `AttesterSlashing`.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct SlashableAttestation {
     /// Lists validator registry indices, not committee indices.
     pub validator_indices: Vec<u64>,

eth2/types/src/test_utils/test_random.rs

@@ -51,3 +51,17 @@ where
         ]
     }
 }
+
+macro_rules! impl_test_random_for_u8_array {
+    ($len: expr) => {
+        impl<T: RngCore> TestRandom<T> for [u8; $len] {
+            fn random_for_test(rng: &mut T) -> Self {
+                let mut bytes = [0; $len];
+                rng.fill_bytes(&mut bytes);
+                bytes
+            }
+        }
+    };
+}
+
+impl_test_random_for_u8_array!(4);

eth2/types/src/transfer.rs

@@ -2,7 +2,7 @@ use super::Slot;
 use crate::test_utils::TestRandom;
 use bls::{PublicKey, Signature};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// The data submitted to the deposit contract.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct Transfer {
     pub from: u64,
     pub to: u64,

eth2/types/src/voluntary_exit.rs

@@ -1,7 +1,7 @@
 use crate::{test_utils::TestRandom, Epoch};
 use bls::Signature;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -9,7 +9,18 @@ use test_random_derive::TestRandom;
 /// An exit voluntarily submitted a validator who wishes to withdraw.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct VoluntaryExit {
     pub epoch: Epoch,
     pub validator_index: u64,

eth2/utils/bls/Cargo.toml

@@ -10,4 +10,5 @@ hashing = { path = "../hashing" }
 hex = "0.3"
 serde = "1.0"
 serde_derive = "1.0"
+serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }

eth2/utils/bls/src/aggregate_signature.rs

@@ -2,7 +2,9 @@ use super::{AggregatePublicKey, Signature};
 use bls_aggregates::{
     AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
 };
+use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
 use ssz::{
     decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
 };
@@ -82,7 +84,19 @@ impl Serialize for AggregateSignature {
     where
         S: Serializer,
     {
-        serializer.serialize_bytes(&ssz_encode(self))
+        serializer.serialize_str(&hex_encode(ssz_encode(self)))
+    }
+}
+
+impl<'de> Deserialize<'de> for AggregateSignature {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let (obj, _) = <_>::ssz_decode(&bytes[..], 0)
+            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+        Ok(obj)
     }
 }
 

eth2/utils/bls/src/public_key.rs

@@ -1,9 +1,8 @@
-use super::serde_vistors::HexVisitor;
 use super::SecretKey;
 use bls_aggregates::PublicKey as RawPublicKey;
-use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
 use ssz::{
     decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
 };
@@ -81,7 +80,7 @@ impl Serialize for PublicKey {
     where
         S: Serializer,
     {
-        serializer.serialize_str(&hex_encode(ssz_encode(self)))
+        serializer.serialize_str(&hex_encode(self.as_raw().as_bytes()))
     }
 }
 
@@ -90,10 +89,10 @@ impl<'de> Deserialize<'de> for PublicKey {
     where
         D: Deserializer<'de>,
     {
-        let bytes = deserializer.deserialize_str(HexVisitor)?;
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
-        let (pubkey, _) = <_>::ssz_decode(&bytes[..], 0)
+        let obj = PublicKey::from_bytes(&bytes[..])
-            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+            .map_err(|e| serde::de::Error::custom(format!("invalid pubkey ({:?})", e)))?;
-        Ok(pubkey)
+        Ok(obj)
     }
 }
 

eth2/utils/boolean-bitfield/Cargo.toml

@@ -5,6 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
+serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
 bit-vec = "0.5.0"
 serde = "1.0"

eth2/utils/boolean-bitfield/src/lib.rs

@@ -3,7 +3,10 @@ extern crate ssz;
 
 use bit_vec::BitVec;
 
+use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode, PrefixedHexVisitor};
+use ssz::Decodable;
 use std::cmp;
 use std::default;
 
@@ -178,11 +181,25 @@ impl ssz::Decodable for BooleanBitfield {
 }
 
 impl Serialize for BooleanBitfield {
+    /// Serde serialization is compliant the Ethereum YAML test format.
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
-        serializer.serialize_bytes(&ssz::ssz_encode(self))
+        serializer.serialize_str(&encode(&ssz::ssz_encode(self)))
+    }
+}
+
+impl<'de> Deserialize<'de> for BooleanBitfield {
+    /// Serde serialization is compliant the Ethereum YAML test format.
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let (bitfield, _) = <_>::ssz_decode(&bytes[..], 0)
+            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+        Ok(bitfield)
     }
 }
 

eth2/utils/serde_hex/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "serde_hex"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+serde = "1.0"
+hex = "0.3"

eth2/utils/serde_hex/src/lib.rs

@@ -0,0 +1,59 @@
+use hex;
+use hex::ToHex;
+use serde::de::{self, Visitor};
+use std::fmt;
+
+pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
+    let mut hex = String::with_capacity(data.as_ref().len() * 2);
+
+    // Writing to a string never errors, so we can unwrap here.
+    data.write_hex(&mut hex).unwrap();
+
+    let mut s = "0x".to_string();
+
+    s.push_str(hex.as_str());
+
+    s
+}
+
+pub struct PrefixedHexVisitor;
+
+impl<'de> Visitor<'de> for PrefixedHexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string with 0x prefix")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.starts_with("0x") {
+            Ok(hex::decode(&value[2..])
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+        } else {
+            Err(de::Error::custom("missing 0x prefix"))
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn encoding() {
+        let bytes = vec![0, 255];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x00ff");
+
+        let bytes = vec![];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x");
+
+        let bytes = vec![1, 2, 3];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x010203");
+    }
+}

eth2/utils/ssz/src/impl_decode.rs

@@ -24,11 +24,30 @@ macro_rules! impl_decodable_for_uint {
     };
 }
 
+macro_rules! impl_decodable_for_u8_array {
+    ($len: expr) => {
+        impl Decodable for [u8; $len] {
+            fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+                if index + $len > bytes.len() {
+                    Err(DecodeError::TooShort)
+                } else {
+                    let mut array: [u8; $len] = [0; $len];
+                    array.copy_from_slice(&bytes[index..index + $len]);
+
+                    Ok((array, index + $len))
+                }
+            }
+        }
+    };
+}
+
 impl_decodable_for_uint!(u16, 16);
 impl_decodable_for_uint!(u32, 32);
 impl_decodable_for_uint!(u64, 64);
 impl_decodable_for_uint!(usize, 64);
 
+impl_decodable_for_u8_array!(4);
+
 impl Decodable for u8 {
     fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
         if index >= bytes.len() {
@@ -246,4 +265,12 @@ mod tests {
         let result: Result<(bool, usize), DecodeError> = decode_ssz(&ssz, 0);
         assert_eq!(result, Err(DecodeError::Invalid));
     }
+
+    #[test]
+    fn test_decode_u8_array() {
+        let ssz = vec![0, 1, 2, 3];
+        let (result, index): ([u8; 4], usize) = decode_ssz(&ssz, 0).unwrap();
+        assert_eq!(index, 4);
+        assert_eq!(result, [0, 1, 2, 3]);
+    }
 }

eth2/utils/ssz/src/impl_encode.rs

@@ -40,12 +40,25 @@ macro_rules! impl_encodable_for_uint {
     };
 }
 
+macro_rules! impl_encodable_for_u8_array {
+    ($len: expr) => {
+        impl Encodable for [u8; $len] {
+            fn ssz_append(&self, s: &mut SszStream) {
+                let bytes: Vec<u8> = self.iter().cloned().collect();
+                s.append_encoded_raw(&bytes);
+            }
+        }
+    };
+}
+
 impl_encodable_for_uint!(u8, 8);
 impl_encodable_for_uint!(u16, 16);
 impl_encodable_for_uint!(u32, 32);
 impl_encodable_for_uint!(u64, 64);
 impl_encodable_for_uint!(usize, 64);
 
+impl_encodable_for_u8_array!(4);
+
 impl Encodable for bool {
     fn ssz_append(&self, s: &mut SszStream) {
         let byte = if *self { 0b1000_0000 } else { 0b0000_0000 };
@@ -77,6 +90,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::ssz_encode;
 
     #[test]
     fn test_ssz_encode_h256() {
@@ -226,4 +240,15 @@ mod tests {
         ssz.append(&x);
         assert_eq!(ssz.drain(), vec![0b1000_0000]);
     }
+
+    #[test]
+    fn test_ssz_encode_u8_array() {
+        let x: [u8; 4] = [0, 1, 7, 8];
+        let ssz = ssz_encode(&x);
+        assert_eq!(ssz, vec![0, 1, 7, 8]);
+
+        let x: [u8; 4] = [255, 255, 255, 255];
+        let ssz = ssz_encode(&x);
+        assert_eq!(ssz, vec![255, 255, 255, 255]);
+    }
 }