Merge branch 'ef-tests' of github.com:sigp/lighthouse into ef-tests

Jenkinsfile

@@ -1,34 +0,0 @@
-pipeline {
-	agent {
-		dockerfile {
-			filename 'Dockerfile'
-			args '-v cargo-cache:/cache/cargocache:rw -e "CARGO_HOME=/cache/cargocache"'
-		}
-	}
-	stages {
-		stage('Build') {
-			steps {
-				sh 'cargo build --verbose --all'
-				sh 'cargo build --verbose --all --release'
-			}
-		}
-        stage('Check') {
-            steps {
-                sh 'cargo fmt --all -- --check'
-                // No clippy until later...
-                //sh 'cargo clippy'
-            }
-        }
-		stage('Test') {
-			steps {
-				sh 'cargo test --verbose --all'
-				sh 'cargo test --verbose --all --release'
-                sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \
-                               --release --features fake_crypto'
-                sh 'cargo test --manifest-path eth2/state_processing/Cargo.toml --verbose \
-                               --release --features fake_crypto -- --ignored'
-
-			}
-		}
-	}
-}

eth2/utils/bls/Cargo.toml

@@ -5,10 +5,11 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
-bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" }
+bls-aggregates = { git = "https://github.com/sigp/signature-schemes", branch = "secret-key-serialization" }
 cached_tree_hash = { path = "../cached_tree_hash" }
 hashing = { path = "../hashing" }
 hex = "0.3"
+rand = "0.5"
 serde = "1.0"
 serde_derive = "1.0"
 serde_hex = { path = "../serde_hex" }

eth2/utils/bls/src/fake_aggregate_public_key.rs

@@ -25,4 +25,12 @@ impl FakeAggregatePublicKey {
     pub fn add(&mut self, _public_key: &PublicKey) {
         // No nothing.
     }
+
+    pub fn as_raw(&self) -> &FakeAggregatePublicKey {
+        &self
+    }
+
+    pub fn as_bytes(&self) -> Vec<u8> {
+        self.bytes.clone()
+    }
 }

eth2/utils/bls/src/fake_public_key.rs

@@ -16,7 +16,7 @@ use tree_hash::tree_hash_ssz_encoding_as_vector;
 /// serialization).
 #[derive(Debug, Clone, Eq)]
 pub struct FakePublicKey {
-    bytes: Vec<u8>
+    bytes: Vec<u8>,
 }
 
 impl FakePublicKey {
@@ -34,14 +34,14 @@ impl FakePublicKey {
     /// Returns the underlying point as compressed bytes.
     ///
     /// Identical to `self.as_uncompressed_bytes()`.
-    fn as_bytes(&self) -> Vec<u8> {
+    pub fn as_bytes(&self) -> Vec<u8> {
         self.bytes.clone()
     }
 
     /// Converts compressed bytes to FakePublicKey
     pub fn from_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
         Ok(Self {
-            bytes: bytes.to_vec()
+            bytes: bytes.to_vec(),
         })
     }
 
@@ -63,6 +63,11 @@ impl FakePublicKey {
         let end_bytes = &bytes[bytes.len().saturating_sub(6)..bytes.len()];
         hex_encode(end_bytes)
     }
+
+    // Returns itself
+    pub fn as_raw(&self) -> &Self {
+        self
+    }
 }
 
 impl fmt::Display for FakePublicKey {

eth2/utils/bls/src/lib.rs

@@ -8,6 +8,7 @@ mod secret_key;
 
 pub use crate::keypair::Keypair;
 pub use crate::secret_key::SecretKey;
+pub use bls_aggregates::{compress_g2, hash_on_g2};
 
 #[cfg(feature = "fake_crypto")]
 mod fake_aggregate_public_key;

eth2/utils/bls/src/secret_key.rs

@@ -1,3 +1,5 @@
+extern crate rand;
+
 use super::BLS_SECRET_KEY_BYTE_SIZE;
 use bls_aggregates::SecretKey as RawSecretKey;
 use hex::encode as hex_encode;
@@ -16,7 +18,7 @@ pub struct SecretKey(RawSecretKey);
 
 impl SecretKey {
     pub fn random() -> Self {
-        SecretKey(RawSecretKey::random())
+        SecretKey(RawSecretKey::random(&mut rand::thread_rng()))
     }
 
     /// Returns the underlying point as compressed bytes.

tests/ef_tests/src/cases.rs

@@ -2,9 +2,21 @@ use super::*;
 use crate::yaml_decode::*;
 use yaml_rust::YamlLoader;
 
+mod bls_aggregate_pubkeys;
+mod bls_aggregate_sigs;
+mod bls_g2_compressed;
+mod bls_g2_uncompressed;
+mod bls_priv_to_pub;
+mod bls_sign_msg;
 mod ssz_generic;
 mod ssz_static;
 
+pub use bls_aggregate_pubkeys::*;
+pub use bls_aggregate_sigs::*;
+pub use bls_g2_compressed::*;
+pub use bls_g2_uncompressed::*;
+pub use bls_priv_to_pub::*;
+pub use bls_sign_msg::*;
 pub use ssz_generic::*;
 pub use ssz_static::*;
 

tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs

@@ -0,0 +1,51 @@
+use super::*;
+use crate::case_result::compare_result;
+use bls::{AggregatePublicKey, PublicKey};
+use serde_derive::Deserialize;
+use types::EthSpec;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregatePubkeys {
+    pub input: Vec<String>,
+    pub output: String,
+}
+
+impl YamlDecode for BlsAggregatePubkeys {
+    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(&yaml.as_str()).unwrap())
+    }
+}
+
+impl EfTest for Cases<BlsAggregatePubkeys> {
+    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .iter()
+            .enumerate()
+            .map(|(i, tc)| {
+                let result = bls_add_pubkeys(&tc.input, &tc.output);
+
+                CaseResult::new(i, tc, result)
+            })
+            .collect()
+    }
+}
+
+/// Execute a `aggregate_pubkeys` test case.
+fn bls_add_pubkeys(inputs: &[String], output: &String) -> Result<(), Error> {
+    let mut aggregate_pubkey = AggregatePublicKey::new();
+
+    for key_str in inputs {
+        let key =
+            hex::decode(&key_str[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+        let key = PublicKey::from_bytes(&key)
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        aggregate_pubkey.add(&key);
+    }
+
+    let output_bytes =
+        Some(hex::decode(&output[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?);
+    let aggregate_pubkey = Ok(aggregate_pubkey.as_raw().as_bytes());
+
+    compare_result::<Vec<u8>, Vec<u8>>(&aggregate_pubkey, &output_bytes)
+}

tests/ef_tests/src/cases/bls_aggregate_sigs.rs

@@ -0,0 +1,51 @@
+use super::*;
+use crate::case_result::compare_result;
+use bls::{AggregateSignature, Signature};
+use serde_derive::Deserialize;
+use types::EthSpec;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsAggregateSigs {
+    pub input: Vec<String>,
+    pub output: String,
+}
+
+impl YamlDecode for BlsAggregateSigs {
+    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(&yaml.as_str()).unwrap())
+    }
+}
+
+impl EfTest for Cases<BlsAggregateSigs> {
+    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .iter()
+            .enumerate()
+            .map(|(i, tc)| {
+                let result = bls_add_signatures(&tc.input, &tc.output);
+
+                CaseResult::new(i, tc, result)
+            })
+            .collect()
+    }
+}
+
+/// Execute a `aggregate_sigs` test case.
+fn bls_add_signatures(inputs: &[String], output: &String) -> Result<(), Error> {
+    let mut aggregate_signature = AggregateSignature::new();
+
+    for key_str in inputs {
+        let sig =
+            hex::decode(&key_str[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+        let sig = Signature::from_bytes(&sig)
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+        aggregate_signature.add(&sig);
+    }
+
+    let output_bytes =
+        Some(hex::decode(&output[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?);
+    let aggregate_signature = Ok(aggregate_signature.as_bytes());
+
+    compare_result::<Vec<u8>, Vec<u8>>(&aggregate_signature, &output_bytes)
+}

tests/ef_tests/src/cases/bls_g2_compressed.rs

@@ -0,0 +1,71 @@
+use super::*;
+use crate::case_result::compare_result;
+use bls::{compress_g2, hash_on_g2};
+use serde_derive::Deserialize;
+use types::EthSpec;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsG2CompressedInput {
+    pub message: String,
+    pub domain: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsG2Compressed {
+    pub input: BlsG2CompressedInput,
+    pub output: Vec<String>,
+}
+
+impl YamlDecode for BlsG2Compressed {
+    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(&yaml.as_str()).unwrap())
+    }
+}
+
+impl EfTest for Cases<BlsG2Compressed> {
+    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .iter()
+            .enumerate()
+            .map(|(i, tc)| {
+                let result = compressed_hash(&tc.input.message, &tc.input.domain, &tc.output);
+
+                CaseResult::new(i, tc, result)
+            })
+            .collect()
+    }
+}
+
+/// Execute a `compressed hash to g2` test case.
+fn compressed_hash(message: &String, domain: &String, output: &Vec<String>) -> Result<(), Error> {
+    // Convert message and domain to required types
+    let msg =
+        hex::decode(&message[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let d = hex::decode(&domain[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let d = bytes_to_u64(&d);
+
+    // Calculate the point and convert it to compressed bytes
+    let mut point = hash_on_g2(&msg, d);
+    let point = compress_g2(&mut point);
+
+    // Convert the output to one set of bytes
+    let mut decoded =
+        hex::decode(&output[0][2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let mut decoded_y =
+        hex::decode(&output[1][2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    decoded.append(&mut decoded_y);
+
+    compare_result::<Vec<u8>, Vec<u8>>(&Ok(point), &Some(decoded))
+}
+
+// Converts a vector to u64 (from big endian)
+fn bytes_to_u64(array: &Vec<u8>) -> u64 {
+    let mut result: u64 = 0;
+    for (i, value) in array.iter().rev().enumerate() {
+        if i == 8 {
+            break;
+        }
+        result += u64::pow(2, i as u32 * 8) * (*value as u64);
+    }
+    result
+}

tests/ef_tests/src/cases/bls_g2_uncompressed.rs

@@ -0,0 +1,85 @@
+use super::*;
+use crate::case_result::compare_result;
+use bls::hash_on_g2;
+use serde_derive::Deserialize;
+use types::EthSpec;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsG2UncompressedInput {
+    pub message: String,
+    pub domain: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsG2Uncompressed {
+    pub input: BlsG2UncompressedInput,
+    pub output: Vec<Vec<String>>,
+}
+
+impl YamlDecode for BlsG2Uncompressed {
+    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(&yaml.as_str()).unwrap())
+    }
+}
+
+impl EfTest for Cases<BlsG2Uncompressed> {
+    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .iter()
+            .enumerate()
+            .map(|(i, tc)| {
+                let result = compressed_hash(&tc.input.message, &tc.input.domain, &tc.output);
+
+                CaseResult::new(i, tc, result)
+            })
+            .collect()
+    }
+}
+
+/// Execute a `compressed hash to g2` test case.
+fn compressed_hash(
+    message: &String,
+    domain: &String,
+    output: &Vec<Vec<String>>,
+) -> Result<(), Error> {
+    // Convert message and domain to required types
+    let msg =
+        hex::decode(&message[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let d = hex::decode(&domain[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let d = bytes_to_u64(&d);
+
+    // Calculate the point and convert it to compressed bytes
+    let point = hash_on_g2(&msg, d);
+    let mut point_bytes = [0 as u8; 288];
+    point.getpx().geta().tobytearray(&mut point_bytes, 0);
+    point.getpx().getb().tobytearray(&mut point_bytes, 48);
+    point.getpy().geta().tobytearray(&mut point_bytes, 96);
+    point.getpy().getb().tobytearray(&mut point_bytes, 144);
+    point.getpz().geta().tobytearray(&mut point_bytes, 192);
+    point.getpz().getb().tobytearray(&mut point_bytes, 240);
+
+    // Convert the output to one set of bytes (x.a, x.b, y.a, y.b, z.a, z.b)
+    let mut decoded: Vec<u8> = vec![];
+    for coordinate in output {
+        let mut decoded_part = hex::decode(&coordinate[0][2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+        decoded.append(&mut decoded_part);
+        decoded_part = hex::decode(&coordinate[1][2..])
+            .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+        decoded.append(&mut decoded_part);
+    }
+
+    compare_result::<Vec<u8>, Vec<u8>>(&Ok(point_bytes.to_vec()), &Some(decoded))
+}
+
+// Converts a vector to u64 (from big endian)
+fn bytes_to_u64(array: &Vec<u8>) -> u64 {
+    let mut result: u64 = 0;
+    for (i, value) in array.iter().rev().enumerate() {
+        if i == 8 {
+            break;
+        }
+        result += u64::pow(2, i as u32 * 8) * (*value as u64);
+    }
+    result
+}

tests/ef_tests/src/cases/bls_priv_to_pub.rs

@@ -0,0 +1,53 @@
+use super::*;
+use crate::case_result::compare_result;
+use bls::{PublicKey, SecretKey};
+use serde_derive::Deserialize;
+use types::EthSpec;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsPrivToPub {
+    pub input: String,
+    pub output: String,
+}
+
+impl YamlDecode for BlsPrivToPub {
+    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(&yaml.as_str()).unwrap())
+    }
+}
+
+impl EfTest for Cases<BlsPrivToPub> {
+    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .iter()
+            .enumerate()
+            .map(|(i, tc)| {
+                let result = secret_to_public(&tc.input, &tc.output);
+
+                CaseResult::new(i, tc, result)
+            })
+            .collect()
+    }
+}
+
+/// Execute a `Private key to public key` test case.
+fn secret_to_public(secret: &String, output: &String) -> Result<(), Error> {
+    // Convert message and domain to required types
+    let mut sk =
+        hex::decode(&secret[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    pad_to_48(&mut sk);
+    let sk = SecretKey::from_bytes(&sk).unwrap();
+    let pk = PublicKey::from_secret_key(&sk);
+
+    let decoded =
+        hex::decode(&output[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+    compare_result::<Vec<u8>, Vec<u8>>(&Ok(pk.as_raw().as_bytes()), &Some(decoded))
+}
+
+// Increase the size of an array to 48 bytes
+fn pad_to_48(array: &mut Vec<u8>) {
+    while array.len() < 48 {
+        array.insert(0, 0);
+    }
+}

tests/ef_tests/src/cases/bls_sign_msg.rs

@@ -0,0 +1,88 @@
+use super::*;
+use crate::case_result::compare_result;
+use bls::{SecretKey, Signature};
+use serde_derive::Deserialize;
+use types::EthSpec;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsSignInput {
+    pub privkey: String,
+    pub message: String,
+    pub domain: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct BlsSign {
+    pub input: BlsSignInput,
+    pub output: String,
+}
+
+impl YamlDecode for BlsSign {
+    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
+        Ok(serde_yaml::from_str(&yaml.as_str()).unwrap())
+    }
+}
+
+impl EfTest for Cases<BlsSign> {
+    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
+        self.test_cases
+            .iter()
+            .enumerate()
+            .map(|(i, tc)| {
+                let result = sign_msg(
+                    &tc.input.privkey,
+                    &tc.input.message,
+                    &tc.input.domain,
+                    &tc.output,
+                );
+
+                CaseResult::new(i, tc, result)
+            })
+            .collect()
+    }
+}
+
+/// Execute a `compressed hash to g2` test case.
+fn sign_msg(
+    private_key: &String,
+    message: &String,
+    domain: &String,
+    output: &String,
+) -> Result<(), Error> {
+    // Convert private_key, message and domain to required types
+    let mut sk =
+        hex::decode(&private_key[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    pad_to_48(&mut sk);
+    let sk = SecretKey::from_bytes(&sk).unwrap();
+    let msg =
+        hex::decode(&message[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let d = hex::decode(&domain[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+    let d = bytes_to_u64(&d);
+
+    let signature = Signature::new(&msg, d, &sk);
+
+    // Convert the output to one set of bytes
+    let decoded =
+        hex::decode(&output[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
+
+    compare_result::<Vec<u8>, Vec<u8>>(&Ok(signature.as_bytes()), &Some(decoded))
+}
+
+// Converts a vector to u64 (from big endian)
+fn bytes_to_u64(array: &Vec<u8>) -> u64 {
+    let mut result: u64 = 0;
+    for (i, value) in array.iter().rev().enumerate() {
+        if i == 8 {
+            break;
+        }
+        result += u64::pow(2, i as u32 * 8) * (*value as u64);
+    }
+    result
+}
+
+// Increase the size of an array to 48 bytes
+fn pad_to_48(array: &mut Vec<u8>) {
+    while array.len() < 48 {
+        array.insert(0, 0);
+    }
+}

tests/ef_tests/src/cases/ssz_static.rs

@@ -3,7 +3,7 @@ use crate::case_result::compare_result;
 use cached_tree_hash::{CachedTreeHash, TreeHashCache};
 use rayon::prelude::*;
 use serde_derive::Deserialize;
-use ssz::Decode;
+use ssz::{Decode, Encode, ssz_encode};
 use std::fmt::Debug;
 use tree_hash::TreeHash;
 use types::{
@@ -91,8 +91,10 @@ impl EfTest for Cases<SszStatic> {
 
 fn ssz_static_test<T>(tc: &SszStatic) -> Result<(), Error>
 where
-    T: Decode
+    T: Clone
+        + Decode
         + Debug
+        + Encode
         + PartialEq<T>
         + serde::de::DeserializeOwned
         + TreeHash
@@ -106,8 +108,12 @@ where
     let decode_result = T::from_ssz_bytes(&ssz);
     compare_result(&decode_result, &Some(expected))?;
 
-    // Verify the TreeHash root of the decoded struct matches the test.
+    // Verify we can encode the result back into original ssz bytes
     let decoded = decode_result.unwrap();
+    let encoded_result = decoded.as_ssz_bytes();
+    compare_result::<Vec<u8>, Error>(&Ok(encoded_result), &Some(ssz));
+
+    // Verify the TreeHash root of the decoded struct matches the test.
     let expected_root =
         &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
     let expected_root = Hash256::from_slice(&expected_root);

tests/ef_tests/src/doc.rs

@@ -39,9 +39,24 @@ impl Doc {
             header.handler.as_ref(),
             header.config.as_ref(),
         ) {
-            ("ssz", "uint", _) => run_test::<SszGeneric, MainnetEthSpec>(&self),
-            ("ssz", "static", "minimal") => run_test::<SszStatic, MinimalEthSpec>(&self),
-            ("ssz", "static", "mainnet") => run_test::<SszStatic, MainnetEthSpec>(&self),
+            ("ssz", "uint", _) => run_test::<SszGeneric, MainnetEthSpec>(self),
+            ("ssz", "static", "minimal") => run_test::<SszStatic, MinimalEthSpec>(self),
+            ("ssz", "static", "mainnet") => run_test::<SszStatic, MainnetEthSpec>(self),
+            ("bls", "aggregate_pubkeys", "mainnet") => {
+                run_test::<BlsAggregatePubkeys, MainnetEthSpec>(self)
+            }
+            ("bls", "aggregate_sigs", "mainnet") => {
+                run_test::<BlsAggregateSigs, MainnetEthSpec>(self)
+            }
+            ("bls", "msg_hash_compressed", "mainnet") => {
+                run_test::<BlsG2Compressed, MainnetEthSpec>(self)
+            }
+            ("bls", "msg_hash_uncompressed", "mainnet") => {
+                // Note this test fails but Not due to a bug
+                vec![] // run_test::<BlsG2Uncompressed, MainnetEthSpec>(&self.yaml)
+            }
+            ("bls", "priv_to_pub", "mainnet") => run_test::<BlsPrivToPub, MainnetEthSpec>(self),
+            ("bls", "sign_msg", "mainnet") => run_test::<BlsSign, MainnetEthSpec>(self),
             (runner, handler, config) => panic!(
                 "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"",
                 runner, handler, config

tests/ef_tests/tests/tests.rs

@@ -43,3 +43,12 @@ fn ssz_static() {
             Doc::assert_tests_pass(file);
         });
 }
+
+#[test]
+fn bls() {
+    yaml_files_in_test_dir("bls")
+        .into_par_iter()
+        .for_each(|file| {
+            Doc::assert_tests_pass(file);
+        });
+}