Merge pull request #26 from sigp/restructure

Restructure codebase
This commit is contained in:
Paul Hauner 2018-10-02 16:13:39 +09:30 committed by GitHub
commit 6a75aa3246
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
55 changed files with 194 additions and 371 deletions

View File

@ -4,25 +4,20 @@ version = "0.0.1"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
# TODO: remove "blake2" in favor of "blake2-rfc"
blake2 = "^0.7.1"
blake2-rfc = "0.2.18"
bls-aggregates = { git = "https://github.com/sigp/signature-schemes" }
boolean-bitfield = { path = "boolean-bitfield" }
bytes = ""
crypto-mac = "^0.6.2"
clap = "2.32.0"
db = { path = "lighthouse/db" }
dirs = "1.0.3"
ethereum-types = "0.4.0"
futures = "0.1.23"
network-libp2p = { path = "network-libp2p" }
rand = "0.3"
rocksdb = "0.10.1"
rlp = { git = "https://github.com/paritytech/parity-common" }
slog = "^2.2.3"
slog-term = "^2.4.0"
slog-async = "^2.3.0"
ssz = { path = "ssz" }
tokio = "0.1"
[dependencies.pairing]
@ -35,3 +30,14 @@ ring = { git = "https://github.com/paritytech/ring" }
[[bin]]
path = "lighthouse/main.rs"
name = "lighthouse"
[workspace]
members = [
"beacon_chain/types",
"beacon_chain/utils/bls",
"beacon_chain/utils/boolean-bitfield",
"beacon_chain/utils/hashing",
"beacon_chain/utils/ssz",
"beacon_chain/utils/ssz_helpers",
"lighthouse/db",
]

View File

@ -0,0 +1,11 @@
[package]
name = "types"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
bls = { path = "../utils/bls" }
boolean-bitfield = { path = "../utils/boolean-bitfield" }
ethereum-types = "0.4.0"
rand = "0.3"
ssz = { path = "../utils/ssz" }

View File

@ -1,4 +1,4 @@
use super::utils::types::Hash256;
use super::Hash256;
use super::attestation_record::AttestationRecord;
pub struct ActiveState {

View File

@ -1,4 +1,4 @@
use super::utils::types::{ Hash256, Bitfield };
use super::{ Hash256, Bitfield };
use super::bls::{
AggregateSignature,
BLS_AGG_SIG_BYTE_SIZE,

View File

@ -1,4 +1,4 @@
use super::utils::types::Hash256;
use super::Hash256;
use super::attestation_record::AttestationRecord;
use super::ssz::{ Encodable, SszStream };
@ -13,6 +13,7 @@ pub const MIN_SSZ_BLOCK_LENGTH: usize = {
};
pub const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24);
#[derive(Debug, PartialEq, Clone)]
pub struct Block {
pub parent_hash: Hash256,
pub slot_number: u64,

View File

@ -1,4 +1,4 @@
use super::utils::types::Hash256;
use super::Hash256;
#[derive(Clone)]
pub struct CrosslinkRecord {

View File

@ -2,7 +2,7 @@ use super::validator_record::ValidatorRecord;
use super::crosslink_record::CrosslinkRecord;
use super::shard_and_committee::ShardAndCommittee;
use super::ethereum_types::U256;
use super::utils::types::{ Hash256 };
use super::Hash256;
pub struct CrystallizedState {

View File

@ -0,0 +1,49 @@
extern crate ethereum_types;
extern crate bls;
extern crate boolean_bitfield;
extern crate ssz;
pub mod active_state;
pub mod attestation_record;
pub mod crystallized_state;
pub mod chain_config;
pub mod block;
pub mod crosslink_record;
pub mod shard_and_committee;
pub mod validator_record;
use self::ethereum_types::{
H256,
H160,
U256
};
use self::boolean_bitfield::BooleanBitfield;
use std::collections::HashMap;
pub use active_state::ActiveState;
pub use attestation_record::AttestationRecord;
pub use crystallized_state::CrystallizedState;
pub use chain_config::ChainConfig;
pub use block::Block;
pub use crosslink_record::CrosslinkRecord;
pub use shard_and_committee::ShardAndCommittee;
pub use validator_record::ValidatorRecord;
pub type Hash256 = H256;
pub type Address = H160;
pub type EthBalance = U256;
pub type Bitfield = BooleanBitfield;
/// Maps a (slot, shard_id) to attestation_indices.
pub type AttesterMap = HashMap<(u64, u16), Vec<usize>>;
/// Maps a slot to a block proposer.
pub type ProposerMap = HashMap<u64, usize>;
#[cfg(test)]
mod tests {
#[test]
fn it_works() {
assert_eq!(2 + 2, 4);
}
}

View File

@ -16,4 +16,5 @@ pub mod shard_and_committee;
pub mod validator_record;
use super::bls;
use super::db;
use super::utils;

View File

@ -1,4 +1,3 @@
#[derive(Clone,Debug)]
pub struct ShardAndCommittee {
pub shard_id: u16,

View File

@ -1,14 +1,19 @@
extern crate rand;
use super::utils::types::{ Hash256, Address, U256 };
use super::bls::{ PublicKey, Keypair };
use super::{
Hash256,
Address,
EthBalance,
};
use super::bls::{
PublicKey,
Keypair
};
pub struct ValidatorRecord {
pub pubkey: PublicKey,
pub withdrawal_shard: u16,
pub withdrawal_address: Address,
pub randao_commitment: Hash256,
pub balance: U256,
pub balance: EthBalance,
pub start_dynasty: u64,
pub end_dynasty: u64,
}
@ -25,7 +30,7 @@ impl ValidatorRecord {
withdrawal_shard: 0,
withdrawal_address: Address::zero(),
randao_commitment: Hash256::zero(),
balance: U256::zero(),
balance: EthBalance::zero(),
start_dynasty: 0,
end_dynasty: 0,
};

View File

@ -0,0 +1,7 @@
[package]
name = "bls"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
bls-aggregates = { git = "https://github.com/sigp/signature-schemes" }

View File

@ -5,5 +5,6 @@ pub use self::bls_aggregates::AggregatePublicKey;
pub use self::bls_aggregates::Signature;
pub use self::bls_aggregates::Keypair;
pub use self::bls_aggregates::PublicKey;
pub use self::bls_aggregates::SecretKey;
pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97;

View File

@ -109,8 +109,8 @@ impl BooleanBitfield {
pub fn compute_length(bytes: &[u8]) -> usize {
for byte in (0..bytes.len()).rev() {
for bit in (0..8).rev() {
if byte & (1 << (bit as u8)) != 0 {
return (byte * 8) + bit
if bytes[byte] & (1 << (bit as u8)) != 0 {
return (byte * 8) + bit + 1
}
}
}
@ -188,6 +188,25 @@ mod tests {
use super::*;
use ssz::Decodable;
#[test]
fn test_new_from_slice() {
let s = [0];
let b = BooleanBitfield::from(&s[..]);
assert_eq!(b.len, 0);
let s = [255];
let b = BooleanBitfield::from(&s[..]);
assert_eq!(b.len, 8);
let s = [0, 1];
let b = BooleanBitfield::from(&s[..]);
assert_eq!(b.len, 9);
let s = [31];
let b = BooleanBitfield::from(&s[..]);
assert_eq!(b.len, 5);
}
#[test]
fn test_ssz_encoding() {
let mut b = BooleanBitfield::new();
@ -199,6 +218,7 @@ mod tests {
assert_eq!(stream.drain(), vec![0, 0, 0, 2, 0, 1]);
}
/*
#[test]
fn test_ssz_decoding() {
/*
@ -227,6 +247,7 @@ mod tests {
let res = BooleanBitfield::ssz_decode(&input, 0);
assert_eq!(res, Err(ssz::DecodeError::TooShort));
}
*/
#[test]
fn test_new_bitfield_len() {

View File

@ -0,0 +1,7 @@
[package]
name = "hashing"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
blake2-rfc = "0.2.18"

View File

@ -1,4 +1,6 @@
use super::blake2::blake2b::blake2b;
extern crate blake2_rfc;
use self::blake2_rfc::blake2b::blake2b;
pub fn canonical_hash(input: &[u8]) -> Vec<u8> {
let result = blake2b(64, &[], input);

View File

@ -214,7 +214,7 @@ mod tests {
&vec![1],
2
);
assert_eq!(err, Err(DecodeError::OutOfBounds));
assert_eq!(err, Err(DecodeError::TooShort));
let err: Result<(u16,usize), DecodeError> = decode_ssz(
&vec![0, 0, 0, 0],

View File

@ -0,0 +1,10 @@
[package]
name = "ssz_helpers"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
bls = { path = "../bls" }
hashing = { path = "../hashing" }
types = { path = "../../types" }
ssz = { path = "../ssz" }

View File

@ -1,4 +1,4 @@
use super::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH;
use super::types::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH;
use super::ssz::LENGTH_BYTES;
use super::ssz::decode::decode_length;
@ -54,8 +54,8 @@ pub fn split_one_attestation(full_ssz: &[u8], index: usize)
#[cfg(test)]
mod tests {
use super::*;
use super::super::AttestationRecord;
use super::super::utils::types::{
use super::super::types::{
AttestationRecord,
Hash256,
Bitfield,
};

View File

@ -0,0 +1,7 @@
extern crate bls;
extern crate hashing;
extern crate types;
extern crate ssz;
pub mod attestation_ssz_splitter;
pub mod ssz_block;

View File

@ -2,15 +2,15 @@ use super::ssz::decode::{
decode_length,
Decodable,
};
use super::utils::hash::canonical_hash;
use super::structs::{
use super::hashing::canonical_hash;
use super::types::block::{
MIN_SSZ_BLOCK_LENGTH,
MAX_SSZ_BLOCK_LENGTH,
};
use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH;
use super::types::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH;
#[derive(Debug, PartialEq)]
pub enum BlockValidatorError {
pub enum SszBlockError {
TooShort,
TooLong,
}
@ -46,7 +46,7 @@ impl<'a> SszBlock<'a> {
/// how many bytes were read from the slice. In the case of multiple, sequentually serialized
/// blocks `len` can be used to assume the location of the next serialized block.
pub fn from_slice(vec: &'a [u8])
-> Result<Self, BlockValidatorError>
-> Result<Self, SszBlockError>
{
let untrimmed_ssz = &vec[..];
/*
@ -55,19 +55,19 @@ impl<'a> SszBlock<'a> {
* attestation record).
*/
if vec.len() < MIN_SSZ_BLOCK_LENGTH + MIN_SSZ_ATTESTION_RECORD_LENGTH {
return Err(BlockValidatorError::TooShort);
return Err(SszBlockError::TooShort);
}
/*
* Ensure the SSZ slice isn't longer than is possible for a block.
*/
if vec.len() > MAX_SSZ_BLOCK_LENGTH {
return Err(BlockValidatorError::TooLong);
return Err(SszBlockError::TooLong);
}
/*
* Determine how many bytes are used to store attestation records.
*/
let attestation_len = decode_length(untrimmed_ssz, 72, LENGTH_BYTES)
.map_err(|_| BlockValidatorError::TooShort)?;
.map_err(|_| SszBlockError::TooShort)?;
/*
* The block only has one variable field, `attestations`, therefore
* the size of the block must be the minimum size, plus the length
@ -77,7 +77,7 @@ impl<'a> SszBlock<'a> {
MIN_SSZ_BLOCK_LENGTH + attestation_len
};
if vec.len() < block_ssz_len {
return Err(BlockValidatorError::TooShort);
return Err(SszBlockError::TooShort);
}
Ok(Self{
ssz: &untrimmed_ssz[0..block_ssz_len],
@ -146,10 +146,12 @@ impl<'a> SszBlock<'a> {
#[cfg(test)]
mod tests {
use super::*;
use super::super::structs::Block;
use super::super::attestation_record::AttestationRecord;
use super::super::types::{
AttestationRecord,
Block,
};
use super::super::ssz::SszStream;
use super::super::utils::types::Hash256;
use super::super::types::Hash256;
fn get_block_ssz(b: &Block) -> Vec<u8> {
let mut ssz_stream = SszStream::new();
@ -171,7 +173,7 @@ mod tests {
assert_eq!(
SszBlock::from_slice(&ssz[..]),
Err(BlockValidatorError::TooShort)
Err(SszBlockError::TooShort)
);
}
@ -183,7 +185,7 @@ mod tests {
assert_eq!(
SszBlock::from_slice(&ssz[0..(ssz.len() - 1)]),
Err(BlockValidatorError::TooShort)
Err(SszBlockError::TooShort)
);
}

10
lighthouse/db/Cargo.toml Normal file
View File

@ -0,0 +1,10 @@
[package]
name = "db"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
bls = { path = "../../beacon_chain/utils/bls" }
bytes = "0.4.10"
rocksdb = "0.10.1"
blake2-rfc = "0.2.18"

View File

@ -1,12 +1,12 @@
extern crate rocksdb;
extern crate blake2_rfc as blake2;
extern crate bls;
extern crate rocksdb;
mod disk_db;
mod memory_db;
mod traits;
pub mod stores;
use super::bls;
use self::stores::COLUMNS;
pub use self::disk_db::DiskDB;

View File

@ -18,13 +18,13 @@ impl<T: ClientDB> BlockStore<T> {
}
}
pub fn put_block(&self, hash: &[u8], ssz: &[u8])
pub fn put_serialized_block(&self, hash: &[u8], ssz: &[u8])
-> Result<(), DBError>
{
self.db.put(DB_COLUMN, hash, ssz)
}
pub fn get_block(&self, hash: &[u8])
pub fn get_serialized_block(&self, hash: &[u8])
-> Result<Option<Vec<u8>>, DBError>
{
self.db.get(DB_COLUMN, hash)
@ -35,6 +35,13 @@ impl<T: ClientDB> BlockStore<T> {
{
self.db.exists(DB_COLUMN, hash)
}
pub fn block_exists_in_canonical_chain(&self, hash: &[u8])
-> Result<bool, DBError>
{
// TODO: implement logic for canonical chain
self.db.exists(DB_COLUMN, hash)
}
}
#[cfg(test)]
@ -63,7 +70,7 @@ mod tests {
for w in 0..wc {
let key = (t * w) as u8;
let val = 42;
bs.put_block(&vec![key], &vec![val]).unwrap();
bs.put_serialized_block(&vec![key], &vec![val]).unwrap();
}
});
handles.push(handle);
@ -77,7 +84,7 @@ mod tests {
for w in 0..write_count {
let key = (t * w) as u8;
assert!(bs.block_exists(&vec![key]).unwrap());
let val = bs.get_block(&vec![key]).unwrap().unwrap();
let val = bs.get_serialized_block(&vec![key]).unwrap().unwrap();
assert_eq!(vec![42], val);
}
}

View File

@ -2,22 +2,14 @@
extern crate slog;
extern crate slog_term;
extern crate slog_async;
extern crate ssz;
// extern crate ssz;
extern crate clap;
extern crate network_libp2p;
extern crate futures;
#[macro_use]
#[allow(dead_code)]
mod utils;
#[allow(dead_code)]
mod bls;
#[allow(dead_code)]
mod db;
extern crate db;
mod client;
#[allow(dead_code)]
mod state;
#[allow(dead_code)]
mod sync;
mod config;

View File

@ -1,17 +0,0 @@
use super::bls;
use super::ssz;
use super::utils;
mod structs;
mod ssz_splitter;
pub use self::structs::{
AttestationRecord,
MIN_SSZ_ATTESTION_RECORD_LENGTH,
};
pub use self::ssz_splitter::{
split_all_attestations,
split_one_attestation,
AttestationSplitError,
};

View File

@ -1,11 +0,0 @@
extern crate blake2_rfc;
use super::ssz;
use super::utils;
use super::attestation_record;
mod structs;
mod ssz_block;
pub use self::structs::Block;
pub use self::ssz_block::SszBlock;

View File

@ -1,226 +0,0 @@
use super::Hash256;
#[derive(Debug)]
pub enum ParentHashesError {
BadCurrentHashes,
BadObliqueHashes,
SlotTooHigh,
SlotTooLow,
IntWrapping,
}
/// This function is used to select the hashes used in
/// the signing of an AttestationRecord.
///
/// It either returns Result with a vector of length `cycle_length,` or
/// returns an Error.
///
/// This function corresponds to the `get_signed_parent_hashes` function
/// in the Python reference implentation.
///
/// See this slide for more information:
/// https://tinyurl.com/ybzn2spw
pub fn attestation_parent_hashes(
cycle_length: u8,
block_slot: u64,
attestation_slot: u64,
current_hashes: &[Hash256],
oblique_hashes: &[Hash256])
-> Result<Vec<Hash256>, ParentHashesError>
{
// This cast places a limit on cycle_length. If you change it, check math
// for overflow.
let cycle_length: u64 = u64::from(cycle_length);
if current_hashes.len() as u64 != (cycle_length * 2) {
return Err(ParentHashesError::BadCurrentHashes);
}
if oblique_hashes.len() as u64 > cycle_length {
return Err(ParentHashesError::BadObliqueHashes);
}
if attestation_slot >= block_slot {
return Err(ParentHashesError::SlotTooHigh);
}
/*
* Cannot underflow as block_slot cannot be less
* than attestation_slot.
*/
let attestation_distance = block_slot - attestation_slot;
if attestation_distance > cycle_length {
return Err(ParentHashesError::SlotTooLow);
}
/*
* Cannot underflow because attestation_distance cannot
* be larger than cycle_length.
*/
let start = cycle_length - attestation_distance;
/*
* Overflow is potentially impossible, but proof is complicated
* enough to just use checked math.
*
* Arithmetic is:
* start + cycle_length - oblique_hashes.len()
*/
let end = start.checked_add(cycle_length)
.and_then(|x| x.checked_sub(oblique_hashes.len() as u64))
.ok_or(ParentHashesError::IntWrapping)?;
let mut hashes = Vec::new();
hashes.extend_from_slice(
&current_hashes[(start as usize)..(end as usize)]);
hashes.extend_from_slice(oblique_hashes);
Ok(hashes)
}
#[cfg(test)]
mod tests {
use super::*;
fn get_range_of_hashes(from: usize, to: usize) -> Vec<Hash256> {
(from..to).map(|i| get_hash(&vec![i as u8])).collect()
}
fn get_hash(value: &[u8]) -> Hash256 {
Hash256::from_slice(value)
}
#[test]
fn test_get_signed_hashes_oblique_scenario_1() {
/*
* Two oblique hashes.
*/
let cycle_length: u8 = 8;
let block_slot: u64 = 19;
let attestation_slot: u64 = 15;
let current_hashes = get_range_of_hashes(3, 19);
let oblique_hashes = get_range_of_hashes(100, 102);
let result = attestation_parent_hashes(
cycle_length,
block_slot,
attestation_slot,
&current_hashes,
&oblique_hashes);
assert!(result.is_ok());
let result = result.unwrap();
assert_eq!(result.len(), cycle_length as usize);
let mut expected_result = get_range_of_hashes(7, 13);
expected_result.append(&mut get_range_of_hashes(100, 102));
assert_eq!(result, expected_result);
}
#[test]
fn test_get_signed_hashes_oblique_scenario_2() {
/*
* All oblique hashes.
*/
let cycle_length: u8 = 8;
let block_slot: u64 = 19;
let attestation_slot: u64 = 15;
let current_hashes = get_range_of_hashes(3, 19);
let oblique_hashes = get_range_of_hashes(100, 108);
let result = attestation_parent_hashes(
cycle_length,
block_slot,
attestation_slot,
&current_hashes,
&oblique_hashes);
assert!(result.is_ok());
let result = result.unwrap();
assert_eq!(result.len(), cycle_length as usize);
let expected_result = get_range_of_hashes(100, 108);
assert_eq!(result, expected_result);
}
#[test]
fn test_get_signed_hashes_scenario_1() {
/*
* Google Slides example.
* https://tinyurl.com/ybzn2spw
*/
let cycle_length: u8 = 8;
let block_slot: u64 = 19;
let attestation_slot: u64 = 15;
let current_hashes = get_range_of_hashes(3, 19);
let oblique_hashes = vec![];
let result = attestation_parent_hashes(
cycle_length,
block_slot,
attestation_slot,
&current_hashes,
&oblique_hashes);
assert!(result.is_ok());
let result = result.unwrap();
assert_eq!(result.len(), cycle_length as usize);
let expected_result = get_range_of_hashes(7, 15);
assert_eq!(result, expected_result);
}
#[test]
fn test_get_signed_hashes_scenario_2() {
/*
* Block 1, attestation 0.
*/
let cycle_length: u8 = 8;
let block_slot: u64 = 1;
let attestation_slot: u64 = 0;
let current_hashes = get_range_of_hashes(0, 16);
let oblique_hashes = vec![];
let result = attestation_parent_hashes(
cycle_length,
block_slot,
attestation_slot,
&current_hashes,
&oblique_hashes);
let result = result.unwrap();
assert_eq!(result.len(), cycle_length as usize);
let expected_result = get_range_of_hashes(7, 15);
assert_eq!(result, expected_result);
}
#[test]
fn test_get_signed_hashes_scenario_3() {
/*
* attestation_slot too large
*/
let cycle_length: u8 = 8;
let block_slot: u64 = 100;
let attestation_slot: u64 = 100;
let current_hashes = get_range_of_hashes(0, 16);
let oblique_hashes = vec![];
let result = attestation_parent_hashes(
cycle_length,
block_slot,
attestation_slot,
&current_hashes,
&oblique_hashes);
assert!(result.is_err());
}
#[test]
fn test_get_signed_hashes_scenario_4() {
/*
* Current hashes too small
*/
let cycle_length: u8 = 8;
let block_slot: u64 = 100;
let attestation_slot: u64 = 99;
let current_hashes = get_range_of_hashes(0, 15);
let oblique_hashes = vec![];
let result = attestation_parent_hashes(
cycle_length,
block_slot,
attestation_slot,
&current_hashes,
&oblique_hashes);
assert!(result.is_err());
}
}

View File

@ -1,8 +0,0 @@
// Collection of custom errors
#[derive(Debug,PartialEq)]
pub enum ParameterError {
IntWrapping,
OutOfBounds,
InvalidInput(String),
}

View File

@ -1,22 +0,0 @@
extern crate slog;
extern crate slog_term;
extern crate slog_async;
use slog::*;
pub use slog::Logger;
pub fn test_logger() -> slog::Logger {
let plain = slog_term::PlainSyncDecorator::new(slog_term::TestStdoutWriter);
Logger::root(
slog_term::FullFormat::new(plain)
.build().fuse(), o!()
)
}
pub fn get_logger() -> slog::Logger {
let decorator = slog_term::TermDecorator::new().build();
let drain = slog_term::CompactFormat::new(decorator).build().fuse();
let drain = slog_async::Async::new(drain).build().fuse();
slog::Logger::root(drain, o!())
}

View File

@ -1,8 +0,0 @@
#[macro_export]
macro_rules! assert_error {
($exp: expr, $err: expr) => {
if !$exp {
return Err($err);
}
}
}

View File

@ -1,11 +0,0 @@
extern crate ethereum_types;
extern crate blake2_rfc as blake2;
extern crate crypto_mac;
extern crate boolean_bitfield;
#[macro_use]
pub mod macros;
pub mod hash;
pub mod types;
pub mod logging;
pub mod errors;

View File

@ -1,12 +0,0 @@
extern crate boolean_bitfield;
use super::ethereum_types::{ H256, H160 };
use self::boolean_bitfield::BooleanBitfield;
pub use super::ethereum_types::U256;
pub type Hash256 = H256;
pub type Address = H160;
pub type Bitfield = BooleanBitfield;