cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'sync'

Browse Source
This commit is contained in:
Paul Hauner 2018-09-13 13:05:19 +10:00
commit 88f9295e10
16 changed files with 876 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
boolean-bitfield/Cargo.toml
View File

@ -4,3 +4,4 @@ version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
[dependencies]
ssz = { path = "../ssz" }

100
boolean-bitfield/src/lib.rs
View File

@ -8,66 +8,78 @@
*/
use std::cmp::max;
#[derive(Eq)]
#[derive(Eq,Clone)]
pub struct BooleanBitfield{
len: usize,
vec: Vec<u8>
}
impl BooleanBitfield {
/// Create a new bitfield with a length of zero.
pub fn new() -> Self {
Self {
len: 0,
vec: vec![]
}
}
/// Create a new bitfield of a certain capacity
pub fn with_capacity(capacity: usize) -> Self {
Self {
len: 0,
vec: Vec::with_capacity(capacity)
vec: Vec::with_capacity(capacity / 8 + 1)
}
}
// Output the bitfield as a big-endian vec of u8.
pub fn to_be_vec(&self) -> Vec<u8> {
let mut o = self.vec.clone();
o.reverse();
o
}
/// Read the value of a bit.
///
/// Will return `true` if the bit has been set to `true`
/// without then being set to `False`.
pub fn get_bit(&self, i: &usize) -> bool {
self.get_bit_on_byte(*i % 8, *i / 8)
}
let bit = |i: &usize| *i % 8;
let byte = |i: &usize| *i / 8;
fn get_bit_on_byte(&self, bit: usize, byte: usize) -> bool {
assert!(bit < 8);
if byte >= self.vec.len() {
if byte(i) >= self.vec.len() {
false
} else {
self.vec[byte] & (1 << (bit as u8)) != 0
self.vec[byte(i)] & (1 << (bit(i) as u8)) != 0
}
}
pub fn set_bit(&mut self, bit: &usize, to: &bool) {
self.len = max(self.len, *bit + 1);
self.set_bit_on_byte(*bit % 8, *bit / 8, to);
}
/// Set the value of a bit.
///
/// If this bit is larger than the length of the underlying byte
/// array it will be extended.
pub fn set_bit(&mut self, i: &usize, to: &bool) {
let bit = |i: &usize| *i % 8;
let byte = |i: &usize| *i / 8;
fn set_bit_on_byte(&mut self, bit: usize, byte: usize, val: &bool) {
assert!(bit < 8);
if byte >= self.vec.len() {
self.vec.resize(byte + 1, 0);
self.len = max(self.len, i + 1);
if byte(i) >= self.vec.len() {
self.vec.resize(byte(i) + 1, 0);
}
match val {
true => self.vec[byte] = self.vec[byte] | (1 << (bit as u8)),
false => self.vec[byte] = self.vec[byte] & !(1 << (bit as u8))
match to {
true => {
self.vec[byte(i)] =
self.vec[byte(i)] | (1 << (bit(i) as u8))
}
false => {
self.vec[byte(i)] =
self.vec[byte(i)] & !(1 << (bit(i) as u8))
}
}
}
/// Return the "length" of this bitfield. Length is defined as
/// the highest bit that has been set.
///
/// Note: this is distinct from the length of the underlying
/// vector.
pub fn len(&self) -> usize { self.len }
// Return the total number of bits set to true.
/// Iterate through the underlying vector and count the number of
/// true bits.
pub fn num_true_bits(&self) -> u64 {
let mut count: u64 = 0;
for byte in &self.vec {
@ -79,6 +91,13 @@ impl BooleanBitfield {
}
count
}
/// Clone and return the underlying byte array (`Vec<u8>`).
pub fn to_be_vec(&self) -> Vec<u8> {
let mut o = self.vec.clone();
o.reverse();
o
}
}
impl PartialEq for BooleanBitfield {
@ -88,34 +107,24 @@ impl PartialEq for BooleanBitfield {
}
}
impl Clone for BooleanBitfield {
fn clone(&self) -> Self {
Self {
vec: self.vec.to_vec(),
..*self
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use super::rlp;
#[test]
fn test_bitfield_set() {
let mut b = BooleanBitfield::new();
b.set_bit(&0, &false);
assert_eq!(b.to_be_vec(), [0]);
b = BooleanBitfield::new();
b.set_bit(&7, &true);
assert_eq!(b.to_be_vec(), [128]);
b.set_bit(&7, &false);
assert_eq!(b.to_be_vec(), [0]);
assert_eq!(b.len(), 8);
b = BooleanBitfield::new();
b.set_bit(&7, &true);
b.set_bit(&0, &true);
@ -123,21 +132,22 @@ mod tests {
b.set_bit(&7, &false);
assert_eq!(b.to_be_vec(), [1]);
assert_eq!(b.len(), 8);
b = BooleanBitfield::new();
b.set_bit(&8, &true);
assert_eq!(b.to_be_vec(), [1, 0]);
assert_eq!(b.len(), 9);
b.set_bit(&8, &false);
assert_eq!(b.to_be_vec(), [0, 0]);
assert_eq!(b.len(), 9);
b = BooleanBitfield::new();
b.set_bit(&15, &true);
assert_eq!(b.to_be_vec(), [128, 0]);
b.set_bit(&15, &false);
assert_eq!(b.to_be_vec(), [0, 0]);
assert_eq!(b.len(), 16);
b = BooleanBitfield::new();
b.set_bit(&8, &true);
b.set_bit(&15, &true);
@ -156,9 +166,9 @@ mod tests {
b.set_bit(&i, &true);
assert_eq!(b.get_bit(&i), true);
b.set_bit(&i, &true);
}
}
}
#[test]
fn test_bitfield_num_true_bits() {
let mut b = BooleanBitfield::new();

4
lighthouse/client/mod.rs
View File

@ -8,7 +8,7 @@ use super::futures::sync::mpsc::{
use super::network_libp2p::service::listen as network_listen;
use super::network_libp2p::state::NetworkState;
use super::slog::Logger;
use super::sync::start_sync;
use super::sync::run_sync_future;
/// Represents the co-ordination of the
/// networking, syncing and RPC (not-yet-implemented) threads.
@ -59,7 +59,7 @@ impl Client {
let sync_log = log.new(o!());
let sync_db = Arc::clone(&db);
let thread = thread::spawn(move || {
start_sync(
run_sync_future(
sync_db,
network_tx.clone(),
network_rx,

12
lighthouse/state/attestation_record.rs
View File

@ -1,5 +1,6 @@
use super::utils::types::{ Hash256, Bitfield };
use super::utils::bls::{ AggregateSignature };
use super::ssz::{ Encodable, SszStream };
pub struct AttestationRecord {
@ -11,6 +12,17 @@ pub struct AttestationRecord {
pub aggregate_sig: Option<AggregateSignature>,
}
impl Encodable for AttestationRecord {
fn ssz_append(&self, s: &mut SszStream) {
s.append(&self.slot);
s.append(&self.shard_id);
s.append_vec(&self.oblique_parent_hashes);
s.append(&self.shard_block_hash);
s.append_vec(&self.attester_bitfield.to_be_vec());
// TODO: add aggregate signature
}
}
impl AttestationRecord {
pub fn zero() -> Self {
Self {

24
lighthouse/state/block.rs
View File

@ -1,6 +1,6 @@
use super::utils::types::Hash256;
use super::attestation_record::AttestationRecord;
use super::ssz;
use super::ssz::{ Encodable, SszStream };
const SSZ_BLOCK_LENGTH: usize = 192;
@ -12,7 +12,7 @@ pub struct Block {
pub pow_chain_ref: Hash256,
pub active_state_root: Hash256,
pub crystallized_state_root: Hash256,
}
}
impl Block {
pub fn zero() -> Self {
@ -27,12 +27,12 @@ impl Block {
}
}
// Not sure if this will be useful, leaving it here for the
// time being.
pub fn ssz_encode_without_attestations(&self)
/// Return the bytes that should be signed in order to
/// attest for this block.
pub fn encode_for_signing(&self)
-> [u8; SSZ_BLOCK_LENGTH]
{
let mut s = ssz::SszStream::new();
let mut s = SszStream::new();
s.append(&self.parent_hash);
s.append(&self.slot_number);
s.append(&self.randao_reveal);
@ -45,6 +45,18 @@ impl Block {
}
}
impl Encodable for Block {
fn ssz_append(&self, s: &mut SszStream) {
s.append(&self.parent_hash);
s.append(&self.slot_number);
s.append(&self.randao_reveal);
s.append_vec(&self.attestations);
s.append(&self.pow_chain_ref);
s.append(&self.active_state_root);
s.append(&self.crystallized_state_root);
}
}
#[cfg(test)]
mod tests {

12
lighthouse/sync/messages.rs Normal file
View File

@ -0,0 +1,12 @@
pub enum SyncEventType {
Invalid,
PeerConnect,
PeerDrop,
ReceiveBlocks,
ReceiveAttestationRecords,
}
pub struct SyncEvent {
event: SyncEventType,
data: Option<Vec<u8>>
}

51
lighthouse/sync/mod.rs
View File

@ -1,50 +1,13 @@
extern crate futures;
extern crate slog;
extern crate tokio;
extern crate network_libp2p;
use self::futures::sync::mpsc::{
UnboundedReceiver,
UnboundedSender,
};
use self::tokio::prelude::*;
use std::sync::{ RwLock, Arc };
use super::network_libp2p::message::{
NetworkEvent,
OutgoingMessage,
};
use super::db::DB;
use slog::Logger;
pub mod messages;
pub mod network;
pub mod sync_future;
pub mod wire_protocol;
type NetworkSender = UnboundedSender<OutgoingMessage>;
type NetworkReceiver = UnboundedReceiver<NetworkEvent>;
pub use self::sync_future::run_sync_future;
type SyncSender = UnboundedSender<Vec<u8>>;
type SyncReceiver = UnboundedReceiver<Vec<u8>>;
/// Start a syncing tokio future.
///
/// This is effectively a stub function being
/// used to test network functionality.
///
/// Expect a full re-write.
pub fn start_sync(
_db: Arc<RwLock<DB>>,
_network_tx: NetworkSender,
network_rx: NetworkReceiver,
_sync_tx: SyncSender,
_sync_rx: SyncReceiver,
log: Logger) {
let rx_future = network_rx
.for_each(move |event| {
debug!(&log, "Sync receive";
"msg" => format!("{:?}", event));
Ok(())
})
.map_err(|_| panic!("rx failed"));
/*
* This is an unfinished stub function.
*/
tokio::run(rx_future);
}
use super::db;

66
lighthouse/sync/network.rs Normal file
View File

@ -0,0 +1,66 @@
use std::sync::{ RwLock, Arc };
use super::db::DB;
use slog::Logger;
use super::network_libp2p::message::{
NetworkEvent,
OutgoingMessage,
NetworkEventType,
};
use super::wire_protocol::{ WireMessageType, message_type };
use super::futures::sync::mpsc::{
UnboundedSender,
};
/// Accept a network event and perform all required processing.
///
/// This function should be called whenever an underlying network
/// (e.g., libp2p) has an event to push up to the sync process.
pub fn handle_network_event(
event: NetworkEvent,
db: Arc<RwLock<DB>>,
network_tx: UnboundedSender<OutgoingMessage>,
log: Logger)
-> Result<(), ()>
{
debug!(&log, "";
"network_event" => format!("{:?}", &event));
match event.event {
NetworkEventType::PeerConnect => Ok(()),
NetworkEventType::PeerDrop => Ok(()),
NetworkEventType::Message => {
if let Some(data) = event.data {
handle_network_message(
data,
db,
network_tx,
log)
} else {
Ok(())
}
}
}
}
/// Accept a message from the network and perform all required
/// processing.
///
/// This function should be called whenever a peer from a network
/// (e.g., libp2p) has sent a message to us.
fn handle_network_message(
message: Vec<u8>,
_db: Arc<RwLock<DB>>,
_network_tx: UnboundedSender<OutgoingMessage>,
_log: Logger)
-> Result<(), ()>
{
match message_type(&message) {
Some(WireMessageType::Blocks) => {
// Do something with inbound blocks.
Ok(())
}
_ => Ok(())
}
}

48
lighthouse/sync/sync_future.rs Normal file
View File

@ -0,0 +1,48 @@
use super::tokio;
use super::futures::{ Future, Stream };
use super::futures::sync::mpsc::{
UnboundedReceiver,
UnboundedSender,
};
use super::network_libp2p::message::{
NetworkEvent,
OutgoingMessage,
};
use super::network::handle_network_event;
use std::sync::{ RwLock, Arc };
use super::db::DB;
use slog::Logger;
type NetworkSender = UnboundedSender<OutgoingMessage>;
type NetworkReceiver = UnboundedReceiver<NetworkEvent>;
type SyncSender = UnboundedSender<Vec<u8>>;
type SyncReceiver = UnboundedReceiver<Vec<u8>>;
/// Start a syncing tokio future.
///
/// Uses green-threading to process messages
/// from the network and the RPC and update
/// the state.
pub fn run_sync_future(
db: Arc<RwLock<DB>>,
network_tx: NetworkSender,
network_rx: NetworkReceiver,
_sync_tx: SyncSender,
_sync_rx: SyncReceiver,
log: Logger)
{
let network_future = {
network_rx
.for_each(move |event| {
handle_network_event(
event,
db.clone(),
network_tx.clone(),
log.clone())
})
.map_err(|_| panic!("rx failed"))
};
tokio::run(network_future);
}

24
lighthouse/sync/wire_protocol.rs Normal file
View File

@ -0,0 +1,24 @@
pub enum WireMessageType {
Status,
NewBlockHashes,
GetBlockHashes,
BlockHashes,
GetBlocks,
Blocks,
NewBlock,
}
/// Determines the message type of some given
/// message.
///
/// Does not check the validity of the message data,
/// it just reads the first byte.
pub fn message_type(message: &Vec<u8>)
-> Option<WireMessageType>
{
match message.get(0) {
Some(0x06) => Some(WireMessageType::Blocks),
_ => None
}
}

26
network-libp2p/src/service.rs
View File

@ -18,7 +18,7 @@ use super::state::NetworkState;
use super::message::{ NetworkEvent, NetworkEventType, OutgoingMessage };
use self::bigint::U512;
use self::futures::{ Future, Stream, Poll };
use self::futures::sync::mpsc::{
use self::futures::sync::mpsc::{
UnboundedSender, UnboundedReceiver
};
use self::libp2p_core::{ AddrComponent, Endpoint, Multiaddr,
@ -39,14 +39,14 @@ pub use self::libp2p_floodsub::Message;
pub fn listen(state: NetworkState,
events_to_app: UnboundedSender<NetworkEvent>,
raw_rx: UnboundedReceiver<OutgoingMessage>,
log: Logger)
log: Logger)
{
let peer_store = state.peer_store;
let peer_id = state.peer_id;
let listen_multiaddr = state.listen_multiaddr;
let listened_addrs = Arc::new(RwLock::new(vec![]));
let rx = ApplicationReciever{ inner: raw_rx };
// Build a tokio core
let mut core = tokio_core::reactor::Core::new().expect("tokio failure.");
// Build a base TCP libp2p transport
@ -65,10 +65,10 @@ pub fn listen(state: NetworkState,
// is stored not the internal addr.
.map(move |out, _, _| {
if let(Some(ref observed), ref listen_multiaddr) =
(out.observed_addr, listen_multiaddr)
(out.observed_addr, listen_multiaddr)
{
if let Some(viewed_from_outside) =
transport.nat_traversal(listen_multiaddr, observed)
if let Some(viewed_from_outside) =
transport.nat_traversal(listen_multiaddr, observed)
{
listened_addrs.write().unwrap()
.push(viewed_from_outside);
@ -79,7 +79,7 @@ pub fn listen(state: NetworkState,
};
// Configure and build a Kademlia upgrade to be applied
// to the base TCP transport.
// to the base TCP transport.
let kad_config = libp2p_kad::KademliaConfig {
parallelism: 3,
record_store: (),
@ -91,10 +91,10 @@ pub fn listen(state: NetworkState,
KademliaControllerPrototype::new(kad_config);
let kad_upgrade = libp2p_kad::
KademliaUpgrade::from_prototype(&kad_ctl_proto);
// Build a floodsub upgrade to allow pushing topic'ed
// messages across the network.
let (floodsub_upgrade, floodsub_rx) =
let (floodsub_upgrade, floodsub_rx) =
FloodSubUpgrade::new(peer_id.clone());
// Combine the Kademlia and Identify upgrades into a single
@ -104,7 +104,7 @@ pub fn listen(state: NetworkState,
floodsub: floodsub_upgrade.clone(),
identify: libp2p_identify::IdentifyProtocolConfig,
};
// Build a Swarm to manage upgrading connections to peers.
let swarm_listened_addrs = listened_addrs.clone();
let swarm_peer_id = peer_id.clone();
@ -166,7 +166,7 @@ pub fn listen(state: NetworkState,
for peer in peers {
let peer_hash = U512::from(peer.hash());
let distance = 512 - (local_hash ^ peer_hash).leading_zeros();
info!(kad_poll_log, "Discovered peer";
info!(kad_poll_log, "Discovered peer";
"distance" => distance,
"peer_id" => peer.to_base58());
let peer_addr = AddrComponent::P2P(peer.into_bytes()).into();
@ -240,7 +240,7 @@ struct ConnectionUpgrader<P, R> {
}
impl<C, P, R, Pc> ConnectionUpgrade<C> for ConnectionUpgrader<P, R>
where
where
C: AsyncRead + AsyncWrite + 'static,
P: Deref<Target = Pc> + Clone + 'static,
for<'r> &'r Pc: libp2p_peerstore::Peerstore,
@ -251,7 +251,7 @@ where
type Output = FinalUpgrade<C>;
type Future = Box<Future<Item = FinalUpgrade<C>, Error = IoError>>;
#[inline]
#[inline]
fn protocol_names(&self) -> Self::NamesIter {
vec![
(Bytes::from("/ipfs/kad/1.0.0"), 0),

135
ssz/src/decode.rs Normal file
View File

@ -0,0 +1,135 @@
use super::{
LENGTH_BYTES,
};
#[derive(Debug, PartialEq)]
pub enum DecodeError {
OutOfBounds,
TooShort,
TooLong,
}
pub trait Decodable: Sized {
fn ssz_decode(bytes: &[u8]) -> Result<Self, DecodeError>;
}
/// Decode the nth element of some ssz list.
///
/// A single ssz encoded value can be considered a list of
/// one element, so this function will work on it too.
pub fn decode_ssz_list_element<T>(ssz_bytes: &[u8], n: usize)
-> Result<T, DecodeError>
where T: Decodable
{
T::ssz_decode(nth_value(ssz_bytes, n)?)
}
/// Return the nth value in some ssz encoded list.
///
/// The four-byte length prefix is not included in the return.
///
/// A single ssz encoded value can be considered a list of
/// one element, so this function will work on it too.
fn nth_value(ssz_bytes: &[u8], n: usize)
-> Result<&[u8], DecodeError>
{
let mut c: usize = 0;
for i in 0..(n + 1) {
let length = decode_length(&ssz_bytes[c..], LENGTH_BYTES)?;
let next = c + LENGTH_BYTES + length;
if i == n {
return Ok(&ssz_bytes[c + LENGTH_BYTES..next]);
} else {
if next >= ssz_bytes.len() {
return Err(DecodeError::OutOfBounds);
} else {
c = next;
}
}
}
Err(DecodeError::OutOfBounds)
}
/// Given some number of bytes, interpret the first four
/// bytes as a 32-bit big-endian integer and return the
/// result.
fn decode_length(bytes: &[u8], length_bytes: usize)
-> Result<usize, DecodeError>
{
if bytes.len() < length_bytes {
return Err(DecodeError::TooShort);
};
let mut len: usize = 0;
for i in 0..length_bytes {
let offset = (length_bytes - i - 1) * 8;
len = ((bytes[i] as usize) << offset) | len;
};
Ok(len)
}
#[cfg(test)]
mod tests {
use super::*;
use super::super::encode::encode_length;
#[test]
fn test_ssz_decode_length() {
let decoded = decode_length(
&vec![0, 0, 1],
LENGTH_BYTES);
assert_eq!(decoded.unwrap(), 1);
let decoded = decode_length(
&vec![0, 1, 0],
LENGTH_BYTES);
assert_eq!(decoded.unwrap(), 256);
let decoded = decode_length(
&vec![0, 1, 255],
LENGTH_BYTES);
assert_eq!(decoded.unwrap(), 511);
let decoded = decode_length(
&vec![255, 255, 255],
LENGTH_BYTES);
assert_eq!(decoded.unwrap(), 16777215);
}
#[test]
fn test_encode_decode_length() {
let params: Vec<usize> = vec![
0, 1, 2, 3, 7, 8, 16,
2^8, 2^8 + 1,
2^16, 2^16 + 1,
2^24, 2^24 + 1,
2^32,
];
for i in params {
let decoded = decode_length(
&encode_length(i, LENGTH_BYTES),
LENGTH_BYTES).unwrap();
assert_eq!(i, decoded);
}
}
#[test]
fn test_ssz_nth_value() {
let ssz = vec![0, 0, 1, 0];
let result = nth_value(&ssz, 0).unwrap();
assert_eq!(result, vec![0].as_slice());
let ssz = vec![0, 0, 4, 1, 2, 3, 4];
let result = nth_value(&ssz, 0).unwrap();
assert_eq!(result, vec![1, 2, 3, 4].as_slice());
let ssz = vec![0, 0, 1, 0, 0, 0, 1, 1];
let result = nth_value(&ssz, 1).unwrap();
assert_eq!(result, vec![1].as_slice());
let mut ssz = vec![0, 1, 255];
ssz.append(&mut vec![42; 511]);
let result = nth_value(&ssz, 0).unwrap();
assert_eq!(result, vec![42; 511].as_slice());
}
}

166
ssz/src/encode.rs Normal file
View File

@ -0,0 +1,166 @@
use super::LENGTH_BYTES;
pub trait Encodable {
fn ssz_append(&self, s: &mut SszStream);
}
/// Provides a buffer for appending ssz-encodable values.
///
/// Use the `append()` fn to add a value to a list, then use
/// the `drain()` method to consume the struct and return the
/// ssz encoded bytes.
pub struct SszStream {
buffer: Vec<u8>
}
impl SszStream {
/// Create a new, empty stream for writing ssz values.
pub fn new() -> Self {
SszStream {
buffer: Vec::new()
}
}
/// Append some ssz encodable value to the stream.
pub fn append<E>(&mut self, value: &E) -> &mut Self
where E: Encodable
{
value.ssz_append(self);
self
}
/// Append some ssz encoded bytes to the stream.
///
/// The length of the supplied bytes will be concatenated
/// to the stream before the supplied bytes.
pub fn append_encoded_val(&mut self, vec: &Vec<u8>) {
self.buffer.extend_from_slice(
&encode_length(vec.len(),
LENGTH_BYTES));
self.buffer.extend_from_slice(&vec);
}
/// Append some vector (list) of encodable values to the stream.
///
/// The length of the list will be concatenated to the stream, then
/// each item in the vector will be encoded and concatenated.
pub fn append_vec<E>(&mut self, vec: &Vec<E>)
where E: Encodable
{
self.buffer.extend_from_slice(&encode_length(vec.len(), LENGTH_BYTES));
for v in vec {
v.ssz_append(self);
}
}
/// Consume the stream and return the underlying bytes.
pub fn drain(self) -> Vec<u8> {
self.buffer
}
}
/// Encode some length into a ssz size prefix.
///
/// The ssz size prefix is 4 bytes, which is treated as a continuious
/// 32bit big-endian integer.
pub fn encode_length(len: usize, length_bytes: usize) -> Vec<u8> {
assert!(length_bytes > 0); // For sanity
assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8));
let mut header: Vec<u8> = vec![0; length_bytes];
for i in 0..length_bytes {
let offset = (length_bytes - i - 1) * 8;
header[i] = ((len >> offset) & 0xff) as u8;
};
header
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[should_panic]
fn test_encode_length_0_bytes_panic() {
encode_length(0, 0);
}
#[test]
fn test_encode_length_4_bytes() {
assert_eq!(
encode_length(0, LENGTH_BYTES),
vec![0; 3]
);
assert_eq!(
encode_length(1, LENGTH_BYTES),
vec![0, 0, 1]
);
assert_eq!(
encode_length(255, LENGTH_BYTES),
vec![0, 0, 255]
);
assert_eq!(
encode_length(256, LENGTH_BYTES),
vec![0, 1, 0]
);
assert_eq!(
encode_length(16777215, LENGTH_BYTES), // 2^(3*8) - 1
vec![255, 255, 255]
);
}
#[test]
#[should_panic]
fn test_encode_length_4_bytes_panic() {
encode_length(16777216, LENGTH_BYTES); // 2^(3*8)
}
/*
#[test]
fn test_encode_struct() {
pub struct TestStruct {
pub one: u32,
pub two: H256,
pub three: u64,
pub four: U256,
}
impl Encodable for TestStruct {
fn ssz_append(&self, s: &mut SszStream) {
s.append(&self.one);
s.append(&self.two);
s.append(&self.three);
s.append(&self.four);
}
}
let t = TestStruct {
one: 1,
two: H256::zero(),
three: 100,
four: U256::zero(),
};
let mut s = SszStream::new();
s.append(&t);
let e = s.drain();
let expected_len = {
3 + 4 +
3 + 32 +
3 + 8 +
3 + 32
};
assert_eq!(e[0..4], [0, 0, 0, 4]);
assert_eq!(e[4..8], [0, 0, 0, 1]);
assert_eq!(e[8..12], [0, 0, 0, 32]);
assert_eq!(e[12..44], [0; 32]);
assert_eq!(e[44..48], [0, 0, 0, 8]);
assert_eq!(e[48..56], [0, 0, 0, 0, 0, 0, 0, 100]);
assert_eq!(e[56..60], [0, 0, 0, 32]);
assert_eq!(e[60..92], [0; 32]);
assert_eq!(e.len(), expected_len);
}
*/
}

114
ssz/src/impl_decode.rs Normal file
View File

@ -0,0 +1,114 @@
use super::{
DecodeError,
Decodable,
};
macro_rules! impl_decodable_for_uint {
($type: ident, $bit_size: expr) => {
impl Decodable for $type {
fn ssz_decode(bytes: &[u8])
-> Result<Self, DecodeError>
{
assert!((0 < $bit_size) &
($bit_size <= 64) &
($bit_size % 8 == 0));
let max_bytes = $bit_size / 8;
if bytes.len() <= max_bytes {
let mut result: $type = 0;
for i in 0..bytes.len() {
let offset = (bytes.len() - i - 1) * 8;
result = ((bytes[i] as $type) << offset) | result;
};
Ok(result)
} else {
Err(DecodeError::TooLong)
}
}
}
}
}
impl_decodable_for_uint!(u16, 16);
impl_decodable_for_uint!(u32, 32);
impl_decodable_for_uint!(u64, 64);
impl_decodable_for_uint!(usize, 64);
#[cfg(test)]
mod tests {
use super::super::{
DecodeError,
decode_ssz_list_element,
};
#[test]
fn test_ssz_decode_u16() {
let ssz = vec![0, 0, 1, 0];
let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 0);
let ssz = vec![0, 0, 1, 16];
let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 16);
let ssz = vec![0, 0, 2, 1, 0];
let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 256);
let ssz = vec![0, 0, 2, 255, 255];
let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 65535);
let ssz = vec![0, 0, 3, 0, 0, 1];
let result: Result<u16, DecodeError> =
decode_ssz_list_element(&ssz, 0);
assert_eq!(result, Err(DecodeError::TooLong));
}
#[test]
fn test_ssz_decode_u32() {
let ssz = vec![0, 0, 1, 0];
let result: u32 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 0);
let ssz = vec![0, 0, 4, 255, 255, 255, 255];
let result: u32 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 4294967295);
let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1];
let result: Result<u32, DecodeError> =
decode_ssz_list_element(&ssz, 0);
assert_eq!(result, Err(DecodeError::TooLong));
}
#[test]
fn test_ssz_decode_u64() {
let ssz = vec![0, 0, 1, 0];
let result: u64 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 0);
let ssz = vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255];
let result: u64 = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 18446744073709551615);
let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1];
let result: Result<u64, DecodeError> =
decode_ssz_list_element(&ssz, 0);
assert_eq!(result, Err(DecodeError::TooLong));
}
#[test]
fn test_ssz_decode_usize() {
let ssz = vec![0, 0, 1, 0];
let result: usize = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 0);
let ssz = vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255];
let result: usize = decode_ssz_list_element(&ssz, 0).unwrap();
assert_eq!(result, 18446744073709551615);
let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1];
let result: Result<usize, DecodeError> =
decode_ssz_list_element(&ssz, 0);
assert_eq!(result, Err(DecodeError::TooLong));
}
}

186
ssz/src/impl_encode.rs Normal file
View File

@ -0,0 +1,186 @@
use super::{
Encodable,
SszStream
};
use super::ethereum_types::{ H256, U256 };
/*
* Note: there is a "to_bytes" function for integers
* in Rust nightly. When it is in stable, we should
* use it instead.
*/
macro_rules! impl_encodable_for_uint {
($type: ident) => {
impl Encodable for $type {
fn ssz_append(&self, s: &mut SszStream)
{
// Number of bits required to represent this integer.
// This could be optimised at the expense of complexity.
let num_bits = {
let mut n = *self;
let mut r: usize = 0;
while n > 0 {
n >>= 1;
r += 1;
}
if r == 0 { 1 } else { r }
};
// Number of bytes required to represent this bit
let num_bytes = (num_bits + 8 - 1) / 8;
let mut ssz_val: Vec<u8> = Vec::with_capacity(num_bytes);
ssz_val.resize(num_bytes, 0);
for i in (0..num_bytes).rev() {
let offset = (num_bytes - i - 1) * 8;
ssz_val[i] = 0_u8 | (self >> offset) as u8
}
s.append_encoded_val(&ssz_val);
}
}
}
}
impl_encodable_for_uint!(u8);
impl_encodable_for_uint!(u16);
impl_encodable_for_uint!(u32);
impl_encodable_for_uint!(u64);
impl_encodable_for_uint!(usize);
impl Encodable for H256 {
fn ssz_append(&self, s: &mut SszStream) {
s.append_encoded_val(&self.to_vec());
}
}
impl Encodable for U256 {
fn ssz_append(&self, s: &mut SszStream) {
let mut a = [0; 32];
self.to_big_endian(&mut a);
s.append_encoded_val(&a.to_vec());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_ssz_encode_u8() {
let x: u16 = 0;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 0]);
let x: u16 = 1;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
let x: u16 = 100;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
let x: u16 = 255;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 255]);
}
#[test]
fn test_ssz_encode_u16() {
let x: u16 = 1;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
let x: u16 = 100;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
let x: u16 = 1 << 8;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 2, 1, 0]);
let x: u16 = 65535;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 2, 255, 255]);
}
#[test]
fn test_ssz_encode_u32() {
let x: u32 = 1;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
let x: u32 = 100;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
let x: u32 = 1 << 16;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 3, 1, 0, 0]);
let x: u32 = 1 << 24;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 4, 1, 0, 0, 0]);
let x: u32 = !0;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 4, 255, 255, 255, 255]);
}
#[test]
fn test_ssz_encode_u64() {
let x: u64 = 1;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
let x: u64 = 100;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
let x: u64 = 1 << 32;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 5, 1, 0, 0, 0, 0]);
let x: u64 = !0;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]);
}
#[test]
fn test_ssz_encode_usize() {
let x: usize = 1;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 1]);
let x: usize = 100;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 1, 100]);
let x: usize = 1 << 32;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 5, 1, 0, 0, 0, 0]);
let x: usize = !0;
let mut ssz = SszStream::new();
ssz.append(&x);
assert_eq!(ssz.drain(), vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]);
}
}

187
ssz/src/lib.rs
View File

@ -1,8 +1,8 @@
/*
* This is a WIP of implementing an alternative
* This is a WIP of implementing an alternative
* serialization strategy. It attempts to follow Vitalik's
* "ssz" format here:
* https://github.com/ethereum/research/tree/master/py_ssz
* "simpleserialize" format here:
* https://github.com/ethereum/beacon_chain/blob/master/beacon_chain/utils/simpleserialize.py
*
* This implementation is not final and would almost certainly
* have issues.
@ -10,172 +10,19 @@
extern crate bytes;
extern crate ethereum_types;
use self::bytes::{ BytesMut, BufMut };
use self::ethereum_types::{ H256, U256 };
mod encode;
mod decode;
mod impl_encode;
mod impl_decode;
pub const LENGTH_BYTES: usize = 4;
pub use decode::{
Decodable,
DecodeError,
decode_ssz_list_element,
};
pub use encode::{
Encodable,
SszStream,
};
pub trait Encodable {
fn ssz_append(&self, s: &mut SszStream);
}
pub struct SszStream {
buffer: Vec<u8>
}
impl SszStream {
pub fn new() -> Self {
SszStream {
buffer: Vec::new()
}
}
pub fn append<E>(&mut self, value: &E) -> &mut Self
where E: Encodable
{
value.ssz_append(self);
self
}
fn append_encoded_vec(&mut self, v: &mut Vec<u8>) {
self.buffer.append(&mut encode_length(v.len(), LENGTH_BYTES));
self.buffer.append(v) ;
}
fn append_encoded_array(&mut self, a: &mut [u8]) {
let len = a.len();
self.buffer.append(&mut encode_length(len, LENGTH_BYTES));
self.buffer.extend_from_slice(&a[0..len]);
}
pub fn drain(self) -> Vec<u8> {
self.buffer
}
}
pub fn encode<E>(value: &E) -> Vec<u8>
where E: Encodable
{
let mut stream = SszStream::new();
stream.append(value);
stream.drain()
}
fn encode_length(len: usize, length_bytes: usize) -> Vec<u8> {
assert!(length_bytes > 0); // For sanity
assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8));
let mut header: Vec<u8> = vec![0; length_bytes];
for i in 0..length_bytes {
let offset = (length_bytes - i - 1) * 8;
header[i] = ((len >> offset) & 0xff) as u8;
};
header
}
/*
* Implementations for various types
*/
impl Encodable for u32 {
fn ssz_append(&self, s: &mut SszStream) {
let mut buf = BytesMut::with_capacity(32/8);
buf.put_u32_be(*self);
s.append_encoded_vec(&mut buf.to_vec());
}
}
impl Encodable for u64 {
fn ssz_append(&self, s: &mut SszStream) {
let mut buf = BytesMut::with_capacity(64/8);
buf.put_u64_be(*self);
s.append_encoded_vec(&mut buf.to_vec());
}
}
impl Encodable for H256 {
fn ssz_append(&self, s: &mut SszStream) {
s.append_encoded_vec(&mut self.to_vec());
}
}
impl Encodable for U256 {
fn ssz_append(&self, s: &mut SszStream) {
let mut a = [0; 32];
self.to_big_endian(&mut a);
s.append_encoded_array(&mut a);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[should_panic]
fn test_encode_length_0_bytes_panic() {
encode_length(0, 0);
}
#[test]
fn test_encode_length_4_bytes() {
assert_eq!(
encode_length(0, 4),
vec![0; 4]
);
assert_eq!(
encode_length(1, 4),
vec![0, 0, 0, 1]
);
assert_eq!(
encode_length(255, 4),
vec![0, 0, 0, 255]
);
assert_eq!(
encode_length(256, 4),
vec![0, 0, 1, 0]
);
assert_eq!(
encode_length(4294967295, 4), // 2^(4*8) - 1
vec![255, 255, 255, 255]
);
}
#[test]
#[should_panic]
fn test_encode_length_4_bytes_panic() {
encode_length(4294967296, 4); // 2^(4*8)
}
#[test]
fn test_serialization() {
pub struct TestStruct {
pub one: u32,
pub two: H256,
pub three: u64,
}
impl Encodable for TestStruct {
fn ssz_append(&self, s: &mut SszStream) {
s.append(&self.one);
s.append(&self.two);
s.append(&self.three);
}
}
let t = TestStruct {
one: 1,
two: H256::zero(),
three: 100
};
let e = encode(&t);
assert_eq!(e[0..4], [0, 0, 0, 4]);
assert_eq!(e[4..8], [0, 0, 0, 1]);
assert_eq!(e[8..12], [0, 0, 0, 32]);
assert_eq!(e[12..44], [0; 32]);
assert_eq!(e[44..48], [0, 0, 0, 8]);
assert_eq!(e[48..56], [0, 0, 0, 0, 0, 0, 0, 100]);
assert_eq!(e.len(), 56);
}
}
pub const LENGTH_BYTES: usize = 3;