Add WIP libp2p floodsub

.gitignore

@@ -1,3 +1,5 @@
 /target
 **/*.rs.bk
 Cargo.lock
+*.pk
+*.sk

Cargo.toml

@@ -4,16 +4,29 @@ version = "0.0.1"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 
 [dependencies]
-ethereum-types = ""
-rand = "0.3"
-bytes = ""
 blake2 = "^0.7.1"
+bls = { git = "https://github.com/sigp/bls" }
+bytes = ""
+crypto-mac = "^0.6.2"
+clap = "2.32.0"
+eth-secp256k1 = { git = "https://github.com/paritytech/rust-secp256k1" }
+ethereum-types = ""
+futures = "0.1.23"
+hex = "0.3.2"
+libp2p-peerstore = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+libp2p-core = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+libp2p-mplex = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+libp2p-tcp-transport = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+libp2p-floodsub = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+libp2p-identify = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+libp2p-kad = { git = "https://github.com/tomaka/libp2p-rs", branch ="zksummit" }
+rand = "0.3"
+rlp = { git = "https://github.com/paritytech/parity-common" }
 slog = "^2.2.3"
 slog-term = "^2.4.0"
 slog-async = "^2.3.0"
-crypto-mac = "^0.6.2"
-bls = { git = "https://github.com/sigp/bls" }
-rlp = { git = "https://github.com/paritytech/parity-common" }
+tokio-io = "0.1"
+tokio-core = "0.1"
 
 [dependencies.pairing]
 git = "https://github.com/mmaker/pairing"

src/lib.rs

@@ -1,6 +0,0 @@
-#[macro_use]
-extern crate slog;
-
-pub mod pubkeystore;
-pub mod state;
-pub mod utils;

src/main.rs

@@ -0,0 +1,54 @@
+#[macro_use]
+extern crate slog;
+extern crate slog_term;
+extern crate slog_async;
+extern crate clap;
+extern crate libp2p_peerstore;
+
+pub mod p2p;
+pub mod pubkeystore;
+pub mod state;
+pub mod utils;
+
+use p2p::keys;
+use p2p::floodsub;
+use slog::Drain;
+use clap::{ App, SubCommand};
+use std::sync::Arc;
+use std::time::Duration;
+use libp2p_peerstore::{ PeerAccess, Peerstore };
+use libp2p_peerstore::memory_peerstore::MemoryPeerstore;
+
+fn main() {
+    let decorator = slog_term::TermDecorator::new().build();
+    let drain = slog_term::CompactFormat::new(decorator).build().fuse();
+    let drain = slog_async::Async::new(drain).build().fuse();
+    let log = slog::Logger::root(drain, o!());
+
+    let matches = App::new("Lighthouse")
+        .version("0.0.1")
+        .author("Paul H. <paul@sigmaprime.io>")
+        .about("Eth 2.0 Client")
+        .subcommand(SubCommand::with_name("generate-keys"))
+            .about("Generates a new set of random keys for p2p dev.")
+        .get_matches();
+
+    if let Some(_) = matches.subcommand_matches("generate-keys") {
+        keys::generate_keys(&log).expect("Failed to generate keys");
+    } else {
+        let (s256k1_public, _s256k1_secret) = keys::load_local_keys(&log);
+        let peer_id = keys::peer_id_from_pub_key(&s256k1_public);
+        let bootstrap_peer_id = 
+            keys::peer_id_from_pub_key(&keys::load_bootstrap_pk(&log));
+        
+       let peer_store = Arc::new(MemoryPeerstore::empty());
+
+       peer_store.peer_or_create(&bootstrap_peer_id).add_addr(
+           "/ip4/127.0.0.1/tcp/10101/ws".parse().unwrap(),
+           Duration::from_secs(3600 * 24 * 356)
+        );
+        
+        floodsub::listen(peer_id, peer_store, &log);
+    }
+    info!(log, "Exiting.");
+}

src/p2p/floodsub.rs

@@ -0,0 +1,187 @@
+extern crate bytes;
+extern crate futures;
+extern crate libp2p_peerstore;
+extern crate libp2p_identify;
+extern crate libp2p_core;
+extern crate libp2p_mplex;
+extern crate libp2p_tcp_transport;
+extern crate libp2p_floodsub;
+extern crate libp2p_kad;
+extern crate slog;
+extern crate tokio_core;
+extern crate tokio_io;
+
+use self::futures::Future;
+use self::libp2p_peerstore::PeerId;
+use self::libp2p_core::{ Endpoint, Multiaddr, Transport, ConnectionUpgrade };
+use self::libp2p_floodsub::{ FloodSubUpgrade, FloodSubFuture };
+use self::libp2p_kad::{ KademliaUpgrade, KademliaProcessingFuture};
+use self::libp2p_identify::{ IdentifyInfo, IdentifyTransport, IdentifyOutput };
+use self::slog::Logger;
+use std::sync::{ Arc, RwLock };
+use std::time::Duration;
+use std::ops::Deref;
+use std::io::Error as IoError;
+use libp2p_peerstore::memory_peerstore::MemoryPeerstore;
+use self::tokio_io::{ AsyncRead, AsyncWrite };
+use self::bytes::Bytes;
+
+pub fn listen(peer_id: PeerId, 
+               peer_store: Arc<MemoryPeerstore>,
+               log: &Logger) 
+{
+    let listen_multiaddr: Multiaddr = "/ip4/0.0.0.0/tcp/10101/ws"
+        .parse::<Multiaddr>().expect("Failed to parse listen multiaddr.");
+    
+    let core =  tokio_core::reactor::Core::new().expect("tokio failure.");
+    let listened_addrs = Arc::new(RwLock::new(vec![]));
+    let transport = libp2p_tcp_transport::TcpConfig::new(core.handle())
+        .with_upgrade(libp2p_core::upgrade::PlainTextConfig)
+        .with_upgrade(libp2p_mplex::BufferedMultiplexConfig::<[_; 256]>::new())
+        .into_connection_reuse();
+
+    let (floodsub_upgrade, floodsub_rx) = FloodSubUpgrade::new(peer_id.clone());
+
+    let transport_sockets = {
+        let listened_addrs = Arc::new(RwLock::new(vec![]));
+        let listen_multiaddr = listen_multiaddr.clone();
+        IdentifyTransport::new(transport.clone(), peer_store.clone())
+            .map(move |out, _, _| {
+                if let(Some(ref observed), ref listen_multiaddr) = (out.observed_addr, listen_multiaddr) {
+                    if let Some(viewed_from_outisde) = transport.nat_traversal(listen_multiaddr, observed) {
+                        listened_addrs.write().unwrap().push(viewed_from_outisde);
+                    }
+                }
+                out.socket
+            })
+    };
+
+    let kad_config = libp2p_kad::KademliaConfig {
+        parallelism: 3,
+        record_store: (),
+        peer_store: peer_store,
+        local_peer_id: peer_id.clone(),
+        timeout: Duration::from_secs(2)
+    };
+
+    let kad_ctl_proto = libp2p_kad::KademliaControllerPrototype::new(kad_config);
+    let kad_upgrade = libp2p_kad::KademliaUpgrade::from_prototype(&kad_ctl_proto);
+
+    let upgrade = ConnectionUpgrader {
+        kad: kad_upgrade.clone(),
+        identify: libp2p_identify::IdentifyProtocolConfig,
+        floodsub: floodsub_upgrade.clone(),
+    };
+   
+    let swarm_listened_addrs = listened_addrs.clone();
+    let (swarm_ctrl, swarm_future) = libp2p_core::swarm(
+        transport_sockets.clone().with_upgrade(upgrade),
+        move |upgrade, client_addr| match upgrade {
+            FinalUpgrade::Kad(kad) => Box::new(kad) as Box<_>,
+            FinalUpgrade::FloodSub(future) => Box::new(future) as Box<_>,
+            FinalUpgrade::Identify(IdentifyOutput::Sender { sender, .. }) => sender.send(
+                IdentifyInfo {
+                    public_key: peer_id.clone().into_bytes(),
+                    agent_version: "lighthouse/1.0.0".to_owned(),
+                    protocol_version: "rust-libp2p/1.0.0".to_owned(),
+                    listen_addrs: swarm_listened_addrs.read().unwrap().to_vec(),
+                    protocols: vec![
+                        "/ipfs/kad/1.0.0".to_owned(),
+                        "/ipfs/id/1.0.0".to_owned(),
+                        "/floodsub/1.0.0".to_owned(),
+                    ]
+                },
+                &client_addr
+            ),
+            FinalUpgrade::Identify(IdentifyOutput::RemoteInfo { .. }) => {
+                unreachable!("Never dial with the identify protocol.")
+            }
+        },
+    );
+
+    let actual_addr = swarm_ctrl
+        .listen_on(listen_multiaddr)
+        .expect("Failed to listen on multiaddr");
+}
+
+#[derive(Clone)]
+struct ConnectionUpgrader<P, R> {
+    kad: KademliaUpgrade<P, R>,
+    identify: libp2p_identify::IdentifyProtocolConfig,
+    floodsub: FloodSubUpgrade
+}
+
+impl<C, P, R, Pc> ConnectionUpgrade<C> for ConnectionUpgrader<P, R>
+where 
+    C: AsyncRead + AsyncWrite + 'static,
+    P: Deref<Target = Pc> + Clone + 'static,
+    for<'r> &'r Pc: libp2p_peerstore::Peerstore,
+    R: 'static
+{
+    type NamesIter = ::std::vec::IntoIter<(Bytes, usize)>;
+    type UpgradeIdentifier = usize;
+    type Output = FinalUpgrade<C>;
+    type Future = Box<Future<Item = FinalUpgrade<C>, Error = IoError>>;
+
+    #[inline] 
+    fn protocol_names(&self) -> Self::NamesIter {
+        vec![
+            (Bytes::from("/ipfs/kad/1.0.0"), 0),
+            (Bytes::from("/ipfs/id/1.0.0"), 1),
+            (Bytes::from("/floodsub/1.0.0"), 2),
+        ].into_iter()
+    }
+
+    fn upgrade(
+        self,
+        socket: C,
+        id: Self::UpgradeIdentifier,
+        ty: Endpoint,
+        remote_addr: &Multiaddr)
+        -> Self::Future
+    {
+        match id {
+            0 => Box::new(
+                self.kad
+                    .upgrade(socket, (), ty, remote_addr)
+                    .map(|upg| upg.into())),
+            1 => Box::new(
+                self.identify
+                    .upgrade(socket, (), ty, remote_addr)
+                    .map(|upg| upg.into())),
+            2 => Box::new(
+                self.floodsub
+                    .upgrade(socket, (), ty, remote_addr)
+                    .map(|upg| upg.into())),
+            _ => unreachable!()
+        }
+
+    }
+}
+
+enum FinalUpgrade<C> {
+    Kad(KademliaProcessingFuture),
+    Identify(IdentifyOutput<C>),
+    FloodSub(FloodSubFuture),
+}
+
+impl<C> From<libp2p_kad::KademliaProcessingFuture> for FinalUpgrade<C> {
+    #[inline]
+    fn from(upgrade: libp2p_kad::KademliaProcessingFuture) -> Self {
+        FinalUpgrade::Kad(upgrade)
+    }
+}
+
+impl<C> From<IdentifyOutput<C>> for FinalUpgrade<C> {
+    #[inline]
+    fn from(upgrade: IdentifyOutput<C>) -> Self {
+        FinalUpgrade::Identify(upgrade)
+    }
+}
+
+impl<C> From<FloodSubFuture> for FinalUpgrade<C> {
+    #[inline]
+    fn from(upgrade: FloodSubFuture) -> Self {
+        FinalUpgrade::FloodSub(upgrade)
+    }
+}

src/p2p/keys.rs

@@ -0,0 +1,73 @@
+extern crate secp256k1;
+extern crate libp2p_peerstore;
+extern crate rand;
+extern crate hex;
+
+use std;
+use std::io::prelude::*;
+use std::fs::File;
+use slog::Logger;
+
+use self::secp256k1::key::{ SecretKey, PublicKey };
+use self::libp2p_peerstore::PeerId;
+
+const LOCAL_PK_FILE: &str = "local.pk";
+const LOCAL_SK_FILE: &str = "local.sk";
+const BOOTSTRAP_PK_FILE: &str = "bootstrap.pk";
+
+pub fn peer_id_from_pub_key(pk: &PublicKey) -> PeerId {
+    let curve = get_curve();
+    PeerId::from_public_key(&pk.serialize_vec(&curve, false))
+}
+
+fn get_curve() -> secp256k1::Secp256k1 { secp256k1::Secp256k1::new() }
+
+/// Generates a new public and secret key pair and writes them to 
+/// individual files.
+pub fn generate_keys(log: &Logger) -> std::io::Result<()> {
+    info!(log, "Generating keys...");
+    let mut rng = rand::thread_rng();
+    let curve = get_curve();
+    let s = SecretKey::new(&curve, &mut rng);
+    let p = PublicKey::from_secret_key(&curve, &s).unwrap();
+    let p_vec = p.serialize_vec(&curve, false);
+    let s_vec = &s[..];
+    let p_string = hex::encode(p_vec);
+    let s_string = hex::encode(s_vec);
+    let mut p_file = File::create(LOCAL_PK_FILE)?;
+    let mut s_file = File::create(LOCAL_SK_FILE)?;
+    info!(log, "Writing public key...");
+    p_file.write(p_string.as_bytes())?;
+    info!(log, "Writing secret key...");
+    s_file.write(s_string.as_bytes())?;
+    Ok(())
+}
+
+pub fn load_bootstrap_pk(log: &Logger) -> PublicKey {
+    info!(log, "Loading boostrap public key from filesystem...");
+    load_pk_from_file(BOOTSTRAP_PK_FILE)
+}
+
+pub fn load_local_keys(log: &Logger) -> (PublicKey, SecretKey) {
+    info!(log, "Loading local keys from filesystem...");
+    (load_pk_from_file(LOCAL_PK_FILE), load_sk_from_file(LOCAL_SK_FILE))
+}
+
+fn load_sk_from_file(file: &str) -> SecretKey {
+    let vec = load_vec_from_hex_file(file);
+    let curve = get_curve();
+    SecretKey::from_slice(&curve, &vec).expect("secret key invalid")
+}
+
+fn load_pk_from_file(file: &str) -> PublicKey {
+    let vec = load_vec_from_hex_file(file);
+    let curve = get_curve();
+    PublicKey::from_slice(&curve, &vec).expect("public key invalid")
+}
+
+fn load_vec_from_hex_file(file: &str) -> Vec<u8> {
+    let mut contents = String::new();
+    let mut file = File::open(file).expect("key not found");
+    file.read_to_string(&mut contents).expect("error reading from file");
+    hex::decode(contents).expect("public key corrupt")
+}

src/p2p/mod.rs

@@ -0,0 +1,2 @@
+pub mod floodsub;
+pub mod keys;