Merge branch 'master' into v0.5.0-state-transition-tests

Signed-off-by: Kirk Baird <baird.k@outlook.com>

Cargo.toml

@@ -30,4 +30,5 @@ members = [
 	"beacon_node/beacon_chain/test_harness",
 	"protos",
 	"validator_client",
+	"account_manager",
 ]

account_manager/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "account_manager"
+version = "0.0.1"
+authors = ["Luke Anderson <luke@sigmaprime.io>"]
+edition = "2018"
+
+[dependencies]
+bls = { path = "../eth2/utils/bls" }
+clap = "2.32.0"
+slog = "^2.2.3"
+slog-term = "^2.4.0"
+slog-async = "^2.3.0"
+validator_client = { path = "../validator_client" }

account_manager/README.md

@@ -0,0 +1,24 @@
+# Lighthouse Accounts Manager
+
+The accounts manager (AM) is a stand-alone binary which allows
+users to generate and manage the cryptographic keys necessary to
+interact with Ethereum Serenity.
+
+## Roles
+
+The AM is responsible for the following tasks:
+- Generation of cryptographic key pairs
+  - Must acquire sufficient entropy to ensure keys are generated securely (TBD)
+- Secure storage of private keys
+  - Keys must be encrypted while at rest on the disk (TBD)
+  - The format is compatible with the validator client
+- Produces messages and transactions necessary to initiate
+staking on Ethereum 1.x (TPD)
+
+
+## Implementation
+
+The AM is not a service, and does not run continuously, nor does it
+interact with any running services.
+It is intended to be executed separately from other Lighthouse binaries
+and produce files which can be consumed by them.

account_manager/src/main.rs

@@ -0,0 +1,58 @@
+use bls::Keypair;
+use clap::{App, Arg, SubCommand};
+use slog::{debug, info, o, Drain};
+use std::path::PathBuf;
+use validator_client::Config as ValidatorClientConfig;
+
+fn main() {
+    // Logging
+    let decorator = slog_term::TermDecorator::new().build();
+    let drain = slog_term::CompactFormat::new(decorator).build().fuse();
+    let drain = slog_async::Async::new(drain).build().fuse();
+    let log = slog::Logger::root(drain, o!());
+
+    // CLI
+    let matches = App::new("Lighthouse Accounts Manager")
+        .version("0.0.1")
+        .author("Sigma Prime <contact@sigmaprime.io>")
+        .about("Eth 2.0 Accounts Manager")
+        .arg(
+            Arg::with_name("datadir")
+                .long("datadir")
+                .value_name("DIR")
+                .help("Data directory for keys and databases.")
+                .takes_value(true),
+        )
+        .subcommand(
+            SubCommand::with_name("generate")
+                .about("Generates a new validator private key")
+                .version("0.0.1")
+                .author("Sigma Prime <contact@sigmaprime.io>"),
+        )
+        .get_matches();
+
+    let config = ValidatorClientConfig::parse_args(&matches, &log)
+        .expect("Unable to build a configuration for the account manager.");
+
+    // Log configuration
+    info!(log, "";
+          "data_dir" => &config.data_dir.to_str());
+
+    match matches.subcommand() {
+        ("generate", Some(_gen_m)) => {
+            let keypair = Keypair::random();
+            let key_path: PathBuf = config
+                .save_key(&keypair)
+                .expect("Unable to save newly generated private key.");
+            debug!(
+                log,
+                "Keypair generated {:?}, saved to: {:?}",
+                keypair.identifier(),
+                key_path.to_string_lossy()
+            );
+        }
+        _ => panic!(
+            "The account manager must be run with a subcommand. See help for more information."
+        ),
+    }
+}

eth2/utils/bls/src/keypair.rs

@@ -14,4 +14,8 @@ impl Keypair {
         let pk = PublicKey::from_secret_key(&sk);
         Keypair { sk, pk }
     }
+
+    pub fn identifier(&self) -> String {
+        self.pk.concatenated_hex_id()
+    }
 }

eth2/utils/boolean-bitfield/src/lib.rs

@@ -6,7 +6,7 @@ use bit_vec::BitVec;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode, PrefixedHexVisitor};
-use ssz::Decodable;
+use ssz::{Decodable, Encodable};
 use std::cmp;
 use std::default;
 
@@ -144,14 +144,13 @@ impl std::ops::BitAnd for BooleanBitfield {
     }
 }
 
-impl ssz::Encodable for BooleanBitfield {
-    // ssz_append encodes Self according to the `ssz` spec.
+impl Encodable for BooleanBitfield {
     fn ssz_append(&self, s: &mut ssz::SszStream) {
         s.append_vec(&self.to_bytes())
     }
 }
 
-impl ssz::Decodable for BooleanBitfield {
+impl Decodable for BooleanBitfield {
     fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), ssz::DecodeError> {
         let len = ssz::decode::decode_length(bytes, index, ssz::LENGTH_BYTES)?;
         if (ssz::LENGTH_BYTES + len) > bytes.len() {

validator_client/Cargo.toml

@@ -4,6 +4,15 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
+[[bin]]
+name = "validator_client"
+path = "src/main.rs"
+
+[lib]
+name = "validator_client"
+path = "src/lib.rs"
+
+
 [dependencies]
 block_proposer = { path = "../eth2/block_proposer" }
 bls = { path = "../eth2/utils/bls" }
@@ -18,3 +27,4 @@ slog = "^2.2.3"
 slog-term = "^2.4.0"
 slog-async = "^2.3.0"
 ssz = { path = "../eth2/utils/ssz" }
+bincode = "^1.1.2"

validator_client/README.md

@@ -57,10 +57,30 @@ complete and return a block from the BN.
 
 ### Configuration
 
-Presently the validator specifics (pubkey, etc.) are randomly generated and the
-chain specification (slot length, BLS domain, etc.) are fixed to foundation
-parameters. This is temporary and will be upgrade so these parameters can be
-read from file (or initialized on first-boot).
+Validator configurations are stored in a separate data directory from the main Beacon Node
+binary. The validator data directory defaults to:
+`$HOME/.lighthouse-validator`, however an alternative can be specified on the command line
+with `--datadir`.
+
+The configuration directory structure looks like:
+```
+~/.lighthouse-validator
+    ├── 3cf4210d58ec
+    │   └── private.key
+    ├── 9b5d8b5be4e7
+    │   └── private.key
+    └── cf6e07188f48
+        └── private.key
+```
+
+Where the hex value of the directory is a portion of the validator public key.
+
+Validator keys must be generated using the separate `accounts_manager` binary, which will
+place the keys into this directory structure in a format compatible with the validator client.
+
+The chain specification (slot length, BLS domain, etc.) defaults to foundation
+parameters, however is temporary and an upgrade will allow these parameters to be
+read from a file (or initialized on first-boot).
 
 ## BN Communication
 

validator_client/src/config.rs

@@ -1,28 +1,39 @@
+use bincode;
+use bls::Keypair;
+use clap::ArgMatches;
+use slog::{debug, error, info};
 use std::fs;
+use std::fs::File;
+use std::io::{Error, ErrorKind};
 use std::path::PathBuf;
 use types::ChainSpec;
 
 /// Stores the core configuration for this validator instance.
 #[derive(Clone)]
-pub struct ClientConfig {
+pub struct Config {
+    /// The data directory, which stores all validator databases
     pub data_dir: PathBuf,
+    /// The server at which the Beacon Node can be contacted
     pub server: String,
+    /// The chain specification that we are connecting to
     pub spec: ChainSpec,
 }
 
-const DEFAULT_LIGHTHOUSE_DIR: &str = ".lighthouse-validators";
+const DEFAULT_PRIVATE_KEY_FILENAME: &str = "private.key";
 
-impl ClientConfig {
-    /// Build a new configuration from defaults.
-    pub fn default() -> Self {
+impl Default for Config {
+    fn default() -> Self {
         let data_dir = {
-            let home = dirs::home_dir().expect("Unable to determine home dir.");
-            home.join(DEFAULT_LIGHTHOUSE_DIR)
+            let home = dirs::home_dir().expect("Unable to determine home directory.");
+            home.join(".lighthouse-validator")
         };
         fs::create_dir_all(&data_dir)
             .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir));
+
         let server = "localhost:50051".to_string();
+
         let spec = ChainSpec::foundation();
+
         Self {
             data_dir,
             server,
@@ -30,3 +41,114 @@ impl ClientConfig {
         }
     }
 }
+
+impl Config {
+    /// Build a new configuration from defaults, which are overrided by arguments provided.
+    pub fn parse_args(args: &ArgMatches, log: &slog::Logger) -> Result<Self, Error> {
+        let mut config = Config::default();
+
+        // Use the specified datadir, or default in the home directory
+        if let Some(datadir) = args.value_of("datadir") {
+            config.data_dir = PathBuf::from(datadir);
+            fs::create_dir_all(&config.data_dir)
+                .unwrap_or_else(|_| panic!("Unable to create {:?}", &config.data_dir));
+            info!(log, "Using custom data dir: {:?}", &config.data_dir);
+        };
+
+        if let Some(srv) = args.value_of("server") {
+            //TODO: I don't think this parses correctly a server & port combo
+            config.server = srv.to_string();
+            info!(log, "Using custom server: {:?}", &config.server);
+        };
+
+        // TODO: Permit loading a custom spec from file.
+        if let Some(spec_str) = args.value_of("spec") {
+            info!(log, "Using custom spec: {:?}", spec_str);
+            config.spec = match spec_str {
+                "foundation" => ChainSpec::foundation(),
+                "few_validators" => ChainSpec::few_validators(),
+                // Should be impossible due to clap's `possible_values(..)` function.
+                _ => unreachable!(),
+            };
+        };
+
+        Ok(config)
+    }
+
+    /// Try to load keys from validator_dir, returning None if none are found or an error.
+    pub fn fetch_keys(&self, log: &slog::Logger) -> Option<Vec<Keypair>> {
+        let key_pairs: Vec<Keypair> = fs::read_dir(&self.data_dir)
+            .unwrap()
+            .filter_map(|validator_dir| {
+                let validator_dir = validator_dir.ok()?;
+
+                if !(validator_dir.file_type().ok()?.is_dir()) {
+                    // Skip non-directories (i.e. no files/symlinks)
+                    return None;
+                }
+
+                let key_filename = validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME);
+
+                if !(key_filename.is_file()) {
+                    info!(
+                        log,
+                        "Private key is not a file: {:?}",
+                        key_filename.to_str()
+                    );
+                    return None;
+                }
+
+                debug!(
+                    log,
+                    "Deserializing private key from file: {:?}",
+                    key_filename.to_str()
+                );
+
+                let mut key_file = File::open(key_filename.clone()).ok()?;
+
+                let key: Keypair = if let Ok(key_ok) = bincode::deserialize_from(&mut key_file) {
+                    key_ok
+                } else {
+                    error!(
+                        log,
+                        "Unable to deserialize the private key file: {:?}", key_filename
+                    );
+                    return None;
+                };
+
+                let ki = key.identifier();
+                if ki != validator_dir.file_name().into_string().ok()? {
+                    error!(
+                        log,
+                        "The validator key ({:?}) did not match the directory filename {:?}.",
+                        ki,
+                        &validator_dir.path().to_string_lossy()
+                    );
+                    return None;
+                }
+                Some(key)
+            })
+            .collect();
+
+        // Check if it's an empty vector, and return none.
+        if key_pairs.is_empty() {
+            None
+        } else {
+            Some(key_pairs)
+        }
+    }
+
+    /// Saves a keypair to a file inside the appropriate validator directory. Returns the saved path filename.
+    pub fn save_key(&self, key: &Keypair) -> Result<PathBuf, Error> {
+        let validator_config_path = self.data_dir.join(key.identifier());
+        let key_path = validator_config_path.join(DEFAULT_PRIVATE_KEY_FILENAME);
+
+        fs::create_dir_all(&validator_config_path)?;
+
+        let mut key_file = File::create(&key_path)?;
+
+        bincode::serialize_into(&mut key_file, &key)
+            .map_err(|e| Error::new(ErrorKind::InvalidData, e))?;
+        Ok(key_path)
+    }
+}

validator_client/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod config;
+
+pub use crate::config::Config;

validator_client/src/main.rs

@@ -1,17 +1,14 @@
 use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService};
 use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap};
-use crate::config::ClientConfig;
+use crate::config::Config;
 use block_proposer::{test_utils::LocalSigner, BlockProducer};
-use bls::Keypair;
 use clap::{App, Arg};
 use grpcio::{ChannelBuilder, EnvBuilder};
 use protos::services_grpc::{BeaconBlockServiceClient, ValidatorServiceClient};
-use slog::{error, info, o, Drain};
+use slog::{info, o, Drain};
 use slot_clock::SystemTimeSlotClock;
-use std::path::PathBuf;
 use std::sync::Arc;
 use std::thread;
-use types::ChainSpec;
 
 mod block_producer_service;
 mod config;
@@ -55,36 +52,11 @@ fn main() {
         )
         .get_matches();
 
-    let mut config = ClientConfig::default();
-
-    // Custom datadir
-    if let Some(dir) = matches.value_of("datadir") {
-        config.data_dir = PathBuf::from(dir.to_string());
-    }
-
-    // Custom server port
-    if let Some(server_str) = matches.value_of("server") {
-        if let Ok(addr) = server_str.parse::<u16>() {
-            config.server = addr.to_string();
-        } else {
-            error!(log, "Invalid address"; "server" => server_str);
-            return;
-        }
-    }
-
-    // TODO: Permit loading a custom spec from file.
-    // Custom spec
-    if let Some(spec_str) = matches.value_of("spec") {
-        match spec_str {
-            "foundation" => config.spec = ChainSpec::foundation(),
-            "few_validators" => config.spec = ChainSpec::few_validators(),
-            // Should be impossible due to clap's `possible_values(..)` function.
-            _ => unreachable!(),
-        };
-    }
+    let config = Config::parse_args(&matches, &log)
+        .expect("Unable to build a configuration for the validator client.");
 
     // Log configuration
-    info!(log, "";
+    info!(log, "Configuration parameters:";
           "data_dir" => &config.data_dir.to_str(),
           "server" => &config.server);
 
@@ -119,13 +91,13 @@ fn main() {
     let poll_interval_millis = spec.seconds_per_slot * 1000 / 10; // 10% epoch time precision.
     info!(log, "Starting block producer service"; "polls_per_epoch" => spec.seconds_per_slot * 1000 / poll_interval_millis);
 
+    let keypairs = config.fetch_keys(&log)
+        .expect("No key pairs found in configuration, they must first be generated with: account_manager generate.");
+
     /*
      * Start threads.
      */
     let mut threads = vec![];
-    // TODO: keypairs are randomly generated; they should be loaded from a file or generated.
-    // https://github.com/sigp/lighthouse/issues/160
-    let keypairs = vec![Keypair::random()];
 
     for keypair in keypairs {
         info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id());