From 2a50550b8738803fc99fd6d85b4102da5fb8ef2e Mon Sep 17 00:00:00 2001 From: Matt Garnett <14004106+c-o-l-o-r@users.noreply.github.com> Date: Sat, 22 Jun 2019 13:57:37 -0400 Subject: [PATCH 01/72] make `hashing` crate wasm compatible --- eth2/utils/hashing/.cargo/config | 2 ++ eth2/utils/hashing/Cargo.toml | 11 ++++++++++- eth2/utils/hashing/src/lib.rs | 33 +++++++++++++++++++++++++------- 3 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 eth2/utils/hashing/.cargo/config diff --git a/eth2/utils/hashing/.cargo/config b/eth2/utils/hashing/.cargo/config new file mode 100644 index 000000000..4ec2f3b86 --- /dev/null +++ b/eth2/utils/hashing/.cargo/config @@ -0,0 +1,2 @@ +[target.wasm32-unknown-unknown] +runner = 'wasm-bindgen-test-runner' diff --git a/eth2/utils/hashing/Cargo.toml b/eth2/utils/hashing/Cargo.toml index 78dd70e43..506b84a6b 100644 --- a/eth2/utils/hashing/Cargo.toml +++ b/eth2/utils/hashing/Cargo.toml @@ -4,5 +4,14 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" -[dependencies] +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] ring = "0.14.6" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +sha2 = "0.8.0" + +[dev-dependencies] +rustc-hex = "2.0.1" + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +wasm-bindgen-test = "0.2.47" diff --git a/eth2/utils/hashing/src/lib.rs b/eth2/utils/hashing/src/lib.rs index a9e286c39..7214c7421 100644 --- a/eth2/utils/hashing/src/lib.rs +++ b/eth2/utils/hashing/src/lib.rs @@ -1,7 +1,17 @@ +#[cfg(not(target_arch = "wasm32"))] use ring::digest::{digest, SHA256}; +#[cfg(target_arch = "wasm32")] +use sha2::{Digest, Sha256}; + pub fn hash(input: &[u8]) -> Vec { - digest(&SHA256, input).as_ref().into() + #[cfg(not(target_arch = "wasm32"))] + let h = digest(&SHA256, input).as_ref().into(); + + #[cfg(target_arch = "wasm32")] + let h = Sha256::digest(input).as_ref().into(); + + h } /// Get merkle root of some hashed values - the input leaf nodes is expected to already be hashed @@ -37,19 +47,24 @@ pub fn merkle_root(values: &[Vec]) -> Option> { #[cfg(test)] mod tests { use super::*; - use ring::test; + use rustc_hex::FromHex; - #[test] + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::*; + + #[cfg_attr(not(target_arch = "wasm32"), test)] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn test_hashing() { let input: Vec = b"hello world".as_ref().into(); let output = hash(input.as_ref()); let expected_hex = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; - let expected: Vec = test::from_hex(expected_hex).unwrap(); + let expected: Vec = expected_hex.from_hex().unwrap(); assert_eq!(expected, output); } - #[test] + #[cfg_attr(not(target_arch = "wasm32"), test)] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn test_merkle_root() { // hash the leaf nodes let mut input = vec![ @@ -79,13 +94,17 @@ mod tests { assert_eq!(&expected[..], output.unwrap().as_slice()); } - #[test] + + #[cfg_attr(not(target_arch = "wasm32"), test)] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn test_empty_input_merkle_root() { let input = vec![]; let output = merkle_root(&input[..]); assert_eq!(None, output); } - #[test] + + #[cfg_attr(not(target_arch = "wasm32"), test)] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn test_odd_leaf_merkle_root() { let input = vec![ hash("a".as_bytes()), From 87e681c6175df08d529599e94b7bb5ec11bc584c Mon Sep 17 00:00:00 2001 From: Matt Garnett <14004106+c-o-l-o-r@users.noreply.github.com> Date: Sat, 22 Jun 2019 14:34:29 -0400 Subject: [PATCH 02/72] make `ssz` crate wasm compatible --- eth2/utils/ssz/src/decode/impls.rs | 5 +++++ eth2/utils/ssz/src/lib.rs | 2 ++ 2 files changed, 7 insertions(+) diff --git a/eth2/utils/ssz/src/decode/impls.rs b/eth2/utils/ssz/src/decode/impls.rs index 0965ee3e5..75dd5d444 100644 --- a/eth2/utils/ssz/src/decode/impls.rs +++ b/eth2/utils/ssz/src/decode/impls.rs @@ -34,6 +34,11 @@ impl_decodable_for_uint!(u8, 8); impl_decodable_for_uint!(u16, 16); impl_decodable_for_uint!(u32, 32); impl_decodable_for_uint!(u64, 64); + +#[cfg(target_pointer_width = "32")] +impl_decodable_for_uint!(usize, 32); + +#[cfg(target_pointer_width = "64")] impl_decodable_for_uint!(usize, 64); impl Decode for bool { diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index fceebcc44..51bafa95e 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -46,6 +46,8 @@ pub use encode::{Encode, SszEncoder}; /// The number of bytes used to represent an offset. pub const BYTES_PER_LENGTH_OFFSET: usize = 4; /// The maximum value that can be represented using `BYTES_PER_LENGTH_OFFSET`. +// allows overflow on 32-bit target +#[allow(const_err)] pub const MAX_LENGTH_VALUE: usize = (1 << (BYTES_PER_LENGTH_OFFSET * 8)) - 1; /// Convenience function to SSZ encode an object supporting ssz::Encode. From d87c84b5c57698cdca418cd2a995f462713888b2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Jun 2019 14:02:19 +1000 Subject: [PATCH 03/72] Update readme --- README.md | 240 +++++++++++++++++++++---------------------- docs/installation.md | 37 +++++++ 2 files changed, 154 insertions(+), 123 deletions(-) create mode 100644 docs/installation.md diff --git a/README.md b/README.md index 2151a0db8..a9bc6df05 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Lighthouse: an Ethereum Serenity client +# Lighthouse: Ethereum 2.0 + +An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime. [![Build Status]][Build Link] [![Doc Status]][Doc Link] [![Gitter Badge]][Gitter Link] @@ -9,24 +11,119 @@ [Doc Status]: https://img.shields.io/badge/docs-master-blue.svg [Doc Link]: http://lighthouse-docs.sigmaprime.io/ -A work-in-progress, open-source implementation of the Serenity Beacon -Chain, maintained by Sigma Prime. +## Overview -The "Serenity" project is also known as "Ethereum 2.0" or "Shasper". +Lighthouse is: -## Lighthouse Client +- Fully open-source, licensed under Apache 2.0. +- Security-focussed, fuzzing has begun and security reviews are planned + for late-2019. +- Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and + excellent performance (comparable to C++). +- Funded by various organisations, including Sigma Prime, the + Ethereum Foundation, Consensys and private individuals. +- Actively working to promote an inter-operable, multi-client Ethereum 2.0. -Lighthouse is an open-source Ethereum Serenity client that is currently under -development. Designed as a Serenity-only client, Lighthouse will not -re-implement the existing proof-of-work protocol. Maintaining a forward-focus -on Ethereum Serenity ensures that Lighthouse avoids reproducing the high-quality -work already undertaken by existing projects. As such, Lighthouse will connect -to existing clients, such as -[Geth](https://github.com/ethereum/go-ethereum) or -[Parity-Ethereum](https://github.com/paritytech/parity-ethereum), via RPC to enable -present-Ethereum functionality. -### Further Reading +## Development Status + +Lighthouse, like all Ethereum 2.0 clients, is a work-in-progress. Instructions +are provided for running the client, however these instructions are designed +for developers and researchers working on the project. We do not (yet) provide +user-facing functionality. + +Current development overview: + +- Specification `v0.6.3` implemented, optimized and passing test vectors. +- Rust-native libp2p integrated, with Gossipsub. +- Discv5 (P2P discovery mechanism) integration started. +- Metrics via Prometheus. +- Basic gRPC API, soon to be replaced with RESTful HTTP/JSON. + +### Roadmap + +- **July 2019**: `lighthouse-0.0.1` release: A stable testnet for developers with a useful + HTTP API. +- **September 2019**: Inter-operability with other Ethereum 2.0 clients. +- **October 2019**: Public, multi-client testnet with user-facing functionality. +- **January 2020**: Production Beacon Chain testnet. + +## Usage + +Lighthouse consists of multiple binaries: + +- [`beacon_node/`](beacon_node/): produces and verifies blocks from the P2P + connected validators and the P2P network. Provides an API for external services to + interact with Ethereum 2.0. +- [`validator_client/`](validator_client/): connects to a `beacon_node` and + performs the role of a proof-of-stake validator. +- [`account_manager/`](account_manager/): a stand-alone component providing key + management and creation for validators. + +### Simple Local Testnet + +**Note: these instructions are intended for developers and researchers. We do +not yet support end-users.** + +In this example we use the `account_manager` to create some keys, launch two +`beacon_node` instances and connect a `validator_client` to one. The two +`beacon_nodes` should stay in sync and build a Beacon Chain. + +First, clone this repository, [setup a development +environment](docs/installation.md) and navigate to the root directory of this repository. + +Then, run `$ cargo build --all --release` and navigate to the `target/release` +directory. + +#### 1. Generate Validator Keys + +Generate 16 validator keys and store them in `~/.lighthouse-validator`: + +``` +$ ./account_manager -d ~/.lighthouse-validator generate_deterministic -i 0 -n 16 +``` + +_Note: these keys are for development only. The secret keys are +deterministically generated from low integers. Assume they are public +knowledge._ + +#### 2. Start a Beacon Node + +This node will act as the boot node and provide an API for the +`validator_client`. + +``` +$ ./beacon_node --recent-genesis --rpc +``` + +_Note: `--recent-genesis` defines the genesis time as either the start of the +current hour, or half-way through the current hour (whichever is most recent). +This makes it very easy to create a testnet, but does not allow nodes to +connect if they were started in separate 30-minute windows._ + +#### 3. Start Another Beacon Node + +In another terminal window, start another boot node that will connect to the +running node. + +``` +$ ./beacon_node -r --boot-nodes /ip4/127.0.0.1/tcp/9000 --listen-address /ip4/127.0.0.1/tcp/9001 +``` + +#### 4. Start a Validator Client + +In a third terminal window, start a validator client: + +``` +$ ./validator-client +``` + +You should be able to observe the validator signing blocks, the boot node +processing these blocks and publishing them to the other node. If you have +issues, try restarting the beacon nodes to ensure they have the same genesis +time. Alternatively, raise an issue and include your terminal output. + +## Further Reading - [About Lighthouse](docs/lighthouse.md): Goals, Ideology and Ethos surrounding this implementation. @@ -37,7 +134,7 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the [company website](https://sigmaprime.io). -### Directory Structure +## Directory Structure - [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively associated with it. @@ -50,112 +147,10 @@ If you'd like some background on Sigma Prime, please see the [Lighthouse Update - [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively associated with it. -### Components -The following list describes some of the components actively under development -by the team: +## Contributing -- **BLS cryptography**: Lighthouse presently use the [Apache - Milagro](https://milagro.apache.org/) cryptography library to create and - verify BLS aggregate signatures. BLS signatures are core to Serenity as they - allow the signatures of many validators to be compressed into a constant 96 - bytes and efficiently verified. The Lighthouse project is presently - maintaining its own [BLS aggregates - library](https://github.com/sigp/signature-schemes), gratefully forked from - [@lovesh](https://github.com/lovesh). -- **DoS-resistant block pre-processing**: Processing blocks in proof-of-stake - is more resource intensive than proof-of-work. As such, clients need to - ensure that bad blocks can be rejected as efficiently as possible. At - present, blocks having 10 million ETH staked can be processed in 0.006 - seconds, and invalid blocks are rejected even more quickly. See - [issue #103](https://github.com/ethereum/beacon_chain/issues/103) on - [ethereum/beacon_chain](https://github.com/ethereum/beacon_chain). -- **P2P networking**: Serenity will likely use the [libp2p - framework](https://libp2p.io/). Lighthouse is working alongside -[Parity](https://www.parity.io/) to ensure -[libp2p-rust](https://github.com/libp2p/rust-libp2p) is fit-for-purpose. -- **Validator duties** : The project involves development of "validator - services" for users who wish to stake ETH. To fulfill their duties, - validators require a consistent view of the chain and the ability to vote - upon blocks from both shard and beacon chains. -- **New serialization formats**: Lighthouse is working alongside researchers - from the Ethereum Foundation to develop *simpleserialize* (SSZ), a - purpose-built serialization format for sending information across a network. - Check out the [SSZ -implementation](https://github.com/ethereum/eth2.0-specs/blob/00aa553fee95963b74fbec84dbd274d7247b8a0e/specs/simple-serialize.md) -and this -[research](https://github.com/sigp/serialization_sandbox/blob/report/report/serialization_report.md) -on serialization formats for more information. -- **Fork-choice**: The current fork choice rule is -[*LMD Ghost*](https://vitalik.ca/general/2018/12/05/cbc_casper.html#lmd-ghost), -which effectively takes the latest messages and forms the canonical chain using -the [GHOST](https://eprint.iacr.org/2013/881.pdf) mechanism. -- **Efficient state transition logic**: State transition logic governs - updates to the validator set as validators log in/out, penalizes/rewards -validators, rotates validators across shards, and implements other core tasks. -- **Fuzzing and testing environments**: Implementation of lab environments with - continuous integration (CI) workflows, providing automated security analysis. - -In addition to these components we are also working on database schemas, RPC -frameworks, specification development, database optimizations (e.g., -bloom-filters), and tons of other interesting stuff (at least we think so). - -### Running - -**NOTE: The cryptography libraries used in this implementation are -experimental. As such all cryptography is assumed to be insecure.** - -This code-base is still very much under-development and does not provide any -user-facing functionality. For developers and researchers, there are several -tests and benchmarks which may be of interest. - -A few basic steps are needed to get set up: - - 1. Install [rustup](https://rustup.rs/). It's a toolchain manager for Rust (Linux | macOS | Windows). For installation, download the script with `$ curl -f https://sh.rustup.rs > rustup.sh`, review its content (e.g. `$ less ./rustup.sh`) and run the script `$ ./rustup.sh` (you may need to change the permissions to allow execution, i.e. `$ chmod +x rustup.sh`) - 2. (Linux & MacOS) To configure your current shell run: `$ source $HOME/.cargo/env` - 3. Use the command `rustup show` to get information about the Rust installation. You should see that the - active toolchain is the stable version. - 4. Run `rustc --version` to check the installation and version of rust. - - Updates can be performed using` rustup update` . - 5. Install build dependencies (Arch packages are listed here, your distribution will likely be similar): - - `clang`: required by RocksDB. - - `protobuf`: required for protobuf serialization (gRPC). - - `cmake`: required for building protobuf - 6. Navigate to the working directory. - 7. Run the test by using command `cargo test --all`. By running, it will pass all the required test cases. - If you are doing it for the first time, then you can grab a coffee in the meantime. Usually, it takes time - to build, compile and pass all test cases. If there is no error then it means everything is working properly - and it's time to get your hands dirty. - In case, if there is an error, then please raise the [issue](https://github.com/sigp/lighthouse/issues). - We will help you. - 8. As an alternative to, or instead of the above step, you may also run benchmarks by using - the command `cargo bench --all` - -##### Note: -Lighthouse presently runs on Rust `stable`, however, benchmarks currently require the -`nightly` version. - -##### Note for Windows users: -Perl may also be required to build lighthouse. You can install [Strawberry Perl](http://strawberryperl.com/), -or alternatively use a choco install command `choco install strawberryperl`. - -Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues compiling in Windows. You can specify -a known working version by editing version in protos/Cargo.toml's "build-dependencies" section to -`protoc-grpcio = "<=0.3.0"`. - -### Contributing - -**Lighthouse welcomes contributors with open-arms.** - -If you would like to learn more about Ethereum Serenity and/or -[Rust](https://www.rust-lang.org/), we are more than happy to on-board you -and assign you some tasks. We aim to be as accepting and understanding as -possible; we are more than happy to up-skill contributors in exchange for their -assistance with the project. - -Alternatively, if you are an ETH/Rust veteran, we'd love your input. We're -always looking for the best way to implement things and welcome all -respectful criticisms. +**Lighthouse welcomes contributors.** If you are looking to contribute, please head to our [onboarding documentation](https://github.com/sigp/lighthouse/blob/master/docs/onboarding.md). @@ -170,10 +165,9 @@ your support! ## Contact The best place for discussion is the [sigp/lighthouse gitter](https://gitter.im/sigp/lighthouse). -Ping @paulhauner or @AgeManning to get the quickest response. -# Donations +## Donations -If you support the cause, we could certainly use donations to help fund development: +If you support the cause, we accept donations to help fund development: `0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b` (donation.sigmaprime.eth) diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000..135304890 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,37 @@ +# Development Environment Setup + +A few basic steps are needed to get set up (skip to #5 if you already have Rust +installed): + + 1. Install [rustup](https://rustup.rs/). It's a toolchain manager for Rust (Linux | macOS | Windows). For installation, download the script with `$ curl -f https://sh.rustup.rs > rustup.sh`, review its content (e.g. `$ less ./rustup.sh`) and run the script `$ ./rustup.sh` (you may need to change the permissions to allow execution, i.e. `$ chmod +x rustup.sh`) + 2. (Linux & MacOS) To configure your current shell run: `$ source $HOME/.cargo/env` + 3. Use the command `rustup show` to get information about the Rust installation. You should see that the + active toolchain is the stable version. + 4. Run `rustc --version` to check the installation and version of rust. + - Updates can be performed using` rustup update` . + 5. Install build dependencies (Arch packages are listed here, your distribution will likely be similar): + - `clang`: required by RocksDB. + - `protobuf`: required for protobuf serialization (gRPC). + - `cmake`: required for building protobuf + 6. Navigate to the working directory. + 7. Run the test by using command `cargo test --all`. By running, it will pass all the required test cases. + If you are doing it for the first time, then you can grab a coffee in the meantime. Usually, it takes time + to build, compile and pass all test cases. If there is no error then it means everything is working properly + and it's time to get your hands dirty. + In case, if there is an error, then please raise the [issue](https://github.com/sigp/lighthouse/issues). + We will help you. + 8. As an alternative to, or instead of the above step, you may also run benchmarks by using + the command `cargo bench --all` + +## Notes: + +Lighthouse targets Rust `stable` but _should_ run on `nightly`. + +### Note for Windows users: + +Perl may also be required to build lighthouse. You can install [Strawberry Perl](http://strawberryperl.com/), +or alternatively use a choco install command `choco install strawberryperl`. + +Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues compiling in Windows. You can specify +a known working version by editing version in protos/Cargo.toml's "build-dependencies" section to +`protoc-grpcio = "<=0.3.0"`. From 906580be153b19887a205791614ba808ab48d77a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 25 Jun 2019 16:05:26 +1000 Subject: [PATCH 04/72] Attempt to catch edge case in syncing --- beacon_node/network/src/sync/simple_sync.rs | 54 +++++++++++++++++---- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 0a082afcf..4e2f5daa9 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -165,6 +165,8 @@ impl SimpleSync { let remote = PeerSyncInfo::from(hello); let local = PeerSyncInfo::from(&self.chain); + let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); + // Disconnect nodes who are on a different network. if local.network_id != remote.network_id { info!( @@ -173,16 +175,14 @@ impl SimpleSync { "reason" => "network_id" ); network.disconnect(peer_id.clone(), GoodbyeReason::IrreleventNetwork); - // Disconnect nodes if our finalized epoch is greater than thieirs, and their finalized + // Disconnect nodes if our finalized epoch is greater than theirs, and their finalized // epoch is not in our chain. Viz., they are on another chain. // // If the local or remote have a `latest_finalized_root == ZERO_HASH`, skips checks about - // the finalized_root. The logic is akward and I think we're better without it. + // the finalized_root. The logic is awkward and I think we're better without it. } else if (local.latest_finalized_epoch >= remote.latest_finalized_epoch) - && (!self - .chain - .rev_iter_block_roots(local.best_slot) - .any(|(root, _slot)| root == remote.latest_finalized_root)) + && self.root_at_slot(start_slot(remote.latest_finalized_epoch)) + != Some(remote.latest_finalized_root) && (local.latest_finalized_root != spec.zero_hash) && (remote.latest_finalized_root != spec.zero_hash) { @@ -197,14 +197,22 @@ impl SimpleSync { info!(self.log, "HandshakeSuccess"; "peer" => format!("{:?}", peer_id)); self.known_peers.insert(peer_id.clone(), remote); - // If we have equal or better finalized epochs and best slots, we require nothing else from - // this peer. + let remote_best_root_is_in_chain = + self.root_at_slot(remote.best_slot) == Some(local.best_root); + + // We require nothing from this peer if: // - // We make an exception when our best slot is 0. Best slot does not indicate wether or + // - Their finalized epoch is less than ours + // - Their finalized root is in our chain (established earlier) + // - Their best slot is less than ours + // - Their best root is in our chain. + // + // We make an exception when our best slot is 0. Best slot does not indicate Wether or // not there is a block at slot zero. if (remote.latest_finalized_epoch <= local.latest_finalized_epoch) && (remote.best_slot <= local.best_slot) && (local.best_slot > 0) + && remote_best_root_is_in_chain { debug!(self.log, "Peer is naive"; "peer" => format!("{:?}", peer_id)); return; @@ -236,6 +244,24 @@ impl SimpleSync { .start_slot(T::EthSpec::slots_per_epoch()); let required_slots = remote.best_slot - start_slot; + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + start_slot, + count: required_slots.into(), + }, + network, + ); + // The remote has a lower best slot, but the root for that slot is not in our chain. + // + // This means the remote is on another chain. + } else if remote.best_slot <= local.best_slot && !remote_best_root_is_in_chain { + debug!(self.log, "Peer has a best slot on a different chain"; "peer" => format!("{:?}", peer_id)); + let start_slot = local + .latest_finalized_epoch + .start_slot(T::EthSpec::slots_per_epoch()); + let required_slots = remote.best_slot - start_slot; + self.request_block_roots( peer_id, BeaconBlockRootsRequest { @@ -245,11 +271,19 @@ impl SimpleSync { network, ); } else { - debug!(self.log, "Nothing to request from peer"; "peer" => format!("{:?}", peer_id)); + warn!(self.log, "Unexpected condition in syncing"; "peer" => format!("{:?}", peer_id)); } } } + fn root_at_slot(&self, target_slot: Slot) -> Option { + self.chain + .rev_iter_block_roots(target_slot) + .take(1) + .find(|(_root, slot)| *slot == target_slot) + .map(|(root, _slot)| root) + } + /// Handle a `BeaconBlockRoots` request from the peer. pub fn on_beacon_block_roots_request( &mut self, From db9dd3dffe36711d3832e279e47b6402eca6a818 Mon Sep 17 00:00:00 2001 From: Matt Garnett <14004106+c-o-l-o-r@users.noreply.github.com> Date: Tue, 25 Jun 2019 09:59:50 -0400 Subject: [PATCH 05/72] fix encoding impl for usize on 32-bit architectures --- eth2/utils/ssz/src/encode/impls.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index 04492a1f2..e0e2d9dbc 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -24,6 +24,11 @@ impl_encodable_for_uint!(u8, 8); impl_encodable_for_uint!(u16, 16); impl_encodable_for_uint!(u32, 32); impl_encodable_for_uint!(u64, 64); + +#[cfg(target_pointer_width = "32")] +impl_encodable_for_uint!(usize, 32); + +#[cfg(target_pointer_width = "64")] impl_encodable_for_uint!(usize, 64); /// The SSZ "union" type. From e93fb94e7ad61b497e66227c74ea39a228ec2241 Mon Sep 17 00:00:00 2001 From: Matt Garnett <14004106+c-o-l-o-r@users.noreply.github.com> Date: Tue, 25 Jun 2019 10:12:49 -0400 Subject: [PATCH 06/72] calculate MAX_LENGTH_VALUE for 32-bit and 64-bit targets --- eth2/utils/ssz/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index 51bafa95e..51dd16523 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -46,9 +46,10 @@ pub use encode::{Encode, SszEncoder}; /// The number of bytes used to represent an offset. pub const BYTES_PER_LENGTH_OFFSET: usize = 4; /// The maximum value that can be represented using `BYTES_PER_LENGTH_OFFSET`. -// allows overflow on 32-bit target -#[allow(const_err)] -pub const MAX_LENGTH_VALUE: usize = (1 << (BYTES_PER_LENGTH_OFFSET * 8)) - 1; +#[cfg(target_pointer_width = "32")] +pub const MAX_LENGTH_VALUE: usize = (std::u32::MAX >> 8 * (4 - BYTES_PER_LENGTH_OFFSET)) as usize; +#[cfg(target_pointer_width = "64")] +pub const MAX_LENGTH_VALUE: usize = (std::u64::MAX >> 8 * (8 - BYTES_PER_LENGTH_OFFSET)) as usize; /// Convenience function to SSZ encode an object supporting ssz::Encode. /// From 8bd9ccdaa03762da3a219479acd44376f01da2e2 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 26 Jun 2019 13:04:05 +1000 Subject: [PATCH 07/72] Fixed account_manager data dir. - Default data directory for the account_manager, now points to ~/.lighthouse-validator, which is the same data dir as the validator binary. --- account_manager/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index 1c8cc8819..08d266d65 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use types::test_utils::generate_deterministic_keypair; use validator_client::Config as ValidatorClientConfig; -pub const DEFAULT_DATA_DIR: &str = ".lighthouse-account-manager"; +pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator"; pub const CLIENT_CONFIG_FILENAME: &str = "account-manager.toml"; fn main() { From df61231bbe4715bfff12fe3896190151c3f2e9cd Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 27 Jun 2019 13:31:07 +1000 Subject: [PATCH 08/72] Small CLI update. - Added the 'd' short version of the 'datadir' flag for the validator client. --- validator_client/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index f74915438..1d94bd533 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -35,6 +35,7 @@ fn main() { .arg( Arg::with_name("datadir") .long("datadir") + .short("d") .value_name("DIR") .help("Data directory for keys and databases.") .takes_value(true), From 2a7122beafc1486f06e5bf5a8376718b8d1c2100 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 27 Jun 2019 18:05:03 +1000 Subject: [PATCH 09/72] Partially refactor simple_sync, makes improvement --- beacon_node/network/src/sync/import_queue.rs | 121 ++++------- beacon_node/network/src/sync/simple_sync.rs | 214 +++++++++---------- beacon_node/store/src/iter.rs | 3 +- 3 files changed, 143 insertions(+), 195 deletions(-) diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 90c354cfd..8cc3dd65d 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -1,7 +1,8 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::PeerId; -use slog::{debug, error}; +use slog::error; +use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; use tree_hash::TreeHash; @@ -22,7 +23,7 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot}; pub struct ImportQueue { pub chain: Arc>, /// Partially imported blocks, keyed by the root of `BeaconBlockBody`. - pub partials: Vec, + partials: HashMap, /// Time before a queue entry is considered state. pub stale_time: Duration, /// Logging @@ -34,7 +35,7 @@ impl ImportQueue { pub fn new(chain: Arc>, stale_time: Duration, log: slog::Logger) -> Self { Self { chain, - partials: vec![], + partials: HashMap::new(), stale_time, log, } @@ -52,7 +53,7 @@ impl ImportQueue { let mut complete: Vec<(Hash256, BeaconBlock, PeerId)> = self .partials .iter() - .filter_map(|partial| partial.clone().complete()) + .filter_map(|(_, partial)| partial.clone().complete()) .collect(); // Sort the completable partials to be in ascending slot order. @@ -61,14 +62,14 @@ impl ImportQueue { complete } + pub fn contains_block_root(&self, block_root: Hash256) -> bool { + self.partials.contains_key(&block_root) + } + /// Removes the first `PartialBeaconBlock` with a matching `block_root`, returning the partial /// if it exists. pub fn remove(&mut self, block_root: Hash256) -> Option { - let position = self - .partials - .iter() - .position(|p| p.block_root == block_root)?; - Some(self.partials.remove(position)) + self.partials.remove(&block_root) } /// Flushes all stale entries from the queue. @@ -76,31 +77,10 @@ impl ImportQueue { /// An entry is stale if it has as a `inserted` time that is more than `self.stale_time` in the /// past. pub fn remove_stale(&mut self) { - let stale_indices: Vec = self - .partials - .iter() - .enumerate() - .filter_map(|(i, partial)| { - if partial.inserted + self.stale_time <= Instant::now() { - Some(i) - } else { - None - } - }) - .collect(); + let stale_time = self.stale_time; - if !stale_indices.is_empty() { - debug!( - self.log, - "ImportQueue removing stale entries"; - "stale_items" => stale_indices.len(), - "stale_time_seconds" => self.stale_time.as_secs() - ); - } - - stale_indices.iter().for_each(|&i| { - self.partials.remove(i); - }); + self.partials + .retain(|_, partial| partial.inserted + stale_time > Instant::now()) } /// Returns `true` if `self.chain` has not yet processed this block. @@ -122,27 +102,30 @@ impl ImportQueue { block_roots: &[BlockRootSlot], sender: PeerId, ) -> Vec { - let new_roots: Vec = block_roots + let new_block_root_slots: Vec = block_roots .iter() + // Ignore any roots already stored in the queue. + .filter(|brs| !self.contains_block_root(brs.block_root)) // Ignore any roots already processed by the chain. .filter(|brs| self.chain_has_not_seen_block(&brs.block_root)) - // Ignore any roots already stored in the queue. - .filter(|brs| !self.partials.iter().any(|p| p.block_root == brs.block_root)) .cloned() .collect(); - new_roots.iter().for_each(|brs| { - self.partials.push(PartialBeaconBlock { - slot: brs.slot, - block_root: brs.block_root, - sender: sender.clone(), - header: None, - body: None, - inserted: Instant::now(), - }) - }); + self.partials.extend( + new_block_root_slots + .iter() + .map(|brs| PartialBeaconBlock { + slot: brs.slot, + block_root: brs.block_root, + sender: sender.clone(), + header: None, + body: None, + inserted: Instant::now(), + }) + .map(|partial| (partial.block_root, partial)), + ); - new_roots + new_block_root_slots } /// Adds the `headers` to the `partials` queue. Returns a list of `Hash256` block roots for @@ -170,7 +153,7 @@ impl ImportQueue { if self.chain_has_not_seen_block(&block_root) { self.insert_header(block_root, header, sender.clone()); - required_bodies.push(block_root) + required_bodies.push(block_root); } } @@ -197,31 +180,20 @@ impl ImportQueue { /// If the header already exists, the `inserted` time is set to `now` and not other /// modifications are made. fn insert_header(&mut self, block_root: Hash256, header: BeaconBlockHeader, sender: PeerId) { - if let Some(i) = self - .partials - .iter() - .position(|p| p.block_root == block_root) - { - // Case 1: there already exists a partial with a matching block root. - // - // The `inserted` time is set to now and the header is replaced, regardless of whether - // it existed or not. - self.partials[i].header = Some(header); - self.partials[i].inserted = Instant::now(); - } else { - // Case 2: there was no partial with a matching block root. - // - // A new partial is added. This case permits adding a header without already known the - // root. - self.partials.push(PartialBeaconBlock { + self.partials + .entry(block_root) + .and_modify(|partial| { + partial.header = Some(header.clone()); + partial.inserted = Instant::now(); + }) + .or_insert_with(|| PartialBeaconBlock { slot: header.slot, block_root, header: Some(header), body: None, inserted: Instant::now(), sender, - }) - } + }); } /// Updates an existing partial with the `body`. @@ -232,7 +204,7 @@ impl ImportQueue { fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) { let body_root = Hash256::from_slice(&body.tree_hash_root()[..]); - self.partials.iter_mut().for_each(|mut p| { + self.partials.iter_mut().for_each(|(_, mut p)| { if let Some(header) = &mut p.header { if body_root == header.block_body_root { p.inserted = Instant::now(); @@ -261,15 +233,10 @@ impl ImportQueue { sender, }; - if let Some(i) = self - .partials - .iter() - .position(|p| p.block_root == block_root) - { - self.partials[i] = partial; - } else { - self.partials.push(partial) - } + self.partials + .entry(block_root) + .and_modify(|existing_partial| *existing_partial = partial.clone()) + .or_insert(partial); } } diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 4e2f5daa9..2382e47a4 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -17,7 +17,7 @@ use types::{ const SLOT_IMPORT_TOLERANCE: u64 = 100; /// The amount of seconds a block (or partial block) may exist in the import queue. -const QUEUE_STALE_SECS: u64 = 600; +const QUEUE_STALE_SECS: u64 = 6; /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it. /// Otherwise we queue it. @@ -72,7 +72,6 @@ pub struct SimpleSync { import_queue: ImportQueue, /// The current state of the syncing protocol. state: SyncState, - /// Sync logger. log: slog::Logger, } @@ -160,119 +159,89 @@ impl SimpleSync { hello: HelloMessage, network: &mut NetworkContext, ) { - let spec = &self.chain.spec; - let remote = PeerSyncInfo::from(hello); let local = PeerSyncInfo::from(&self.chain); let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); - // Disconnect nodes who are on a different network. if local.network_id != remote.network_id { + // The node is on a different network, disconnect them. info!( self.log, "HandshakeFailure"; "peer" => format!("{:?}", peer_id), "reason" => "network_id" ); + network.disconnect(peer_id.clone(), GoodbyeReason::IrreleventNetwork); - // Disconnect nodes if our finalized epoch is greater than theirs, and their finalized - // epoch is not in our chain. Viz., they are on another chain. - // - // If the local or remote have a `latest_finalized_root == ZERO_HASH`, skips checks about - // the finalized_root. The logic is awkward and I think we're better without it. - } else if (local.latest_finalized_epoch >= remote.latest_finalized_epoch) - && self.root_at_slot(start_slot(remote.latest_finalized_epoch)) - != Some(remote.latest_finalized_root) - && (local.latest_finalized_root != spec.zero_hash) - && (remote.latest_finalized_root != spec.zero_hash) + } else if remote.latest_finalized_epoch <= local.latest_finalized_epoch + && remote.latest_finalized_root != self.chain.spec.zero_hash + && local.latest_finalized_root != self.chain.spec.zero_hash + && (self.root_at_slot(start_slot(remote.latest_finalized_epoch)) + != Some(remote.latest_finalized_root)) { + // The remotes finalized epoch is less than or greater than ours, but the block root is + // different to the one in our chain. + // + // Therefore, the node is on a different chain and we should not communicate with them. info!( self.log, "HandshakeFailure"; "peer" => format!("{:?}", peer_id), - "reason" => "wrong_finalized_chain" + "reason" => "different finalized chain" ); network.disconnect(peer_id.clone(), GoodbyeReason::IrreleventNetwork); - // Process handshakes from peers that seem to be on our chain. + } else if remote.latest_finalized_epoch < local.latest_finalized_epoch { + // The node has a lower finalized epoch, their chain is not useful to us. There are two + // cases where a node can have a lower finalized epoch: + // + // ## The node is on the same chain + // + // If a node is on the same chain but has a lower finalized epoch, their head must be + // lower than ours. Therefore, we have nothing to request from them. + // + // ## The node is on a fork + // + // If a node is on a fork that has a lower finalized epoch, switching to that fork would + // cause us to revert a finalized block. This is not permitted, therefore we have no + // interest in their blocks. + debug!( + self.log, + "NaivePeer"; + "peer" => format!("{:?}", peer_id), + "reason" => "lower finalized epoch" + ); + } else if self + .chain + .store + .exists::(&remote.best_root) + .unwrap_or_else(|_| false) + { + // If the node's best-block is already known to us, we have nothing to request. + debug!( + self.log, + "NaivePeer"; + "peer" => format!("{:?}", peer_id), + "reason" => "best block is known" + ); } else { - info!(self.log, "HandshakeSuccess"; "peer" => format!("{:?}", peer_id)); - self.known_peers.insert(peer_id.clone(), remote); - - let remote_best_root_is_in_chain = - self.root_at_slot(remote.best_slot) == Some(local.best_root); - - // We require nothing from this peer if: + // The remote node has an equal or great finalized epoch and we don't know it's head. // - // - Their finalized epoch is less than ours - // - Their finalized root is in our chain (established earlier) - // - Their best slot is less than ours - // - Their best root is in our chain. - // - // We make an exception when our best slot is 0. Best slot does not indicate Wether or - // not there is a block at slot zero. - if (remote.latest_finalized_epoch <= local.latest_finalized_epoch) - && (remote.best_slot <= local.best_slot) - && (local.best_slot > 0) - && remote_best_root_is_in_chain - { - debug!(self.log, "Peer is naive"; "peer" => format!("{:?}", peer_id)); - return; - } + // Therefore, there are some blocks between the local finalized epoch and the remote + // head that are worth downloading. + debug!(self.log, "UsefulPeer"; "peer" => format!("{:?}", peer_id)); - // If the remote has a higher finalized epoch, request all block roots from our finalized - // epoch through to its best slot. - if remote.latest_finalized_epoch > local.latest_finalized_epoch { - debug!(self.log, "Peer has high finalized epoch"; "peer" => format!("{:?}", peer_id)); - let start_slot = local - .latest_finalized_epoch - .start_slot(T::EthSpec::slots_per_epoch()); - let required_slots = remote.best_slot - start_slot; + let start_slot = local + .latest_finalized_epoch + .start_slot(T::EthSpec::slots_per_epoch()); + let required_slots = remote.best_slot - start_slot; - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot, - count: required_slots.into(), - }, - network, - ); - // If the remote has a greater best slot, request the roots between our best slot and their - // best slot. - } else if remote.best_slot > local.best_slot { - debug!(self.log, "Peer has higher best slot"; "peer" => format!("{:?}", peer_id)); - let start_slot = local - .latest_finalized_epoch - .start_slot(T::EthSpec::slots_per_epoch()); - let required_slots = remote.best_slot - start_slot; - - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot, - count: required_slots.into(), - }, - network, - ); - // The remote has a lower best slot, but the root for that slot is not in our chain. - // - // This means the remote is on another chain. - } else if remote.best_slot <= local.best_slot && !remote_best_root_is_in_chain { - debug!(self.log, "Peer has a best slot on a different chain"; "peer" => format!("{:?}", peer_id)); - let start_slot = local - .latest_finalized_epoch - .start_slot(T::EthSpec::slots_per_epoch()); - let required_slots = remote.best_slot - start_slot; - - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot, - count: required_slots.into(), - }, - network, - ); - } else { - warn!(self.log, "Unexpected condition in syncing"; "peer" => format!("{:?}", peer_id)); - } + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + start_slot, + count: required_slots.into(), + }, + network, + ); } } @@ -309,11 +278,13 @@ impl SimpleSync { .collect(); if roots.len() as u64 != req.count { - debug!( + warn!( self.log, "BlockRootsRequest"; "peer" => format!("{:?}", peer_id), "msg" => "Failed to return all requested hashes", + "start_slot" => req.start_slot, + "current_slot" => self.chain.current_state().slot, "requested" => req.count, "returned" => roots.len(), ); @@ -385,7 +356,7 @@ impl SimpleSync { BeaconBlockHeadersRequest { start_root: first.block_root, start_slot: first.slot, - max_headers: (last.slot - first.slot + 1).as_u64(), + max_headers: (last.slot - first.slot).as_u64(), skip_slots: 0, }, network, @@ -467,7 +438,9 @@ impl SimpleSync { .import_queue .enqueue_headers(res.headers, peer_id.clone()); - self.request_block_bodies(peer_id, BeaconBlockBodiesRequest { block_roots }, network); + if !block_roots.is_empty() { + self.request_block_bodies(peer_id, BeaconBlockBodiesRequest { block_roots }, network); + } } /// Handle a `BeaconBlockBodies` request from the peer. @@ -552,10 +525,28 @@ impl SimpleSync { { match outcome { BlockProcessingOutcome::Processed { .. } => SHOULD_FORWARD_GOSSIP_BLOCK, - BlockProcessingOutcome::ParentUnknown { .. } => { + BlockProcessingOutcome::ParentUnknown { parent } => { + // Clean the stale entries from the queue. + self.import_queue.remove_stale(); + + // Add this block to the queue self.import_queue .enqueue_full_blocks(vec![block], peer_id.clone()); + // Unless the parent is in the queue, request the parent block from the peer. + // + // It is likely that this is duplicate work, given we already send a hello + // request. However, I believe there are some edge-cases where the hello + // message doesn't suffice, so we perform this request as well. + if !self.import_queue.contains_block_root(parent) { + // Send a hello to learn of the clients best slot so we can then sync the required + // parent(s). + network.send_rpc_request( + peer_id.clone(), + RPCRequest::Hello(hello_message(&self.chain)), + ); + } + SHOULD_FORWARD_GOSSIP_BLOCK } BlockProcessingOutcome::FutureSlot { @@ -730,7 +721,7 @@ impl SimpleSync { if let Ok(outcome) = processing_result { match outcome { BlockProcessingOutcome::Processed { block_root } => { - info!( + debug!( self.log, "Imported block from network"; "source" => source, "slot" => block.slot, @@ -747,28 +738,19 @@ impl SimpleSync { "peer" => format!("{:?}", peer_id), ); - // Send a hello to learn of the clients best slot so we can then sync the require - // parent(s). - network.send_rpc_request( - peer_id.clone(), - RPCRequest::Hello(hello_message(&self.chain)), - ); - - // Explicitly request the parent block from the peer. + // Unless the parent is in the queue, request the parent block from the peer. // // It is likely that this is duplicate work, given we already send a hello // request. However, I believe there are some edge-cases where the hello // message doesn't suffice, so we perform this request as well. - self.request_block_headers( - peer_id, - BeaconBlockHeadersRequest { - start_root: parent, - start_slot: block.slot - 1, - max_headers: 1, - skip_slots: 0, - }, - network, - ) + if !self.import_queue.contains_block_root(parent) { + // Send a hello to learn of the clients best slot so we can then sync the require + // parent(s). + network.send_rpc_request( + peer_id.clone(), + RPCRequest::Hello(hello_message(&self.chain)), + ); + } } BlockProcessingOutcome::FutureSlot { present_slot, diff --git a/beacon_node/store/src/iter.rs b/beacon_node/store/src/iter.rs index cf50d671b..76807ce8f 100644 --- a/beacon_node/store/src/iter.rs +++ b/beacon_node/store/src/iter.rs @@ -139,8 +139,7 @@ impl<'a, T: EthSpec, U: Store> Iterator for BlockRootsIterator<'a, T, U> { Err(BeaconStateError::SlotOutOfBounds) => { // Read a `BeaconState` from the store that has access to prior historical root. let beacon_state: BeaconState = { - // Load the earlier state from disk. Skip forward one slot, because a state - // doesn't return it's own state root. + // Load the earliest state from disk. let new_state_root = self.beacon_state.get_oldest_state_root().ok()?; self.store.get(&new_state_root).ok()? From 38d2d03e3a98bc88ddf747a0eab3b72a2163623f Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 17 Jun 2019 18:07:14 +1000 Subject: [PATCH 10/72] op_pool: use max cover algorithm, refactor --- eth2/operation_pool/Cargo.toml | 1 + eth2/operation_pool/src/attestation.rs | 91 +++++++++++ eth2/operation_pool/src/attestation_id.rs | 35 ++++ eth2/operation_pool/src/lib.rs | 139 +++------------- eth2/operation_pool/src/max_cover.rs | 189 ++++++++++++++++++++++ eth2/utils/boolean-bitfield/src/lib.rs | 3 +- 6 files changed, 338 insertions(+), 120 deletions(-) create mode 100644 eth2/operation_pool/src/attestation.rs create mode 100644 eth2/operation_pool/src/attestation_id.rs create mode 100644 eth2/operation_pool/src/max_cover.rs diff --git a/eth2/operation_pool/Cargo.toml b/eth2/operation_pool/Cargo.toml index 67d13013c..99c7bfc06 100644 --- a/eth2/operation_pool/Cargo.toml +++ b/eth2/operation_pool/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Michael Sproul "] edition = "2018" [dependencies] +boolean-bitfield = { path = "../utils/boolean-bitfield" } int_to_bytes = { path = "../utils/int_to_bytes" } itertools = "0.8" parking_lot = "0.7" diff --git a/eth2/operation_pool/src/attestation.rs b/eth2/operation_pool/src/attestation.rs new file mode 100644 index 000000000..a2f71c3a4 --- /dev/null +++ b/eth2/operation_pool/src/attestation.rs @@ -0,0 +1,91 @@ +use crate::max_cover::MaxCover; +use boolean_bitfield::BooleanBitfield; +use types::{Attestation, BeaconState, EthSpec}; + +pub struct AttMaxCover<'a> { + /// Underlying attestation. + att: &'a Attestation, + /// Bitfield of validators that are covered by this attestation. + fresh_validators: BooleanBitfield, +} + +impl<'a> AttMaxCover<'a> { + pub fn new(att: &'a Attestation, fresh_validators: BooleanBitfield) -> Self { + Self { + att, + fresh_validators, + } + } +} + +impl<'a> MaxCover for AttMaxCover<'a> { + type Object = Attestation; + type Set = BooleanBitfield; + + fn object(&self) -> Attestation { + self.att.clone() + } + + fn covering_set(&self) -> &BooleanBitfield { + &self.fresh_validators + } + + /// Sneaky: we keep all the attestations together in one bucket, even though + /// their aggregation bitfields refer to different committees. In order to avoid + /// confusing committees when updating covering sets, we update only those attestations + /// whose shard and epoch match the attestation being included in the solution, by the logic + /// that a shard and epoch uniquely identify a committee. + fn update_covering_set( + &mut self, + best_att: &Attestation, + covered_validators: &BooleanBitfield, + ) { + if self.att.data.shard == best_att.data.shard + && self.att.data.target_epoch == best_att.data.target_epoch + { + self.fresh_validators.difference_inplace(covered_validators); + } + } + + fn score(&self) -> usize { + self.fresh_validators.num_set_bits() + } +} + +/// Extract the validators for which `attestation` would be their earliest in the epoch. +/// +/// The reward paid to a proposer for including an attestation is proportional to the number +/// of validators for which the included attestation is their first in the epoch. The attestation +/// is judged against the state's `current_epoch_attestations` or `previous_epoch_attestations` +/// depending on when it was created, and all those validators who have already attested are +/// removed from the `aggregation_bitfield` before returning it. +// TODO: This could be optimised with a map from validator index to whether that validator has +// attested in each of the current and previous epochs. Currently quadratic in number of validators. +pub fn earliest_attestation_validators( + attestation: &Attestation, + state: &BeaconState, +) -> BooleanBitfield { + // Bitfield of validators whose attestations are new/fresh. + let mut new_validators = attestation.aggregation_bitfield.clone(); + + let state_attestations = if attestation.data.target_epoch == state.current_epoch() { + &state.current_epoch_attestations + } else if attestation.data.target_epoch == state.previous_epoch() { + &state.previous_epoch_attestations + } else { + return BooleanBitfield::from_elem(attestation.aggregation_bitfield.len(), false); + }; + + state_attestations + .iter() + // In a single epoch, an attester should only be attesting for one shard. + // TODO: we avoid including slashable attestations in the state here, + // but maybe we should do something else with them (like construct slashings). + .filter(|existing_attestation| existing_attestation.data.shard == attestation.data.shard) + .for_each(|existing_attestation| { + // Remove the validators who have signed the existing attestation (they are not new) + new_validators.difference_inplace(&existing_attestation.aggregation_bitfield); + }); + + new_validators +} diff --git a/eth2/operation_pool/src/attestation_id.rs b/eth2/operation_pool/src/attestation_id.rs new file mode 100644 index 000000000..ba25174e6 --- /dev/null +++ b/eth2/operation_pool/src/attestation_id.rs @@ -0,0 +1,35 @@ +use int_to_bytes::int_to_bytes8; +use ssz::ssz_encode; +use types::{AttestationData, BeaconState, ChainSpec, Domain, Epoch, EthSpec}; + +/// Serialized `AttestationData` augmented with a domain to encode the fork info. +#[derive(PartialEq, Eq, Clone, Hash, Debug)] +pub struct AttestationId(Vec); + +/// Number of domain bytes that the end of an attestation ID is padded with. +const DOMAIN_BYTES_LEN: usize = 8; + +impl AttestationId { + pub fn from_data( + attestation: &AttestationData, + state: &BeaconState, + spec: &ChainSpec, + ) -> Self { + let mut bytes = ssz_encode(attestation); + let epoch = attestation.target_epoch; + bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, state, spec)); + AttestationId(bytes) + } + + pub fn compute_domain_bytes( + epoch: Epoch, + state: &BeaconState, + spec: &ChainSpec, + ) -> Vec { + int_to_bytes8(spec.get_domain(epoch, Domain::Attestation, &state.fork)) + } + + pub fn domain_bytes_match(&self, domain_bytes: &[u8]) -> bool { + &self.0[self.0.len() - DOMAIN_BYTES_LEN..] == domain_bytes + } +} diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index ec7d5aa90..7157ec9ee 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -1,7 +1,12 @@ -use int_to_bytes::int_to_bytes8; +mod attestation; +mod attestation_id; +mod max_cover; + +use attestation::{earliest_attestation_validators, AttMaxCover}; +use attestation_id::AttestationId; use itertools::Itertools; +use max_cover::maximum_cover; use parking_lot::RwLock; -use ssz::ssz_encode; use state_processing::per_block_processing::errors::{ AttestationValidationError, AttesterSlashingValidationError, DepositValidationError, ExitValidationError, ProposerSlashingValidationError, TransferValidationError, @@ -16,10 +21,9 @@ use state_processing::per_block_processing::{ }; use std::collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet}; use std::marker::PhantomData; -use types::chain_spec::Domain; use types::{ - Attestation, AttestationData, AttesterSlashing, BeaconState, ChainSpec, Deposit, Epoch, - EthSpec, ProposerSlashing, Transfer, Validator, VoluntaryExit, + Attestation, AttesterSlashing, BeaconState, ChainSpec, Deposit, EthSpec, ProposerSlashing, + Transfer, Validator, VoluntaryExit, }; #[derive(Default)] @@ -43,71 +47,6 @@ pub struct OperationPool { _phantom: PhantomData, } -/// Serialized `AttestationData` augmented with a domain to encode the fork info. -#[derive(PartialEq, Eq, Clone, Hash, Debug)] -struct AttestationId(Vec); - -/// Number of domain bytes that the end of an attestation ID is padded with. -const DOMAIN_BYTES_LEN: usize = 8; - -impl AttestationId { - fn from_data( - attestation: &AttestationData, - state: &BeaconState, - spec: &ChainSpec, - ) -> Self { - let mut bytes = ssz_encode(attestation); - let epoch = attestation.target_epoch; - bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, state, spec)); - AttestationId(bytes) - } - - fn compute_domain_bytes( - epoch: Epoch, - state: &BeaconState, - spec: &ChainSpec, - ) -> Vec { - int_to_bytes8(spec.get_domain(epoch, Domain::Attestation, &state.fork)) - } - - fn domain_bytes_match(&self, domain_bytes: &[u8]) -> bool { - &self.0[self.0.len() - DOMAIN_BYTES_LEN..] == domain_bytes - } -} - -/// Compute a fitness score for an attestation. -/// -/// The score is calculated by determining the number of *new* attestations that -/// the aggregate attestation introduces, and is proportional to the size of the reward we will -/// receive for including it in a block. -// TODO: this could be optimised with a map from validator index to whether that validator has -// attested in each of the current and previous epochs. Currently quadractic in number of validators. -fn attestation_score(attestation: &Attestation, state: &BeaconState) -> usize { - // Bitfield of validators whose attestations are new/fresh. - let mut new_validators = attestation.aggregation_bitfield.clone(); - - let state_attestations = if attestation.data.target_epoch == state.current_epoch() { - &state.current_epoch_attestations - } else if attestation.data.target_epoch == state.previous_epoch() { - &state.previous_epoch_attestations - } else { - return 0; - }; - - state_attestations - .iter() - // In a single epoch, an attester should only be attesting for one shard. - // TODO: we avoid including slashable attestations in the state here, - // but maybe we should do something else with them (like construct slashings). - .filter(|current_attestation| current_attestation.data.shard == attestation.data.shard) - .for_each(|current_attestation| { - // Remove the validators who have signed the existing attestation (they are not new) - new_validators.difference_inplace(¤t_attestation.aggregation_bitfield); - }); - - new_validators.num_set_bits() -} - #[derive(Debug, PartialEq, Clone)] pub enum DepositInsertStatus { /// The deposit was not already in the pool. @@ -176,29 +115,19 @@ impl OperationPool { let current_epoch = state.current_epoch(); let prev_domain_bytes = AttestationId::compute_domain_bytes(prev_epoch, state, spec); let curr_domain_bytes = AttestationId::compute_domain_bytes(current_epoch, state, spec); - self.attestations - .read() + let reader = self.attestations.read(); + let valid_attestations = reader .iter() .filter(|(key, _)| { key.domain_bytes_match(&prev_domain_bytes) || key.domain_bytes_match(&curr_domain_bytes) }) .flat_map(|(_, attestations)| attestations) - // That are not superseded by an attestation included in the state... - .filter(|attestation| !superior_attestation_exists_in_state(state, attestation)) // That are valid... .filter(|attestation| validate_attestation(state, attestation, spec).is_ok()) - // Scored by the number of new attestations they introduce (descending) - // TODO: need to consider attestations introduced in THIS block - .map(|att| (att, attestation_score(att, state))) - // Don't include any useless attestations (score 0) - .filter(|&(_, score)| score != 0) - .sorted_by_key(|&(_, score)| std::cmp::Reverse(score)) - // Limited to the maximum number of attestations per block - .take(spec.max_attestations as usize) - .map(|(att, _)| att) - .cloned() - .collect() + .map(|att| AttMaxCover::new(att, earliest_attestation_validators(att, state))); + + maximum_cover(valid_attestations, spec.max_attestations as usize) } /// Remove attestations which are too old to be included in a block. @@ -484,34 +413,6 @@ impl OperationPool { } } -/// Returns `true` if the state already contains a `PendingAttestation` that is superior to the -/// given `attestation`. -/// -/// A validator has nothing to gain from re-including an attestation and it adds load to the -/// network. -/// -/// An existing `PendingAttestation` is superior to an existing `attestation` if: -/// -/// - Their `AttestationData` is equal. -/// - `attestation` does not contain any signatures that `PendingAttestation` does not have. -fn superior_attestation_exists_in_state( - state: &BeaconState, - attestation: &Attestation, -) -> bool { - state - .current_epoch_attestations - .iter() - .chain(state.previous_epoch_attestations.iter()) - .any(|existing_attestation| { - let bitfield = &attestation.aggregation_bitfield; - let existing_bitfield = &existing_attestation.aggregation_bitfield; - - existing_attestation.data == attestation.data - && bitfield.intersection(existing_bitfield).num_set_bits() - == bitfield.num_set_bits() - }) -} - /// Filter up to a maximum number of operations out of an iterator. fn filter_limit_operations<'a, T: 'a, I, F>(operations: I, filter: F, limit: u64) -> Vec where @@ -734,15 +635,13 @@ mod tests { state_builder.teleport_to_slot(slot); state_builder.build_caches(&spec).unwrap(); let (state, keypairs) = state_builder.build(); - (state, keypairs, MainnetEthSpec::default_spec()) } #[test] - fn test_attestation_score() { + fn test_earliest_attestation() { let (ref mut state, ref keypairs, ref spec) = attestation_test_state::(1); - let slot = state.slot - 1; let committees = state .get_crosslink_committees_at_slot(slot) @@ -775,9 +674,8 @@ mod tests { assert_eq!( att1.aggregation_bitfield.num_set_bits(), - attestation_score(&att1, state) + earliest_attestation_validators(&att1, state).num_set_bits() ); - state.current_epoch_attestations.push(PendingAttestation { aggregation_bitfield: att1.aggregation_bitfield.clone(), data: att1.data.clone(), @@ -785,7 +683,10 @@ mod tests { proposer_index: 0, }); - assert_eq!(cc.committee.len() - 2, attestation_score(&att2, state)); + assert_eq!( + cc.committee.len() - 2, + earliest_attestation_validators(&att2, state).num_set_bits() + ); } } diff --git a/eth2/operation_pool/src/max_cover.rs b/eth2/operation_pool/src/max_cover.rs new file mode 100644 index 000000000..75ac14054 --- /dev/null +++ b/eth2/operation_pool/src/max_cover.rs @@ -0,0 +1,189 @@ +/// Trait for types that we can compute a maximum cover for. +/// +/// Terminology: +/// * `item`: something that implements this trait +/// * `element`: something contained in a set, and covered by the covering set of an item +/// * `object`: something extracted from an item in order to comprise a solution +/// See: https://en.wikipedia.org/wiki/Maximum_coverage_problem +pub trait MaxCover { + /// The result type, of which we would eventually like a collection of maximal quality. + type Object; + /// The type used to represent sets. + type Set: Clone; + + /// Extract an object for inclusion in a solution. + fn object(&self) -> Self::Object; + + /// Get the set of elements covered. + fn covering_set(&self) -> &Self::Set; + /// Update the set of items covered, for the inclusion of some object in the solution. + fn update_covering_set(&mut self, max_obj: &Self::Object, max_set: &Self::Set); + /// The quality of this item's covering set, usually its cardinality. + fn score(&self) -> usize; +} + +/// Helper struct to track which items of the input are still available for inclusion. +/// Saves removing elements from the work vector. +struct MaxCoverItem { + item: T, + available: bool, +} + +impl MaxCoverItem { + fn new(item: T) -> Self { + MaxCoverItem { + item, + available: true, + } + } +} + +/// Compute an approximate maximum cover using a greedy algorithm. +/// +/// * Time complexity: `O(limit * items_iter.len())` +/// * Space complexity: `O(item_iter.len())` +pub fn maximum_cover<'a, I, T>(items_iter: I, limit: usize) -> Vec +where + I: IntoIterator, + T: MaxCover, +{ + // Construct an initial vec of all items, marked available. + let mut all_items: Vec<_> = items_iter + .into_iter() + .map(MaxCoverItem::new) + .filter(|x| x.item.score() != 0) + .collect(); + + let mut result = vec![]; + + for _ in 0..limit { + // Select the item with the maximum score. + let (best_item, best_cover) = match all_items + .iter_mut() + .filter(|x| x.available && x.item.score() != 0) + .max_by_key(|x| x.item.score()) + { + Some(x) => { + x.available = false; + (x.item.object(), x.item.covering_set().clone()) + } + None => return result, + }; + + // Update the covering sets of the other items, for the inclusion of the selected item. + // Items covered by the selected item can't be re-covered. + all_items + .iter_mut() + .filter(|x| x.available && x.item.score() != 0) + .for_each(|x| x.item.update_covering_set(&best_item, &best_cover)); + + result.push(best_item); + } + + result +} + +#[cfg(test)] +mod test { + use super::*; + use std::iter::FromIterator; + use std::{collections::HashSet, hash::Hash}; + + impl MaxCover for HashSet + where + T: Clone + Eq + Hash, + { + type Object = Self; + type Set = Self; + + fn object(&self) -> Self { + self.clone() + } + + fn covering_set(&self) -> &Self { + &self + } + + fn update_covering_set(&mut self, _: &Self, other: &Self) { + let mut difference = &*self - other; + std::mem::swap(self, &mut difference); + } + + fn score(&self) -> usize { + self.len() + } + } + + fn example_system() -> Vec> { + vec![ + HashSet::from_iter(vec![3]), + HashSet::from_iter(vec![1, 2, 4, 5]), + HashSet::from_iter(vec![1, 2, 4, 5]), + HashSet::from_iter(vec![1]), + HashSet::from_iter(vec![2, 4, 5]), + ] + } + + #[test] + fn zero_limit() { + let cover = maximum_cover(example_system(), 0); + assert_eq!(cover.len(), 0); + } + + #[test] + fn one_limit() { + let sets = example_system(); + let cover = maximum_cover(sets.clone(), 1); + assert_eq!(cover.len(), 1); + assert_eq!(cover[0], sets[1]); + } + + // Check that even if the limit provides room, we don't include useless items in the soln. + #[test] + fn exclude_zero_score() { + let sets = example_system(); + for k in 2..10 { + let cover = maximum_cover(sets.clone(), k); + assert_eq!(cover.len(), 2); + assert_eq!(cover[0], sets[1]); + assert_eq!(cover[1], sets[0]); + } + } + + fn quality(solution: &[HashSet]) -> usize { + solution.iter().map(HashSet::len).sum() + } + + // Optimal solution is the first three sets (quality 15) but our greedy algorithm + // will select the last three (quality 11). The comment at the end of each line + // shows that set's score at each iteration, with a * indicating that it will be chosen. + #[test] + fn suboptimal() { + let sets = vec![ + HashSet::from_iter(vec![0, 1, 8, 11, 14]), // 5, 3, 2 + HashSet::from_iter(vec![2, 3, 7, 9, 10]), // 5, 3, 2 + HashSet::from_iter(vec![4, 5, 6, 12, 13]), // 5, 4, 2 + HashSet::from_iter(vec![9, 10]), // 4, 4, 2* + HashSet::from_iter(vec![5, 6, 7, 8]), // 4, 4* + HashSet::from_iter(vec![0, 1, 2, 3, 4]), // 5* + ]; + let cover = maximum_cover(sets.clone(), 3); + assert_eq!(quality(&cover), 11); + } + + #[test] + fn intersecting_ok() { + let sets = vec![ + HashSet::from_iter(vec![1, 2, 3, 4, 5, 6, 7, 8]), + HashSet::from_iter(vec![1, 2, 3, 9, 10, 11]), + HashSet::from_iter(vec![4, 5, 6, 12, 13, 14]), + HashSet::from_iter(vec![7, 8, 15, 16, 17, 18]), + HashSet::from_iter(vec![1, 2, 9, 10]), + HashSet::from_iter(vec![1, 5, 6, 8]), + HashSet::from_iter(vec![1, 7, 11, 19]), + ]; + let cover = maximum_cover(sets.clone(), 5); + assert_eq!(quality(&cover), 19); + assert_eq!(cover.len(), 5); + } +} diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index 08e56e7c3..ac6ffa89a 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -13,7 +13,7 @@ use std::default; /// A BooleanBitfield represents a set of booleans compactly stored as a vector of bits. /// The BooleanBitfield is given a fixed size during construction. Reads outside of the current size return an out-of-bounds error. Writes outside of the current size expand the size of the set. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Hash)] pub struct BooleanBitfield(BitVec); /// Error represents some reason a request against a bitfield was not satisfied @@ -170,6 +170,7 @@ impl cmp::PartialEq for BooleanBitfield { ssz::ssz_encode(self) == ssz::ssz_encode(other) } } +impl Eq for BooleanBitfield {} /// Create a new bitfield that is a union of two other bitfields. /// From 604fe2d97f4f107f25ea6a3b9a0640059374e1e7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 18 Jun 2019 17:55:18 +1000 Subject: [PATCH 11/72] op_pool: partial persistence support --- beacon_node/beacon_chain/src/beacon_chain.rs | 7 +- .../src/persisted_beacon_chain.rs | 3 +- eth2/operation_pool/Cargo.toml | 1 + eth2/operation_pool/src/attestation_id.rs | 11 +- eth2/operation_pool/src/lib.rs | 3 + eth2/operation_pool/src/persistence.rs | 112 ++++++++++++++++++ 6 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 eth2/operation_pool/src/persistence.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 0137a0746..07801eb2a 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -6,7 +6,7 @@ use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; use lmd_ghost::LmdGhost; use log::trace; use operation_pool::DepositInsertStatus; -use operation_pool::OperationPool; +use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::{RwLock, RwLockReadGuard}; use slot_clock::SlotClock; use state_processing::per_block_processing::errors::{ @@ -147,11 +147,13 @@ impl BeaconChain { let last_finalized_root = p.canonical_head.beacon_state.finalized_root; let last_finalized_block = &p.canonical_head.beacon_block; + let op_pool = p.op_pool.into_operation_pool(&p.state, &spec); + Ok(Some(BeaconChain { spec, slot_clock, fork_choice: ForkChoice::new(store.clone(), last_finalized_block, last_finalized_root), - op_pool: OperationPool::default(), + op_pool, canonical_head: RwLock::new(p.canonical_head), state: RwLock::new(p.state), genesis_block_root: p.genesis_block_root, @@ -164,6 +166,7 @@ impl BeaconChain { pub fn persist(&self) -> Result<(), Error> { let p: PersistedBeaconChain = PersistedBeaconChain { canonical_head: self.canonical_head.read().clone(), + op_pool: PersistedOperationPool::from_operation_pool(&self.op_pool), genesis_block_root: self.genesis_block_root, state: self.state.read().clone(), }; diff --git a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs index f5bdfdee1..479e1cd8e 100644 --- a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs +++ b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs @@ -1,4 +1,5 @@ use crate::{BeaconChainTypes, CheckPoint}; +use operation_pool::PersistedOperationPool; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use store::{DBColumn, Error as StoreError, StoreItem}; @@ -10,7 +11,7 @@ pub const BEACON_CHAIN_DB_KEY: &str = "PERSISTEDBEACONCHAINPERSISTEDBEA"; #[derive(Encode, Decode)] pub struct PersistedBeaconChain { pub canonical_head: CheckPoint, - // TODO: operations pool. + pub op_pool: PersistedOperationPool, pub genesis_block_root: Hash256, pub state: BeaconState, } diff --git a/eth2/operation_pool/Cargo.toml b/eth2/operation_pool/Cargo.toml index 99c7bfc06..770843b75 100644 --- a/eth2/operation_pool/Cargo.toml +++ b/eth2/operation_pool/Cargo.toml @@ -12,3 +12,4 @@ parking_lot = "0.7" types = { path = "../types" } state_processing = { path = "../state_processing" } ssz = { path = "../utils/ssz" } +ssz_derive = { path = "../utils/ssz_derive" } diff --git a/eth2/operation_pool/src/attestation_id.rs b/eth2/operation_pool/src/attestation_id.rs index ba25174e6..a79023a69 100644 --- a/eth2/operation_pool/src/attestation_id.rs +++ b/eth2/operation_pool/src/attestation_id.rs @@ -1,10 +1,13 @@ use int_to_bytes::int_to_bytes8; use ssz::ssz_encode; +use ssz_derive::{Decode, Encode}; use types::{AttestationData, BeaconState, ChainSpec, Domain, Epoch, EthSpec}; /// Serialized `AttestationData` augmented with a domain to encode the fork info. -#[derive(PartialEq, Eq, Clone, Hash, Debug)] -pub struct AttestationId(Vec); +#[derive(PartialEq, Eq, Clone, Hash, Debug, PartialOrd, Ord, Encode, Decode)] +pub struct AttestationId { + v: Vec, +} /// Number of domain bytes that the end of an attestation ID is padded with. const DOMAIN_BYTES_LEN: usize = 8; @@ -18,7 +21,7 @@ impl AttestationId { let mut bytes = ssz_encode(attestation); let epoch = attestation.target_epoch; bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, state, spec)); - AttestationId(bytes) + AttestationId { v: bytes } } pub fn compute_domain_bytes( @@ -30,6 +33,6 @@ impl AttestationId { } pub fn domain_bytes_match(&self, domain_bytes: &[u8]) -> bool { - &self.0[self.0.len() - DOMAIN_BYTES_LEN..] == domain_bytes + &self.v[self.v.len() - DOMAIN_BYTES_LEN..] == domain_bytes } } diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index 7157ec9ee..d19d68080 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -1,6 +1,9 @@ mod attestation; mod attestation_id; mod max_cover; +mod persistence; + +pub use persistence::PersistedOperationPool; use attestation::{earliest_attestation_validators, AttMaxCover}; use attestation_id::AttestationId; diff --git a/eth2/operation_pool/src/persistence.rs b/eth2/operation_pool/src/persistence.rs new file mode 100644 index 000000000..78fe3ea2e --- /dev/null +++ b/eth2/operation_pool/src/persistence.rs @@ -0,0 +1,112 @@ +use crate::attestation_id::AttestationId; +use crate::OperationPool; +use itertools::Itertools; +use parking_lot::RwLock; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use types::*; + +/// Tuples for SSZ +#[derive(Encode, Decode)] +struct SszPair { + x: X, + y: Y, +} + +impl SszPair { + fn new(x: X, y: Y) -> Self { + Self { x, y } + } +} + +impl From<(X, Y)> for SszPair +where + X: Encode + Decode, + Y: Encode + Decode, +{ + fn from((x, y): (X, Y)) -> Self { + Self { x, y } + } +} + +impl Into<(X, Y)> for SszPair +where + X: Encode + Decode, + Y: Encode + Decode, +{ + fn into(self) -> (X, Y) { + (self.x, self.y) + } +} + +#[derive(Encode, Decode)] +pub struct PersistedOperationPool { + /// Mapping from attestation ID to attestation mappings, sorted by ID. + // TODO: we could save space by not storing the attestation ID, but it might + // be difficult to make that roundtrip due to eager aggregation. + attestations: Vec>>, + deposits: Vec, + /// Attester slashings sorted by their pair of attestation IDs (not stored). + attester_slashings: Vec, +} + +impl PersistedOperationPool { + pub fn from_operation_pool(operation_pool: &OperationPool) -> Self { + let attestations = operation_pool + .attestations + .read() + .iter() + .map(|(att_id, att)| SszPair::new(att_id.clone(), att.clone())) + .sorted_by(|att1, att2| Ord::cmp(&att1.x, &att2.x)) + .collect(); + + let deposits = operation_pool + .deposits + .read() + .iter() + .map(|(_, d)| d.clone()) + .collect(); + + let attester_slashings = operation_pool + .attester_slashings + .read() + .iter() + .sorted_by(|(id1, _), (id2, _)| Ord::cmp(&id1, &id2)) + .map(|(_, slashing)| slashing.clone()) + .collect(); + + Self { + attestations, + deposits, + attester_slashings, + } + } + + pub fn into_operation_pool( + self, + state: &BeaconState, + spec: &ChainSpec, + ) -> OperationPool { + let attestations = RwLock::new(self.attestations.into_iter().map(SszPair::into).collect()); + let deposits = RwLock::new(self.deposits.into_iter().map(|d| (d.index, d)).collect()); + let attester_slashings = RwLock::new( + self.attester_slashings + .into_iter() + .map(|slashing| { + ( + OperationPool::attester_slashing_id(&slashing, state, spec), + slashing, + ) + }) + .collect(), + ); + + OperationPool { + attestations, + deposits, + attester_slashings, + // TODO + ..OperationPool::new() + } + } +} From 7fe458af45c037037c287dbd8b2001948f2f29fa Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 26 Jun 2019 13:04:54 +1000 Subject: [PATCH 12/72] op_pool: re-jig deposit handling (needs more work) --- beacon_node/beacon_chain/src/beacon_chain.rs | 3 +- eth2/operation_pool/src/lib.rs | 40 +++++--------------- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 07801eb2a..2d8282270 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -509,8 +509,7 @@ impl BeaconChain { &self, deposit: Deposit, ) -> Result { - self.op_pool - .insert_deposit(deposit, &*self.state.read(), &self.spec) + self.op_pool.insert_deposit(deposit) } /// Accept some exit and queue it for inclusion in an appropriate block. diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index d19d68080..c19b501ee 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -14,8 +14,6 @@ use state_processing::per_block_processing::errors::{ AttestationValidationError, AttesterSlashingValidationError, DepositValidationError, ExitValidationError, ProposerSlashingValidationError, TransferValidationError, }; -#[cfg(not(test))] -use state_processing::per_block_processing::verify_deposit_merkle_proof; use state_processing::per_block_processing::{ get_slashable_indices_modular, validate_attestation, validate_attestation_time_independent_only, verify_attester_slashing, verify_exit, @@ -151,20 +149,14 @@ impl OperationPool { /// Add a deposit to the pool. /// /// No two distinct deposits should be added with the same index. - #[cfg_attr(test, allow(unused_variables))] pub fn insert_deposit( &self, deposit: Deposit, - state: &BeaconState, - spec: &ChainSpec, ) -> Result { use DepositInsertStatus::*; match self.deposits.write().entry(deposit.index) { Entry::Vacant(entry) => { - // TODO: fix tests to generate valid merkle proofs - #[cfg(not(test))] - verify_deposit_merkle_proof(state, &deposit, spec)?; entry.insert(deposit); Ok(Fresh) } @@ -172,9 +164,6 @@ impl OperationPool { if entry.get() == &deposit { Ok(Duplicate) } else { - // TODO: fix tests to generate valid merkle proofs - #[cfg(not(test))] - verify_deposit_merkle_proof(state, &deposit, spec)?; Ok(Replaced(Box::new(entry.insert(deposit)))) } } @@ -185,7 +174,9 @@ impl OperationPool { /// /// Take at most the maximum number of deposits, beginning from the current deposit index. pub fn get_deposits(&self, state: &BeaconState, spec: &ChainSpec) -> Vec { - // TODO: might want to re-check the Merkle proof to account for Eth1 forking + // TODO: We need to update the Merkle proofs for existing deposits as more deposits + // are added. It probably makes sense to construct the proofs from scratch when forming + // a block, using fresh info from the ETH1 chain for the current deposit root. let start_idx = state.deposit_index; (start_idx..start_idx + spec.max_deposits) .map(|idx| self.deposits.read().get(&idx).cloned()) @@ -461,22 +452,15 @@ mod tests { #[test] fn insert_deposit() { let rng = &mut XorShiftRng::from_seed([42; 16]); - let (ref spec, ref state) = test_state(rng); - let op_pool = OperationPool::new(); + let op_pool = OperationPool::::new(); let deposit1 = make_deposit(rng); let mut deposit2 = make_deposit(rng); deposit2.index = deposit1.index; + assert_eq!(op_pool.insert_deposit(deposit1.clone()), Ok(Fresh)); + assert_eq!(op_pool.insert_deposit(deposit1.clone()), Ok(Duplicate)); assert_eq!( - op_pool.insert_deposit(deposit1.clone(), state, spec), - Ok(Fresh) - ); - assert_eq!( - op_pool.insert_deposit(deposit1.clone(), state, spec), - Ok(Duplicate) - ); - assert_eq!( - op_pool.insert_deposit(deposit2, state, spec), + op_pool.insert_deposit(deposit2), Ok(Replaced(Box::new(deposit1))) ); } @@ -495,10 +479,7 @@ mod tests { let deposits = dummy_deposits(rng, start, max_deposits + extra); for deposit in &deposits { - assert_eq!( - op_pool.insert_deposit(deposit.clone(), &state, &spec), - Ok(Fresh) - ); + assert_eq!(op_pool.insert_deposit(deposit.clone()), Ok(Fresh)); } state.deposit_index = start + offset; @@ -514,8 +495,7 @@ mod tests { #[test] fn prune_deposits() { let rng = &mut XorShiftRng::from_seed([42; 16]); - let (spec, state) = test_state(rng); - let op_pool = OperationPool::new(); + let op_pool = OperationPool::::new(); let start1 = 100; // test is super slow in debug mode if this parameter is too high @@ -527,7 +507,7 @@ mod tests { let deposits2 = dummy_deposits(rng, start2, count); for d in deposits1.into_iter().chain(deposits2) { - assert!(op_pool.insert_deposit(d, &state, &spec).is_ok()); + assert!(op_pool.insert_deposit(d).is_ok()); } assert_eq!(op_pool.num_deposits(), 2 * count as usize); From 73c4171b526bb08407aedee2c3e5eebcb115da5a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 26 Jun 2019 13:06:08 +1000 Subject: [PATCH 13/72] op_pool: finish persistence support --- beacon_node/beacon_chain/Cargo.toml | 3 + beacon_node/beacon_chain/src/test_utils.rs | 6 +- beacon_node/beacon_chain/tests/tests.rs | 52 +++++- eth2/operation_pool/src/lib.rs | 14 +- eth2/operation_pool/src/persistence.rs | 191 +++++++++++++-------- 5 files changed, 183 insertions(+), 83 deletions(-) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index abf1bc647..b6f8d1ab8 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -26,3 +26,6 @@ state_processing = { path = "../../eth2/state_processing" } tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } lmd_ghost = { path = "../../eth2/lmd_ghost" } + +[dev-dependencies] +rand = "0.5.5" diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 164857e5d..9b3f7c1cb 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -14,6 +14,8 @@ use types::{ Hash256, Keypair, RelativeEpoch, SecretKey, Signature, Slot, }; +pub use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; + /// Indicates how the `BeaconChainHarness` should produce blocks. #[derive(Clone, Copy, Debug)] pub enum BlockStrategy { @@ -68,8 +70,8 @@ where E: EthSpec, { pub chain: BeaconChain>, - keypairs: Vec, - spec: ChainSpec, + pub keypairs: Vec, + pub spec: ChainSpec, } impl BeaconChainHarness diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index 17e373ad6..882d9f235 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -1,16 +1,21 @@ #![cfg(not(debug_assertions))] -use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}; +use beacon_chain::test_utils::{ + AttestationStrategy, BeaconChainHarness, BlockStrategy, CommonTypes, PersistedBeaconChain, + BEACON_CHAIN_DB_KEY, +}; use lmd_ghost::ThreadSafeReducedTree; -use store::MemoryStore; -use types::{EthSpec, MinimalEthSpec, Slot}; +use rand::Rng; +use store::{MemoryStore, Store}; +use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; +use types::{Deposit, EthSpec, Hash256, MinimalEthSpec, Slot}; // Should ideally be divisible by 3. pub const VALIDATOR_COUNT: usize = 24; -fn get_harness( - validator_count: usize, -) -> BeaconChainHarness, MinimalEthSpec> { +type TestForkChoice = ThreadSafeReducedTree; + +fn get_harness(validator_count: usize) -> BeaconChainHarness { let harness = BeaconChainHarness::new(validator_count); // Move past the zero slot. @@ -225,3 +230,38 @@ fn does_not_finalize_without_attestation() { "no epoch should have been finalized" ); } + +#[test] +fn roundtrip_operation_pool() { + let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5; + + let harness = get_harness(VALIDATOR_COUNT); + + // Add some attestations + harness.extend_chain( + num_blocks_produced as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ); + assert!(harness.chain.op_pool.num_attestations() > 0); + + // Add some deposits + let rng = &mut XorShiftRng::from_seed([66; 16]); + for _ in 0..rng.gen_range(1, VALIDATOR_COUNT) { + harness + .chain + .process_deposit(Deposit::random_for_test(rng)) + .unwrap(); + } + + // TODO: could add some other operations + harness.chain.persist().unwrap(); + + let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes()); + let p: PersistedBeaconChain> = + harness.chain.store.get(&key).unwrap().unwrap(); + + let restored_op_pool = p.op_pool.into_operation_pool(&p.state, &harness.spec); + + assert_eq!(harness.chain.op_pool, restored_op_pool); +} diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index c19b501ee..6c6f1e752 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -27,7 +27,7 @@ use types::{ Transfer, Validator, VoluntaryExit, }; -#[derive(Default)] +#[derive(Default, Debug)] pub struct OperationPool { /// Map from attestation ID (see below) to vectors of attestations. attestations: RwLock>>, @@ -442,6 +442,18 @@ fn prune_validator_hash_map( }); } +/// Compare two operation pools. +impl PartialEq for OperationPool { + fn eq(&self, other: &Self) -> bool { + *self.attestations.read() == *other.attestations.read() + && *self.deposits.read() == *other.deposits.read() + && *self.attester_slashings.read() == *other.attester_slashings.read() + && *self.proposer_slashings.read() == *other.proposer_slashings.read() + && *self.voluntary_exits.read() == *other.voluntary_exits.read() + && *self.transfers.read() == *other.transfers.read() + } +} + #[cfg(test)] mod tests { use super::DepositInsertStatus::*; diff --git a/eth2/operation_pool/src/persistence.rs b/eth2/operation_pool/src/persistence.rs index 78fe3ea2e..992c14555 100644 --- a/eth2/operation_pool/src/persistence.rs +++ b/eth2/operation_pool/src/persistence.rs @@ -1,12 +1,127 @@ use crate::attestation_id::AttestationId; use crate::OperationPool; -use itertools::Itertools; use parking_lot::RwLock; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use types::*; -/// Tuples for SSZ +/// SSZ-serializable version of `OperationPool`. +/// +/// Operations are stored in arbitrary order, so it's not a good idea to compare instances +/// of this type (or its encoded form) for equality. Convert back to an `OperationPool` first. +#[derive(Encode, Decode)] +pub struct PersistedOperationPool { + /// Mapping from attestation ID to attestation mappings. + // We could save space by not storing the attestation ID, but it might + // be difficult to make that roundtrip due to eager aggregation. + attestations: Vec>>, + deposits: Vec, + /// Attester slashings. + attester_slashings: Vec, + /// Proposer slashings. + proposer_slashings: Vec, + /// Voluntary exits. + voluntary_exits: Vec, + /// Transfers. + transfers: Vec, +} + +impl PersistedOperationPool { + /// Convert an `OperationPool` into serializable form. + pub fn from_operation_pool(operation_pool: &OperationPool) -> Self { + let attestations = operation_pool + .attestations + .read() + .iter() + .map(|(att_id, att)| SszPair::new(att_id.clone(), att.clone())) + .collect(); + + let deposits = operation_pool + .deposits + .read() + .iter() + .map(|(_, d)| d.clone()) + .collect(); + + let attester_slashings = operation_pool + .attester_slashings + .read() + .iter() + .map(|(_, slashing)| slashing.clone()) + .collect(); + + let proposer_slashings = operation_pool + .proposer_slashings + .read() + .iter() + .map(|(_, slashing)| slashing.clone()) + .collect(); + + let voluntary_exits = operation_pool + .voluntary_exits + .read() + .iter() + .map(|(_, exit)| exit.clone()) + .collect(); + + let transfers = operation_pool.transfers.read().iter().cloned().collect(); + + Self { + attestations, + deposits, + attester_slashings, + proposer_slashings, + voluntary_exits, + transfers, + } + } + + /// Reconstruct an `OperationPool`. + pub fn into_operation_pool( + self, + state: &BeaconState, + spec: &ChainSpec, + ) -> OperationPool { + let attestations = RwLock::new(self.attestations.into_iter().map(SszPair::into).collect()); + let deposits = RwLock::new(self.deposits.into_iter().map(|d| (d.index, d)).collect()); + let attester_slashings = RwLock::new( + self.attester_slashings + .into_iter() + .map(|slashing| { + ( + OperationPool::attester_slashing_id(&slashing, state, spec), + slashing, + ) + }) + .collect(), + ); + let proposer_slashings = RwLock::new( + self.proposer_slashings + .into_iter() + .map(|slashing| (slashing.proposer_index, slashing)) + .collect(), + ); + let voluntary_exits = RwLock::new( + self.voluntary_exits + .into_iter() + .map(|exit| (exit.validator_index, exit)) + .collect(), + ); + let transfers = RwLock::new(self.transfers.into_iter().collect()); + + OperationPool { + attestations, + deposits, + attester_slashings, + proposer_slashings, + voluntary_exits, + transfers, + _phantom: Default::default(), + } + } +} + +/// Tuples for SSZ. #[derive(Encode, Decode)] struct SszPair { x: X, @@ -38,75 +153,3 @@ where (self.x, self.y) } } - -#[derive(Encode, Decode)] -pub struct PersistedOperationPool { - /// Mapping from attestation ID to attestation mappings, sorted by ID. - // TODO: we could save space by not storing the attestation ID, but it might - // be difficult to make that roundtrip due to eager aggregation. - attestations: Vec>>, - deposits: Vec, - /// Attester slashings sorted by their pair of attestation IDs (not stored). - attester_slashings: Vec, -} - -impl PersistedOperationPool { - pub fn from_operation_pool(operation_pool: &OperationPool) -> Self { - let attestations = operation_pool - .attestations - .read() - .iter() - .map(|(att_id, att)| SszPair::new(att_id.clone(), att.clone())) - .sorted_by(|att1, att2| Ord::cmp(&att1.x, &att2.x)) - .collect(); - - let deposits = operation_pool - .deposits - .read() - .iter() - .map(|(_, d)| d.clone()) - .collect(); - - let attester_slashings = operation_pool - .attester_slashings - .read() - .iter() - .sorted_by(|(id1, _), (id2, _)| Ord::cmp(&id1, &id2)) - .map(|(_, slashing)| slashing.clone()) - .collect(); - - Self { - attestations, - deposits, - attester_slashings, - } - } - - pub fn into_operation_pool( - self, - state: &BeaconState, - spec: &ChainSpec, - ) -> OperationPool { - let attestations = RwLock::new(self.attestations.into_iter().map(SszPair::into).collect()); - let deposits = RwLock::new(self.deposits.into_iter().map(|d| (d.index, d)).collect()); - let attester_slashings = RwLock::new( - self.attester_slashings - .into_iter() - .map(|slashing| { - ( - OperationPool::attester_slashing_id(&slashing, state, spec), - slashing, - ) - }) - .collect(), - ); - - OperationPool { - attestations, - deposits, - attester_slashings, - // TODO - ..OperationPool::new() - } - } -} From 44ed3228b9da15a9984c46e8192e022aadabb2f5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 1 Jul 2019 14:54:34 +1000 Subject: [PATCH 14/72] ssz: implement Encode and Decode for tuples --- eth2/utils/ssz/src/decode/impls.rs | 158 +++++++++++++++++++++++++++++ eth2/utils/ssz/src/encode/impls.rs | 155 ++++++++++++++++++++++++++++ eth2/utils/ssz/tests/tests.rs | 30 ++++++ 3 files changed, 343 insertions(+) diff --git a/eth2/utils/ssz/src/decode/impls.rs b/eth2/utils/ssz/src/decode/impls.rs index 75dd5d444..6f7986945 100644 --- a/eth2/utils/ssz/src/decode/impls.rs +++ b/eth2/utils/ssz/src/decode/impls.rs @@ -41,6 +41,153 @@ impl_decodable_for_uint!(usize, 32); #[cfg(target_pointer_width = "64")] impl_decodable_for_uint!(usize, 64); +macro_rules! impl_decode_for_tuples { + ($( + $Tuple:ident { + $(($idx:tt) -> $T:ident)+ + } + )+) => { + $( + impl<$($T: Decode),+> Decode for ($($T,)+) { + fn is_ssz_fixed_len() -> bool { + $( + <$T as Decode>::is_ssz_fixed_len() && + )* + true + } + + fn ssz_fixed_len() -> usize { + if ::is_ssz_fixed_len() { + $( + <$T as Decode>::ssz_fixed_len() + + )* + 0 + } else { + BYTES_PER_LENGTH_OFFSET + } + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let mut builder = SszDecoderBuilder::new(bytes); + + $( + builder.register_type::<$T>()?; + )* + + let mut decoder = builder.build()?; + + Ok(($( + decoder.decode_next::<$T>()?, + )* + )) + } + } + )+ + } +} + +impl_decode_for_tuples! { + Tuple2 { + (0) -> A + (1) -> B + } + Tuple3 { + (0) -> A + (1) -> B + (2) -> C + } + Tuple4 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + } + Tuple5 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + } + Tuple6 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + } + Tuple7 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + } + Tuple8 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + } + Tuple9 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + } + Tuple10 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + } + Tuple11 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + (10) -> K + } + Tuple12 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + (10) -> K + (11) -> L + } +} + impl Decode for bool { fn is_ssz_fixed_len() -> bool { true @@ -520,4 +667,15 @@ mod tests { }) ); } + + #[test] + fn tuple() { + assert_eq!(<(u16, u16)>::from_ssz_bytes(&[0, 0, 0, 0]), Ok((0, 0))); + assert_eq!(<(u16, u16)>::from_ssz_bytes(&[16, 0, 17, 0]), Ok((16, 17))); + assert_eq!(<(u16, u16)>::from_ssz_bytes(&[0, 1, 2, 0]), Ok((256, 2))); + assert_eq!( + <(u16, u16)>::from_ssz_bytes(&[255, 255, 0, 0]), + Ok((65535, 0)) + ); + } } diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index e0e2d9dbc..3d68d8911 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -31,6 +31,154 @@ impl_encodable_for_uint!(usize, 32); #[cfg(target_pointer_width = "64")] impl_encodable_for_uint!(usize, 64); +// Based on the `tuple_impls` macro from the standard library. +macro_rules! impl_encode_for_tuples { + ($( + $Tuple:ident { + $(($idx:tt) -> $T:ident)+ + } + )+) => { + $( + impl<$($T: Encode),+> Encode for ($($T,)+) { + fn is_ssz_fixed_len() -> bool { + $( + <$T as Encode>::is_ssz_fixed_len() && + )* + true + } + + fn ssz_fixed_len() -> usize { + if ::is_ssz_fixed_len() { + $( + <$T as Encode>::ssz_fixed_len() + + )* + 0 + } else { + BYTES_PER_LENGTH_OFFSET + } + } + + fn ssz_append(&self, buf: &mut Vec) { + let offset = $( + <$T as Encode>::ssz_fixed_len() + + )* + 0; + + let mut encoder = SszEncoder::container(buf, offset); + + $( + encoder.append(&self.$idx); + )* + + encoder.finalize(); + } + } + )+ + } +} + +impl_encode_for_tuples! { + Tuple2 { + (0) -> A + (1) -> B + } + Tuple3 { + (0) -> A + (1) -> B + (2) -> C + } + Tuple4 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + } + Tuple5 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + } + Tuple6 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + } + Tuple7 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + } + Tuple8 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + } + Tuple9 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + } + Tuple10 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + } + Tuple11 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + (10) -> K + } + Tuple12 { + (0) -> A + (1) -> B + (2) -> C + (3) -> D + (4) -> E + (5) -> F + (6) -> G + (7) -> H + (8) -> I + (9) -> J + (10) -> K + (11) -> L + } +} + /// The SSZ "union" type. impl Encode for Option { fn is_ssz_fixed_len() -> bool { @@ -292,4 +440,11 @@ mod tests { assert_eq!([1, 0, 0, 0].as_ssz_bytes(), vec![1, 0, 0, 0]); assert_eq!([1, 2, 3, 4].as_ssz_bytes(), vec![1, 2, 3, 4]); } + + #[test] + fn tuple() { + assert_eq!((10u8, 11u8).as_ssz_bytes(), vec![10, 11]); + assert_eq!((10u32, 11u8).as_ssz_bytes(), vec![10, 0, 0, 0, 11]); + assert_eq!((10u8, 11u8, 12u8).as_ssz_bytes(), vec![10, 11, 12]); + } } diff --git a/eth2/utils/ssz/tests/tests.rs b/eth2/utils/ssz/tests/tests.rs index 9447cf537..c19e36662 100644 --- a/eth2/utils/ssz/tests/tests.rs +++ b/eth2/utils/ssz/tests/tests.rs @@ -346,4 +346,34 @@ mod round_trip { round_trip(vec); } + + #[test] + fn tuple_u8_u16() { + let vec: Vec<(u8, u16)> = vec![ + (0, 0), + (0, 1), + (1, 0), + (u8::max_value(), u16::max_value()), + (0, u16::max_value()), + (u8::max_value(), 0), + (42, 12301), + ]; + + round_trip(vec); + } + + #[test] + fn tuple_vec_vec() { + let vec: Vec<(u64, Vec, Vec>)> = vec![ + (0, vec![], vec![vec![]]), + (99, vec![101], vec![vec![], vec![]]), + ( + 42, + vec![12, 13, 14], + vec![vec![99, 98, 97, 96], vec![42, 44, 46, 48, 50]], + ), + ]; + + round_trip(vec); + } } From a04b1f981e0a67d4cae9b395513b63723b05f67a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 1 Jul 2019 15:21:11 +1000 Subject: [PATCH 15/72] op_pool: remove SszPair --- eth2/operation_pool/src/persistence.rs | 40 ++------------------------ 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/eth2/operation_pool/src/persistence.rs b/eth2/operation_pool/src/persistence.rs index 992c14555..aa6df597c 100644 --- a/eth2/operation_pool/src/persistence.rs +++ b/eth2/operation_pool/src/persistence.rs @@ -1,7 +1,6 @@ use crate::attestation_id::AttestationId; use crate::OperationPool; use parking_lot::RwLock; -use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use types::*; @@ -14,7 +13,7 @@ pub struct PersistedOperationPool { /// Mapping from attestation ID to attestation mappings. // We could save space by not storing the attestation ID, but it might // be difficult to make that roundtrip due to eager aggregation. - attestations: Vec>>, + attestations: Vec<(AttestationId, Vec)>, deposits: Vec, /// Attester slashings. attester_slashings: Vec, @@ -33,7 +32,7 @@ impl PersistedOperationPool { .attestations .read() .iter() - .map(|(att_id, att)| SszPair::new(att_id.clone(), att.clone())) + .map(|(att_id, att)| (att_id.clone(), att.clone())) .collect(); let deposits = operation_pool @@ -82,7 +81,7 @@ impl PersistedOperationPool { state: &BeaconState, spec: &ChainSpec, ) -> OperationPool { - let attestations = RwLock::new(self.attestations.into_iter().map(SszPair::into).collect()); + let attestations = RwLock::new(self.attestations.into_iter().collect()); let deposits = RwLock::new(self.deposits.into_iter().map(|d| (d.index, d)).collect()); let attester_slashings = RwLock::new( self.attester_slashings @@ -120,36 +119,3 @@ impl PersistedOperationPool { } } } - -/// Tuples for SSZ. -#[derive(Encode, Decode)] -struct SszPair { - x: X, - y: Y, -} - -impl SszPair { - fn new(x: X, y: Y) -> Self { - Self { x, y } - } -} - -impl From<(X, Y)> for SszPair -where - X: Encode + Decode, - Y: Encode + Decode, -{ - fn from((x, y): (X, Y)) -> Self { - Self { x, y } - } -} - -impl Into<(X, Y)> for SszPair -where - X: Encode + Decode, - Y: Encode + Decode, -{ - fn into(self) -> (X, Y) { - (self.x, self.y) - } -} From 027f0a539d50004b4d81c89c7a0a84a5a365960b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 3 Jul 2019 16:06:20 +1000 Subject: [PATCH 16/72] Prepare ssz for publishing on crates.io --- beacon_node/beacon_chain/Cargo.toml | 4 ++-- beacon_node/client/Cargo.toml | 2 +- beacon_node/eth2-libp2p/Cargo.toml | 4 ++-- beacon_node/http_server/Cargo.toml | 2 +- beacon_node/network/Cargo.toml | 2 +- beacon_node/rpc/Cargo.toml | 2 +- beacon_node/store/Cargo.toml | 4 ++-- eth2/lmd_ghost/Cargo.toml | 2 +- eth2/operation_pool/Cargo.toml | 4 ++-- eth2/state_processing/Cargo.toml | 6 +++--- eth2/types/Cargo.toml | 4 ++-- eth2/utils/bls/Cargo.toml | 2 +- eth2/utils/boolean-bitfield/Cargo.toml | 2 +- eth2/utils/boolean-bitfield/fuzz/Cargo.toml | 2 +- eth2/utils/fixed_len_vec/Cargo.toml | 2 +- eth2/utils/ssz/Cargo.toml | 10 +++++++--- eth2/utils/ssz/src/lib.rs | 2 +- eth2/utils/ssz_derive/Cargo.toml | 9 +++++---- eth2/validator_change/Cargo.toml | 2 +- tests/ef_tests/Cargo.toml | 2 +- validator_client/Cargo.toml | 2 +- 21 files changed, 38 insertions(+), 33 deletions(-) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index b6f8d1ab8..793ce79cd 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -20,8 +20,8 @@ serde = "1.0" serde_derive = "1.0" serde_json = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } -ssz = { path = "../../eth2/utils/ssz" } -ssz_derive = { path = "../../eth2/utils/ssz_derive" } +eth2_ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz_derive = { path = "../../eth2/utils/ssz_derive" } state_processing = { path = "../../eth2/state_processing" } tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index f97302a7c..7c8ee9c7c 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -19,7 +19,7 @@ serde = "1.0" serde_derive = "1.0" error-chain = "0.12.0" slog = "^2.2.3" -ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz = { path = "../../eth2/utils/ssz" } tokio = "0.1.15" clap = "2.32.0" dirs = "1.0.3" diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index cc6393e38..ada5faebb 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -12,8 +12,8 @@ libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" -ssz = { path = "../../eth2/utils/ssz" } -ssz_derive = { path = "../../eth2/utils/ssz_derive" } +eth2_ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz_derive = { path = "../../eth2/utils/ssz_derive" } slog = "2.4.1" version = { path = "../version" } tokio = "0.1.16" diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml index 45e0349f5..9a7a4b0a5 100644 --- a/beacon_node/http_server/Cargo.toml +++ b/beacon_node/http_server/Cargo.toml @@ -13,7 +13,7 @@ network = { path = "../network" } eth2-libp2p = { path = "../eth2-libp2p" } version = { path = "../version" } types = { path = "../../eth2/types" } -ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz = { path = "../../eth2/utils/ssz" } slot_clock = { path = "../../eth2/utils/slot_clock" } protos = { path = "../../protos" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index ebf71aa4e..695adc9bd 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -14,7 +14,7 @@ eth2-libp2p = { path = "../eth2-libp2p" } version = { path = "../version" } types = { path = "../../eth2/types" } slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } -ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz = { path = "../../eth2/utils/ssz" } tree_hash = { path = "../../eth2/utils/tree_hash" } futures = "0.1.25" error-chain = "0.12.0" diff --git a/beacon_node/rpc/Cargo.toml b/beacon_node/rpc/Cargo.toml index d707cc36d..a37492796 100644 --- a/beacon_node/rpc/Cargo.toml +++ b/beacon_node/rpc/Cargo.toml @@ -11,7 +11,7 @@ network = { path = "../network" } eth2-libp2p = { path = "../eth2-libp2p" } version = { path = "../version" } types = { path = "../../eth2/types" } -ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz = { path = "../../eth2/utils/ssz" } slot_clock = { path = "../../eth2/utils/slot_clock" } protos = { path = "../../protos" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } diff --git a/beacon_node/store/Cargo.toml b/beacon_node/store/Cargo.toml index a95dafa90..6dcb771d2 100644 --- a/beacon_node/store/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -14,7 +14,7 @@ bytes = "0.4.10" db-key = "0.0.5" leveldb = "0.8.4" parking_lot = "0.7" -ssz = { path = "../../eth2/utils/ssz" } -ssz_derive = { path = "../../eth2/utils/ssz_derive" } +eth2_ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz_derive = { path = "../../eth2/utils/ssz_derive" } tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/eth2/lmd_ghost/Cargo.toml b/eth2/lmd_ghost/Cargo.toml index 788708faa..c21af693e 100644 --- a/eth2/lmd_ghost/Cargo.toml +++ b/eth2/lmd_ghost/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] parking_lot = "0.7" store = { path = "../../beacon_node/store" } -ssz = { path = "../utils/ssz" } +eth2_ssz = { path = "../utils/ssz" } state_processing = { path = "../state_processing" } types = { path = "../types" } log = "0.4.6" diff --git a/eth2/operation_pool/Cargo.toml b/eth2/operation_pool/Cargo.toml index 770843b75..d1fd18191 100644 --- a/eth2/operation_pool/Cargo.toml +++ b/eth2/operation_pool/Cargo.toml @@ -11,5 +11,5 @@ itertools = "0.8" parking_lot = "0.7" types = { path = "../types" } state_processing = { path = "../state_processing" } -ssz = { path = "../utils/ssz" } -ssz_derive = { path = "../utils/ssz_derive" } +eth2_ssz = { path = "../utils/ssz" } +eth2_ssz_derive = { path = "../utils/ssz_derive" } diff --git a/eth2/state_processing/Cargo.toml b/eth2/state_processing/Cargo.toml index fa42671d9..e1f98260b 100644 --- a/eth2/state_processing/Cargo.toml +++ b/eth2/state_processing/Cargo.toml @@ -24,12 +24,12 @@ integer-sqrt = "0.1" itertools = "0.8" log = "0.4" merkle_proof = { path = "../utils/merkle_proof" } -ssz = { path = "../utils/ssz" } -ssz_derive = { path = "../utils/ssz_derive" } +eth2_ssz = { path = "../utils/ssz" } +eth2_ssz_derive = { path = "../utils/ssz_derive" } tree_hash = { path = "../utils/tree_hash" } tree_hash_derive = { path = "../utils/tree_hash_derive" } types = { path = "../types" } rayon = "1.0" [features] -fake_crypto = ["bls/fake_crypto"] \ No newline at end of file +fake_crypto = ["bls/fake_crypto"] diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index fa1fe6a6d..ed6307684 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -26,8 +26,8 @@ serde_derive = "1.0" serde_json = "1.0" serde_yaml = "0.8" slog = "^2.2.3" -ssz = { path = "../utils/ssz" } -ssz_derive = { path = "../utils/ssz_derive" } +eth2_ssz = { path = "../utils/ssz" } +eth2_ssz_derive = { path = "../utils/ssz_derive" } swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } test_random_derive = { path = "../utils/test_random_derive" } tree_hash = { path = "../utils/tree_hash" } diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 4fb1246be..127589463 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -13,7 +13,7 @@ rand = "^0.5" serde = "1.0" serde_derive = "1.0" serde_hex = { path = "../serde_hex" } -ssz = { path = "../ssz" } +eth2_ssz = { path = "../ssz" } tree_hash = { path = "../tree_hash" } [features] diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml index dfc97ce77..ceb04a55a 100644 --- a/eth2/utils/boolean-bitfield/Cargo.toml +++ b/eth2/utils/boolean-bitfield/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] cached_tree_hash = { path = "../cached_tree_hash" } serde_hex = { path = "../serde_hex" } -ssz = { path = "../ssz" } +eth2_ssz = { path = "../ssz" } bit-vec = "0.5.0" bit_reverse = "0.1" serde = "1.0" diff --git a/eth2/utils/boolean-bitfield/fuzz/Cargo.toml b/eth2/utils/boolean-bitfield/fuzz/Cargo.toml index 9769fc50e..6a664ee60 100644 --- a/eth2/utils/boolean-bitfield/fuzz/Cargo.toml +++ b/eth2/utils/boolean-bitfield/fuzz/Cargo.toml @@ -9,7 +9,7 @@ publish = false cargo-fuzz = true [dependencies] -ssz = { path = "../../ssz" } +eth2_ssz = { path = "../../ssz" } [dependencies.boolean-bitfield] path = ".." diff --git a/eth2/utils/fixed_len_vec/Cargo.toml b/eth2/utils/fixed_len_vec/Cargo.toml index ddfc33103..2750d3acd 100644 --- a/eth2/utils/fixed_len_vec/Cargo.toml +++ b/eth2/utils/fixed_len_vec/Cargo.toml @@ -9,5 +9,5 @@ cached_tree_hash = { path = "../cached_tree_hash" } tree_hash = { path = "../tree_hash" } serde = "1.0" serde_derive = "1.0" -ssz = { path = "../ssz" } +eth2_ssz = { path = "../ssz" } typenum = "1.10" diff --git a/eth2/utils/ssz/Cargo.toml b/eth2/utils/ssz/Cargo.toml index 0423b1a8b..5fd007264 100644 --- a/eth2/utils/ssz/Cargo.toml +++ b/eth2/utils/ssz/Cargo.toml @@ -1,8 +1,12 @@ [package] -name = "ssz" +name = "eth2_ssz" version = "0.1.0" -authors = ["Paul Hauner "] +authors = ["Paul Hauner "] edition = "2018" +description = "SimpleSerialize (SSZ) as used in Ethereum 2.0" + +[lib] +name = "ssz" [[bench]] name = "benches" @@ -10,7 +14,7 @@ harness = false [dev-dependencies] criterion = "0.2" -ssz_derive = { path = "../ssz_derive" } +eth2_ssz_derive = { path = "../ssz_derive" } [dependencies] bytes = "0.4.9" diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index 51dd16523..bcb9f525c 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -2,7 +2,7 @@ //! format designed for use in Ethereum 2.0. //! //! Conforms to -//! [v0.6.1](https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/simple-serialize.md) of the +//! [v0.7.1](https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/simple-serialize.md) of the //! Ethereum 2.0 specification. //! //! ## Example diff --git a/eth2/utils/ssz_derive/Cargo.toml b/eth2/utils/ssz_derive/Cargo.toml index 3e58d752b..0b76f6153 100644 --- a/eth2/utils/ssz_derive/Cargo.toml +++ b/eth2/utils/ssz_derive/Cargo.toml @@ -1,14 +1,15 @@ [package] -name = "ssz_derive" +name = "eth2_ssz_derive" version = "0.1.0" -authors = ["Paul Hauner "] +authors = ["Paul Hauner "] edition = "2018" -description = "Procedural derive macros for SSZ encoding and decoding." +description = "Procedural derive macros to accompany the eth2_ssz crate." [lib] +name = "ssz_derive" proc-macro = true [dependencies] syn = "0.15" quote = "0.6" -ssz = { path = "../ssz" } +eth2_ssz = { path = "../ssz" } diff --git a/eth2/validator_change/Cargo.toml b/eth2/validator_change/Cargo.toml index a1c499340..725799612 100644 --- a/eth2/validator_change/Cargo.toml +++ b/eth2/validator_change/Cargo.toml @@ -7,5 +7,5 @@ edition = "2018" [dependencies] bytes = "0.4.10" hashing = { path = "../utils/hashing" } -ssz = { path = "../utils/ssz" } +eth2_ssz = { path = "../utils/ssz" } types = { path = "../types" } diff --git a/tests/ef_tests/Cargo.toml b/tests/ef_tests/Cargo.toml index e199f4cb6..e8d6b0f2f 100644 --- a/tests/ef_tests/Cargo.toml +++ b/tests/ef_tests/Cargo.toml @@ -17,7 +17,7 @@ serde = "1.0" serde_derive = "1.0" serde_repr = "0.1" serde_yaml = "0.8" -ssz = { path = "../../eth2/utils/ssz" } +eth2_ssz = { path = "../../eth2/utils/ssz" } tree_hash = { path = "../../eth2/utils/tree_hash" } cached_tree_hash = { path = "../../eth2/utils/cached_tree_hash" } state_processing = { path = "../../eth2/state_processing" } diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 1784bdcb1..fdb4c33c0 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -14,7 +14,7 @@ path = "src/lib.rs" [dependencies] bls = { path = "../eth2/utils/bls" } -ssz = { path = "../eth2/utils/ssz" } +eth2_ssz = { path = "../eth2/utils/ssz" } eth2_config = { path = "../eth2/utils/eth2_config" } tree_hash = { path = "../eth2/utils/tree_hash" } clap = "2.32.0" From 54bda210e209c55b692d0d4256990e6ca8525e4f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 4 Jul 2019 13:22:33 +1000 Subject: [PATCH 17/72] Update ssz_derive for publishing to crates.io --- eth2/utils/ssz_derive/Cargo.toml | 2 +- eth2/utils/ssz_derive/src/lib.rs | 13 +++++++++++++ eth2/utils/ssz_derive/tests/tests.rs | 22 ---------------------- 3 files changed, 14 insertions(+), 23 deletions(-) delete mode 100644 eth2/utils/ssz_derive/tests/tests.rs diff --git a/eth2/utils/ssz_derive/Cargo.toml b/eth2/utils/ssz_derive/Cargo.toml index 0b76f6153..db25e16be 100644 --- a/eth2/utils/ssz_derive/Cargo.toml +++ b/eth2/utils/ssz_derive/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" description = "Procedural derive macros to accompany the eth2_ssz crate." +license = "Apache-2.0" [lib] name = "ssz_derive" @@ -12,4 +13,3 @@ proc-macro = true [dependencies] syn = "0.15" quote = "0.6" -eth2_ssz = { path = "../ssz" } diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index ef6e2440f..47d96859e 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -1,4 +1,7 @@ #![recursion_limit = "128"] +//! Provides procedural derive macros for the `Encode` and `Decode` traits of the `eth2_ssz` crate. +//! +//! Supports field attributes, see each derive macro for more information. extern crate proc_macro; @@ -61,6 +64,10 @@ fn should_skip_serializing(field: &syn::Field) -> bool { /// Implements `ssz::Encode` for some `struct`. /// /// Fields are encoded in the order they are defined. +/// +/// ## Field attributes +/// +/// - `#[ssz(skip_serializing)]`: the field will not be serialized. #[proc_macro_derive(Encode, attributes(ssz))] pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); @@ -132,6 +139,12 @@ fn should_skip_deserializing(field: &syn::Field) -> bool { /// Implements `ssz::Decode` for some `struct`. /// /// Fields are decoded in the order they are defined. +/// +/// ## Field attributes +/// +/// - `#[ssz(skip_deserializing)]`: during de-serialization the field will be instantiated from a +/// `Default` implementation. The decoder will assume that the field was not serialized at all +/// (e.g., if it has been serialized, an error will be raised instead of `Default` overriding it). #[proc_macro_derive(Decode)] pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as DeriveInput); diff --git a/eth2/utils/ssz_derive/tests/tests.rs b/eth2/utils/ssz_derive/tests/tests.rs deleted file mode 100644 index d58db3b62..000000000 --- a/eth2/utils/ssz_derive/tests/tests.rs +++ /dev/null @@ -1,22 +0,0 @@ -use ssz::Encode; -use ssz_derive::Encode; - -#[derive(Debug, PartialEq, Encode)] -pub struct Foo { - a: u16, - b: Vec, - c: u16, -} - -#[test] -fn encode() { - let foo = Foo { - a: 42, - b: vec![0, 1, 2, 3], - c: 11, - }; - - let bytes = vec![42, 0, 8, 0, 0, 0, 11, 0, 0, 1, 2, 3]; - - assert_eq!(foo.as_ssz_bytes(), bytes); -} From 4dc274858e684a8956bbde8d77bb4e3b86ee428e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 4 Jul 2019 13:32:41 +1000 Subject: [PATCH 18/72] Update SSZ for publishing to crates.io --- eth2/utils/ssz/Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eth2/utils/ssz/Cargo.toml b/eth2/utils/ssz/Cargo.toml index 5fd007264..9002dd6e7 100644 --- a/eth2/utils/ssz/Cargo.toml +++ b/eth2/utils/ssz/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" description = "SimpleSerialize (SSZ) as used in Ethereum 2.0" +license = "Apache-2.0" [lib] name = "ssz" @@ -14,12 +15,10 @@ harness = false [dev-dependencies] criterion = "0.2" -eth2_ssz_derive = { path = "../ssz_derive" } +eth2_ssz_derive = "0.1.0" [dependencies] bytes = "0.4.9" ethereum-types = "0.5" -hashing = { path = "../hashing" } -int_to_bytes = { path = "../int_to_bytes" } hex = "0.3" yaml-rust = "0.4" From c7bd02caaf1086558d83babd01df1458d6270e02 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 11:20:13 +1100 Subject: [PATCH 19/72] Propogate valid attestations accross the network --- beacon_node/rpc/src/attestation.rs | 7 +++---- beacon_node/rpc/src/lib.rs | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index 0f585b7e7..a64ec620f 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -136,11 +136,10 @@ impl AttestationService for AttestationServiceInstance { "type" => "valid_attestation", ); - // TODO: Obtain topics from the network service properly. - let topic = types::TopicBuilder::new("beacon_chain".to_string()).build(); + // valid attestation, propagate to the network + let topic = types::TopicBuilder::new("attestations".to_string()).build(); let message = PubsubMessage::Attestation(attestation); - // Publish the attestation to the p2p network via gossipsub. self.network_chan .send(NetworkMessage::Publish { topics: vec![topic], @@ -150,7 +149,7 @@ impl AttestationService for AttestationServiceInstance { error!( self.log, "PublishAttestation"; - "type" => "failed to publish to gossipsub", + "type" => "failed to publish attestation to gossipsub", "error" => format!("{:?}", e) ); }); diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index f2f1b2abf..4506d90fc 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -60,6 +60,7 @@ pub fn start_server( }; let attestation_service = { let instance = AttestationServiceInstance { + network_chan, chain: beacon_chain.clone(), network_chan, log: log.clone(), From 4e24c8e651fd1c8dfc7641bc5c8e4352c60d71ae Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 12:25:05 +1100 Subject: [PATCH 20/72] Add topics to chain id --- beacon_node/client/src/client_config.rs | 15 +++++++++++++++ beacon_node/eth2-libp2p/src/config.rs | 7 +++++-- eth2/types/src/chain_spec.rs | 9 ++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 166725b61..9ed2a7c6e 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -18,6 +18,21 @@ pub struct ClientConfig { impl Default for ClientConfig { fn default() -> Self { + let data_dir = { + let home = dirs::home_dir().expect("Unable to determine home dir."); + home.join(".lighthouse/") + }; + fs::create_dir_all(&data_dir) + .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); + + let default_spec = ChainSpec::lighthouse_testnet(); + let default_pubsub_topics = vec![ + default_spec.beacon_chain_topic.clone(), + default_spec.shard_topic_prefix.clone(), + ]; // simple singular attestation topic for now. + let default_net_conf = + NetworkConfig::new(default_spec.boot_nodes.clone(), default_pubsub_topics); + Self { data_dir: PathBuf::from(".lighthouse"), db_type: "disk".to_string(), diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index ee2add75e..97559343a 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -2,6 +2,7 @@ use clap::ArgMatches; use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; use serde_derive::{Deserialize, Serialize}; use types::multiaddr::{Error as MultiaddrError, Multiaddr}; +//use std::time::Duration; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] @@ -30,19 +31,21 @@ impl Default for Config { listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000".to_string()], gs_config: GossipsubConfigBuilder::new() .max_gossip_size(4_000_000) + // .inactivity_timeout(Duration::from_secs(90)) .build(), identify_config: IdentifyConfig::default(), boot_nodes: vec![], client_version: version::version(), - topics: vec![String::from("beacon_chain")], + topics: Vec::new(), } } } impl Config { - pub fn new(boot_nodes: Vec) -> Self { + pub fn new(boot_nodes: Vec, topics: Vec) -> Self { let mut conf = Config::default(); conf.boot_nodes = boot_nodes; + conf.topics = topics; conf } diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 74ce40671..4fa79adcd 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -106,10 +106,11 @@ pub struct ChainSpec { /* * Network specific parameters - * */ pub boot_nodes: Vec, pub chain_id: u8, + pub beacon_chain_topic: String, + pub shard_topic_prefix: String, } impl ChainSpec { @@ -216,10 +217,12 @@ impl ChainSpec { domain_transfer: 5, /* - * Boot nodes + * Network specific */ boot_nodes: vec![], - chain_id: 1, // mainnet chain id + chain_id: 1, // foundation chain id + beacon_chain_topic: String::from("beacon_chain"), + shard_topic_prefix: String::from("attestations"), // simple single attestation topic for now } } From a31d6bcb22f6010a9b737efe780cff1fef940f02 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 13:50:11 +1100 Subject: [PATCH 21/72] RPC methods get pubsub topics from chain spec --- beacon_node/network/src/sync/simple_sync.rs | 3 ++- beacon_node/rpc/src/attestation.rs | 5 ++++- beacon_node/rpc/src/beacon_block.rs | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 2382e47a4..0e2103900 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -4,7 +4,8 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; -use slog::{debug, error, info, o, warn}; +use slog::{debug, error, info, o, trace, warn}; +use ssz::TreeHash; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index a64ec620f..c89cbbbb0 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -136,8 +136,11 @@ impl AttestationService for AttestationServiceInstance { "type" => "valid_attestation", ); + // get the network topic to send on + let topic_string = self.chain.get_spec().shard_topic_prefix.clone(); + // valid attestation, propagate to the network - let topic = types::TopicBuilder::new("attestations".to_string()).build(); + let topic = types::TopicBuilder::new(topic_string).build(); let message = PubsubMessage::Attestation(attestation); self.network_chan diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index 533fd285a..791c5008d 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -104,9 +104,9 @@ impl BeaconBlockService for BeaconBlockServiceInstance { "block_root" => format!("{}", block_root), ); - // TODO: Obtain topics from the network service properly. - let topic = - types::TopicBuilder::new("beacon_chain".to_string()).build(); + // get the network topic to send on + let topic_string = self.chain.get_spec().beacon_chain_topic.clone(); + let topic = types::TopicBuilder::new(topic_string).build(); let message = PubsubMessage::Block(block); // Publish the block to the p2p network via gossipsub. From 64abd0bc5b0f4c862c118a17536ebdeeb561be45 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 16:00:09 +1100 Subject: [PATCH 22/72] Removes network parameters from chain spec --- beacon_node/client/src/client_config.rs | 9 ++-- beacon_node/eth2-libp2p/src/behaviour.rs | 2 +- beacon_node/eth2-libp2p/src/config.rs | 57 +++++++++++++++++++++--- beacon_node/eth2-libp2p/src/lib.rs | 5 ++- beacon_node/eth2-libp2p/src/service.rs | 14 ++++-- beacon_node/network/src/service.rs | 2 +- beacon_node/rpc/src/attestation.rs | 3 +- eth2/types/src/chain_spec.rs | 6 --- eth2/types/src/lib.rs | 3 -- 9 files changed, 71 insertions(+), 30 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 9ed2a7c6e..685f58c61 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -25,13 +25,10 @@ impl Default for ClientConfig { fs::create_dir_all(&data_dir) .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); + // currently lighthouse spec let default_spec = ChainSpec::lighthouse_testnet(); - let default_pubsub_topics = vec![ - default_spec.beacon_chain_topic.clone(), - default_spec.shard_topic_prefix.clone(), - ]; // simple singular attestation topic for now. - let default_net_conf = - NetworkConfig::new(default_spec.boot_nodes.clone(), default_pubsub_topics); + // builds a chain-specific network config + let net_conf = NetworkConfig::from(default_spec.chain_id); Self { data_dir: PathBuf::from(".lighthouse"), diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 10b140c3b..f362f5795 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -1,5 +1,6 @@ use crate::rpc::{RPCEvent, RPCMessage, Rpc}; use crate::NetworkConfig; +use crate::{Topic, TopicHash}; use futures::prelude::*; use libp2p::{ core::{ @@ -15,7 +16,6 @@ use libp2p::{ use slog::{debug, o, trace, warn}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; use types::{Attestation, BeaconBlock}; -use types::{Topic, TopicHash}; /// Builds the network behaviour for the libp2p Swarm. /// Implements gossipsub message routing. diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 97559343a..88f315d0c 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -20,8 +20,12 @@ pub struct Config { boot_nodes: Vec, /// Client version pub client_version: String, - /// List of topics to subscribe to as strings + /// List of extra topics to initially subscribe to as strings. pub topics: Vec, + /// Shard pubsub topic prefix. + pub shard_prefix: String, + /// The main beacon chain topic to subscribe to. + pub beacon_chain_topic: String, } impl Default for Config { @@ -37,17 +41,16 @@ impl Default for Config { boot_nodes: vec![], client_version: version::version(), topics: Vec::new(), + beacon_chain_topic: String::from("beacon_chain"), + shard_prefix: String::from("attestations"), // single topic for all attestation for the moment. } } } +/// Generates a default Config. impl Config { - pub fn new(boot_nodes: Vec, topics: Vec) -> Self { - let mut conf = Config::default(); - conf.boot_nodes = boot_nodes; - conf.topics = topics; - - conf + pub fn new() -> Self { + Config::default() } pub fn listen_addresses(&self) -> Result, MultiaddrError> { @@ -90,3 +93,43 @@ impl Default for IdentifyConfig { } } } + +/// Creates a standard network config from a chain_id. +/// +/// This creates specified network parameters for each chain type. +impl From for Config { + fn from(chain_type: ChainType) -> Self { + match chain_type { + ChainType::Foundation => Config::default(), + + ChainType::LighthouseTestnet => { + let boot_nodes = vec!["/ip4/127.0.0.1/tcp/9000" + .parse() + .expect("correct multiaddr")]; + Self { + boot_nodes, + ..Config::default() + } + } + + ChainType::Other => Config::default(), + } + } +} + +pub enum ChainType { + Foundation, + LighthouseTestnet, + Other, +} + +/// Maps a chain id to a ChainType. +impl From for ChainType { + fn from(chain_id: u8) -> Self { + match chain_id { + 1 => ChainType::Foundation, + 2 => ChainType::LighthouseTestnet, + _ => ChainType::Other, + } + } +} diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 659d6b01c..4bd775802 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -10,6 +10,9 @@ mod service; pub use behaviour::PubsubMessage; pub use config::Config as NetworkConfig; +pub use libp2p::floodsub::{Topic, TopicBuilder, TopicHash}; +pub use libp2p::multiaddr; +pub use libp2p::Multiaddr; pub use libp2p::{ gossipsub::{GossipsubConfig, GossipsubConfigBuilder}, PeerId, @@ -17,5 +20,3 @@ pub use libp2p::{ pub use rpc::RPCEvent; pub use service::Libp2pEvent; pub use service::Service; -pub use types::multiaddr; -pub use types::Multiaddr; diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 18f7ca98c..99de38de6 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -3,6 +3,7 @@ use crate::error; use crate::multiaddr::Protocol; use crate::rpc::RPCEvent; use crate::NetworkConfig; +use crate::{TopicBuilder, TopicHash}; use futures::prelude::*; use futures::Stream; use libp2p::core::{ @@ -17,7 +18,6 @@ use libp2p::{core, secio, PeerId, Swarm, Transport}; use slog::{debug, info, trace, warn}; use std::io::{Error, ErrorKind}; use std::time::Duration; -use types::{TopicBuilder, TopicHash}; type Libp2pStream = Boxed<(PeerId, StreamMuxerBox), Error>; type Libp2pBehaviour = Behaviour>; @@ -85,9 +85,17 @@ impl Service { } // subscribe to default gossipsub topics + let mut topics = vec![]; + //TODO: Handle multiple shard attestations. For now we simply use a separate topic for + //attestations + topics.push(config.shard_prefix); + topics.push(config.beacon_chain_topic); + + topics.append(&mut config.topics.clone()); + let mut subscribed_topics = vec![]; - for topic in config.topics { - let t = TopicBuilder::new(topic.to_string()).build(); + for topic in topics { + let t = TopicBuilder::new(topic.clone()).build(); if swarm.subscribe(t) { trace!(log, "Subscribed to topic: {:?}", topic); subscribed_topics.push(topic); diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 9c71a60f7..c19aef004 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -4,6 +4,7 @@ use crate::NetworkConfig; use beacon_chain::{BeaconChain, BeaconChainTypes}; use crossbeam_channel::{unbounded as channel, Sender, TryRecvError}; use eth2_libp2p::Service as LibP2PService; +use eth2_libp2p::Topic; use eth2_libp2p::{Libp2pEvent, PeerId}; use eth2_libp2p::{PubsubMessage, RPCEvent}; use futures::prelude::*; @@ -13,7 +14,6 @@ use slog::{debug, info, o, trace}; use std::marker::PhantomData; use std::sync::Arc; use tokio::runtime::TaskExecutor; -use types::Topic; /// Service that handles communication between internal services and the eth2_libp2p network service. pub struct Service { diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index c89cbbbb0..b9b05b7cd 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,5 +1,6 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::PubsubMessage; +use eth2_libp2p::TopicBuilder; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use network::NetworkMessage; @@ -140,7 +141,7 @@ impl AttestationService for AttestationServiceInstance { let topic_string = self.chain.get_spec().shard_topic_prefix.clone(); // valid attestation, propagate to the network - let topic = types::TopicBuilder::new(topic_string).build(); + let topic = TopicBuilder::new(topic_string).build(); let message = PubsubMessage::Attestation(attestation); self.network_chan diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 4fa79adcd..8e4bd9c9c 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -107,10 +107,7 @@ pub struct ChainSpec { /* * Network specific parameters */ - pub boot_nodes: Vec, pub chain_id: u8, - pub beacon_chain_topic: String, - pub shard_topic_prefix: String, } impl ChainSpec { @@ -219,10 +216,7 @@ impl ChainSpec { /* * Network specific */ - boot_nodes: vec![], chain_id: 1, // foundation chain id - beacon_chain_topic: String::from("beacon_chain"), - shard_topic_prefix: String::from("attestations"), // simple single attestation topic for now } } diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 4d0ec5fae..2406c3a18 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -82,6 +82,3 @@ pub type ProposerMap = HashMap; pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, SecretKey, Signature}; pub use fixed_len_vec::{typenum, typenum::Unsigned, FixedLenVec}; -pub use libp2p::floodsub::{Topic, TopicBuilder, TopicHash}; -pub use libp2p::multiaddr; -pub use libp2p::Multiaddr; From 7920f8098f1098db85bc90f945af1cbb6f675fe3 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 16:33:12 +1100 Subject: [PATCH 23/72] Complete moving network logc into beacon node --- beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/client_config.rs | 9 ++++++++- beacon_node/eth2-libp2p/src/config.rs | 10 ++++------ beacon_node/eth2-libp2p/src/lib.rs | 2 +- beacon_node/eth2-libp2p/src/service.rs | 5 +++-- beacon_node/network/src/lib.rs | 2 +- beacon_node/rpc/src/attestation.rs | 6 ++---- beacon_node/rpc/src/beacon_block.rs | 6 +++--- 8 files changed, 23 insertions(+), 18 deletions(-) diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 7c8ee9c7c..3b2e3ead8 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -9,6 +9,7 @@ beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } store = { path = "../store" } http_server = { path = "../http_server" } +eth2-libp2p = { path = "../eth2-libp2p" } rpc = { path = "../rpc" } prometheus = "^0.6" types = { path = "../../eth2/types" } diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 685f58c61..c533cbcc8 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -1,7 +1,13 @@ use clap::ArgMatches; +use eth2_libp2p::multiaddr::Protocol; +use eth2_libp2p::multiaddr::ToMultiaddr; +use eth2_libp2p::Multiaddr; +use fork_choice::ForkChoiceAlgorithm; use http_server::HttpServerConfig; use network::NetworkConfig; +use network::{ChainType, NetworkConfig}; use serde_derive::{Deserialize, Serialize}; +use slog::error; use std::fs; use std::path::PathBuf; @@ -27,8 +33,9 @@ impl Default for ClientConfig { // currently lighthouse spec let default_spec = ChainSpec::lighthouse_testnet(); + let chain_type = ChainType::from(default_spec.chain_id); // builds a chain-specific network config - let net_conf = NetworkConfig::from(default_spec.chain_id); + let net_conf = NetworkConfig::from(chain_type); Self { data_dir: PathBuf::from(".lighthouse"), diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 88f315d0c..0757f1cbb 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -4,6 +4,10 @@ use serde_derive::{Deserialize, Serialize}; use types::multiaddr::{Error as MultiaddrError, Multiaddr}; //use std::time::Duration; +/// The beacon node topic string to subscribe to. +pub const BEACON_PUBSUB_TOPIC: &str = "beacon_node"; +pub const SHARD_TOPIC_PREFIX: &str = "attestations"; // single topic for all attestation for the moment. + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] /// Network configuration for lighthouse. @@ -22,10 +26,6 @@ pub struct Config { pub client_version: String, /// List of extra topics to initially subscribe to as strings. pub topics: Vec, - /// Shard pubsub topic prefix. - pub shard_prefix: String, - /// The main beacon chain topic to subscribe to. - pub beacon_chain_topic: String, } impl Default for Config { @@ -41,8 +41,6 @@ impl Default for Config { boot_nodes: vec![], client_version: version::version(), topics: Vec::new(), - beacon_chain_topic: String::from("beacon_chain"), - shard_prefix: String::from("attestations"), // single topic for all attestation for the moment. } } } diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 4bd775802..5597f9107 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -9,7 +9,7 @@ pub mod rpc; mod service; pub use behaviour::PubsubMessage; -pub use config::Config as NetworkConfig; +pub use config::{ChainType, Config as NetworkConfig, BEACON_PUBSUB_TOPIC, SHARD_TOPIC_PREFIX}; pub use libp2p::floodsub::{Topic, TopicBuilder, TopicHash}; pub use libp2p::multiaddr; pub use libp2p::Multiaddr; diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 99de38de6..9cbceda8d 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -4,6 +4,7 @@ use crate::multiaddr::Protocol; use crate::rpc::RPCEvent; use crate::NetworkConfig; use crate::{TopicBuilder, TopicHash}; +use crate::{BEACON_PUBSUB_TOPIC, SHARD_TOPIC_PREFIX}; use futures::prelude::*; use futures::Stream; use libp2p::core::{ @@ -88,8 +89,8 @@ impl Service { let mut topics = vec![]; //TODO: Handle multiple shard attestations. For now we simply use a separate topic for //attestations - topics.push(config.shard_prefix); - topics.push(config.beacon_chain_topic); + topics.push(SHARD_TOPIC_PREFIX.to_string()); + topics.push(BEACON_PUBSUB_TOPIC.to_string()); topics.append(&mut config.topics.clone()); diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index b805c1d75..d00c16292 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -4,6 +4,6 @@ pub mod message_handler; pub mod service; pub mod sync; -pub use eth2_libp2p::NetworkConfig; +pub use eth2_libp2p::{ChainType, NetworkConfig}; pub use service::NetworkMessage; pub use service::Service; diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index b9b05b7cd..86f4331f1 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,6 +1,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::PubsubMessage; use eth2_libp2p::TopicBuilder; +use eth2_libp2p::SHARD_TOPIC_PREFIX; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use network::NetworkMessage; @@ -137,11 +138,8 @@ impl AttestationService for AttestationServiceInstance { "type" => "valid_attestation", ); - // get the network topic to send on - let topic_string = self.chain.get_spec().shard_topic_prefix.clone(); - // valid attestation, propagate to the network - let topic = TopicBuilder::new(topic_string).build(); + let topic = TopicBuilder::new(SHARD_TOPIC_PREFIX).build(); let message = PubsubMessage::Attestation(attestation); self.network_chan diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index 791c5008d..cdf46a1ab 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -1,6 +1,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use crossbeam_channel; -use eth2_libp2p::PubsubMessage; +use eth2_libp2p::BEACON_PUBSUB_TOPIC; +use eth2_libp2p::{PubsubMessage, TopicBuilder}; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use network::NetworkMessage; @@ -105,8 +106,7 @@ impl BeaconBlockService for BeaconBlockServiceInstance { ); // get the network topic to send on - let topic_string = self.chain.get_spec().beacon_chain_topic.clone(); - let topic = types::TopicBuilder::new(topic_string).build(); + let topic = TopicBuilder::new(BEACON_PUBSUB_TOPIC).build(); let message = PubsubMessage::Block(block); // Publish the block to the p2p network via gossipsub. From 2d710f19fc883fc13439d745f84b50cde2463af6 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 17:13:55 +1100 Subject: [PATCH 24/72] Update to latest libp2p --- beacon_node/eth2-libp2p/Cargo.toml | 2 +- beacon_node/network/src/sync/simple_sync.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index ada5faebb..264d77d76 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" # SigP repository until PR is merged -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc89079499cc6e8a6bab0bffc3" } +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "fb852bcc2b9b3935555cc93930e913cbec2b0688" } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 0e2103900..99e427c8c 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -533,6 +533,11 @@ impl SimpleSync { // Add this block to the queue self.import_queue .enqueue_full_blocks(vec![block], peer_id.clone()); + trace!( + self.log, + "NewGossipBlock"; + "peer" => format!("{:?}", peer_id), + ); // Unless the parent is in the queue, request the parent block from the peer. // From be6ebb5ffa3fc9a1def43407eb7ba1d6f37f6284 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 3 Apr 2019 17:16:32 +1100 Subject: [PATCH 25/72] Add custom inactivity timeout to gossipsub --- beacon_node/eth2-libp2p/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 0757f1cbb..1a3f3ad3d 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -1,8 +1,8 @@ use clap::ArgMatches; use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; use serde_derive::{Deserialize, Serialize}; +use std::time::Duration; use types::multiaddr::{Error as MultiaddrError, Multiaddr}; -//use std::time::Duration; /// The beacon node topic string to subscribe to. pub const BEACON_PUBSUB_TOPIC: &str = "beacon_node"; @@ -35,7 +35,7 @@ impl Default for Config { listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000".to_string()], gs_config: GossipsubConfigBuilder::new() .max_gossip_size(4_000_000) - // .inactivity_timeout(Duration::from_secs(90)) + .inactivity_timeout(Duration::from_secs(90)) .build(), identify_config: IdentifyConfig::default(), boot_nodes: vec![], From a38f4c4cd1dc1a3c9a22b2c8747364b7316daed1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 9 Apr 2019 09:15:11 +1000 Subject: [PATCH 26/72] Adds Kademlia for peer discovery --- beacon_node/eth2-libp2p/Cargo.toml | 1 + beacon_node/eth2-libp2p/src/behaviour.rs | 72 ++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 264d77d76..31c0a7f2d 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -19,3 +19,4 @@ version = { path = "../version" } tokio = "0.1.16" futures = "0.1.25" error-chain = "0.12.0" +tokio-timer = "0.2.10" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index f362f5795..591d93bb5 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -9,14 +9,21 @@ use libp2p::{ }, gossipsub::{Gossipsub, GossipsubEvent}, identify::{protocol::IdentifyInfo, Identify, IdentifyEvent}, + kad::{Kademlia, KademliaOut}, ping::{Ping, PingEvent}, tokio_io::{AsyncRead, AsyncWrite}, NetworkBehaviour, PeerId, }; use slog::{debug, o, trace, warn}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; +use std::time::{Duration, Instant}; +use tokio_timer::Delay; +>>>>>>> Adds Kademlia for peer discovery use types::{Attestation, BeaconBlock}; +//TODO: Make this dynamic +const TIME_BETWEEN_KAD_REQUESTS: Duration = Duration::from_secs(30); + /// Builds the network behaviour for the libp2p Swarm. /// Implements gossipsub message routing. #[derive(NetworkBehaviour)] @@ -24,17 +31,20 @@ use types::{Attestation, BeaconBlock}; pub struct Behaviour { /// The routing pub-sub mechanism for eth2. gossipsub: Gossipsub, - // TODO: Add Kademlia for peer discovery /// The events generated by this behaviour to be consumed in the swarm poll. serenity_rpc: Rpc, /// Allows discovery of IP addresses for peers on the network. identify: Identify, /// Keep regular connection to peers and disconnect if absent. - // TODO: Keepalive, likely remove this later. - // TODO: Make the ping time customizeable. ping: Ping, + /// Kademlia for peer discovery. + kad: Kademlia, + /// Queue of behaviour events to be processed. #[behaviour(ignore)] events: Vec, + /// The delay until we next search for more peers. + #[behaviour(ignore)] + kad_delay: Delay, /// Logger for behaviour actions. #[behaviour(ignore)] log: slog::Logger, @@ -121,6 +131,33 @@ impl NetworkBehaviourEventProcess } } +// implement the kademlia behaviour +impl NetworkBehaviourEventProcess + for Behaviour +{ + fn inject_event(&mut self, out: KademliaOut) { + match out { + KademliaOut::Discovered { .. } => { + // send this to our topology behaviour + } + KademliaOut::KBucketAdded { .. } => { + // send this to our topology behaviour + } + KademliaOut::FindNodeResult { closer_peers, .. } => { + debug!( + self.log, + "Kademlia query found {} peers", + closer_peers.len() + ); + if closer_peers.is_empty() { + warn!(self.log, "Kademlia random query yielded empty results"); + } + } + KademliaOut::GetProvidersResult { .. } => (), + } + } +} + impl Behaviour { pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { let local_peer_id = local_public_key.clone().into_peer_id(); @@ -128,8 +165,9 @@ impl Behaviour { let behaviour_log = log.new(o!()); Behaviour { - gossipsub: Gossipsub::new(local_peer_id, net_conf.gs_config.clone()), serenity_rpc: Rpc::new(log), + gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), + kad: Kademlia::new(local_peer_id), identify: Identify::new( identify_config.version, identify_config.user_agent, @@ -137,6 +175,7 @@ impl Behaviour { ), ping: Ping::new(), events: Vec::new(), + kad_delay: Delay::new(Instant::now()), log: behaviour_log, } } @@ -149,6 +188,19 @@ impl Behaviour { return Async::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0))); } + // check to see if it's time to search for me peers with kademlia + loop { + match self.kad_delay.poll() { + Ok(Async::Ready(_)) => { + self.get_kad_peers(); + } + Ok(Async::NotReady) => break, + Err(e) => { + warn!(self.log, "Error getting peers from Kademlia. Err: {:?}", e); + } + } + } + Async::NotReady } } @@ -172,6 +224,18 @@ impl Behaviour { self.gossipsub.publish(topic, message_bytes.clone()); } } + + /// Queries for more peers randomly using Kademlia. + pub fn get_kad_peers(&mut self) { + // pick a random PeerId + let random_peer = PeerId::random(); + debug!(self.log, "Running kademlia random peer query"); + self.kad.find_node(random_peer); + + // update the kademlia timeout + self.kad_delay + .reset(Instant::now() + TIME_BETWEEN_KAD_REQUESTS); + } } /// The types of events than can be obtained from polling the behaviour. From e36fa3152d7ee975c323f0a3084faa3aacde8b88 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 15 Apr 2019 11:29:08 +1000 Subject: [PATCH 27/72] Adds verbosity cli flag --- beacon_node/Cargo.toml | 1 + beacon_node/client/Cargo.toml | 4 +++- beacon_node/client/src/client_config.rs | 11 ++------- beacon_node/eth2-libp2p/src/behaviour.rs | 3 ++- beacon_node/src/main.rs | 30 +++++++++++++++++------- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 309f162e5..0bc7de4cc 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -22,3 +22,4 @@ tokio-timer = "0.2.10" futures = "0.1.25" exit-future = "0.1.3" state_processing = { path = "../eth2/state_processing" } +slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 3b2e3ead8..7c5a67b89 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -19,8 +19,10 @@ slot_clock = { path = "../../eth2/utils/slot_clock" } serde = "1.0" serde_derive = "1.0" error-chain = "0.12.0" -slog = "^2.2.3" eth2_ssz = { path = "../../eth2/utils/ssz" } +slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } +slog-term = "^2.4.0" +slog-async = "^2.3.0" tokio = "0.1.15" clap = "2.32.0" dirs = "1.0.3" diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index c533cbcc8..4d0e286b0 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -7,7 +7,7 @@ use http_server::HttpServerConfig; use network::NetworkConfig; use network::{ChainType, NetworkConfig}; use serde_derive::{Deserialize, Serialize}; -use slog::error; +use slog::{error, o, Drain, Level}; use std::fs; use std::path::PathBuf; @@ -57,13 +57,6 @@ impl ClientConfig { .and_then(|path| Some(path.join(&self.db_name))) } - /// Returns the core path for the client. - pub fn data_dir(&self) -> Option { - let path = dirs::home_dir()?.join(&self.data_dir); - fs::create_dir_all(&path).ok()?; - Some(path) - } - /// Apply the following arguments to `self`, replacing values if they are specified in `args`. /// /// Returns an error if arguments are obviously invalid. May succeed even if some values are @@ -81,6 +74,6 @@ impl ClientConfig { self.rpc.apply_cli_args(args)?; self.http.apply_cli_args(args)?; - Ok(()) + Ok(log) } } diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 591d93bb5..b80881853 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -137,7 +137,8 @@ impl NetworkBehaviourEventProcess { + KademliaOut::Discovered { peer_id, .. } => { + debug!(self.log, "Kademlia peer discovered: {:?}", peer_id); // send this to our topology behaviour } KademliaOut::KBucketAdded { .. } => { diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index d6274befc..60b51303a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -1,5 +1,3 @@ -extern crate slog; - mod run; use clap::{App, Arg}; @@ -14,11 +12,6 @@ pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn main() { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::CompactFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - let logger = slog::Logger::root(drain, o!()); - let matches = App::new("Lighthouse") .version(version::version().as_str()) .author("Sigma Prime ") @@ -116,8 +109,29 @@ fn main() { .short("r") .help("When present, genesis will be within 30 minutes prior. Only for testing"), ) + .arg( + Arg::with_name("verbosity") + .short("v") + .multiple(true) + .help("Sets the verbosity level") + .takes_value(true), + ) .get_matches(); + // build the initial logger + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::CompactFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build(); + + let drain = match matches.occurrences_of("verbosity") { + 0 => drain.filter_level(Level::Info), + 1 => drain.filter_level(Level::Debug), + 2 => drain.filter_level(Level::Trace), + _ => drain.filter_level(Level::Info), + }; + + let logger = slog::Logger::root(drain.fuse(), o!()); + let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { Ok(dir) => dir, Err(e) => { @@ -128,7 +142,7 @@ fn main() { let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME); - // Attempt to lead the `ClientConfig` from disk. + // Attempt to load the `ClientConfig` from disk. // // If file doesn't exist, create a new, default one. let mut client_config = match read_from_file::(client_config_path.clone()) { From d2f80e3b2a649826ec3df6685b331a13460388a7 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 15 Apr 2019 13:58:41 +1000 Subject: [PATCH 28/72] Adds env logger to output libp2p logs --- beacon_node/Cargo.toml | 1 + beacon_node/src/main.rs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 0bc7de4cc..43e75d0a6 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -23,3 +23,4 @@ futures = "0.1.25" exit-future = "0.1.3" state_processing = { path = "../eth2/state_processing" } slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } +env_logger = "0.6.1" diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 60b51303a..96353ddf9 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -2,6 +2,7 @@ mod run; use clap::{App, Arg}; use client::{ClientConfig, Eth2Config}; +use env_logger::{Builder, Env}; use eth2_config::{get_data_dir, read_from_file, write_to_file}; use slog::{crit, o, Drain}; use std::path::PathBuf; @@ -12,6 +13,9 @@ pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn main() { + // debugging output for libp2p and external crates + Builder::from_env(Env::default()).init(); + let matches = App::new("Lighthouse") .version(version::version().as_str()) .author("Sigma Prime ") From f80c34b74fa1ca3ba1cfb48113c816170733770a Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 15 Apr 2019 18:29:49 +1000 Subject: [PATCH 29/72] Builds on discovery. Adds identify to discovery --- beacon_node/eth2-libp2p/Cargo.toml | 3 +- beacon_node/eth2-libp2p/src/behaviour.rs | 71 ++------- beacon_node/eth2-libp2p/src/config.rs | 1 + beacon_node/eth2-libp2p/src/discovery.rs | 182 +++++++++++++++++++++++ beacon_node/eth2-libp2p/src/lib.rs | 1 + 5 files changed, 199 insertions(+), 59 deletions(-) create mode 100644 beacon_node/eth2-libp2p/src/discovery.rs diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 31c0a7f2d..aded78b02 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -8,7 +8,8 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" # SigP repository until PR is merged -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "fb852bcc2b9b3935555cc93930e913cbec2b0688" } +#libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "fb852bcc2b9b3935555cc93930e913cbec2b0688" } +libp2p = { path = "../../../sharding/rust-libp2p" } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index b80881853..e952d1f81 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -1,3 +1,4 @@ +use crate::discovery::Discovery; use crate::rpc::{RPCEvent, RPCMessage, Rpc}; use crate::NetworkConfig; use crate::{Topic, TopicHash}; @@ -9,7 +10,7 @@ use libp2p::{ }, gossipsub::{Gossipsub, GossipsubEvent}, identify::{protocol::IdentifyInfo, Identify, IdentifyEvent}, - kad::{Kademlia, KademliaOut}, + kad::KademliaOut, ping::{Ping, PingEvent}, tokio_io::{AsyncRead, AsyncWrite}, NetworkBehaviour, PeerId, @@ -18,12 +19,8 @@ use slog::{debug, o, trace, warn}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; use std::time::{Duration, Instant}; use tokio_timer::Delay; ->>>>>>> Adds Kademlia for peer discovery use types::{Attestation, BeaconBlock}; -//TODO: Make this dynamic -const TIME_BETWEEN_KAD_REQUESTS: Duration = Duration::from_secs(30); - /// Builds the network behaviour for the libp2p Swarm. /// Implements gossipsub message routing. #[derive(NetworkBehaviour)] @@ -38,13 +35,10 @@ pub struct Behaviour { /// Keep regular connection to peers and disconnect if absent. ping: Ping, /// Kademlia for peer discovery. - kad: Kademlia, + discovery: Discovery, /// Queue of behaviour events to be processed. #[behaviour(ignore)] events: Vec, - /// The delay until we next search for more peers. - #[behaviour(ignore)] - kad_delay: Delay, /// Logger for behaviour actions. #[behaviour(ignore)] log: slog::Logger, @@ -116,6 +110,12 @@ impl NetworkBehaviourEventProcess format!("{:?}", peer_id), "Addresses" => format!("{:?}", info.listen_addrs)); + // inject the found addresses into our discovery behaviour + for address in &info.listen_addrs { + self.discovery + .add_connected_address(&peer_id, address.clone()); + } } IdentifyEvent::Error { .. } => {} IdentifyEvent::SendBack { .. } => {} @@ -131,31 +131,12 @@ impl NetworkBehaviourEventProcess } } -// implement the kademlia behaviour +// implement the discovery behaviour (currently kademlia) impl NetworkBehaviourEventProcess for Behaviour { - fn inject_event(&mut self, out: KademliaOut) { - match out { - KademliaOut::Discovered { peer_id, .. } => { - debug!(self.log, "Kademlia peer discovered: {:?}", peer_id); - // send this to our topology behaviour - } - KademliaOut::KBucketAdded { .. } => { - // send this to our topology behaviour - } - KademliaOut::FindNodeResult { closer_peers, .. } => { - debug!( - self.log, - "Kademlia query found {} peers", - closer_peers.len() - ); - if closer_peers.is_empty() { - warn!(self.log, "Kademlia random query yielded empty results"); - } - } - KademliaOut::GetProvidersResult { .. } => (), - } + fn inject_event(&mut self, _out: KademliaOut) { + // not interested in kademlia results at the moment } } @@ -168,7 +149,7 @@ impl Behaviour { Behaviour { serenity_rpc: Rpc::new(log), gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), - kad: Kademlia::new(local_peer_id), + discovery: Discovery::new(local_peer_id, log), identify: Identify::new( identify_config.version, identify_config.user_agent, @@ -176,7 +157,6 @@ impl Behaviour { ), ping: Ping::new(), events: Vec::new(), - kad_delay: Delay::new(Instant::now()), log: behaviour_log, } } @@ -189,19 +169,6 @@ impl Behaviour { return Async::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0))); } - // check to see if it's time to search for me peers with kademlia - loop { - match self.kad_delay.poll() { - Ok(Async::Ready(_)) => { - self.get_kad_peers(); - } - Ok(Async::NotReady) => break, - Err(e) => { - warn!(self.log, "Error getting peers from Kademlia. Err: {:?}", e); - } - } - } - Async::NotReady } } @@ -225,18 +192,6 @@ impl Behaviour { self.gossipsub.publish(topic, message_bytes.clone()); } } - - /// Queries for more peers randomly using Kademlia. - pub fn get_kad_peers(&mut self) { - // pick a random PeerId - let random_peer = PeerId::random(); - debug!(self.log, "Running kademlia random peer query"); - self.kad.find_node(random_peer); - - // update the kademlia timeout - self.kad_delay - .reset(Instant::now() + TIME_BETWEEN_KAD_REQUESTS); - } } /// The types of events than can be obtained from polling the behaviour. diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 1a3f3ad3d..b6857cd37 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -36,6 +36,7 @@ impl Default for Config { gs_config: GossipsubConfigBuilder::new() .max_gossip_size(4_000_000) .inactivity_timeout(Duration::from_secs(90)) + .heartbeat_interval(Duration::from_secs(20)) .build(), identify_config: IdentifyConfig::default(), boot_nodes: vec![], diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs new file mode 100644 index 000000000..232590c05 --- /dev/null +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -0,0 +1,182 @@ +/// This manages the discovery and management of peers. +/// +/// Currently using Kademlia for peer discovery. +/// +use futures::prelude::*; +use libp2p::core::swarm::{ + ConnectedPoint, NetworkBehaviour, NetworkBehaviourAction, PollParameters, +}; +use libp2p::core::{Multiaddr, PeerId, ProtocolsHandler}; +use libp2p::kad::{Kademlia, KademliaOut}; +use slog::{debug, o, warn}; +use std::collections::HashMap; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_timer::Delay; + +//TODO: Make this dynamic +const TIME_BETWEEN_KAD_REQUESTS: Duration = Duration::from_secs(30); + +/// Maintains a list of discovered peers and implements the discovery protocol to discover new +/// peers. +pub struct Discovery { + /// Queue of events to processed. + // TODO: Re-implement as discovery protocol grows + // events: Vec>, + /// The discovery behaviour used to discover new peers. + discovery: Kademlia, + /// The delay between peer discovery searches. + peer_discovery_delay: Delay, + /// Mapping of known addresses for peer ids. + known_peers: HashMap>, + /// Logger for the discovery behaviour. + log: slog::Logger, +} + +impl Discovery { + pub fn new(local_peer_id: PeerId, log: &slog::Logger) -> Self { + let log = log.new(o!("Service" => "Libp2p-Discovery")); + Self { + // events: Vec::new(), + discovery: Kademlia::new(local_peer_id), + peer_discovery_delay: Delay::new(Instant::now()), + known_peers: HashMap::new(), + log, + } + } + + /// Uses discovery to search for new peers. + pub fn find_peers(&mut self) { + // pick a random PeerId + let random_peer = PeerId::random(); + debug!(self.log, "Searching for peers..."); + self.discovery.find_node(random_peer); + + // update the kademlia timeout + self.peer_discovery_delay + .reset(Instant::now() + TIME_BETWEEN_KAD_REQUESTS); + } + + /// We have discovered an address for a peer, add it to known peers. + pub fn add_connected_address(&mut self, peer_id: &PeerId, address: Multiaddr) { + let known_peers = self + .known_peers + .entry(peer_id.clone()) + .or_insert_with(|| vec![]); + if !known_peers.contains(&address) { + known_peers.push(address.clone()); + } + // pass the address on to kademlia + self.discovery.add_connected_address(peer_id, address); + } +} + +// Redirect all behaviour event to underlying discovery behaviour. +impl NetworkBehaviour for Discovery +where + TSubstream: AsyncRead + AsyncWrite, +{ + type ProtocolsHandler = as NetworkBehaviour>::ProtocolsHandler; + type OutEvent = as NetworkBehaviour>::OutEvent; + + fn new_handler(&mut self) -> Self::ProtocolsHandler { + NetworkBehaviour::new_handler(&mut self.discovery) + } + + // TODO: we store all peers in known_peers, when upgrading to discv5 we will avoid duplication + // of peer storage. + fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec { + if let Some(addresses) = self.known_peers.get(peer_id) { + addresses.clone() + } else { + debug!( + self.log, + "Tried to dial: {:?} but no address stored", peer_id + ); + Vec::new() + } + } + + fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { + NetworkBehaviour::inject_connected(&mut self.discovery, peer_id, endpoint) + } + + fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { + NetworkBehaviour::inject_disconnected(&mut self.discovery, peer_id, endpoint) + } + + fn inject_replaced(&mut self, peer_id: PeerId, closed: ConnectedPoint, opened: ConnectedPoint) { + NetworkBehaviour::inject_replaced(&mut self.discovery, peer_id, closed, opened) + } + + fn inject_node_event( + &mut self, + peer_id: PeerId, + event: ::OutEvent, + ) { + // TODO: Upgrade to discv5 + NetworkBehaviour::inject_node_event(&mut self.discovery, peer_id, event) + } + + fn poll( + &mut self, + params: &mut PollParameters, + ) -> Async< + NetworkBehaviourAction< + ::InEvent, + Self::OutEvent, + >, + > { + // check to see if it's time to search for peers + loop { + match self.peer_discovery_delay.poll() { + Ok(Async::Ready(_)) => { + self.find_peers(); + } + Ok(Async::NotReady) => break, + Err(e) => { + warn!( + self.log, + "Error getting peers from discovery behaviour. Err: {:?}", e + ); + } + } + } + // Poll discovery + match self.discovery.poll(params) { + Async::Ready(action) => { + match &action { + NetworkBehaviourAction::GenerateEvent(disc_output) => match disc_output { + KademliaOut::Discovered { + peer_id, addresses, .. + } => { + debug!(self.log, "Kademlia peer discovered"; "Peer"=> format!("{:?}", peer_id), "Addresses" => format!("{:?}", addresses)); + (*self + .known_peers + .entry(peer_id.clone()) + .or_insert_with(|| vec![])) + .extend(addresses.clone()); + } + KademliaOut::FindNodeResult { closer_peers, .. } => { + debug!( + self.log, + "Kademlia query found {} peers", + closer_peers.len() + ); + if closer_peers.is_empty() { + debug!(self.log, "Kademlia random query yielded empty results"); + } + return Async::Ready(action); + } + _ => {} + }, + _ => {} + }; + return Async::Ready(action); + } + Async::NotReady => (), + } + + Async::NotReady + } +} diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 5597f9107..197c074df 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -4,6 +4,7 @@ /// This crate builds and manages the libp2p services required by the beacon node. pub mod behaviour; mod config; +mod discovery; pub mod error; pub mod rpc; mod service; From cb7d5eba1c3457409601329ed4889f8c6b1531f5 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 18 Apr 2019 15:26:30 +1000 Subject: [PATCH 30/72] Discovery and gossip bug fixes --- beacon_node/Cargo.toml | 2 +- beacon_node/eth2-libp2p/src/discovery.rs | 20 ++++---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 43e75d0a6..783cdcda3 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -22,5 +22,5 @@ tokio-timer = "0.2.10" futures = "0.1.25" exit-future = "0.1.3" state_processing = { path = "../eth2/state_processing" } -slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } +slog = "^2.2.3" env_logger = "0.6.1" diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 232590c05..d6fd43ef4 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -83,18 +83,9 @@ where NetworkBehaviour::new_handler(&mut self.discovery) } - // TODO: we store all peers in known_peers, when upgrading to discv5 we will avoid duplication - // of peer storage. fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec { - if let Some(addresses) = self.known_peers.get(peer_id) { - addresses.clone() - } else { - debug!( - self.log, - "Tried to dial: {:?} but no address stored", peer_id - ); - Vec::new() - } + // Let discovery track possible known peers. + self.discovery.addresses_of_peer(peer_id) } fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { @@ -151,11 +142,6 @@ where peer_id, addresses, .. } => { debug!(self.log, "Kademlia peer discovered"; "Peer"=> format!("{:?}", peer_id), "Addresses" => format!("{:?}", addresses)); - (*self - .known_peers - .entry(peer_id.clone()) - .or_insert_with(|| vec![])) - .extend(addresses.clone()); } KademliaOut::FindNodeResult { closer_peers, .. } => { debug!( @@ -163,6 +149,8 @@ where "Kademlia query found {} peers", closer_peers.len() ); + debug!(self.log, "Kademlia peers discovered"; "Peer"=> format!("{:?}", closer_peers)); + if closer_peers.is_empty() { debug!(self.log, "Kademlia random query yielded empty results"); } From b33ce5dd104dbd0caea2c2fab05ffbd35be07ea5 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 30 Apr 2019 15:12:57 +1000 Subject: [PATCH 31/72] Initial core grouping of libp2p behaviours --- beacon_node/eth2-libp2p/Cargo.toml | 2 +- beacon_node/eth2-libp2p/src/behaviour.rs | 48 +-- beacon_node/eth2-libp2p/src/core-behaviour.rs | 279 ++++++++++++++++++ beacon_node/eth2-libp2p/src/discovery.rs | 13 +- beacon_node/network/Cargo.toml | 2 +- 5 files changed, 310 insertions(+), 34 deletions(-) create mode 100644 beacon_node/eth2-libp2p/src/core-behaviour.rs diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index aded78b02..ef404e8b8 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -15,7 +15,7 @@ serde = "1.0" serde_derive = "1.0" eth2_ssz = { path = "../../eth2/utils/ssz" } eth2_ssz_derive = { path = "../../eth2/utils/ssz_derive" } -slog = "2.4.1" +slog = { version = "^2.4.1" , features = ["max_level_trace", "release_max_level_trace"] } version = { path = "../version" } tokio = "0.1.16" futures = "0.1.25" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index e952d1f81..7ddbd95b7 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -3,6 +3,7 @@ use crate::rpc::{RPCEvent, RPCMessage, Rpc}; use crate::NetworkConfig; use crate::{Topic, TopicHash}; use futures::prelude::*; +use libp2p::Multiaddr; use libp2p::{ core::{ swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess}, @@ -19,16 +20,18 @@ use slog::{debug, o, trace, warn}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; use std::time::{Duration, Instant}; use tokio_timer::Delay; +use std::collections::HashMap; use types::{Attestation, BeaconBlock}; -/// Builds the network behaviour for the libp2p Swarm. -/// Implements gossipsub message routing. +/// Builds the network behaviour that manages the core protocols of eth2. +/// This core behaviour is managed by `Behaviour` which adds peer management to all core +/// behaviours. #[derive(NetworkBehaviour)] -#[behaviour(out_event = "BehaviourEvent", poll_method = "poll")] -pub struct Behaviour { +#[behaviour(out_event = "CoreBehaviourEvent", poll_method = "poll")] +pub struct CoreCoreBehaviourTSubstream: AsyncRead + AsyncWrite> { /// The routing pub-sub mechanism for eth2. gossipsub: Gossipsub, - /// The events generated by this behaviour to be consumed in the swarm poll. + /// The serenity RPC specified in the wire-0 protocol. serenity_rpc: Rpc, /// Allows discovery of IP addresses for peers on the network. identify: Identify, @@ -36,9 +39,9 @@ pub struct Behaviour { ping: Ping, /// Kademlia for peer discovery. discovery: Discovery, - /// Queue of behaviour events to be processed. #[behaviour(ignore)] - events: Vec, + /// The events generated by this behaviour to be consumed in the swarm poll. + events: Vec, /// Logger for behaviour actions. #[behaviour(ignore)] log: slog::Logger, @@ -46,7 +49,7 @@ pub struct Behaviour { // Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for Behaviour impl NetworkBehaviourEventProcess - for Behaviour + for CoreBehaviourTSubstream> { fn inject_event(&mut self, event: GossipsubEvent) { match event { @@ -79,7 +82,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for Behaviour + for CoreBehaviourTSubstream> { fn inject_event(&mut self, event: RPCMessage) { match event { @@ -94,7 +97,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for Behaviour + for CoreBehaviourTSubstream> { fn inject_event(&mut self, event: IdentifyEvent) { match event { @@ -124,7 +127,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for Behaviour + for CoreBehaviourTSubstream> { fn inject_event(&mut self, _event: PingEvent) { // not interested in ping responses at the moment. @@ -133,14 +136,14 @@ impl NetworkBehaviourEventProcess // implement the discovery behaviour (currently kademlia) impl NetworkBehaviourEventProcess - for Behaviour + for CoreBehaviourTSubstream> { fn inject_event(&mut self, _out: KademliaOut) { // not interested in kademlia results at the moment } } -impl Behaviour { +impl CoreBehaviourTSubstream> { pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { let local_peer_id = local_public_key.clone().into_peer_id(); let identify_config = net_conf.identify_config.clone(); @@ -174,17 +177,14 @@ impl Behaviour { } /// Implements the combined behaviour for the libp2p service. -impl Behaviour { +impl CoreBehaviourTSubstream> { + /* Pubsub behaviour functions */ + /// Subscribes to a gossipsub topic. pub fn subscribe(&mut self, topic: Topic) -> bool { self.gossipsub.subscribe(topic) } - /// Sends an RPC Request/Response via the RPC protocol. - pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { - self.serenity_rpc.send_rpc(peer_id, rpc_event); - } - /// Publishes a message on the pubsub (gossipsub) behaviour. pub fn publish(&mut self, topics: Vec, message: PubsubMessage) { let message_bytes = ssz_encode(&message); @@ -192,10 +192,18 @@ impl Behaviour { self.gossipsub.publish(topic, message_bytes.clone()); } } + + /* Eth2 RPC behaviour functions */ + + /// Sends an RPC Request/Response via the RPC protocol. + pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { + self.serenity_rpc.send_rpc(peer_id, rpc_event); + } + } /// The types of events than can be obtained from polling the behaviour. -pub enum BehaviourEvent { +pub enum CoreBehaviourEvent { RPC(PeerId, RPCEvent), PeerDialed(PeerId), Identified(PeerId, Box), diff --git a/beacon_node/eth2-libp2p/src/core-behaviour.rs b/beacon_node/eth2-libp2p/src/core-behaviour.rs new file mode 100644 index 000000000..e59183b4c --- /dev/null +++ b/beacon_node/eth2-libp2p/src/core-behaviour.rs @@ -0,0 +1,279 @@ +use crate::discovery::Discovery; +use crate::rpc::{RPCEvent, RPCMessage, Rpc}; +use crate::NetworkConfig; +use crate::{Topic, TopicHash}; +use futures::prelude::*; +use libp2p::Multiaddr; +use libp2p::{ + core::{ + swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess}, + PublicKey, + }, + gossipsub::{Gossipsub, GossipsubEvent}, + identify::{protocol::IdentifyInfo, Identify, IdentifyEvent}, + kad::KademliaOut, + ping::{Ping, PingEvent}, + tokio_io::{AsyncRead, AsyncWrite}, + NetworkBehaviour, PeerId, +}; +use slog::{debug, o, trace, warn}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +use std::collections::HashMap; +use types::{Attestation, BeaconBlock}; + +/// Builds the network behaviour that manages the core protocols of eth2. +/// This core behaviour is managed by `Behaviour` which adds peer management to all core +/// behaviours. +#[derive(NetworkBehaviour)] +#[behaviour(out_event = "CoreBehaviourEvent", poll_method = "poll")] +pub struct CoreBehaviour { + /// The routing pub-sub mechanism for eth2. + gossipsub: Gossipsub, + /// The serenity RPC specified in the wire-0 protocol. + serenity_rpc: Rpc, + /// Allows discovery of IP addresses for peers on the network. + identify: Identify, + /// Keep regular connection to peers and disconnect if absent. + ping: Ping, + /// Kademlia for peer discovery. + discovery: Discovery, + #[behaviour(ignore)] + /// The events generated by this behaviour to be consumed by the global behaviour. + events: Vec, + /// Logger for behaviour actions. + #[behaviour(ignore)] + log: slog::Logger, +} + +impl CoreBehaviour { + pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { + let local_peer_id = local_public_key.clone().into_peer_id(); + let identify_config = net_conf.identify_config.clone(); + let behaviour_log = log.new(o!()); + + CoreBehaviour { + serenity_rpc: Rpc::new(log), + gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), + discovery: Discovery::new(local_peer_id, log), + identify: Identify::new( + identify_config.version, + identify_config.user_agent, + local_public_key, + ), + ping: Ping::new(), + events: Vec::new(), + log: behaviour_log, + } + } + + /// Consumes the events list when polled. + fn poll( + &mut self, + ) -> Async> { + if !self.events.is_empty() { + return Async::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0))); + } + + Async::NotReady + } +} + +// Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for CoreBehaviour +impl NetworkBehaviourEventProcess + for CoreBehaviour +{ + fn inject_event(&mut self, event: GossipsubEvent) { + match event { + GossipsubEvent::Message(gs_msg) => { + trace!(self.log, "Received GossipEvent"; "msg" => format!("{:?}", gs_msg)); + + let pubsub_message = match PubsubMessage::ssz_decode(&gs_msg.data, 0) { + //TODO: Punish peer on error + Err(e) => { + warn!( + self.log, + "Received undecodable message from Peer {:?} error", gs_msg.source; + "error" => format!("{:?}", e) + ); + return; + } + Ok((msg, _index)) => msg, + }; + + self.events.push(BehaviourEvent::GossipMessage { + source: gs_msg.source, + topics: gs_msg.topics, + message: pubsub_message, + }); + } + GossipsubEvent::Subscribed { + peer_id: _, + topic: _, + } + | GossipsubEvent::Unsubscribed { + peer_id: _, + topic: _, + } => {} + } + } +} + +impl NetworkBehaviourEventProcess + for CoreBehaviour +{ + fn inject_event(&mut self, event: RPCMessage) { + match event { + RPCMessage::PeerDialed(peer_id) => { + self.events.push(BehaviourEvent::PeerDialed(peer_id)) + } + RPCMessage::RPC(peer_id, rpc_event) => { + self.events.push(BehaviourEvent::RPC(peer_id, rpc_event)) + } + } + } +} + +impl NetworkBehaviourEventProcess + for CoreBehaviour +{ + fn inject_event(&mut self, event: IdentifyEvent) { + match event { + IdentifyEvent::Identified { + peer_id, mut info, .. + } => { + if info.listen_addrs.len() > 20 { + debug!( + self.log, + "More than 20 peers have been identified, truncating" + ); + info.listen_addrs.truncate(20); + } + trace!(self.log, "Found addresses"; "Peer Id" => format!("{:?}", peer_id), "Addresses" => format!("{:?}", info.listen_addrs)); + // inject the found addresses into our discovery behaviour + for address in &info.listen_addrs { + self.discovery + .add_connected_address(&peer_id, address.clone()); + } + self.events.push(BehaviourEvent::Identified(peer_id, info)); + } + IdentifyEvent::Error { .. } => {} + IdentifyEvent::SendBack { .. } => {} + } + } +} + +impl NetworkBehaviourEventProcess + for CoreBehaviour +{ + fn inject_event(&mut self, _event: PingEvent) { + // not interested in ping responses at the moment. + } +} + +// implement the discovery behaviour (currently kademlia) +impl NetworkBehaviourEventProcess + for CoreBehaviour +{ + fn inject_event(&mut self, _out: KademliaOut) { + // not interested in kademlia results at the moment + } +} + +/// Implements the combined behaviour for the libp2p service. +impl CoreBehaviour { + /* Pubsub behaviour functions */ + + /// Subscribes to a gossipsub topic. + pub fn subscribe(&mut self, topic: Topic) -> bool { + self.gossipsub.subscribe(topic) + } + + /// Publishes a message on the pubsub (gossipsub) behaviour. + pub fn publish(&mut self, topics: Vec, message: PubsubMessage) { + let message_bytes = ssz_encode(&message); + for topic in topics { + self.gossipsub.publish(topic, message_bytes.clone()); + } + } + + /* Eth2 RPC behaviour functions */ + + /// Sends an RPC Request/Response via the RPC protocol. + pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { + self.serenity_rpc.send_rpc(peer_id, rpc_event); + } +} + +/// The types of events than can be obtained from polling the behaviour. +pub enum CoreBehaviourEvent { + RPC(PeerId, RPCEvent), + PeerDialed(PeerId), + Identified(PeerId, IdentifyInfo), + // TODO: This is a stub at the moment + GossipMessage { + source: PeerId, + topics: Vec, + message: PubsubMessage, + }, +} + +/// Messages that are passed to and from the pubsub (Gossipsub) behaviour. +#[derive(Debug, Clone, PartialEq)] +pub enum PubsubMessage { + /// Gossipsub message providing notification of a new block. + Block(BeaconBlock), + /// Gossipsub message providing notification of a new attestation. + Attestation(Attestation), +} + +//TODO: Correctly encode/decode enums. Prefixing with integer for now. +impl Encodable for PubsubMessage { + fn ssz_append(&self, s: &mut SszStream) { + match self { + PubsubMessage::Block(block_gossip) => { + 0u32.ssz_append(s); + block_gossip.ssz_append(s); + } + PubsubMessage::Attestation(attestation_gossip) => { + 1u32.ssz_append(s); + attestation_gossip.ssz_append(s); + } + } + } +} + +impl Decodable for PubsubMessage { + fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { + let (id, index) = u32::ssz_decode(bytes, index)?; + match id { + 0 => { + let (block, index) = BeaconBlock::ssz_decode(bytes, index)?; + Ok((PubsubMessage::Block(block), index)) + } + 1 => { + let (attestation, index) = Attestation::ssz_decode(bytes, index)?; + Ok((PubsubMessage::Attestation(attestation), index)) + } + _ => Err(DecodeError::Invalid), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use types::*; + + #[test] + fn ssz_encoding() { + let original = PubsubMessage::Block(BeaconBlock::empty(&ChainSpec::foundation())); + + let encoded = ssz_encode(&original); + + println!("{:?}", encoded); + + let (decoded, _i) = PubsubMessage::ssz_decode(&encoded, 0).unwrap(); + + assert_eq!(original, decoded); + } +} diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index d6fd43ef4..dc91b487c 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -9,7 +9,6 @@ use libp2p::core::swarm::{ use libp2p::core::{Multiaddr, PeerId, ProtocolsHandler}; use libp2p::kad::{Kademlia, KademliaOut}; use slog::{debug, o, warn}; -use std::collections::HashMap; use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_timer::Delay; @@ -27,8 +26,6 @@ pub struct Discovery { discovery: Kademlia, /// The delay between peer discovery searches. peer_discovery_delay: Delay, - /// Mapping of known addresses for peer ids. - known_peers: HashMap>, /// Logger for the discovery behaviour. log: slog::Logger, } @@ -37,10 +34,8 @@ impl Discovery { pub fn new(local_peer_id: PeerId, log: &slog::Logger) -> Self { let log = log.new(o!("Service" => "Libp2p-Discovery")); Self { - // events: Vec::new(), discovery: Kademlia::new(local_peer_id), peer_discovery_delay: Delay::new(Instant::now()), - known_peers: HashMap::new(), log, } } @@ -59,13 +54,6 @@ impl Discovery { /// We have discovered an address for a peer, add it to known peers. pub fn add_connected_address(&mut self, peer_id: &PeerId, address: Multiaddr) { - let known_peers = self - .known_peers - .entry(peer_id.clone()) - .or_insert_with(|| vec![]); - if !known_peers.contains(&address) { - known_peers.push(address.clone()); - } // pass the address on to kademlia self.discovery.add_connected_address(peer_id, address); } @@ -160,6 +148,7 @@ where }, _ => {} }; + // propagate result upwards return Async::Ready(action); } Async::NotReady => (), diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 695adc9bd..23fbdd7d9 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -13,7 +13,7 @@ store = { path = "../store" } eth2-libp2p = { path = "../eth2-libp2p" } version = { path = "../version" } types = { path = "../../eth2/types" } -slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } +slog = { version = "^2.2.3" } eth2_ssz = { path = "../../eth2/utils/ssz" } tree_hash = { path = "../../eth2/utils/tree_hash" } futures = "0.1.25" From 43135484ca7f3831ea9e7c63e22e330a931c4644 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 13 May 2019 17:50:11 +1000 Subject: [PATCH 32/72] Update to lastest libp2p --- beacon_node/client/src/client_config.rs | 2 +- beacon_node/eth2-libp2p/src/behaviour.rs | 42 +-- beacon_node/eth2-libp2p/src/core-behaviour.rs | 279 ------------------ beacon_node/eth2-libp2p/src/service.rs | 9 +- 4 files changed, 31 insertions(+), 301 deletions(-) delete mode 100644 beacon_node/eth2-libp2p/src/core-behaviour.rs diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 4d0e286b0..74115c547 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -1,6 +1,5 @@ use clap::ArgMatches; use eth2_libp2p::multiaddr::Protocol; -use eth2_libp2p::multiaddr::ToMultiaddr; use eth2_libp2p::Multiaddr; use fork_choice::ForkChoiceAlgorithm; use http_server::HttpServerConfig; @@ -51,6 +50,7 @@ impl Default for ClientConfig { } impl ClientConfig { +<<<<<<< HEAD /// Returns the path to which the client may initialize an on-disk database. pub fn db_path(&self) -> Option { self.data_dir() diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 7ddbd95b7..58f603276 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -3,7 +3,6 @@ use crate::rpc::{RPCEvent, RPCMessage, Rpc}; use crate::NetworkConfig; use crate::{Topic, TopicHash}; use futures::prelude::*; -use libp2p::Multiaddr; use libp2p::{ core::{ swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess}, @@ -12,23 +11,24 @@ use libp2p::{ gossipsub::{Gossipsub, GossipsubEvent}, identify::{protocol::IdentifyInfo, Identify, IdentifyEvent}, kad::KademliaOut, - ping::{Ping, PingEvent}, + ping::{Ping, PingConfig, PingEvent}, tokio_io::{AsyncRead, AsyncWrite}, NetworkBehaviour, PeerId, }; use slog::{debug, o, trace, warn}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; +use std::collections::HashMap; +use std::num::NonZeroU32; use std::time::{Duration, Instant}; use tokio_timer::Delay; -use std::collections::HashMap; use types::{Attestation, BeaconBlock}; /// Builds the network behaviour that manages the core protocols of eth2. /// This core behaviour is managed by `Behaviour` which adds peer management to all core /// behaviours. #[derive(NetworkBehaviour)] -#[behaviour(out_event = "CoreBehaviourEvent", poll_method = "poll")] -pub struct CoreCoreBehaviourTSubstream: AsyncRead + AsyncWrite> { +#[behaviour(out_event = "BehaviourEvent", poll_method = "poll")] +pub struct Behaviour { /// The routing pub-sub mechanism for eth2. gossipsub: Gossipsub, /// The serenity RPC specified in the wire-0 protocol. @@ -41,7 +41,7 @@ pub struct CoreCoreBehaviourTSubstream: AsyncRead + AsyncWrite> { discovery: Discovery, #[behaviour(ignore)] /// The events generated by this behaviour to be consumed in the swarm poll. - events: Vec, + events: Vec, /// Logger for behaviour actions. #[behaviour(ignore)] log: slog::Logger, @@ -49,7 +49,7 @@ pub struct CoreCoreBehaviourTSubstream: AsyncRead + AsyncWrite> { // Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for Behaviour impl NetworkBehaviourEventProcess - for CoreBehaviourTSubstream> + for Behaviour { fn inject_event(&mut self, event: GossipsubEvent) { match event { @@ -82,7 +82,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for CoreBehaviourTSubstream> + for Behaviour { fn inject_event(&mut self, event: RPCMessage) { match event { @@ -97,7 +97,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for CoreBehaviourTSubstream> + for Behaviour { fn inject_event(&mut self, event: IdentifyEvent) { match event { @@ -127,7 +127,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for CoreBehaviourTSubstream> + for Behaviour { fn inject_event(&mut self, _event: PingEvent) { // not interested in ping responses at the moment. @@ -136,19 +136,28 @@ impl NetworkBehaviourEventProcess // implement the discovery behaviour (currently kademlia) impl NetworkBehaviourEventProcess - for CoreBehaviourTSubstream> + for Behaviour { fn inject_event(&mut self, _out: KademliaOut) { // not interested in kademlia results at the moment } } -impl CoreBehaviourTSubstream> { +impl Behaviour { pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { let local_peer_id = local_public_key.clone().into_peer_id(); - let identify_config = net_conf.identify_config.clone(); let behaviour_log = log.new(o!()); + // identify configuration + let identify_config = net_conf.identify_config.clone(); + + // ping configuration + let ping_config = PingConfig::new() + .with_timeout(Duration::from_secs(30)) + .with_interval(Duration::from_secs(20)) + .with_max_failures(NonZeroU32::new(2).expect("2 != 0")) + .with_keep_alive(false); + Behaviour { serenity_rpc: Rpc::new(log), gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), @@ -158,7 +167,7 @@ impl CoreBehaviourTSubstream> { identify_config.user_agent, local_public_key, ), - ping: Ping::new(), + ping: Ping::new(ping_config), events: Vec::new(), log: behaviour_log, } @@ -177,7 +186,7 @@ impl CoreBehaviourTSubstream> { } /// Implements the combined behaviour for the libp2p service. -impl CoreBehaviourTSubstream> { +impl Behaviour { /* Pubsub behaviour functions */ /// Subscribes to a gossipsub topic. @@ -199,11 +208,10 @@ impl CoreBehaviourTSubstream> { pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { self.serenity_rpc.send_rpc(peer_id, rpc_event); } - } /// The types of events than can be obtained from polling the behaviour. -pub enum CoreBehaviourEvent { +pub enum BehaviourEvent { RPC(PeerId, RPCEvent), PeerDialed(PeerId), Identified(PeerId, Box), diff --git a/beacon_node/eth2-libp2p/src/core-behaviour.rs b/beacon_node/eth2-libp2p/src/core-behaviour.rs deleted file mode 100644 index e59183b4c..000000000 --- a/beacon_node/eth2-libp2p/src/core-behaviour.rs +++ /dev/null @@ -1,279 +0,0 @@ -use crate::discovery::Discovery; -use crate::rpc::{RPCEvent, RPCMessage, Rpc}; -use crate::NetworkConfig; -use crate::{Topic, TopicHash}; -use futures::prelude::*; -use libp2p::Multiaddr; -use libp2p::{ - core::{ - swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess}, - PublicKey, - }, - gossipsub::{Gossipsub, GossipsubEvent}, - identify::{protocol::IdentifyInfo, Identify, IdentifyEvent}, - kad::KademliaOut, - ping::{Ping, PingEvent}, - tokio_io::{AsyncRead, AsyncWrite}, - NetworkBehaviour, PeerId, -}; -use slog::{debug, o, trace, warn}; -use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; -use std::collections::HashMap; -use types::{Attestation, BeaconBlock}; - -/// Builds the network behaviour that manages the core protocols of eth2. -/// This core behaviour is managed by `Behaviour` which adds peer management to all core -/// behaviours. -#[derive(NetworkBehaviour)] -#[behaviour(out_event = "CoreBehaviourEvent", poll_method = "poll")] -pub struct CoreBehaviour { - /// The routing pub-sub mechanism for eth2. - gossipsub: Gossipsub, - /// The serenity RPC specified in the wire-0 protocol. - serenity_rpc: Rpc, - /// Allows discovery of IP addresses for peers on the network. - identify: Identify, - /// Keep regular connection to peers and disconnect if absent. - ping: Ping, - /// Kademlia for peer discovery. - discovery: Discovery, - #[behaviour(ignore)] - /// The events generated by this behaviour to be consumed by the global behaviour. - events: Vec, - /// Logger for behaviour actions. - #[behaviour(ignore)] - log: slog::Logger, -} - -impl CoreBehaviour { - pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { - let local_peer_id = local_public_key.clone().into_peer_id(); - let identify_config = net_conf.identify_config.clone(); - let behaviour_log = log.new(o!()); - - CoreBehaviour { - serenity_rpc: Rpc::new(log), - gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), - discovery: Discovery::new(local_peer_id, log), - identify: Identify::new( - identify_config.version, - identify_config.user_agent, - local_public_key, - ), - ping: Ping::new(), - events: Vec::new(), - log: behaviour_log, - } - } - - /// Consumes the events list when polled. - fn poll( - &mut self, - ) -> Async> { - if !self.events.is_empty() { - return Async::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0))); - } - - Async::NotReady - } -} - -// Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for CoreBehaviour -impl NetworkBehaviourEventProcess - for CoreBehaviour -{ - fn inject_event(&mut self, event: GossipsubEvent) { - match event { - GossipsubEvent::Message(gs_msg) => { - trace!(self.log, "Received GossipEvent"; "msg" => format!("{:?}", gs_msg)); - - let pubsub_message = match PubsubMessage::ssz_decode(&gs_msg.data, 0) { - //TODO: Punish peer on error - Err(e) => { - warn!( - self.log, - "Received undecodable message from Peer {:?} error", gs_msg.source; - "error" => format!("{:?}", e) - ); - return; - } - Ok((msg, _index)) => msg, - }; - - self.events.push(BehaviourEvent::GossipMessage { - source: gs_msg.source, - topics: gs_msg.topics, - message: pubsub_message, - }); - } - GossipsubEvent::Subscribed { - peer_id: _, - topic: _, - } - | GossipsubEvent::Unsubscribed { - peer_id: _, - topic: _, - } => {} - } - } -} - -impl NetworkBehaviourEventProcess - for CoreBehaviour -{ - fn inject_event(&mut self, event: RPCMessage) { - match event { - RPCMessage::PeerDialed(peer_id) => { - self.events.push(BehaviourEvent::PeerDialed(peer_id)) - } - RPCMessage::RPC(peer_id, rpc_event) => { - self.events.push(BehaviourEvent::RPC(peer_id, rpc_event)) - } - } - } -} - -impl NetworkBehaviourEventProcess - for CoreBehaviour -{ - fn inject_event(&mut self, event: IdentifyEvent) { - match event { - IdentifyEvent::Identified { - peer_id, mut info, .. - } => { - if info.listen_addrs.len() > 20 { - debug!( - self.log, - "More than 20 peers have been identified, truncating" - ); - info.listen_addrs.truncate(20); - } - trace!(self.log, "Found addresses"; "Peer Id" => format!("{:?}", peer_id), "Addresses" => format!("{:?}", info.listen_addrs)); - // inject the found addresses into our discovery behaviour - for address in &info.listen_addrs { - self.discovery - .add_connected_address(&peer_id, address.clone()); - } - self.events.push(BehaviourEvent::Identified(peer_id, info)); - } - IdentifyEvent::Error { .. } => {} - IdentifyEvent::SendBack { .. } => {} - } - } -} - -impl NetworkBehaviourEventProcess - for CoreBehaviour -{ - fn inject_event(&mut self, _event: PingEvent) { - // not interested in ping responses at the moment. - } -} - -// implement the discovery behaviour (currently kademlia) -impl NetworkBehaviourEventProcess - for CoreBehaviour -{ - fn inject_event(&mut self, _out: KademliaOut) { - // not interested in kademlia results at the moment - } -} - -/// Implements the combined behaviour for the libp2p service. -impl CoreBehaviour { - /* Pubsub behaviour functions */ - - /// Subscribes to a gossipsub topic. - pub fn subscribe(&mut self, topic: Topic) -> bool { - self.gossipsub.subscribe(topic) - } - - /// Publishes a message on the pubsub (gossipsub) behaviour. - pub fn publish(&mut self, topics: Vec, message: PubsubMessage) { - let message_bytes = ssz_encode(&message); - for topic in topics { - self.gossipsub.publish(topic, message_bytes.clone()); - } - } - - /* Eth2 RPC behaviour functions */ - - /// Sends an RPC Request/Response via the RPC protocol. - pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { - self.serenity_rpc.send_rpc(peer_id, rpc_event); - } -} - -/// The types of events than can be obtained from polling the behaviour. -pub enum CoreBehaviourEvent { - RPC(PeerId, RPCEvent), - PeerDialed(PeerId), - Identified(PeerId, IdentifyInfo), - // TODO: This is a stub at the moment - GossipMessage { - source: PeerId, - topics: Vec, - message: PubsubMessage, - }, -} - -/// Messages that are passed to and from the pubsub (Gossipsub) behaviour. -#[derive(Debug, Clone, PartialEq)] -pub enum PubsubMessage { - /// Gossipsub message providing notification of a new block. - Block(BeaconBlock), - /// Gossipsub message providing notification of a new attestation. - Attestation(Attestation), -} - -//TODO: Correctly encode/decode enums. Prefixing with integer for now. -impl Encodable for PubsubMessage { - fn ssz_append(&self, s: &mut SszStream) { - match self { - PubsubMessage::Block(block_gossip) => { - 0u32.ssz_append(s); - block_gossip.ssz_append(s); - } - PubsubMessage::Attestation(attestation_gossip) => { - 1u32.ssz_append(s); - attestation_gossip.ssz_append(s); - } - } - } -} - -impl Decodable for PubsubMessage { - fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> { - let (id, index) = u32::ssz_decode(bytes, index)?; - match id { - 0 => { - let (block, index) = BeaconBlock::ssz_decode(bytes, index)?; - Ok((PubsubMessage::Block(block), index)) - } - 1 => { - let (attestation, index) = Attestation::ssz_decode(bytes, index)?; - Ok((PubsubMessage::Attestation(attestation), index)) - } - _ => Err(DecodeError::Invalid), - } - } -} - -#[cfg(test)] -mod test { - use super::*; - use types::*; - - #[test] - fn ssz_encoding() { - let original = PubsubMessage::Block(BeaconBlock::empty(&ChainSpec::foundation())); - - let encoded = ssz_encode(&original); - - println!("{:?}", encoded); - - let (decoded, _i) = PubsubMessage::ssz_decode(&encoded, 0).unwrap(); - - assert_eq!(original, decoded); - } -} diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 9cbceda8d..68ca72620 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -63,11 +63,12 @@ impl Service { .map_err(|e| format!("Invalid listen multiaddr: {}", e))? { match Swarm::listen_on(&mut swarm, address.clone()) { - Ok(mut listen_addr) => { - listen_addr.append(Protocol::P2p(local_peer_id.clone().into())); - info!(log, "Listening on: {}", listen_addr); + Ok(_) => { + let mut log_address = address.clone(); + log_address.push(Protocol::P2p(local_peer_id.clone().into())); + info!(log, "Listening on: {}", log_address); } - Err(err) => warn!(log, "Cannot listen on: {} : {:?}", address, err), + Err(err) => warn!(log, "Cannot listen on: {} because: {:?}", address, err), }; } // connect to boot nodes - these are currently stored as multiaddrs From c7e17c86414b56457ca94cdd52bc81735e91e781 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 20 Jun 2019 22:43:50 +1000 Subject: [PATCH 33/72] Updates for latest master --- beacon_node/client/src/client_config.rs | 23 ++++++++--------------- beacon_node/eth2-libp2p/Cargo.toml | 5 ++--- beacon_node/eth2-libp2p/src/behaviour.rs | 4 ---- eth2/types/src/chain_spec.rs | 4 +++- 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 74115c547..93ff5f7eb 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -23,19 +23,6 @@ pub struct ClientConfig { impl Default for ClientConfig { fn default() -> Self { - let data_dir = { - let home = dirs::home_dir().expect("Unable to determine home dir."); - home.join(".lighthouse/") - }; - fs::create_dir_all(&data_dir) - .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); - - // currently lighthouse spec - let default_spec = ChainSpec::lighthouse_testnet(); - let chain_type = ChainType::from(default_spec.chain_id); - // builds a chain-specific network config - let net_conf = NetworkConfig::from(chain_type); - Self { data_dir: PathBuf::from(".lighthouse"), db_type: "disk".to_string(), @@ -50,13 +37,19 @@ impl Default for ClientConfig { } impl ClientConfig { -<<<<<<< HEAD /// Returns the path to which the client may initialize an on-disk database. pub fn db_path(&self) -> Option { self.data_dir() .and_then(|path| Some(path.join(&self.db_name))) } + /// Returns the core path for the client. + pub fn data_dir(&self) -> Option { + let path = dirs::home_dir()?.join(&self.data_dir); + fs::create_dir_all(&path).ok()?; + Some(path) + } + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. /// /// Returns an error if arguments are obviously invalid. May succeed even if some values are @@ -74,6 +67,6 @@ impl ClientConfig { self.rpc.apply_cli_args(args)?; self.http.apply_cli_args(args)?; - Ok(log) + Ok(()) } } diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index ef404e8b8..3fc327e99 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -7,9 +7,8 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" -# SigP repository until PR is merged -#libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "fb852bcc2b9b3935555cc93930e913cbec2b0688" } -libp2p = { path = "../../../sharding/rust-libp2p" } +# SigP repository +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "8d5e5bbbe32d07ad271d6a2e15fde0347894061a" } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 58f603276..ed466bb75 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -147,11 +147,7 @@ impl Behaviour { pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { let local_peer_id = local_public_key.clone().into_peer_id(); let behaviour_log = log.new(o!()); - - // identify configuration let identify_config = net_conf.identify_config.clone(); - - // ping configuration let ping_config = PingConfig::new() .with_timeout(Duration::from_secs(30)) .with_interval(Duration::from_secs(20)) diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 8e4bd9c9c..d35f696b9 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -107,6 +107,7 @@ pub struct ChainSpec { /* * Network specific parameters */ + pub boot_nodes: Vec, pub chain_id: u8, } @@ -216,7 +217,8 @@ impl ChainSpec { /* * Network specific */ - chain_id: 1, // foundation chain id + boot_nodes: vec![], + chain_id: 1, // mainnet chain id } } From 6ee2b4df34bf84a2c9701279d4b196043a446312 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Sun, 23 Jun 2019 12:34:00 +1000 Subject: [PATCH 34/72] Complete merging of network addition branch --- beacon_node/Cargo.toml | 1 - beacon_node/client/src/client_config.rs | 3 +-- beacon_node/eth2-libp2p/Cargo.toml | 2 +- beacon_node/eth2-libp2p/src/behaviour.rs | 6 ++++-- beacon_node/eth2-libp2p/src/config.rs | 2 +- beacon_node/eth2-libp2p/src/discovery.rs | 2 +- beacon_node/network/src/sync/simple_sync.rs | 1 - beacon_node/rpc/src/lib.rs | 1 - beacon_node/src/main.rs | 2 +- eth2/types/Cargo.toml | 1 - eth2/types/src/chain_spec.rs | 13 +++---------- 11 files changed, 12 insertions(+), 22 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 783cdcda3..7e43a13df 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -22,5 +22,4 @@ tokio-timer = "0.2.10" futures = "0.1.25" exit-future = "0.1.3" state_processing = { path = "../eth2/state_processing" } -slog = "^2.2.3" env_logger = "0.6.1" diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 93ff5f7eb..f2f356daf 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -3,7 +3,6 @@ use eth2_libp2p::multiaddr::Protocol; use eth2_libp2p::Multiaddr; use fork_choice::ForkChoiceAlgorithm; use http_server::HttpServerConfig; -use network::NetworkConfig; use network::{ChainType, NetworkConfig}; use serde_derive::{Deserialize, Serialize}; use slog::{error, o, Drain, Level}; @@ -29,7 +28,7 @@ impl Default for ClientConfig { db_name: "chain_db".to_string(), // Note: there are no default bootnodes specified. // Once bootnodes are established, add them here. - network: NetworkConfig::new(vec![]), + network: NetworkConfig::new(), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), } diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 3fc327e99..6fd141028 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" # SigP repository -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "8d5e5bbbe32d07ad271d6a2e15fde0347894061a" } +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "71744d4090ebd93a993d1b390787919add4098fd" } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index ed466bb75..c711a2134 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -111,14 +111,16 @@ impl NetworkBehaviourEventProcess format!("{:?}", peer_id), "Addresses" => format!("{:?}", info.listen_addrs)); // inject the found addresses into our discovery behaviour + for address in &info.listen_addrs { self.discovery .add_connected_address(&peer_id, address.clone()); } + + self.events + .push(BehaviourEvent::Identified(peer_id, Box::new(info))); } IdentifyEvent::Error { .. } => {} IdentifyEvent::SendBack { .. } => {} diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index b6857cd37..baa1bf47f 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -1,8 +1,8 @@ use clap::ArgMatches; use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; +use libp2p::multiaddr::{Error as MultiaddrError, Multiaddr}; use serde_derive::{Deserialize, Serialize}; use std::time::Duration; -use types::multiaddr::{Error as MultiaddrError, Multiaddr}; /// The beacon node topic string to subscribe to. pub const BEACON_PUBSUB_TOPIC: &str = "beacon_node"; diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index dc91b487c..c3a02b16f 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -55,7 +55,7 @@ impl Discovery { /// We have discovered an address for a peer, add it to known peers. pub fn add_connected_address(&mut self, peer_id: &PeerId, address: Multiaddr) { // pass the address on to kademlia - self.discovery.add_connected_address(peer_id, address); + self.discovery.add_address(peer_id, address); } } diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 99e427c8c..5899e5aea 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -5,7 +5,6 @@ use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, error, info, o, trace, warn}; -use ssz::TreeHash; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index 4506d90fc..3e6fd3e73 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -62,7 +62,6 @@ pub fn start_server( let instance = AttestationServiceInstance { network_chan, chain: beacon_chain.clone(), - network_chan, log: log.clone(), }; create_attestation_service(instance) diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 96353ddf9..07215119f 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -4,7 +4,7 @@ use clap::{App, Arg}; use client::{ClientConfig, Eth2Config}; use env_logger::{Builder, Env}; use eth2_config::{get_data_dir, read_from_file, write_to_file}; -use slog::{crit, o, Drain}; +use slog::{crit, o, Drain, Level}; use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index ed6307684..fd6578340 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -32,7 +32,6 @@ swap_or_not_shuffle = { path = "../utils/swap_or_not_shuffle" } test_random_derive = { path = "../utils/test_random_derive" } tree_hash = { path = "../utils/tree_hash" } tree_hash_derive = { path = "../utils/tree_hash_derive" } -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc89079499cc6e8a6bab0bffc3" } [dev-dependencies] env_logger = "0.6.0" diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index d35f696b9..6073fb32e 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -104,10 +104,7 @@ pub struct ChainSpec { domain_voluntary_exit: u32, domain_transfer: u32, - /* - * Network specific parameters - */ - pub boot_nodes: Vec, + pub boot_nodes: Vec, pub chain_id: u8, } @@ -230,12 +227,8 @@ impl ChainSpec { pub fn minimal() -> Self { let genesis_slot = Slot::new(0); - // Note: these bootnodes are placeholders. - // - // Should be updated once static bootnodes exist. - let boot_nodes = vec!["/ip4/127.0.0.1/tcp/9000" - .parse() - .expect("correct multiaddr")]; + // Note: bootnodes to be updated when static nodes exist. + let boot_nodes = vec![]; Self { target_committee_size: 4, From 44c9058477792d1fc15e6e0d68b44464d4a55d81 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 25 Jun 2019 14:51:45 +1000 Subject: [PATCH 35/72] Integrate discv5 into lighthouse --- beacon_node/client/Cargo.toml | 2 +- beacon_node/client/src/client_config.rs | 71 ------- beacon_node/client/src/lib.rs | 4 +- beacon_node/eth2-libp2p/Cargo.toml | 3 +- beacon_node/eth2-libp2p/src/behaviour.rs | 116 ++++------- beacon_node/eth2-libp2p/src/config.rs | 143 ++++++------- beacon_node/eth2-libp2p/src/discovery.rs | 246 ++++++++++++++++------- beacon_node/eth2-libp2p/src/lib.rs | 4 +- beacon_node/eth2-libp2p/src/service.rs | 46 +---- beacon_node/network/src/lib.rs | 2 +- beacon_node/src/main.rs | 24 ++- beacon_node/src/run.rs | 2 +- 12 files changed, 312 insertions(+), 351 deletions(-) delete mode 100644 beacon_node/client/src/client_config.rs diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 7c5a67b89..94a529ea7 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -16,7 +16,7 @@ types = { path = "../../eth2/types" } tree_hash = { path = "../../eth2/utils/tree_hash" } eth2_config = { path = "../../eth2/utils/eth2_config" } slot_clock = { path = "../../eth2/utils/slot_clock" } -serde = "1.0" +serde = "1.0.93" serde_derive = "1.0" error-chain = "0.12.0" eth2_ssz = { path = "../../eth2/utils/ssz" } diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs deleted file mode 100644 index f2f356daf..000000000 --- a/beacon_node/client/src/client_config.rs +++ /dev/null @@ -1,71 +0,0 @@ -use clap::ArgMatches; -use eth2_libp2p::multiaddr::Protocol; -use eth2_libp2p::Multiaddr; -use fork_choice::ForkChoiceAlgorithm; -use http_server::HttpServerConfig; -use network::{ChainType, NetworkConfig}; -use serde_derive::{Deserialize, Serialize}; -use slog::{error, o, Drain, Level}; -use std::fs; -use std::path::PathBuf; - -/// The core configuration of a Lighthouse beacon node. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ClientConfig { - pub data_dir: PathBuf, - pub db_type: String, - db_name: String, - pub network: network::NetworkConfig, - pub rpc: rpc::RPCConfig, - pub http: HttpServerConfig, -} - -impl Default for ClientConfig { - fn default() -> Self { - Self { - data_dir: PathBuf::from(".lighthouse"), - db_type: "disk".to_string(), - db_name: "chain_db".to_string(), - // Note: there are no default bootnodes specified. - // Once bootnodes are established, add them here. - network: NetworkConfig::new(), - rpc: rpc::RPCConfig::default(), - http: HttpServerConfig::default(), - } - } -} - -impl ClientConfig { - /// Returns the path to which the client may initialize an on-disk database. - pub fn db_path(&self) -> Option { - self.data_dir() - .and_then(|path| Some(path.join(&self.db_name))) - } - - /// Returns the core path for the client. - pub fn data_dir(&self) -> Option { - let path = dirs::home_dir()?.join(&self.data_dir); - fs::create_dir_all(&path).ok()?; - Some(path) - } - - /// Apply the following arguments to `self`, replacing values if they are specified in `args`. - /// - /// Returns an error if arguments are obviously invalid. May succeed even if some values are - /// invalid. - pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { - if let Some(dir) = args.value_of("datadir") { - self.data_dir = PathBuf::from(dir); - }; - - if let Some(dir) = args.value_of("db") { - self.db_type = dir.to_string(); - } - - self.network.apply_cli_args(args)?; - self.rpc.apply_cli_args(args)?; - self.http.apply_cli_args(args)?; - - Ok(()) - } -} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 18ddef7bb..7eee8ac0a 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -1,7 +1,7 @@ extern crate slog; mod beacon_chain_types; -mod client_config; +mod config; pub mod error; pub mod notifier; @@ -21,7 +21,7 @@ use tokio::timer::Interval; pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; -pub use client_config::ClientConfig; +pub use config::Config as ClientConfig; pub use eth2_config::Eth2Config; /// Main beacon node client service. This provides the connection and initialisation of the clients diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 6fd141028..fd7162767 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -8,7 +8,8 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" # SigP repository -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "71744d4090ebd93a993d1b390787919add4098fd" } +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "f018f5c443ed5a93de890048dbc6755393373e72" } +enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "f018f5c443ed5a93de890048dbc6755393373e72", features = ["serde"] } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index c711a2134..4e4cf24f3 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -1,26 +1,23 @@ use crate::discovery::Discovery; use crate::rpc::{RPCEvent, RPCMessage, Rpc}; -use crate::NetworkConfig; +use crate::{error, NetworkConfig}; use crate::{Topic, TopicHash}; use futures::prelude::*; use libp2p::{ core::{ + identity::Keypair, swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess}, - PublicKey, }, + discv5::Discv5Event, gossipsub::{Gossipsub, GossipsubEvent}, - identify::{protocol::IdentifyInfo, Identify, IdentifyEvent}, - kad::KademliaOut, ping::{Ping, PingConfig, PingEvent}, tokio_io::{AsyncRead, AsyncWrite}, NetworkBehaviour, PeerId, }; -use slog::{debug, o, trace, warn}; +use slog::{o, trace, warn}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; -use std::collections::HashMap; use std::num::NonZeroU32; -use std::time::{Duration, Instant}; -use tokio_timer::Delay; +use std::time::Duration; use types::{Attestation, BeaconBlock}; /// Builds the network behaviour that manages the core protocols of eth2. @@ -33,8 +30,6 @@ pub struct Behaviour { gossipsub: Gossipsub, /// The serenity RPC specified in the wire-0 protocol. serenity_rpc: Rpc, - /// Allows discovery of IP addresses for peers on the network. - identify: Identify, /// Keep regular connection to peers and disconnect if absent. ping: Ping, /// Kademlia for peer discovery. @@ -47,6 +42,31 @@ pub struct Behaviour { log: slog::Logger, } +impl Behaviour { + pub fn new( + local_key: &Keypair, + net_conf: &NetworkConfig, + log: &slog::Logger, + ) -> error::Result { + let local_peer_id = local_key.public().clone().into_peer_id(); + let behaviour_log = log.new(o!()); + let ping_config = PingConfig::new() + .with_timeout(Duration::from_secs(30)) + .with_interval(Duration::from_secs(20)) + .with_max_failures(NonZeroU32::new(2).expect("2 != 0")) + .with_keep_alive(false); + + Ok(Behaviour { + serenity_rpc: Rpc::new(log), + gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), + discovery: Discovery::new(local_key, net_conf, log)?, + ping: Ping::new(ping_config), + events: Vec::new(), + log: behaviour_log, + }) + } +} + // Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for Behaviour impl NetworkBehaviourEventProcess for Behaviour @@ -96,38 +116,6 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for Behaviour -{ - fn inject_event(&mut self, event: IdentifyEvent) { - match event { - IdentifyEvent::Identified { - peer_id, mut info, .. - } => { - if info.listen_addrs.len() > 20 { - debug!( - self.log, - "More than 20 peers have been identified, truncating" - ); - info.listen_addrs.truncate(20); - } - trace!(self.log, "Found addresses"; "Peer Id" => format!("{:?}", peer_id), "Addresses" => format!("{:?}", info.listen_addrs)); - // inject the found addresses into our discovery behaviour - - for address in &info.listen_addrs { - self.discovery - .add_connected_address(&peer_id, address.clone()); - } - - self.events - .push(BehaviourEvent::Identified(peer_id, Box::new(info))); - } - IdentifyEvent::Error { .. } => {} - IdentifyEvent::SendBack { .. } => {} - } - } -} - impl NetworkBehaviourEventProcess for Behaviour { @@ -136,41 +124,7 @@ impl NetworkBehaviourEventProcess } } -// implement the discovery behaviour (currently kademlia) -impl NetworkBehaviourEventProcess - for Behaviour -{ - fn inject_event(&mut self, _out: KademliaOut) { - // not interested in kademlia results at the moment - } -} - impl Behaviour { - pub fn new(local_public_key: PublicKey, net_conf: &NetworkConfig, log: &slog::Logger) -> Self { - let local_peer_id = local_public_key.clone().into_peer_id(); - let behaviour_log = log.new(o!()); - let identify_config = net_conf.identify_config.clone(); - let ping_config = PingConfig::new() - .with_timeout(Duration::from_secs(30)) - .with_interval(Duration::from_secs(20)) - .with_max_failures(NonZeroU32::new(2).expect("2 != 0")) - .with_keep_alive(false); - - Behaviour { - serenity_rpc: Rpc::new(log), - gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()), - discovery: Discovery::new(local_peer_id, log), - identify: Identify::new( - identify_config.version, - identify_config.user_agent, - local_public_key, - ), - ping: Ping::new(ping_config), - events: Vec::new(), - log: behaviour_log, - } - } - /// Consumes the events list when polled. fn poll( &mut self, @@ -183,6 +137,14 @@ impl Behaviour { } } +impl NetworkBehaviourEventProcess + for Behaviour +{ + fn inject_event(&mut self, _event: Discv5Event) { + // discv5 has no events to inject + } +} + /// Implements the combined behaviour for the libp2p service. impl Behaviour { /* Pubsub behaviour functions */ @@ -212,8 +174,6 @@ impl Behaviour { pub enum BehaviourEvent { RPC(PeerId, RPCEvent), PeerDialed(PeerId), - Identified(PeerId, Box), - // TODO: This is a stub at the moment GossipMessage { source: PeerId, topics: Vec, diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index baa1bf47f..00a8ed51e 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -1,29 +1,44 @@ use clap::ArgMatches; -use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; -use libp2p::multiaddr::{Error as MultiaddrError, Multiaddr}; +use enr::Enr; +use libp2p::{ + gossipsub::{GossipsubConfig, GossipsubConfigBuilder}, + multiaddr::Multiaddr, +}; use serde_derive::{Deserialize, Serialize}; use std::time::Duration; /// The beacon node topic string to subscribe to. -pub const BEACON_PUBSUB_TOPIC: &str = "beacon_node"; -pub const SHARD_TOPIC_PREFIX: &str = "attestations"; // single topic for all attestation for the moment. +pub const BEACON_PUBSUB_TOPIC: &str = "beacon_block"; +pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation"; +//TODO: Implement shard subnets +pub const SHARD_TOPIC_PREFIX: &str = "shard"; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] /// Network configuration for lighthouse. pub struct Config { /// IP address to listen on. - listen_addresses: Vec, + pub listen_addresses: Vec, + + /// Specifies the IP address that the discovery protocol will listen on. + pub discovery_address: std::net::IpAddr, + + /// UDP port that discovery listens on. + pub discovery_port: u16, + + /// Target number of connected peers. + pub max_peers: usize, + /// Gossipsub configuration parameters. #[serde(skip)] pub gs_config: GossipsubConfig, - /// Configuration parameters for node identification protocol. - #[serde(skip)] - pub identify_config: IdentifyConfig, + /// List of nodes to initially connect to. - boot_nodes: Vec, + pub boot_nodes: Vec, + /// Client version pub client_version: String, + /// List of extra topics to initially subscribe to as strings. pub topics: Vec, } @@ -32,13 +47,16 @@ impl Default for Config { /// Generate a default network configuration. fn default() -> Self { Config { - listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000".to_string()], + listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000".parse().expect("vaild multiaddr")], + discovery_address: "0.0.0.0".parse().expect("valid ip address"), + discovery_port: 9000, + max_peers: 10, + //TODO: Set realistic values for production gs_config: GossipsubConfigBuilder::new() .max_gossip_size(4_000_000) .inactivity_timeout(Duration::from_secs(90)) .heartbeat_interval(Duration::from_secs(20)) .build(), - identify_config: IdentifyConfig::default(), boot_nodes: vec![], client_version: version::version(), topics: Vec::new(), @@ -52,83 +70,42 @@ impl Config { Config::default() } - pub fn listen_addresses(&self) -> Result, MultiaddrError> { - self.listen_addresses.iter().map(|s| s.parse()).collect() - } - - pub fn boot_nodes(&self) -> Result, MultiaddrError> { - self.boot_nodes.iter().map(|s| s.parse()).collect() - } - - pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> { if let Some(listen_address_str) = args.value_of("listen-address") { - let listen_addresses = listen_address_str.split(',').map(Into::into).collect(); - self.listen_addresses = listen_addresses; + self.listen_addresses = listen_address_str + .split(',') + .map(|a| { + a.parse::() + .map_err(|_| format!("Invalid Listen address: {:?}", a)) + }) + .collect::, _>>()?; } - if let Some(boot_addresses_str) = args.value_of("boot-nodes") { - let boot_addresses = boot_addresses_str.split(',').map(Into::into).collect(); - self.boot_nodes = boot_addresses; + if let Some(max_peers_str) = args.value_of("maxpeers") { + self.max_peers = max_peers_str + .parse::() + .map_err(|_| format!("Invalid number of max peers: {}", max_peers_str))?; + } + + if let Some(discovery_address_str) = args.value_of("disc-listen-address") { + self.discovery_address = discovery_address_str + .parse::() + .map_err(|_| format!("Invalid discovery address: {:?}", discovery_address_str))?; + } + + if let Some(boot_enr_str) = args.value_of("boot-nodes") { + self.boot_nodes = boot_enr_str + .split(',') + .map(|enr| enr.parse().map_err(|_| format!("Invalid ENR: {}", enr))) + .collect::, _>>()?; + } + + if let Some(disc_port_str) = args.value_of("disc-port") { + self.discovery_port = disc_port_str + .parse::() + .map_err(|_| format!("Invalid discovery port: {}", disc_port_str))?; } Ok(()) } } - -/// The configuration parameters for the Identify protocol -#[derive(Debug, Clone)] -pub struct IdentifyConfig { - /// The protocol version to listen on. - pub version: String, - /// The client's name and version for identification. - pub user_agent: String, -} - -impl Default for IdentifyConfig { - fn default() -> Self { - Self { - version: "/eth/serenity/1.0".to_string(), - user_agent: version::version(), - } - } -} - -/// Creates a standard network config from a chain_id. -/// -/// This creates specified network parameters for each chain type. -impl From for Config { - fn from(chain_type: ChainType) -> Self { - match chain_type { - ChainType::Foundation => Config::default(), - - ChainType::LighthouseTestnet => { - let boot_nodes = vec!["/ip4/127.0.0.1/tcp/9000" - .parse() - .expect("correct multiaddr")]; - Self { - boot_nodes, - ..Config::default() - } - } - - ChainType::Other => Config::default(), - } - } -} - -pub enum ChainType { - Foundation, - LighthouseTestnet, - Other, -} - -/// Maps a chain id to a ChainType. -impl From for ChainType { - fn from(chain_id: u8) -> Self { - match chain_id { - 1 => ChainType::Foundation, - 2 => ChainType::LighthouseTestnet, - _ => ChainType::Other, - } - } -} diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index c3a02b16f..9a1e75691 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -1,3 +1,4 @@ +use crate::{error, NetworkConfig}; /// This manages the discovery and management of peers. /// /// Currently using Kademlia for peer discovery. @@ -6,66 +7,154 @@ use futures::prelude::*; use libp2p::core::swarm::{ ConnectedPoint, NetworkBehaviour, NetworkBehaviourAction, PollParameters, }; -use libp2p::core::{Multiaddr, PeerId, ProtocolsHandler}; -use libp2p::kad::{Kademlia, KademliaOut}; -use slog::{debug, o, warn}; +use libp2p::core::{identity::Keypair, Multiaddr, PeerId, ProtocolsHandler}; +use libp2p::discv5::{Discv5, Discv5Event}; +use libp2p::enr::{Enr, EnrBuilder, NodeId}; +use libp2p::multiaddr::Protocol; +use slog::{debug, error, info, o, warn}; +use std::collections::HashSet; use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_timer::Delay; -//TODO: Make this dynamic -const TIME_BETWEEN_KAD_REQUESTS: Duration = Duration::from_secs(30); +/// Maximum seconds before searching for extra peers. +const MAX_TIME_BETWEEN_PEER_SEARCHES: u64 = 60; -/// Maintains a list of discovered peers and implements the discovery protocol to discover new -/// peers. +/// Lighthouse discovery behaviour. This provides peer management and discovery using the Discv5 +/// libp2p protocol. pub struct Discovery { - /// Queue of events to processed. - // TODO: Re-implement as discovery protocol grows - // events: Vec>, - /// The discovery behaviour used to discover new peers. - discovery: Kademlia, + /// The peers currently connected to libp2p streams. + connected_peers: HashSet, + + /// The target number of connected peers on the libp2p interface. + max_peers: usize, + /// The delay between peer discovery searches. peer_discovery_delay: Delay, + + /// Tracks the last discovery delay. The delay is doubled each round until the max + /// time is reached. + past_discovery_delay: u64, + + /// The TCP port for libp2p. Used to convert an updated IP address to a multiaddr. Note: This + /// assumes that the external TCP port is the same as the internal TCP port if behind a NAT. + //TODO: Improve NAT handling limit the above restriction + tcp_port: u16, + + /// The discovery behaviour used to discover new peers. + discovery: Discv5, + /// Logger for the discovery behaviour. log: slog::Logger, } impl Discovery { - pub fn new(local_peer_id: PeerId, log: &slog::Logger) -> Self { + pub fn new( + local_key: &Keypair, + net_conf: &NetworkConfig, + log: &slog::Logger, + ) -> error::Result { let log = log.new(o!("Service" => "Libp2p-Discovery")); - Self { - discovery: Kademlia::new(local_peer_id), - peer_discovery_delay: Delay::new(Instant::now()), - log, + + // Build the local ENR. + // The first TCP listening address is used for the ENR record. This will inform our peers to + // connect to this TCP port and establish libp2p streams. + // Note: Discovery should update the ENR record's IP to the external IP as seen by the + // majority of our peers. + let tcp_multiaddr = net_conf + .listen_addresses + .iter() + .filter(|a| { + if let Some(Protocol::Tcp(_)) = a.iter().last() { + true + } else { + false + } + }) + .next() + .ok_or_else(|| "No valid TCP addresses")?; + + let ip: std::net::IpAddr = match tcp_multiaddr.iter().next() { + Some(Protocol::Ip4(ip)) => ip.into(), + Some(Protocol::Ip6(ip)) => ip.into(), + _ => { + error!(log, "Multiaddr has an invalid IP address"); + return Err(format!("Invalid IP Address: {}", tcp_multiaddr).into()); + } + }; + + let tcp_port = match tcp_multiaddr.iter().last() { + Some(Protocol::Tcp(tcp)) => tcp, + _ => unreachable!(), + }; + + let local_enr = EnrBuilder::new() + .ip(ip.into()) + .tcp(tcp_port) + .udp(net_conf.discovery_port) + .build(&local_key) + .map_err(|e| format!("Could not build Local ENR: {:?}", e))?; + info!(log, "Local ENR: {}", local_enr.to_base64()); + + let mut discovery = Discv5::new(local_enr, local_key.clone(), net_conf.discovery_address) + .map_err(|e| format!("Discv5 service failed: {:?}", e))?; + + // Add bootnodes to routing table + for bootnode_enr in net_conf.boot_nodes.clone() { + discovery.add_enr(bootnode_enr); } + + Ok(Self { + connected_peers: HashSet::new(), + max_peers: net_conf.max_peers, + peer_discovery_delay: Delay::new(Instant::now()), + past_discovery_delay: 1, + tcp_port, + discovery, + log, + }) } - /// Uses discovery to search for new peers. - pub fn find_peers(&mut self) { - // pick a random PeerId - let random_peer = PeerId::random(); + /// Manually search for peers. This restarts the discovery round, sparking multiple rapid + /// queries. + pub fn discover_peers(&mut self) { + self.past_discovery_delay = 1; + self.find_peers(); + } + + /// Add an Enr to the routing table of the discovery mechanism. + pub fn add_enr(&mut self, enr: Enr) { + self.discovery.add_enr(enr); + } + + /// Search for new peers using the underlying discovery mechanism. + fn find_peers(&mut self) { + // pick a random NodeId + let random_node = NodeId::random(); debug!(self.log, "Searching for peers..."); - self.discovery.find_node(random_peer); + self.discovery.find_node(random_node); - // update the kademlia timeout + // update the time until next discovery + let delay = { + if self.past_discovery_delay < MAX_TIME_BETWEEN_PEER_SEARCHES { + self.past_discovery_delay *= 2; + self.past_discovery_delay + } else { + MAX_TIME_BETWEEN_PEER_SEARCHES + } + }; self.peer_discovery_delay - .reset(Instant::now() + TIME_BETWEEN_KAD_REQUESTS); - } - - /// We have discovered an address for a peer, add it to known peers. - pub fn add_connected_address(&mut self, peer_id: &PeerId, address: Multiaddr) { - // pass the address on to kademlia - self.discovery.add_address(peer_id, address); + .reset(Instant::now() + Duration::from_secs(delay)); } } -// Redirect all behaviour event to underlying discovery behaviour. +// Redirect all behaviour events to underlying discovery behaviour. impl NetworkBehaviour for Discovery where TSubstream: AsyncRead + AsyncWrite, { - type ProtocolsHandler = as NetworkBehaviour>::ProtocolsHandler; - type OutEvent = as NetworkBehaviour>::OutEvent; + type ProtocolsHandler = as NetworkBehaviour>::ProtocolsHandler; + type OutEvent = as NetworkBehaviour>::OutEvent; fn new_handler(&mut self) -> Self::ProtocolsHandler { NetworkBehaviour::new_handler(&mut self.discovery) @@ -76,25 +165,29 @@ where self.discovery.addresses_of_peer(peer_id) } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - NetworkBehaviour::inject_connected(&mut self.discovery, peer_id, endpoint) + fn inject_connected(&mut self, peer_id: PeerId, _endpoint: ConnectedPoint) { + self.connected_peers.insert(peer_id); } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - NetworkBehaviour::inject_disconnected(&mut self.discovery, peer_id, endpoint) + fn inject_disconnected(&mut self, peer_id: &PeerId, _endpoint: ConnectedPoint) { + self.connected_peers.remove(peer_id); } - fn inject_replaced(&mut self, peer_id: PeerId, closed: ConnectedPoint, opened: ConnectedPoint) { - NetworkBehaviour::inject_replaced(&mut self.discovery, peer_id, closed, opened) + fn inject_replaced( + &mut self, + _peer_id: PeerId, + _closed: ConnectedPoint, + _opened: ConnectedPoint, + ) { + // discv5 doesn't implement } fn inject_node_event( &mut self, - peer_id: PeerId, - event: ::OutEvent, + _peer_id: PeerId, + _event: ::OutEvent, ) { - // TODO: Upgrade to discv5 - NetworkBehaviour::inject_node_event(&mut self.discovery, peer_id, event) + // discv5 doesn't implement } fn poll( @@ -106,7 +199,7 @@ where Self::OutEvent, >, > { - // check to see if it's time to search for peers + // search of peers if it is time loop { match self.peer_discovery_delay.poll() { Ok(Async::Ready(_)) => { @@ -114,46 +207,49 @@ where } Ok(Async::NotReady) => break, Err(e) => { - warn!( - self.log, - "Error getting peers from discovery behaviour. Err: {:?}", e - ); + warn!(self.log, "Discovery peer search failed: {:?}", e); } } } - // Poll discovery - match self.discovery.poll(params) { - Async::Ready(action) => { - match &action { - NetworkBehaviourAction::GenerateEvent(disc_output) => match disc_output { - KademliaOut::Discovered { - peer_id, addresses, .. - } => { - debug!(self.log, "Kademlia peer discovered"; "Peer"=> format!("{:?}", peer_id), "Addresses" => format!("{:?}", addresses)); - } - KademliaOut::FindNodeResult { closer_peers, .. } => { - debug!( - self.log, - "Kademlia query found {} peers", - closer_peers.len() - ); - debug!(self.log, "Kademlia peers discovered"; "Peer"=> format!("{:?}", closer_peers)); - if closer_peers.is_empty() { - debug!(self.log, "Kademlia random query yielded empty results"); + // Poll discovery + loop { + match self.discovery.poll(params) { + Async::Ready(NetworkBehaviourAction::GenerateEvent(event)) => { + match event { + Discv5Event::Discovered(enr) => { + debug!(self.log, "Discv5: Peer discovered"; "Peer"=> format!("{:?}", enr.peer_id()), "Addresses" => format!("{:?}", enr.multiaddr())); + + let peer_id = enr.peer_id(); + // if we need more peers, attempt a connection + if self.connected_peers.len() < self.max_peers + && self.connected_peers.get(&peer_id).is_none() + { + return Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }); + } + } + Discv5Event::SocketUpdated(socket) => { + info!(self.log, "Address updated"; "IP" => format!("{}",socket.ip())); + let mut address = Multiaddr::from(socket.ip()); + address.push(Protocol::Tcp(self.tcp_port)); + return Async::Ready(NetworkBehaviourAction::ReportObservedAddr { + address, + }); + } + Discv5Event::FindNodeResult { closer_peers, .. } => { + debug!(self.log, "Discv5 query found {} peers", closer_peers.len()); + if closer_peers.is_empty() { + debug!(self.log, "Discv5 random query yielded empty results"); } - return Async::Ready(action); } _ => {} - }, - _ => {} - }; - // propagate result upwards - return Async::Ready(action); + } + } + // discv5 does not output any other NetworkBehaviourAction + Async::Ready(_) => {} + Async::NotReady => break, } - Async::NotReady => (), } - Async::NotReady } } diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 197c074df..7a3b2e632 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -10,7 +10,9 @@ pub mod rpc; mod service; pub use behaviour::PubsubMessage; -pub use config::{ChainType, Config as NetworkConfig, BEACON_PUBSUB_TOPIC, SHARD_TOPIC_PREFIX}; +pub use config::{ + Config as NetworkConfig, BEACON_ATTESTATION_TOPIC, BEACON_PUBSUB_TOPIC, SHARD_TOPIC_PREFIX, +}; pub use libp2p::floodsub::{Topic, TopicBuilder, TopicHash}; pub use libp2p::multiaddr; pub use libp2p::Multiaddr; diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 68ca72620..780b1453f 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -4,7 +4,7 @@ use crate::multiaddr::Protocol; use crate::rpc::RPCEvent; use crate::NetworkConfig; use crate::{TopicBuilder, TopicHash}; -use crate::{BEACON_PUBSUB_TOPIC, SHARD_TOPIC_PREFIX}; +use crate::{BEACON_ATTESTATION_TOPIC, BEACON_PUBSUB_TOPIC}; use futures::prelude::*; use futures::Stream; use libp2p::core::{ @@ -36,32 +36,24 @@ pub struct Service { impl Service { pub fn new(config: NetworkConfig, log: slog::Logger) -> error::Result { - debug!(log, "Libp2p Service starting"); + debug!(log, "Network-libp2p Service starting"); - // TODO: Currently using secp256k1 key pairs. Wire protocol specifies RSA. Waiting for this - // PR to be merged to generate RSA keys: https://github.com/briansmith/ring/pull/733 // TODO: Save and recover node key from disk + // TODO: Currently using secp256k1 keypairs - currently required for discv5 let local_private_key = identity::Keypair::generate_secp256k1(); - - let local_public_key = local_private_key.public(); let local_peer_id = PeerId::from(local_private_key.public()); info!(log, "Local peer id: {:?}", local_peer_id); let mut swarm = { - // Set up the transport - let transport = build_transport(local_private_key); - // Set up gossipsub routing - let behaviour = Behaviour::new(local_public_key.clone(), &config, &log); - // Set up Topology - let topology = local_peer_id.clone(); - Swarm::new(transport, behaviour, topology) + // Set up the transport - tcp/ws with secio and mplex/yamux + let transport = build_transport(local_private_key.clone()); + // Lighthouse network behaviour + let behaviour = Behaviour::new(&local_private_key, &config, &log)?; + Swarm::new(transport, behaviour, local_peer_id.clone()) }; // listen on all addresses - for address in config - .listen_addresses() - .map_err(|e| format!("Invalid listen multiaddr: {}", e))? - { + for address in config.listen_addresses { match Swarm::listen_on(&mut swarm, address.clone()) { Ok(_) => { let mut log_address = address.clone(); @@ -71,28 +63,13 @@ impl Service { Err(err) => warn!(log, "Cannot listen on: {} because: {:?}", address, err), }; } - // connect to boot nodes - these are currently stored as multiaddrs - // Once we have discovery, can set to peerId - for bootnode in config - .boot_nodes() - .map_err(|e| format!("Invalid boot node multiaddr: {:?}", e))? - { - match Swarm::dial_addr(&mut swarm, bootnode.clone()) { - Ok(()) => debug!(log, "Dialing bootnode: {}", bootnode), - Err(err) => debug!( - log, - "Could not connect to bootnode: {} error: {:?}", bootnode, err - ), - }; - } // subscribe to default gossipsub topics let mut topics = vec![]; //TODO: Handle multiple shard attestations. For now we simply use a separate topic for //attestations - topics.push(SHARD_TOPIC_PREFIX.to_string()); + topics.push(BEACON_ATTESTATION_TOPIC.to_string()); topics.push(BEACON_PUBSUB_TOPIC.to_string()); - topics.append(&mut config.topics.clone()); let mut subscribed_topics = vec![]; @@ -145,9 +122,6 @@ impl Stream for Service { BehaviourEvent::PeerDialed(peer_id) => { return Ok(Async::Ready(Some(Libp2pEvent::PeerDialed(peer_id)))); } - BehaviourEvent::Identified(peer_id, info) => { - return Ok(Async::Ready(Some(Libp2pEvent::Identified(peer_id, info)))); - } }, Ok(Async::Ready(None)) => unreachable!("Swarm stream shouldn't end"), Ok(Async::NotReady) => break, diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index d00c16292..b805c1d75 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -4,6 +4,6 @@ pub mod message_handler; pub mod service; pub mod sync; -pub use eth2_libp2p::{ChainType, NetworkConfig}; +pub use eth2_libp2p::NetworkConfig; pub use service::NetworkMessage; pub use service::Service; diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 07215119f..51d3a58f9 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -37,11 +37,33 @@ fn main() { .help("One or more comma-delimited multi-addresses to listen for p2p connections.") .takes_value(true), ) + .arg( + Arg::with_name("maxpeers") + .long("maxpeers") + .value_name("Max Peers") + .help("The maximum number of peers (default 10)") + .takes_value(true), + ) .arg( Arg::with_name("boot-nodes") .long("boot-nodes") + .allow_hyphen_values(true) .value_name("BOOTNODES") - .help("One or more comma-delimited multi-addresses to bootstrap the p2p network.") + .help("One or more comma-delimited base64-encoded ENR's to bootstrap the p2p network.") + .takes_value(true), + ) + .arg( + Arg::with_name("disc-listen-address") + .long("disc-listen_address") + .value_name("DISCPORT") + .help("The IP address that the discovery protocol will listen on. Defaults to 0.0.0.0") + .takes_value(true), + ) + .arg( + Arg::with_name("discovery-port") + .long("disc-port") + .value_name("DISCPORT") + .help("Listen UDP port for the discovery process") .takes_value(true), ) // rpc related arguments diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 834f9a428..15883d974 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -84,7 +84,7 @@ pub fn run_beacon_node( info!( log, "Started beacon node"; - "p2p_listen_addresses" => format!("{:?}", &other_client_config.network.listen_addresses()), + "p2p_listen_addresses" => format!("{:?}", &other_client_config.network.listen_addresses), "data_dir" => format!("{:?}", other_client_config.data_dir()), "spec_constants" => &spec_constants, "db_type" => &other_client_config.db_type, From 0952a36a21f270f895c743668baecba269527293 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 25 Jun 2019 18:02:11 +1000 Subject: [PATCH 36/72] Initial integration of discovery v5 --- beacon_node/client/src/config.rs | 67 +++++++++++++++++++++ beacon_node/eth2-libp2p/Cargo.toml | 4 +- beacon_node/eth2-libp2p/src/config.rs | 45 ++++++++------ beacon_node/eth2-libp2p/src/discovery.rs | 77 ++++++++++-------------- beacon_node/eth2-libp2p/src/service.rs | 30 +++++---- beacon_node/src/main.rs | 33 +++++----- beacon_node/src/run.rs | 20 +++--- 7 files changed, 172 insertions(+), 104 deletions(-) create mode 100644 beacon_node/client/src/config.rs diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs new file mode 100644 index 000000000..415ef0ec9 --- /dev/null +++ b/beacon_node/client/src/config.rs @@ -0,0 +1,67 @@ +use clap::ArgMatches; +use http_server::HttpServerConfig; +use network::NetworkConfig; +use serde_derive::{Deserialize, Serialize}; +use std::fs; +use std::path::PathBuf; + +/// The core configuration of a Lighthouse beacon node. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + pub data_dir: PathBuf, + pub db_type: String, + db_name: String, + pub network: network::NetworkConfig, + pub rpc: rpc::RPCConfig, + pub http: HttpServerConfig, +} + +impl Default for Config { + fn default() -> Self { + Self { + data_dir: PathBuf::from(".lighthouse"), + db_type: "disk".to_string(), + db_name: "chain_db".to_string(), + // Note: there are no default bootnodes specified. + // Once bootnodes are established, add them here. + network: NetworkConfig::new(), + rpc: rpc::RPCConfig::default(), + http: HttpServerConfig::default(), + } + } +} + +impl Config { + /// Returns the path to which the client may initialize an on-disk database. + pub fn db_path(&self) -> Option { + self.data_dir() + .and_then(|path| Some(path.join(&self.db_name))) + } + + /// Returns the core path for the client. + pub fn data_dir(&self) -> Option { + let path = dirs::home_dir()?.join(&self.data_dir); + fs::create_dir_all(&path).ok()?; + Some(path) + } + + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. + /// + /// Returns an error if arguments are obviously invalid. May succeed even if some values are + /// invalid. + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> { + if let Some(dir) = args.value_of("datadir") { + self.data_dir = PathBuf::from(dir); + }; + + if let Some(dir) = args.value_of("db") { + self.db_type = dir.to_string(); + } + + self.network.apply_cli_args(args)?; + self.rpc.apply_cli_args(args)?; + self.http.apply_cli_args(args)?; + + Ok(()) + } +} diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index fd7162767..d66127df9 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" # SigP repository -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "f018f5c443ed5a93de890048dbc6755393373e72" } -enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "f018f5c443ed5a93de890048dbc6755393373e72", features = ["serde"] } + libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "1c2aa97a338fc9dfd8e804cc47497fb9b8e7ad04" } +enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "1c2aa97a338fc9dfd8e804cc47497fb9b8e7ad04", features = ["serde"] } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 00a8ed51e..cf9422520 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -1,9 +1,6 @@ use clap::ArgMatches; use enr::Enr; -use libp2p::{ - gossipsub::{GossipsubConfig, GossipsubConfigBuilder}, - multiaddr::Multiaddr, -}; +use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; use serde_derive::{Deserialize, Serialize}; use std::time::Duration; @@ -18,9 +15,12 @@ pub const SHARD_TOPIC_PREFIX: &str = "shard"; /// Network configuration for lighthouse. pub struct Config { /// IP address to listen on. - pub listen_addresses: Vec, + pub listen_address: std::net::IpAddr, - /// Specifies the IP address that the discovery protocol will listen on. + /// The TCP port that libp2p listens on. + pub libp2p_port: u16, + + /// The address to broadcast to peers about which address we are listening on. pub discovery_address: std::net::IpAddr, /// UDP port that discovery listens on. @@ -47,8 +47,9 @@ impl Default for Config { /// Generate a default network configuration. fn default() -> Self { Config { - listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000".parse().expect("vaild multiaddr")], - discovery_address: "0.0.0.0".parse().expect("valid ip address"), + listen_address: "127.0.0.1".parse().expect("vaild ip address"), + libp2p_port: 9000, + discovery_address: "127.0.0.1".parse().expect("valid ip address"), discovery_port: 9000, max_peers: 10, //TODO: Set realistic values for production @@ -72,13 +73,11 @@ impl Config { pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> { if let Some(listen_address_str) = args.value_of("listen-address") { - self.listen_addresses = listen_address_str - .split(',') - .map(|a| { - a.parse::() - .map_err(|_| format!("Invalid Listen address: {:?}", a)) - }) - .collect::, _>>()?; + let listen_address = listen_address_str + .parse() + .map_err(|_| format!("Invalid listen address: {:?}", listen_address_str))?; + self.listen_address = listen_address; + self.discovery_address = listen_address; } if let Some(max_peers_str) = args.value_of("maxpeers") { @@ -87,10 +86,12 @@ impl Config { .map_err(|_| format!("Invalid number of max peers: {}", max_peers_str))?; } - if let Some(discovery_address_str) = args.value_of("disc-listen-address") { - self.discovery_address = discovery_address_str - .parse::() - .map_err(|_| format!("Invalid discovery address: {:?}", discovery_address_str))?; + if let Some(port_str) = args.value_of("port") { + let port = port_str + .parse::() + .map_err(|_| format!("Invalid port: {}", port_str))?; + self.libp2p_port = port; + self.discovery_port = port; } if let Some(boot_enr_str) = args.value_of("boot-nodes") { @@ -100,6 +101,12 @@ impl Config { .collect::, _>>()?; } + if let Some(discovery_address_str) = args.value_of("discovery-address") { + self.discovery_address = discovery_address_str + .parse() + .map_err(|_| format!("Invalid discovery address: {:?}", discovery_address_str))? + } + if let Some(disc_port_str) = args.value_of("disc-port") { self.discovery_port = disc_port_str .parse::() diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 9a1e75691..1d4563552 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -11,7 +11,7 @@ use libp2p::core::{identity::Keypair, Multiaddr, PeerId, ProtocolsHandler}; use libp2p::discv5::{Discv5, Discv5Event}; use libp2p::enr::{Enr, EnrBuilder, NodeId}; use libp2p::multiaddr::Protocol; -use slog::{debug, error, info, o, warn}; +use slog::{debug, info, o, warn}; use std::collections::HashSet; use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite}; @@ -19,6 +19,8 @@ use tokio_timer::Delay; /// Maximum seconds before searching for extra peers. const MAX_TIME_BETWEEN_PEER_SEARCHES: u64 = 60; +/// Initial delay between peer searches. +const INITIAL_SEARCH_DELAY: u64 = 5; /// Lighthouse discovery behaviour. This provides peer management and discovery using the Discv5 /// libp2p protocol. @@ -57,50 +59,27 @@ impl Discovery { let log = log.new(o!("Service" => "Libp2p-Discovery")); // Build the local ENR. - // The first TCP listening address is used for the ENR record. This will inform our peers to - // connect to this TCP port and establish libp2p streams. // Note: Discovery should update the ENR record's IP to the external IP as seen by the // majority of our peers. - let tcp_multiaddr = net_conf - .listen_addresses - .iter() - .filter(|a| { - if let Some(Protocol::Tcp(_)) = a.iter().last() { - true - } else { - false - } - }) - .next() - .ok_or_else(|| "No valid TCP addresses")?; - - let ip: std::net::IpAddr = match tcp_multiaddr.iter().next() { - Some(Protocol::Ip4(ip)) => ip.into(), - Some(Protocol::Ip6(ip)) => ip.into(), - _ => { - error!(log, "Multiaddr has an invalid IP address"); - return Err(format!("Invalid IP Address: {}", tcp_multiaddr).into()); - } - }; - - let tcp_port = match tcp_multiaddr.iter().last() { - Some(Protocol::Tcp(tcp)) => tcp, - _ => unreachable!(), - }; let local_enr = EnrBuilder::new() - .ip(ip.into()) - .tcp(tcp_port) + .ip(net_conf.discovery_address.into()) + .tcp(net_conf.libp2p_port) .udp(net_conf.discovery_port) .build(&local_key) .map_err(|e| format!("Could not build Local ENR: {:?}", e))?; info!(log, "Local ENR: {}", local_enr.to_base64()); - let mut discovery = Discv5::new(local_enr, local_key.clone(), net_conf.discovery_address) + let mut discovery = Discv5::new(local_enr, local_key.clone(), net_conf.listen_address) .map_err(|e| format!("Discv5 service failed: {:?}", e))?; // Add bootnodes to routing table for bootnode_enr in net_conf.boot_nodes.clone() { + debug!( + log, + "Adding node to routing table: {}", + bootnode_enr.node_id() + ); discovery.add_enr(bootnode_enr); } @@ -108,8 +87,8 @@ impl Discovery { connected_peers: HashSet::new(), max_peers: net_conf.max_peers, peer_discovery_delay: Delay::new(Instant::now()), - past_discovery_delay: 1, - tcp_port, + past_discovery_delay: INITIAL_SEARCH_DELAY, + tcp_port: net_conf.libp2p_port, discovery, log, }) @@ -118,7 +97,7 @@ impl Discovery { /// Manually search for peers. This restarts the discovery round, sparking multiple rapid /// queries. pub fn discover_peers(&mut self) { - self.past_discovery_delay = 1; + self.past_discovery_delay = INITIAL_SEARCH_DELAY; self.find_peers(); } @@ -203,7 +182,9 @@ where loop { match self.peer_discovery_delay.poll() { Ok(Async::Ready(_)) => { - self.find_peers(); + if self.connected_peers.len() < self.max_peers { + self.find_peers(); + } } Ok(Async::NotReady) => break, Err(e) => { @@ -217,16 +198,9 @@ where match self.discovery.poll(params) { Async::Ready(NetworkBehaviourAction::GenerateEvent(event)) => { match event { - Discv5Event::Discovered(enr) => { - debug!(self.log, "Discv5: Peer discovered"; "Peer"=> format!("{:?}", enr.peer_id()), "Addresses" => format!("{:?}", enr.multiaddr())); - - let peer_id = enr.peer_id(); - // if we need more peers, attempt a connection - if self.connected_peers.len() < self.max_peers - && self.connected_peers.get(&peer_id).is_none() - { - return Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }); - } + Discv5Event::Discovered(_enr) => { + // not concerned about FINDNODE results, rather the result of an entire + // query. } Discv5Event::SocketUpdated(socket) => { info!(self.log, "Address updated"; "IP" => format!("{}",socket.ip())); @@ -241,6 +215,17 @@ where if closer_peers.is_empty() { debug!(self.log, "Discv5 random query yielded empty results"); } + for peer_id in closer_peers { + // if we need more peers, attempt a connection + if self.connected_peers.len() < self.max_peers + && self.connected_peers.get(&peer_id).is_none() + { + debug!(self.log, "Discv5: Peer discovered"; "Peer"=> format!("{:?}", peer_id)); + return Async::Ready(NetworkBehaviourAction::DialPeer { + peer_id, + }); + } + } } _ => {} } diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 780b1453f..1db855cd4 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -9,6 +9,7 @@ use futures::prelude::*; use futures::Stream; use libp2p::core::{ identity, + multiaddr::Multiaddr, muxing::StreamMuxerBox, nodes::Substream, transport::boxed::Boxed, @@ -52,17 +53,24 @@ impl Service { Swarm::new(transport, behaviour, local_peer_id.clone()) }; - // listen on all addresses - for address in config.listen_addresses { - match Swarm::listen_on(&mut swarm, address.clone()) { - Ok(_) => { - let mut log_address = address.clone(); - log_address.push(Protocol::P2p(local_peer_id.clone().into())); - info!(log, "Listening on: {}", log_address); - } - Err(err) => warn!(log, "Cannot listen on: {} because: {:?}", address, err), - }; - } + // listen on the specified address + let listen_multiaddr = { + let mut m = Multiaddr::from(config.listen_address); + m.push(Protocol::Tcp(config.libp2p_port)); + m + }; + + match Swarm::listen_on(&mut swarm, listen_multiaddr.clone()) { + Ok(_) => { + let mut log_address = listen_multiaddr; + log_address.push(Protocol::P2p(local_peer_id.clone().into())); + info!(log, "Listening on: {}", log_address); + } + Err(err) => warn!( + log, + "Cannot listen on: {} because: {:?}", listen_multiaddr, err + ), + }; // subscribe to default gossipsub topics let mut topics = vec![]; diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 51d3a58f9..f7b92275a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -33,15 +33,14 @@ fn main() { .arg( Arg::with_name("listen-address") .long("listen-address") - .value_name("Listen Address") - .help("One or more comma-delimited multi-addresses to listen for p2p connections.") + .value_name("Address") + .help("The address lighthouse will listen for UDP and TCP connections. (default 127.0.0.1).") .takes_value(true), ) .arg( Arg::with_name("maxpeers") .long("maxpeers") - .value_name("Max Peers") - .help("The maximum number of peers (default 10)") + .help("The maximum number of peers (default 10).") .takes_value(true), ) .arg( @@ -53,17 +52,24 @@ fn main() { .takes_value(true), ) .arg( - Arg::with_name("disc-listen-address") - .long("disc-listen_address") - .value_name("DISCPORT") - .help("The IP address that the discovery protocol will listen on. Defaults to 0.0.0.0") + Arg::with_name("port") + .long("port") + .value_name("Lighthouse Port") + .help("The TCP/UDP port to listen on. The UDP port can be modified by the --discovery-port flag.") .takes_value(true), ) .arg( Arg::with_name("discovery-port") .long("disc-port") - .value_name("DISCPORT") - .help("Listen UDP port for the discovery process") + .value_name("DiscoveryPort") + .help("The discovery UDP port.") + .takes_value(true), + ) + .arg( + Arg::with_name("discovery-address") + .long("discovery-address") + .value_name("Address") + .help("The address to broadcast to other peers on how to reach this node.") .takes_value(true), ) // rpc related arguments @@ -77,14 +83,13 @@ fn main() { .arg( Arg::with_name("rpc-address") .long("rpc-address") - .value_name("RPCADDRESS") + .value_name("Address") .help("Listen address for RPC endpoint.") .takes_value(true), ) .arg( Arg::with_name("rpc-port") .long("rpc-port") - .value_name("RPCPORT") .help("Listen port for RPC endpoint.") .takes_value(true), ) @@ -92,21 +97,19 @@ fn main() { .arg( Arg::with_name("http") .long("http") - .value_name("HTTP") .help("Enable the HTTP server.") .takes_value(false), ) .arg( Arg::with_name("http-address") .long("http-address") - .value_name("HTTPADDRESS") + .value_name("Address") .help("Listen address for the HTTP server.") .takes_value(true), ) .arg( Arg::with_name("http-port") .long("http-port") - .value_name("HTTPPORT") .help("Listen port for the HTTP server.") .takes_value(true), ) diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 15883d974..51fa16154 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -41,6 +41,15 @@ pub fn run_beacon_node( "This software is EXPERIMENTAL and provides no guarantees or warranties." ); + info!( + log, + "Starting beacon node"; + "p2p_listen_address" => format!("{:?}", &other_client_config.network.listen_address), + "data_dir" => format!("{:?}", other_client_config.data_dir()), + "spec_constants" => &spec_constants, + "db_type" => &other_client_config.db_type, + ); + let result = match (db_type.as_str(), spec_constants.as_str()) { ("disk", "minimal") => run::>( &db_path, @@ -80,17 +89,6 @@ pub fn run_beacon_node( } }; - if result.is_ok() { - info!( - log, - "Started beacon node"; - "p2p_listen_addresses" => format!("{:?}", &other_client_config.network.listen_addresses), - "data_dir" => format!("{:?}", other_client_config.data_dir()), - "spec_constants" => &spec_constants, - "db_type" => &other_client_config.db_type, - ); - } - result } From 7dc5e2f959cb1e7b6a2725b46b60ca05b1144be4 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 25 Jun 2019 18:57:11 +1000 Subject: [PATCH 37/72] Update to latest libp2p --- beacon_node/eth2-libp2p/Cargo.toml | 4 ++-- beacon_node/eth2-libp2p/src/discovery.rs | 2 +- beacon_node/eth2-libp2p/src/rpc/mod.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index d66127df9..550486e98 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" # SigP repository - libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "1c2aa97a338fc9dfd8e804cc47497fb9b8e7ad04" } -enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "1c2aa97a338fc9dfd8e804cc47497fb9b8e7ad04", features = ["serde"] } + libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "8ff9f2001de0aea1175c1442f22bfbf382dc0e8b" } +enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "8ff9f2001de0aea1175c1442f22bfbf382dc0e8b", features = ["serde"] } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 1d4563552..b69f45be7 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -171,7 +171,7 @@ where fn poll( &mut self, - params: &mut PollParameters, + params: &mut impl PollParameters, ) -> Async< NetworkBehaviourAction< ::InEvent, diff --git a/beacon_node/eth2-libp2p/src/rpc/mod.rs b/beacon_node/eth2-libp2p/src/rpc/mod.rs index 57d7dadbe..2d303469c 100644 --- a/beacon_node/eth2-libp2p/src/rpc/mod.rs +++ b/beacon_node/eth2-libp2p/src/rpc/mod.rs @@ -94,7 +94,7 @@ where fn poll( &mut self, - _: &mut PollParameters<'_>, + _: &mut impl PollParameters, ) -> Async< NetworkBehaviourAction< ::InEvent, From af28d5e20cf5b8484ba55739e3ed04d008ec8142 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 1 Jul 2019 16:38:42 +1000 Subject: [PATCH 38/72] Add persistent network identification --- account_manager/Cargo.toml | 1 + account_manager/src/main.rs | 19 +++-- beacon_node/Cargo.toml | 1 + beacon_node/eth2-libp2p/Cargo.toml | 7 +- beacon_node/eth2-libp2p/src/config.rs | 14 +++ beacon_node/eth2-libp2p/src/discovery.rs | 103 +++++++++++++++++++---- beacon_node/eth2-libp2p/src/service.rs | 65 ++++++++++++-- beacon_node/network/src/service.rs | 6 -- beacon_node/src/main.rs | 41 +++++---- eth2/utils/eth2_config/src/lib.rs | 13 --- validator_client/Cargo.toml | 1 + validator_client/src/main.rs | 20 +++-- 12 files changed, 220 insertions(+), 71 deletions(-) diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 48504d89a..b3c687eef 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -13,3 +13,4 @@ slog-async = "^2.3.0" validator_client = { path = "../validator_client" } types = { path = "../eth2/types" } eth2_config = { path = "../eth2/utils/eth2_config" } +dirs = "2.0.1" diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index 1c8cc8819..e242e8ae4 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -1,7 +1,7 @@ use bls::Keypair; use clap::{App, Arg, SubCommand}; -use eth2_config::get_data_dir; use slog::{crit, debug, info, o, Drain}; +use std::fs; use std::path::PathBuf; use types::test_utils::generate_deterministic_keypair; use validator_client::Config as ValidatorClientConfig; @@ -61,13 +61,22 @@ fn main() { ) .get_matches(); - let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { - Ok(dir) => dir, + let mut default_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); + default_dir.push(DEFAULT_DATA_DIR); + + let data_dir = &matches + .value_of("datadir") + .and_then(|v| Some(PathBuf::from(v))) + .unwrap_or_else(|| PathBuf::from(default_dir)); + + // create the directory if needed + match fs::create_dir_all(&data_dir) { + Ok(_) => {} Err(e) => { - crit!(log, "Failed to initialize data dir"; "error" => format!("{:?}", e)); + crit!(log, "Failed to initialize data dir"; "error" => format!("{}", e)); return; } - }; + } let mut client_config = ValidatorClientConfig::default(); diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 7e43a13df..9e96f8484 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -23,3 +23,4 @@ futures = "0.1.25" exit-future = "0.1.3" state_processing = { path = "../eth2/state_processing" } env_logger = "0.6.1" +dirs = "2.0.1" diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 550486e98..1fbd30872 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -7,9 +7,9 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } clap = "2.32.0" -# SigP repository - libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "8ff9f2001de0aea1175c1442f22bfbf382dc0e8b" } -enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "8ff9f2001de0aea1175c1442f22bfbf382dc0e8b", features = ["serde"] } +#SigP repository +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "be5710bbde69d8c5be732c13ba64239e2f370a7b" } +enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "be5710bbde69d8c5be732c13ba64239e2f370a7b", features = ["serde"] } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" @@ -21,3 +21,4 @@ tokio = "0.1.16" futures = "0.1.25" error-chain = "0.12.0" tokio-timer = "0.2.10" +dirs = "2.0.1" diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index cf9422520..e881408ea 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -2,6 +2,7 @@ use clap::ArgMatches; use enr::Enr; use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; use serde_derive::{Deserialize, Serialize}; +use std::path::PathBuf; use std::time::Duration; /// The beacon node topic string to subscribe to. @@ -14,6 +15,9 @@ pub const SHARD_TOPIC_PREFIX: &str = "shard"; #[serde(default)] /// Network configuration for lighthouse. pub struct Config { + /// Data directory where node's keyfile is stored + pub network_dir: PathBuf, + /// IP address to listen on. pub listen_address: std::net::IpAddr, @@ -46,7 +50,11 @@ pub struct Config { impl Default for Config { /// Generate a default network configuration. fn default() -> Self { + let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); + network_dir.push(".lighthouse"); + network_dir.push("network"); Config { + network_dir, listen_address: "127.0.0.1".parse().expect("vaild ip address"), libp2p_port: 9000, discovery_address: "127.0.0.1".parse().expect("valid ip address"), @@ -72,6 +80,12 @@ impl Config { } pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> { + dbg!(self.network_dir.clone()); + if let Some(dir) = args.value_of("datadir") { + self.network_dir = PathBuf::from(dir).join("network"); + }; + dbg!(self.network_dir.clone()); + if let Some(listen_address_str) = args.value_of("listen-address") { let listen_address = listen_address_str .parse() diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index b69f45be7..104cd0285 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -1,7 +1,7 @@ use crate::{error, NetworkConfig}; /// This manages the discovery and management of peers. /// -/// Currently using Kademlia for peer discovery. +/// Currently using discv5 for peer discovery. /// use futures::prelude::*; use libp2p::core::swarm::{ @@ -13,6 +13,9 @@ use libp2p::enr::{Enr, EnrBuilder, NodeId}; use libp2p::multiaddr::Protocol; use slog::{debug, info, o, warn}; use std::collections::HashSet; +use std::fs::File; +use std::io::prelude::*; +use std::str::FromStr; use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_timer::Delay; @@ -21,6 +24,8 @@ use tokio_timer::Delay; const MAX_TIME_BETWEEN_PEER_SEARCHES: u64 = 60; /// Initial delay between peer searches. const INITIAL_SEARCH_DELAY: u64 = 5; +/// Local ENR storage filename. +const ENR_FILENAME: &str = "enr.dat"; /// Lighthouse discovery behaviour. This provides peer management and discovery using the Discv5 /// libp2p protocol. @@ -53,28 +58,22 @@ pub struct Discovery { impl Discovery { pub fn new( local_key: &Keypair, - net_conf: &NetworkConfig, + config: &NetworkConfig, log: &slog::Logger, ) -> error::Result { let log = log.new(o!("Service" => "Libp2p-Discovery")); - // Build the local ENR. - // Note: Discovery should update the ENR record's IP to the external IP as seen by the - // majority of our peers. + // checks if current ENR matches that found on disk + let local_enr = load_enr(local_key, config, &log)?; - let local_enr = EnrBuilder::new() - .ip(net_conf.discovery_address.into()) - .tcp(net_conf.libp2p_port) - .udp(net_conf.discovery_port) - .build(&local_key) - .map_err(|e| format!("Could not build Local ENR: {:?}", e))?; info!(log, "Local ENR: {}", local_enr.to_base64()); + debug!(log, "Local Node Id: {}", local_enr.node_id()); - let mut discovery = Discv5::new(local_enr, local_key.clone(), net_conf.listen_address) + let mut discovery = Discv5::new(local_enr, local_key.clone(), config.listen_address) .map_err(|e| format!("Discv5 service failed: {:?}", e))?; // Add bootnodes to routing table - for bootnode_enr in net_conf.boot_nodes.clone() { + for bootnode_enr in config.boot_nodes.clone() { debug!( log, "Adding node to routing table: {}", @@ -85,10 +84,10 @@ impl Discovery { Ok(Self { connected_peers: HashSet::new(), - max_peers: net_conf.max_peers, + max_peers: config.max_peers, peer_discovery_delay: Delay::new(Instant::now()), past_discovery_delay: INITIAL_SEARCH_DELAY, - tcp_port: net_conf.libp2p_port, + tcp_port: config.libp2p_port, discovery, log, }) @@ -238,3 +237,77 @@ where Async::NotReady } } + +/// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none +/// exists, generates a new one. +/// +/// If an ENR exists, with the same NodeId and IP addresses, we use the disk-generated one as it's +/// ENR sequence will be equal or higher than a newly generated one. +fn load_enr( + local_key: &Keypair, + config: &NetworkConfig, + log: &slog::Logger, +) -> Result { + // Build the local ENR. + // Note: Discovery should update the ENR record's IP to the external IP as seen by the + // majority of our peers. + let mut local_enr = EnrBuilder::new() + .ip(config.discovery_address.into()) + .tcp(config.libp2p_port) + .udp(config.discovery_port) + .build(&local_key) + .map_err(|e| format!("Could not build Local ENR: {:?}", e))?; + + let enr_f = config.network_dir.join(ENR_FILENAME); + if let Ok(mut enr_file) = File::open(enr_f.clone()) { + let mut enr_string = String::new(); + match enr_file.read_to_string(&mut enr_string) { + Err(_) => debug!(log, "Could not read ENR from file"), + Ok(_) => { + match Enr::from_str(&enr_string) { + Ok(enr) => { + debug!(log, "ENR found in file: {:?}", enr_f); + + if enr.node_id() == local_enr.node_id() { + if enr.ip() == config.discovery_address.into() + && enr.tcp() == Some(config.libp2p_port) + && enr.udp() == Some(config.discovery_port) + { + debug!(log, "ENR loaded from file"); + // the stored ENR has the same configuration, use it + return Ok(enr); + } + + // same node id, different configuration - update the sequence number + let new_seq_no = enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?; + local_enr.set_seq(new_seq_no, local_key).map_err(|e| { + format!("Could not update ENR sequence number: {:?}", e) + })?; + debug!(log, "ENR sequence number increased to: {}", new_seq_no); + } + } + Err(e) => { + warn!(log, "ENR from file could not be decoded: {:?}", e); + } + } + } + } + } + + // write ENR to disk + let _ = std::fs::create_dir_all(&config.network_dir); + match File::create(enr_f.clone()) + .and_then(|mut f| f.write_all(&local_enr.to_base64().as_bytes())) + { + Ok(_) => { + debug!(log, "ENR written to disk"); + } + Err(e) => { + warn!( + log, + "Could not write ENR to file: {:?}. Error: {}", enr_f, e + ); + } + } + Ok(local_enr) +} diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 1db855cd4..69f8a1ca5 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -8,22 +8,25 @@ use crate::{BEACON_ATTESTATION_TOPIC, BEACON_PUBSUB_TOPIC}; use futures::prelude::*; use futures::Stream; use libp2p::core::{ - identity, + identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, nodes::Substream, transport::boxed::Boxed, upgrade::{InboundUpgradeExt, OutboundUpgradeExt}, }; -use libp2p::identify::protocol::IdentifyInfo; use libp2p::{core, secio, PeerId, Swarm, Transport}; use slog::{debug, info, trace, warn}; +use std::fs::File; +use std::io::prelude::*; use std::io::{Error, ErrorKind}; use std::time::Duration; type Libp2pStream = Boxed<(PeerId, StreamMuxerBox), Error>; type Libp2pBehaviour = Behaviour>; +const NETWORK_KEY_FILENAME: &str = "key"; + /// The configuration and state of the libp2p components for the beacon node. pub struct Service { /// The libp2p Swarm handler. @@ -39,9 +42,9 @@ impl Service { pub fn new(config: NetworkConfig, log: slog::Logger) -> error::Result { debug!(log, "Network-libp2p Service starting"); - // TODO: Save and recover node key from disk - // TODO: Currently using secp256k1 keypairs - currently required for discv5 - let local_private_key = identity::Keypair::generate_secp256k1(); + // load the private key from CLI flag, disk or generate a new one + let local_private_key = load_private_key(&config, &log); + let local_peer_id = PeerId::from(local_private_key.public()); info!(log, "Local peer id: {:?}", local_peer_id); @@ -142,7 +145,7 @@ impl Stream for Service { /// The implementation supports TCP/IP, WebSockets over TCP/IP, secio as the encryption layer, and /// mplex or yamux as the multiplexing layer. -fn build_transport(local_private_key: identity::Keypair) -> Boxed<(PeerId, StreamMuxerBox), Error> { +fn build_transport(local_private_key: Keypair) -> Boxed<(PeerId, StreamMuxerBox), Error> { // TODO: The Wire protocol currently doesn't specify encryption and this will need to be customised // in the future. let transport = libp2p::tcp::TcpConfig::new(); @@ -179,8 +182,6 @@ pub enum Libp2pEvent { RPC(PeerId, RPCEvent), /// Initiated the connection to a new peer. PeerDialed(PeerId), - /// Received information about a peer on the network. - Identified(PeerId, Box), /// Received pubsub message. PubsubMessage { source: PeerId, @@ -188,3 +189,51 @@ pub enum Libp2pEvent { message: Box, }, } + +/// Loads a private key from disk. If this fails, a new key is +/// generated and is then saved to disk. +/// +/// Currently only secp256k1 keys are allowed, as these are the only keys supported by discv5. +fn load_private_key(config: &NetworkConfig, log: &slog::Logger) -> Keypair { + // TODO: Currently using secp256k1 keypairs - currently required for discv5 + // check for key from disk + let network_key_f = config.network_dir.join(NETWORK_KEY_FILENAME); + if let Ok(mut network_key_file) = File::open(network_key_f.clone()) { + let mut key_bytes: Vec = Vec::with_capacity(36); + match network_key_file.read_to_end(&mut key_bytes) { + Err(_) => debug!(log, "Could not read network key file"), + Ok(_) => { + // only accept secp256k1 keys for now + if let Ok(secret_key) = + libp2p::core::identity::secp256k1::SecretKey::from_bytes(&mut key_bytes) + { + let kp: libp2p::core::identity::secp256k1::Keypair = secret_key.into(); + debug!(log, "Loaded network key from disk."); + return Keypair::Secp256k1(kp); + } else { + debug!(log, "Network key file is not a valid secp256k1 key"); + } + } + } + } + + // if a key could not be loaded from disk, generate a new one and save it + let local_private_key = Keypair::generate_secp256k1(); + if let Keypair::Secp256k1(key) = local_private_key.clone() { + let _ = std::fs::create_dir_all(&config.network_dir); + match File::create(network_key_f.clone()) + .and_then(|mut f| f.write_all(&key.secret().to_bytes())) + { + Ok(_) => { + debug!(log, "New network key generated and written to disk"); + } + Err(e) => { + warn!( + log, + "Could not write node key to file: {:?}. Error: {}", network_key_f, e + ); + } + } + } + local_private_key +} diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index c19aef004..b2ecc1a0b 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -126,12 +126,6 @@ fn network_service( .send(HandlerMessage::PeerDialed(peer_id)) .map_err(|_| "failed to send rpc to handler")?; } - Libp2pEvent::Identified(peer_id, info) => { - debug!( - log, - "We have identified peer: {:?} with {:?}", peer_id, info - ); - } Libp2pEvent::PubsubMessage { source, message, .. } => { diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index f7b92275a..651ad8e0c 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -3,8 +3,9 @@ mod run; use clap::{App, Arg}; use client::{ClientConfig, Eth2Config}; use env_logger::{Builder, Env}; -use eth2_config::{get_data_dir, read_from_file, write_to_file}; +use eth2_config::{read_from_file, write_to_file}; use slog::{crit, o, Drain, Level}; +use std::fs; use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; @@ -27,7 +28,6 @@ fn main() { .value_name("DIR") .help("Data directory for keys and databases.") .takes_value(true) - .default_value(DEFAULT_DATA_DIR), ) // network related arguments .arg( @@ -69,7 +69,7 @@ fn main() { Arg::with_name("discovery-address") .long("discovery-address") .value_name("Address") - .help("The address to broadcast to other peers on how to reach this node.") + .help("The IP address to broadcast to other peers on how to reach this node.") .takes_value(true), ) // rpc related arguments @@ -159,15 +159,24 @@ fn main() { _ => drain.filter_level(Level::Info), }; - let logger = slog::Logger::root(drain.fuse(), o!()); + let log = slog::Logger::root(drain.fuse(), o!()); - let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { - Ok(dir) => dir, + let mut default_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); + default_dir.push(DEFAULT_DATA_DIR); + + let data_dir = &matches + .value_of("datadir") + .and_then(|v| Some(PathBuf::from(v))) + .unwrap_or_else(|| PathBuf::from(default_dir)); + + // create the directory if needed + match fs::create_dir_all(&data_dir) { + Ok(_) => {} Err(e) => { - crit!(logger, "Failed to initialize data dir"; "error" => format!("{:?}", e)); + crit!(log, "Failed to initialize data dir"; "error" => format!("{}", e)); return; } - }; + } let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME); @@ -179,13 +188,13 @@ fn main() { Ok(None) => { let default = ClientConfig::default(); if let Err(e) = write_to_file(client_config_path, &default) { - crit!(logger, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e)); + crit!(log, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e)); return; } default } Err(e) => { - crit!(logger, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e)); + crit!(log, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e)); return; } }; @@ -197,7 +206,7 @@ fn main() { match client_config.apply_cli_args(&matches) { Ok(()) => (), Err(s) => { - crit!(logger, "Failed to parse ClientConfig CLI arguments"; "error" => s); + crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); return; } }; @@ -216,13 +225,13 @@ fn main() { _ => unreachable!(), // Guarded by slog. }; if let Err(e) = write_to_file(eth2_config_path, &default) { - crit!(logger, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); + crit!(log, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); return; } default } Err(e) => { - crit!(logger, "Failed to load/generate an Eth2Config"; "error" => format!("{:?}", e)); + crit!(log, "Failed to load/generate an Eth2Config"; "error" => format!("{:?}", e)); return; } }; @@ -231,13 +240,13 @@ fn main() { match eth2_config.apply_cli_args(&matches) { Ok(()) => (), Err(s) => { - crit!(logger, "Failed to parse Eth2Config CLI arguments"; "error" => s); + crit!(log, "Failed to parse Eth2Config CLI arguments"; "error" => s); return; } }; - match run::run_beacon_node(client_config, eth2_config, &logger) { + match run::run_beacon_node(client_config, eth2_config, &log) { Ok(_) => {} - Err(e) => crit!(logger, "Beacon node failed to start"; "reason" => format!("{:}", e)), + Err(e) => crit!(log, "Beacon node failed to start"; "reason" => format!("{:}", e)), } } diff --git a/eth2/utils/eth2_config/src/lib.rs b/eth2/utils/eth2_config/src/lib.rs index 9d50a95c1..f6ad54c21 100644 --- a/eth2/utils/eth2_config/src/lib.rs +++ b/eth2/utils/eth2_config/src/lib.rs @@ -1,6 +1,5 @@ use clap::ArgMatches; use serde_derive::{Deserialize, Serialize}; -use std::fs; use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; @@ -105,15 +104,3 @@ where Ok(None) } } - -pub fn get_data_dir(args: &ArgMatches, default_data_dir: PathBuf) -> Result { - if let Some(data_dir) = args.value_of("data_dir") { - Ok(PathBuf::from(data_dir)) - } else { - let path = dirs::home_dir() - .ok_or_else(|| "Unable to locate home directory")? - .join(&default_data_dir); - fs::create_dir_all(&path).map_err(|_| "Unable to create data_dir")?; - Ok(path) - } -} diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index fdb4c33c0..1972f870c 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -34,3 +34,4 @@ toml = "^0.5" error-chain = "0.12.0" bincode = "^1.1.2" futures = "0.1.25" +dirs = "2.0.1" diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index f74915438..43ef2f994 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -9,9 +9,10 @@ mod signer; use crate::config::Config as ValidatorClientConfig; use crate::service::Service as ValidatorService; use clap::{App, Arg}; -use eth2_config::{get_data_dir, read_from_file, write_to_file, Eth2Config}; +use eth2_config::{read_from_file, write_to_file, Eth2Config}; use protos::services_grpc::ValidatorServiceClient; use slog::{crit, error, info, o, Drain}; +use std::fs; use std::path::PathBuf; use types::{Keypair, MainnetEthSpec, MinimalEthSpec}; @@ -66,13 +67,22 @@ fn main() { ) .get_matches(); - let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { - Ok(dir) => dir, + let mut default_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); + default_dir.push(DEFAULT_DATA_DIR); + + let data_dir = &matches + .value_of("datadir") + .and_then(|v| Some(PathBuf::from(v))) + .unwrap_or_else(|| PathBuf::from(default_dir)); + + // create the directory if needed + match fs::create_dir_all(&data_dir) { + Ok(_) => {} Err(e) => { - crit!(log, "Failed to initialize data dir"; "error" => format!("{:?}", e)); + crit!(log, "Failed to initialize data dir"; "error" => format!("{}", e)); return; } - }; + } let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME); From 177daf26094a04d740d4d58b37d78a045371cd19 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 1 Jul 2019 16:40:35 +1000 Subject: [PATCH 39/72] Typo fixes --- beacon_node/eth2-libp2p/src/discovery.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 104cd0285..44b4e655b 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -177,7 +177,7 @@ where Self::OutEvent, >, > { - // search of peers if it is time + // search for peers if it is time loop { match self.peer_discovery_delay.poll() { Ok(Async::Ready(_)) => { @@ -241,7 +241,7 @@ where /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none /// exists, generates a new one. /// -/// If an ENR exists, with the same NodeId and IP addresses, we use the disk-generated one as it's +/// If an ENR exists, with the same NodeId and IP address, we use the disk-generated one as its /// ENR sequence will be equal or higher than a newly generated one. fn load_enr( local_key: &Keypair, From bffe6c327f19bf91de45199cb8e78ec3b9ee4fc3 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 1 Jul 2019 17:23:14 +1000 Subject: [PATCH 40/72] Removes left-over debugging statements --- beacon_node/eth2-libp2p/src/config.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index e881408ea..c5e02d5e3 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -80,11 +80,9 @@ impl Config { } pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> { - dbg!(self.network_dir.clone()); if let Some(dir) = args.value_of("datadir") { self.network_dir = PathBuf::from(dir).join("network"); }; - dbg!(self.network_dir.clone()); if let Some(listen_address_str) = args.value_of("listen-address") { let listen_address = listen_address_str From dd410535cb8a2676592369f179616567ce5b5f9d Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 2 Jul 2019 11:15:35 +1000 Subject: [PATCH 41/72] Remove Phase 1 TODO --- beacon_node/eth2-libp2p/src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index c5e02d5e3..ea87075b7 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -8,7 +8,6 @@ use std::time::Duration; /// The beacon node topic string to subscribe to. pub const BEACON_PUBSUB_TOPIC: &str = "beacon_block"; pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation"; -//TODO: Implement shard subnets pub const SHARD_TOPIC_PREFIX: &str = "shard"; #[derive(Clone, Debug, Serialize, Deserialize)] From 1aeec12b780fe8d66f3deef9eb841cc284987ab5 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 2 Jul 2019 17:32:14 +1000 Subject: [PATCH 42/72] Improve error handling of default directory --- account_manager/src/main.rs | 21 ++++++++++++++++----- beacon_node/eth2-libp2p/src/rpc/protocol.rs | 1 - beacon_node/src/main.rs | 21 ++++++++++++++++----- validator_client/src/main.rs | 21 ++++++++++++++++----- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index e242e8ae4..ee0e86d60 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -61,13 +61,24 @@ fn main() { ) .get_matches(); - let mut default_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); - default_dir.push(DEFAULT_DATA_DIR); - - let data_dir = &matches + let data_dir = match matches .value_of("datadir") .and_then(|v| Some(PathBuf::from(v))) - .unwrap_or_else(|| PathBuf::from(default_dir)); + { + Some(v) => v, + None => { + // use the default + let mut default_dir = match dirs::home_dir() { + Some(v) => v, + None => { + crit!(log, "Failed to find a home directory"); + return; + } + }; + default_dir.push(DEFAULT_DATA_DIR); + PathBuf::from(default_dir) + } + }; // create the directory if needed match fs::create_dir_all(&data_dir) { diff --git a/beacon_node/eth2-libp2p/src/rpc/protocol.rs b/beacon_node/eth2-libp2p/src/rpc/protocol.rs index 2f461988a..7afded3ac 100644 --- a/beacon_node/eth2-libp2p/src/rpc/protocol.rs +++ b/beacon_node/eth2-libp2p/src/rpc/protocol.rs @@ -11,7 +11,6 @@ use tokio::io::{AsyncRead, AsyncWrite}; const MAX_READ_SIZE: usize = 4_194_304; // 4M /// Implementation of the `ConnectionUpgrade` for the rpc protocol. - #[derive(Debug, Clone)] pub struct RPCProtocol; diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 651ad8e0c..791feae54 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -161,13 +161,24 @@ fn main() { let log = slog::Logger::root(drain.fuse(), o!()); - let mut default_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); - default_dir.push(DEFAULT_DATA_DIR); - - let data_dir = &matches + let data_dir = match matches .value_of("datadir") .and_then(|v| Some(PathBuf::from(v))) - .unwrap_or_else(|| PathBuf::from(default_dir)); + { + Some(v) => v, + None => { + // use the default + let mut default_dir = match dirs::home_dir() { + Some(v) => v, + None => { + crit!(log, "Failed to find a home directory"); + return; + } + }; + default_dir.push(DEFAULT_DATA_DIR); + PathBuf::from(default_dir) + } + }; // create the directory if needed match fs::create_dir_all(&data_dir) { diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 43ef2f994..e37b6530e 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -67,13 +67,24 @@ fn main() { ) .get_matches(); - let mut default_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); - default_dir.push(DEFAULT_DATA_DIR); - - let data_dir = &matches + let data_dir = match matches .value_of("datadir") .and_then(|v| Some(PathBuf::from(v))) - .unwrap_or_else(|| PathBuf::from(default_dir)); + { + Some(v) => v, + None => { + // use the default + let mut default_dir = match dirs::home_dir() { + Some(v) => v, + None => { + crit!(log, "Failed to find a home directory"); + return; + } + }; + default_dir.push(DEFAULT_DATA_DIR); + PathBuf::from(default_dir) + } + }; // create the directory if needed match fs::create_dir_all(&data_dir) { From 74e42313570c86ddc4127ab5135c114541d849f3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 4 Jul 2019 15:05:57 +1000 Subject: [PATCH 43/72] Apply suggestions from code review Co-Authored-By: Age Manning --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a9bc6df05..d48c397d9 100644 --- a/README.md +++ b/README.md @@ -103,13 +103,16 @@ connect if they were started in separate 30-minute windows._ #### 3. Start Another Beacon Node -In another terminal window, start another boot node that will connect to the +In another terminal window, start another boot that will connect to the running node. +The running node will display it's ENR as a base64 string. This ENR, by default, has a target address of `127.0.0.1` meaning that any new node will connect to this node via `127.0.0.1`. If a boot node should be connected to on a different address, it should be run with the `--discovery-address` CLI flag to specify how other nodes may connect to it. ``` -$ ./beacon_node -r --boot-nodes /ip4/127.0.0.1/tcp/9000 --listen-address /ip4/127.0.0.1/tcp/9001 +$ ./beacon_node -r --boot-nodes --listen-address 127.0.0.1 --port 9001 ``` +Here is the ENR string displayed in the terminal from the first node. The ENR can also be obtained from it's default directory `.lighthouse/network/enr.dat`. +Note that all future created nodes can use the same boot-node ENR. Once connected to the boot node, all nodes should discover and connect with each other. #### 4. Start a Validator Client In a third terminal window, start a validator client: From 0ad51d466ce78a517c8acc9ee07856c49e4c2e53 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 4 Jul 2019 15:24:36 +1000 Subject: [PATCH 44/72] Add datadir flag to readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d48c397d9..1f71315f5 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ First, clone this repository, [setup a development environment](docs/installation.md) and navigate to the root directory of this repository. Then, run `$ cargo build --all --release` and navigate to the `target/release` -directory. +directory and follow the steps: #### 1. Generate Validator Keys @@ -108,10 +108,14 @@ running node. The running node will display it's ENR as a base64 string. This ENR, by default, has a target address of `127.0.0.1` meaning that any new node will connect to this node via `127.0.0.1`. If a boot node should be connected to on a different address, it should be run with the `--discovery-address` CLI flag to specify how other nodes may connect to it. ``` -$ ./beacon_node -r --boot-nodes --listen-address 127.0.0.1 --port 9001 +$ ./beacon_node -r --boot-nodes --listen-address 127.0.0.1 --port 9001 --datadir /tmp/.lighthouse ``` Here is the ENR string displayed in the terminal from the first node. The ENR can also be obtained from it's default directory `.lighthouse/network/enr.dat`. +The `--datadir` flag tells this Beacon Node to store it's files in a different +directory. If you're on a system that doesn't have a `/tmp` dir (e.g., Mac, +Windows), substitute this with any directory that has write access. + Note that all future created nodes can use the same boot-node ENR. Once connected to the boot node, all nodes should discover and connect with each other. #### 4. Start a Validator Client From 5943e176cf497442beb3fa3bea97edf95a2cda65 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 5 Jul 2019 17:33:20 +1000 Subject: [PATCH 45/72] Add ssz_types crate --- Cargo.toml | 1 + eth2/utils/ssz_types/Cargo.toml | 19 + eth2/utils/ssz_types/src/bit_vector.rs | 229 +++++++++ eth2/utils/ssz_types/src/bitfield.rs | 570 ++++++++++++++++++++++ eth2/utils/ssz_types/src/fixed_vector.rs | 335 +++++++++++++ eth2/utils/ssz_types/src/lib.rs | 23 + eth2/utils/ssz_types/src/variable_list.rs | 321 ++++++++++++ 7 files changed, 1498 insertions(+) create mode 100644 eth2/utils/ssz_types/Cargo.toml create mode 100644 eth2/utils/ssz_types/src/bit_vector.rs create mode 100644 eth2/utils/ssz_types/src/bitfield.rs create mode 100644 eth2/utils/ssz_types/src/fixed_vector.rs create mode 100644 eth2/utils/ssz_types/src/lib.rs create mode 100644 eth2/utils/ssz_types/src/variable_list.rs diff --git a/Cargo.toml b/Cargo.toml index ef17b431e..22ec6fd98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "eth2/utils/slot_clock", "eth2/utils/ssz", "eth2/utils/ssz_derive", + "eth2/utils/ssz_types", "eth2/utils/swap_or_not_shuffle", "eth2/utils/tree_hash", "eth2/utils/tree_hash_derive", diff --git a/eth2/utils/ssz_types/Cargo.toml b/eth2/utils/ssz_types/Cargo.toml new file mode 100644 index 000000000..31d567d49 --- /dev/null +++ b/eth2/utils/ssz_types/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "ssz_types" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +bit_reverse = "0.1" +bit-vec = "0.5.0" +cached_tree_hash = { path = "../cached_tree_hash" } +tree_hash = { path = "../tree_hash" } +serde = "1.0" +serde_derive = "1.0" +serde_hex = { path = "../serde_hex" } +ssz = { path = "../ssz" } +typenum = "1.10" + +[dev-dependencies] +serde_yaml = "0.8" diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs new file mode 100644 index 000000000..48c91f94d --- /dev/null +++ b/eth2/utils/ssz_types/src/bit_vector.rs @@ -0,0 +1,229 @@ +use crate::bitfield::{Bitfield, Error}; +use crate::{FixedSizedError, VariableSizedError}; +use std::marker::PhantomData; +use typenum::Unsigned; + +/// Provides a common `impl` for structs that wrap a `Bitfield`. +macro_rules! common_impl { + ($name: ident, $error: ident) => { + impl $name { + /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. + /// + /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` + /// regardless of `bit`. + pub fn from_elem(initial_len: usize, bit: bool) -> Result { + let bitfield = Bitfield::from_elem(initial_len, bit); + Self::from_bitfield(bitfield) + } + + /// Create a new BitList using the supplied `bytes` as input + pub fn from_bytes(bytes: &[u8]) -> Result { + let bitfield = Bitfield::from_bytes(bytes); + Self::from_bitfield(bitfield) + } + + /// Returns a vector of bytes representing the bitfield + pub fn to_bytes(&self) -> Vec { + self.bitfield.to_bytes() + } + + /// Read the value of a bit. + /// + /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0. + /// If the index is out of bounds, we return an error to that extent. + pub fn get(&self, i: usize) -> Result { + self.bitfield.get(i) + } + + fn capacity() -> usize { + N::to_usize() + } + + /// Set the value of a bit. + /// + /// Returns an `Err` if `i` is outside of the maximum permitted length. + pub fn set(&mut self, i: usize, value: bool) -> Result<(), VariableSizedError> { + if i < Self::capacity() { + self.bitfield.set(i, value); + Ok(()) + } else { + Err(VariableSizedError::ExceedsMaxLength { + len: Self::capacity() + 1, + max_len: Self::capacity(), + }) + } + } + + /// Returns the number of bits in this bitfield. + pub fn len(&self) -> usize { + self.bitfield.len() + } + + /// Returns true if `self.len() == 0` + pub fn is_empty(&self) -> bool { + self.bitfield.is_empty() + } + + /// Returns true if all bits are set to 0. + pub fn is_zero(&self) -> bool { + self.bitfield.is_zero() + } + + /// Returns the number of bytes required to represent this bitfield. + pub fn num_bytes(&self) -> usize { + self.bitfield.num_bytes() + } + + /// Returns the number of `1` bits in the bitfield + pub fn num_set_bits(&self) -> usize { + self.bitfield.num_set_bits() + } + } + }; +} + +/// Emulates a SSZ `Bitvector`. +/// +/// An ordered, heap-allocated, fixed-length, collection of `bool` values, with `N` values. +pub struct BitVector { + bitfield: Bitfield, + _phantom: PhantomData, +} + +common_impl!(BitVector, FixedSizedError); + +impl BitVector { + /// Create a new bitfield. + pub fn new() -> Self { + Self { + bitfield: Bitfield::with_capacity(N::to_usize()), + _phantom: PhantomData, + } + } + + fn from_bitfield(bitfield: Bitfield) -> Result { + if bitfield.len() != Self::capacity() { + Err(FixedSizedError::InvalidLength { + len: bitfield.len(), + fixed_len: Self::capacity(), + }) + } else { + Ok(Self { + bitfield, + _phantom: PhantomData, + }) + } + } +} + +/// Emulates a SSZ `Bitlist`. +/// +/// An ordered, heap-allocated, variable-length, collection of `bool` values, limited to `N` +/// values. +pub struct BitList { + bitfield: Bitfield, + _phantom: PhantomData, +} + +common_impl!(BitList, VariableSizedError); + +impl BitList { + /// Create a new, empty BitList. + pub fn new() -> Self { + Self { + bitfield: Bitfield::default(), + _phantom: PhantomData, + } + } + + /// Create a new BitList list with `initial_len` bits all set to `false`. + pub fn with_capacity(initial_len: usize) -> Result { + Self::from_elem(initial_len, false) + } + + /// The maximum possible number of bits. + pub fn max_len() -> usize { + N::to_usize() + } + + fn from_bitfield(bitfield: Bitfield) -> Result { + if bitfield.len() > Self::max_len() { + Err(VariableSizedError::ExceedsMaxLength { + len: bitfield.len(), + max_len: Self::max_len(), + }) + } else { + Ok(Self { + bitfield, + _phantom: PhantomData, + }) + } + } + + /// Compute the intersection (binary-and) of this bitfield with another + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn intersection(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let bitfield = self.bitfield.intersection(&other.bitfield); + Self::from_bitfield(bitfield).expect( + "An intersection of two same-sized sets cannot be larger than one of the initial sets", + ) + } + + /// Like `intersection` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn intersection_inplace(&mut self, other: &Self) { + self.bitfield.intersection_inplace(&other.bitfield); + } + + /// Compute the union (binary-or) of this bitfield with another. Lengths must match. + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn union(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let bitfield = self.bitfield.union(&other.bitfield); + Self::from_bitfield(bitfield) + .expect("A union of two same-sized sets cannot be larger than one of the initial sets") + } + + /// Like `union` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn union_inplace(&mut self, other: &Self) { + self.bitfield.union_inplace(&other.bitfield) + } + + /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. + /// + /// Computes `self - other`. + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn difference(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let bitfield = self.bitfield.difference(&other.bitfield); + Self::from_bitfield(bitfield).expect( + "A difference of two same-sized sets cannot be larger than one of the initial sets", + ) + } + + /// Like `difference` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn difference_inplace(&mut self, other: &Self) { + self.bitfield.difference_inplace(&other.bitfield) + } +} diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs new file mode 100644 index 000000000..d77f63cf7 --- /dev/null +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -0,0 +1,570 @@ +use bit_reverse::LookupReverse; +use bit_vec::BitVec; +use cached_tree_hash::cached_tree_hash_bytes_as_list; +use serde::de::{Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode, PrefixedHexVisitor}; +use ssz::{Decode, Encode}; +use std::cmp; +use std::default; + +/// A Bitfield represents a set of booleans compactly stored as a vector of bits. +/// The Bitfield is given a fixed size during construction. Reads outside of the current size return an out-of-bounds error. Writes outside of the current size expand the size of the set. +#[derive(Debug, Clone)] +pub struct Bitfield(BitVec); + +/// Error represents some reason a request against a bitfield was not satisfied +#[derive(Debug, PartialEq)] +pub enum Error { + /// OutOfBounds refers to indexing into a bitfield where no bits exist; returns the illegal index and the current size of the bitfield, respectively + OutOfBounds(usize, usize), +} + +impl Bitfield { + pub fn with_capacity(initial_len: usize) -> Self { + Self::from_elem(initial_len, false) + } + + /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. + /// + /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` + /// regardless of `bit`. + pub fn from_elem(initial_len: usize, bit: bool) -> Self { + // BitVec can panic if we don't set the len to be a multiple of 8. + let full_len = ((initial_len + 7) / 8) * 8; + let mut bitfield = BitVec::from_elem(full_len, false); + + if bit { + for i in 0..initial_len { + bitfield.set(i, true); + } + } + + Self { 0: bitfield } + } + + /// Create a new bitfield using the supplied `bytes` as input + pub fn from_bytes(bytes: &[u8]) -> Self { + Self { + 0: BitVec::from_bytes(&reverse_bit_order(bytes.to_vec())), + } + } + + /// Returns a vector of bytes representing the bitfield + pub fn to_bytes(&self) -> Vec { + reverse_bit_order(self.0.to_bytes().to_vec()) + } + + /// Read the value of a bit. + /// + /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0. + /// If the index is out of bounds, we return an error to that extent. + pub fn get(&self, i: usize) -> Result { + match self.0.get(i) { + Some(value) => Ok(value), + None => Err(Error::OutOfBounds(i, self.0.len())), + } + } + + /// Set the value of a bit. + /// + /// If the index is out of bounds, we expand the size of the underlying set to include the new index. + /// Returns the previous value if there was one. + pub fn set(&mut self, i: usize, value: bool) -> Option { + let previous = match self.get(i) { + Ok(previous) => Some(previous), + Err(Error::OutOfBounds(_, len)) => { + let new_len = i - len + 1; + self.0.grow(new_len, false); + None + } + }; + self.0.set(i, value); + previous + } + + /// Returns the number of bits in this bitfield. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns true if `self.len() == 0` + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns true if all bits are set to 0. + pub fn is_zero(&self) -> bool { + self.0.none() + } + + /// Returns the number of bytes required to represent this bitfield. + pub fn num_bytes(&self) -> usize { + self.to_bytes().len() + } + + /// Returns the number of `1` bits in the bitfield + pub fn num_set_bits(&self) -> usize { + self.0.iter().filter(|&bit| bit).count() + } + + /// Compute the intersection (binary-and) of this bitfield with another. Lengths must match. + pub fn intersection(&self, other: &Self) -> Self { + let mut res = self.clone(); + res.intersection_inplace(other); + res + } + + /// Like `intersection` but in-place (updates `self`). + pub fn intersection_inplace(&mut self, other: &Self) { + self.0.intersect(&other.0); + } + + /// Compute the union (binary-or) of this bitfield with another. Lengths must match. + pub fn union(&self, other: &Self) -> Self { + let mut res = self.clone(); + res.union_inplace(other); + res + } + + /// Like `union` but in-place (updates `self`). + pub fn union_inplace(&mut self, other: &Self) { + self.0.union(&other.0); + } + + /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. + /// + /// Computes `self - other`. + pub fn difference(&self, other: &Self) -> Self { + let mut res = self.clone(); + res.difference_inplace(other); + res + } + + /// Like `difference` but in-place (updates `self`). + pub fn difference_inplace(&mut self, other: &Self) { + self.0.difference(&other.0); + } +} + +impl default::Default for Bitfield { + /// default provides the "empty" bitfield + /// Note: the empty bitfield is set to the `0` byte. + fn default() -> Self { + Self::from_elem(8, false) + } +} + +impl cmp::PartialEq for Bitfield { + /// Determines equality by comparing the `ssz` encoding of the two candidates. + /// This method ensures that the presence of high-order (empty) bits in the highest byte do not exclude equality when they are in fact representing the same information. + fn eq(&self, other: &Self) -> bool { + ssz::ssz_encode(self) == ssz::ssz_encode(other) + } +} + +/// Create a new bitfield that is a union of two other bitfields. +/// +/// For example `union(0101, 1000) == 1101` +// TODO: length-independent intersection for BitAnd +impl std::ops::BitOr for Bitfield { + type Output = Self; + + fn bitor(self, other: Self) -> Self { + let (biggest, smallest) = if self.len() > other.len() { + (&self, &other) + } else { + (&other, &self) + }; + let mut new = biggest.clone(); + for i in 0..smallest.len() { + if let Ok(true) = smallest.get(i) { + new.set(i, true); + } + } + new + } +} + +impl Encode for Bitfield { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_append(&self, buf: &mut Vec) { + buf.append(&mut self.to_bytes()) + } +} + +impl Decode for Bitfield { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + Ok(Bitfield::from_bytes(bytes)) + } +} + +// Reverse the bit order of a whole byte vec, so that the ith bit +// of the input vec is placed in the (N - i)th bit of the output vec. +// This function is necessary for converting bitfields to and from YAML, +// as the BitVec library and the hex-parser use opposing bit orders. +fn reverse_bit_order(mut bytes: Vec) -> Vec { + bytes.reverse(); + bytes.into_iter().map(LookupReverse::swap_bits).collect() +} + +impl Serialize for Bitfield { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&encode(self.to_bytes())) + } +} + +impl<'de> Deserialize<'de> for Bitfield { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + Ok(Bitfield::from_bytes(&bytes)) + } +} + +impl tree_hash::TreeHash for Bitfield { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.to_bytes().tree_hash_root() + } +} + +cached_tree_hash_bytes_as_list!(Bitfield); + +#[cfg(test)] +mod tests { + use super::*; + use serde_yaml; + use ssz::ssz_encode; + use tree_hash::TreeHash; + + impl Bitfield { + /// Create a new bitfield. + pub fn new() -> Self { + Default::default() + } + } + + #[test] + pub fn test_cached_tree_hash() { + let original = Bitfield::from_bytes(&vec![18; 12][..]); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let modified = Bitfield::from_bytes(&vec![2; 1][..]); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } + + #[test] + fn test_new_bitfield() { + let mut field = Bitfield::new(); + let original_len = field.len(); + + for i in 0..100 { + if i < original_len { + assert!(!field.get(i).unwrap()); + } else { + assert!(field.get(i).is_err()); + } + let previous = field.set(i, true); + if i < original_len { + assert!(!previous.unwrap()); + } else { + assert!(previous.is_none()); + } + } + } + + #[test] + fn test_empty_bitfield() { + let mut field = Bitfield::from_elem(0, false); + let original_len = field.len(); + + assert_eq!(original_len, 0); + + for i in 0..100 { + if i < original_len { + assert!(!field.get(i).unwrap()); + } else { + assert!(field.get(i).is_err()); + } + let previous = field.set(i, true); + if i < original_len { + assert!(!previous.unwrap()); + } else { + assert!(previous.is_none()); + } + } + + assert_eq!(field.len(), 100); + assert_eq!(field.num_set_bits(), 100); + } + + const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; + + #[test] + fn test_get_from_bitfield() { + let field = Bitfield::from_bytes(INPUT); + let unset = field.get(0).unwrap(); + assert!(!unset); + let set = field.get(6).unwrap(); + assert!(set); + let set = field.get(14).unwrap(); + assert!(set); + } + + #[test] + fn test_set_for_bitfield() { + let mut field = Bitfield::from_bytes(INPUT); + let previous = field.set(10, true).unwrap(); + assert!(!previous); + let previous = field.get(10).unwrap(); + assert!(previous); + let previous = field.set(6, false).unwrap(); + assert!(previous); + let previous = field.get(6).unwrap(); + assert!(!previous); + } + + #[test] + fn test_len() { + let field = Bitfield::from_bytes(INPUT); + assert_eq!(field.len(), 16); + + let field = Bitfield::new(); + assert_eq!(field.len(), 8); + } + + #[test] + fn test_num_set_bits() { + let field = Bitfield::from_bytes(INPUT); + assert_eq!(field.num_set_bits(), 2); + + let field = Bitfield::new(); + assert_eq!(field.num_set_bits(), 0); + } + + #[test] + fn test_to_bytes() { + let field = Bitfield::from_bytes(INPUT); + assert_eq!(field.to_bytes(), INPUT); + + let field = Bitfield::new(); + assert_eq!(field.to_bytes(), vec![0]); + } + + #[test] + fn test_out_of_bounds() { + let mut field = Bitfield::from_bytes(INPUT); + + let out_of_bounds_index = field.len(); + assert!(field.set(out_of_bounds_index, true).is_none()); + assert!(field.len() == out_of_bounds_index + 1); + assert!(field.get(out_of_bounds_index).unwrap()); + + for i in 0..100 { + if i <= out_of_bounds_index { + assert!(field.set(i, true).is_some()); + } else { + assert!(field.set(i, true).is_none()); + } + } + } + + #[test] + fn test_grows_with_false() { + let input_all_set: &[u8] = &[0b1111_1111, 0b1111_1111]; + let mut field = Bitfield::from_bytes(input_all_set); + + // Define `a` and `b`, where both are out of bounds and `b` is greater than `a`. + let a = field.len(); + let b = a + 1; + + // Ensure `a` is out-of-bounds for test integrity. + assert!(field.get(a).is_err()); + + // Set `b` to `true`. Also, for test integrity, ensure it was previously out-of-bounds. + assert!(field.set(b, true).is_none()); + + // Ensure that `a` wasn't also set to `true` during the grow. + assert_eq!(field.get(a), Ok(false)); + assert_eq!(field.get(b), Ok(true)); + } + + #[test] + fn test_num_bytes() { + let field = Bitfield::from_bytes(INPUT); + assert_eq!(field.num_bytes(), 2); + + let field = Bitfield::from_elem(2, true); + assert_eq!(field.num_bytes(), 1); + + let field = Bitfield::from_elem(13, true); + assert_eq!(field.num_bytes(), 2); + } + + #[test] + fn test_ssz_encode() { + let field = create_test_bitfield(); + assert_eq!(field.as_ssz_bytes(), vec![0b0000_0011, 0b1000_0111]); + + let field = Bitfield::from_elem(18, true); + assert_eq!( + field.as_ssz_bytes(), + vec![0b0000_0011, 0b1111_1111, 0b1111_1111] + ); + + let mut b = Bitfield::new(); + b.set(1, true); + assert_eq!(ssz_encode(&b), vec![0b0000_0010]); + } + + fn create_test_bitfield() -> Bitfield { + let count = 2 * 8; + let mut field = Bitfield::with_capacity(count); + + let indices = &[0, 1, 2, 7, 8, 9]; + for &i in indices { + field.set(i, true); + } + field + } + + #[test] + fn test_ssz_decode() { + let encoded = vec![0b0000_0011, 0b1000_0111]; + let field = Bitfield::from_ssz_bytes(&encoded).unwrap(); + let expected = create_test_bitfield(); + assert_eq!(field, expected); + + let encoded = vec![255, 255, 3]; + let field = Bitfield::from_ssz_bytes(&encoded).unwrap(); + let expected = Bitfield::from_bytes(&[255, 255, 3]); + assert_eq!(field, expected); + } + + #[test] + fn test_serialize_deserialize() { + use serde_yaml::Value; + + let data: &[(_, &[_])] = &[ + ("0x01", &[0b00000001]), + ("0xf301", &[0b11110011, 0b00000001]), + ]; + for (hex_data, bytes) in data { + let bitfield = Bitfield::from_bytes(bytes); + assert_eq!( + serde_yaml::from_str::(hex_data).unwrap(), + bitfield + ); + assert_eq!( + serde_yaml::to_value(&bitfield).unwrap(), + Value::String(hex_data.to_string()) + ); + } + } + + #[test] + fn test_ssz_round_trip() { + let original = Bitfield::from_bytes(&vec![18; 12][..]); + let ssz = ssz_encode(&original); + let decoded = Bitfield::from_ssz_bytes(&ssz).unwrap(); + assert_eq!(original, decoded); + } + + #[test] + fn test_bitor() { + let a = Bitfield::from_bytes(&vec![2, 8, 1][..]); + let b = Bitfield::from_bytes(&vec![4, 8, 16][..]); + let c = Bitfield::from_bytes(&vec![6, 8, 17][..]); + assert_eq!(c, a | b); + } + + #[test] + fn test_is_zero() { + let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]]; + for bytes in yes_data { + assert!(Bitfield::from_bytes(bytes).is_zero()); + } + let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]]; + for bytes in no_data { + assert!(!Bitfield::from_bytes(bytes).is_zero()); + } + } + + #[test] + fn test_intersection() { + let a = Bitfield::from_bytes(&[0b1100, 0b0001]); + let b = Bitfield::from_bytes(&[0b1011, 0b1001]); + let c = Bitfield::from_bytes(&[0b1000, 0b0001]); + assert_eq!(a.intersection(&b), c); + assert_eq!(b.intersection(&a), c); + assert_eq!(a.intersection(&c), c); + assert_eq!(b.intersection(&c), c); + assert_eq!(a.intersection(&a), a); + assert_eq!(b.intersection(&b), b); + assert_eq!(c.intersection(&c), c); + } + + #[test] + fn test_union() { + let a = Bitfield::from_bytes(&[0b1100, 0b0001]); + let b = Bitfield::from_bytes(&[0b1011, 0b1001]); + let c = Bitfield::from_bytes(&[0b1111, 0b1001]); + assert_eq!(a.union(&b), c); + assert_eq!(b.union(&a), c); + assert_eq!(a.union(&a), a); + assert_eq!(b.union(&b), b); + assert_eq!(c.union(&c), c); + } + + #[test] + fn test_difference() { + let a = Bitfield::from_bytes(&[0b1100, 0b0001]); + let b = Bitfield::from_bytes(&[0b1011, 0b1001]); + let a_b = Bitfield::from_bytes(&[0b0100, 0b0000]); + let b_a = Bitfield::from_bytes(&[0b0011, 0b1000]); + assert_eq!(a.difference(&b), a_b); + assert_eq!(b.difference(&a), b_a); + assert!(a.difference(&a).is_zero()); + } +} diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs new file mode 100644 index 000000000..071532e22 --- /dev/null +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -0,0 +1,335 @@ +use crate::FixedSizedError as Error; +use serde_derive::{Deserialize, Serialize}; +use std::marker::PhantomData; +use std::ops::{Deref, Index, IndexMut}; +use std::slice::SliceIndex; +use typenum::Unsigned; + +pub use typenum; + +/// Emulates a SSZ `Vector` (distinct from a Rust `Vec`). +/// +/// An ordered, heap-allocated, fixed-length, homogeneous collection of `T`, with `N` values. +/// +/// This struct is backed by a Rust `Vec` but constrained such that it must be instantiated with a +/// fixed number of elements and you may not add or remove elements, only modify. +/// +/// The length of this struct is fixed at the type-level using +/// [typenum](https://crates.io/crates/typenum). +/// +/// ## Note +/// +/// Whilst it is possible with this library, SSZ declares that a `FixedVector` with a length of `0` +/// is illegal. +/// +/// ## Example +/// +/// ``` +/// use ssz_types::{FixedVector, typenum}; +/// +/// let base: Vec = vec![1, 2, 3, 4]; +/// +/// // Create a `FixedVector` from a `Vec` that has the expected length. +/// let exact: FixedVector<_, typenum::U4> = FixedVector::from(base.clone()); +/// assert_eq!(&exact[..], &[1, 2, 3, 4]); +/// +/// // Create a `FixedVector` from a `Vec` that is too long and the `Vec` is truncated. +/// let short: FixedVector<_, typenum::U3> = FixedVector::from(base.clone()); +/// assert_eq!(&short[..], &[1, 2, 3]); +/// +/// // Create a `FixedVector` from a `Vec` that is too short and the missing values are created +/// // using `std::default::Default`. +/// let long: FixedVector<_, typenum::U5> = FixedVector::from(base); +/// assert_eq!(&long[..], &[1, 2, 3, 4, 0]); +/// ``` +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct FixedVector { + vec: Vec, + _phantom: PhantomData, +} + +impl FixedVector { + /// Returns `Some` if the given `vec` equals the fixed length of `Self`. Otherwise returns + /// `None`. + pub fn new(vec: Vec) -> Result { + if vec.len() == Self::capacity() { + Ok(Self { + vec, + _phantom: PhantomData, + }) + } else { + Err(Error::InvalidLength { + len: vec.len(), + fixed_len: Self::capacity(), + }) + } + } + + /// Identical to `self.capacity`, returns the type-level constant length. + /// + /// Exists for compatibility with `Vec`. + pub fn len(&self) -> usize { + self.vec.len() + } + + /// True if the type-level constant length of `self` is zero. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the type-level constant length. + pub fn capacity() -> usize { + N::to_usize() + } +} + +impl From> for FixedVector { + fn from(mut vec: Vec) -> Self { + vec.resize_with(Self::capacity(), Default::default); + + Self { + vec, + _phantom: PhantomData, + } + } +} + +impl Into> for FixedVector { + fn into(self) -> Vec { + self.vec + } +} + +impl Default for FixedVector { + fn default() -> Self { + Self { + vec: Vec::default(), + _phantom: PhantomData, + } + } +} + +impl> Index for FixedVector { + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&self.vec, index) + } +} + +impl> IndexMut for FixedVector { + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut self.vec, index) + } +} + +impl Deref for FixedVector { + type Target = [T]; + + fn deref(&self) -> &[T] { + &self.vec[..] + } +} + +#[cfg(test)] +mod test { + use super::*; + use typenum::*; + + #[test] + fn new() { + let vec = vec![42; 5]; + let fixed: Result, _> = FixedVector::new(vec.clone()); + assert!(fixed.is_err()); + + let vec = vec![42; 3]; + let fixed: Result, _> = FixedVector::new(vec.clone()); + assert!(fixed.is_err()); + + let vec = vec![42; 4]; + let fixed: Result, _> = FixedVector::new(vec.clone()); + assert!(fixed.is_ok()); + } + + #[test] + fn indexing() { + let vec = vec![1, 2]; + + let mut fixed: FixedVector = vec.clone().into(); + + assert_eq!(fixed[0], 1); + assert_eq!(&fixed[0..1], &vec[0..1]); + assert_eq!((&fixed[..]).len(), 8192); + + fixed[1] = 3; + assert_eq!(fixed[1], 3); + } + + #[test] + fn length() { + let vec = vec![42; 5]; + let fixed: FixedVector = FixedVector::from(vec.clone()); + assert_eq!(&fixed[..], &vec[0..4]); + + let vec = vec![42; 3]; + let fixed: FixedVector = FixedVector::from(vec.clone()); + assert_eq!(&fixed[0..3], &vec[..]); + assert_eq!(&fixed[..], &vec![42, 42, 42, 0][..]); + + let vec = vec![]; + let fixed: FixedVector = FixedVector::from(vec.clone()); + assert_eq!(&fixed[..], &vec![0, 0, 0, 0][..]); + } + + #[test] + fn deref() { + let vec = vec![0, 2, 4, 6]; + let fixed: FixedVector = FixedVector::from(vec); + + assert_eq!(fixed.get(0), Some(&0)); + assert_eq!(fixed.get(3), Some(&6)); + assert_eq!(fixed.get(4), None); + } +} + +impl tree_hash::TreeHash for FixedVector +where + T: tree_hash::TreeHash, +{ + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::impls::vec_tree_hash_root(&self.vec) + } +} + +impl cached_tree_hash::CachedTreeHash for FixedVector +where + T: cached_tree_hash::CachedTreeHash + tree_hash::TreeHash, +{ + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let (cache, _overlay) = cached_tree_hash::vec::new_tree_hash_cache(&self.vec, depth)?; + + Ok(cache) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + cached_tree_hash::vec::produce_schema(&self.vec, depth) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + cached_tree_hash::vec::update_tree_hash_cache(&self.vec, cache)?; + + Ok(()) + } +} + +impl ssz::Encode for FixedVector +where + T: ssz::Encode, +{ + fn is_ssz_fixed_len() -> bool { + true + } + + fn ssz_fixed_len() -> usize { + if ::is_ssz_fixed_len() { + T::ssz_fixed_len() * N::to_usize() + } else { + ssz::BYTES_PER_LENGTH_OFFSET + } + } + + fn ssz_append(&self, buf: &mut Vec) { + if T::is_ssz_fixed_len() { + buf.reserve(T::ssz_fixed_len() * self.len()); + + for item in &self.vec { + item.ssz_append(buf); + } + } else { + let mut encoder = ssz::SszEncoder::list(buf, self.len() * ssz::BYTES_PER_LENGTH_OFFSET); + + for item in &self.vec { + encoder.append(item); + } + + encoder.finalize(); + } + } +} + +impl ssz::Decode for FixedVector +where + T: ssz::Decode + Default, +{ + fn is_ssz_fixed_len() -> bool { + T::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + if ::is_ssz_fixed_len() { + T::ssz_fixed_len() * N::to_usize() + } else { + ssz::BYTES_PER_LENGTH_OFFSET + } + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + if bytes.is_empty() { + Ok(FixedVector::from(vec![])) + } else if T::is_ssz_fixed_len() { + bytes + .chunks(T::ssz_fixed_len()) + .map(|chunk| T::from_ssz_bytes(chunk)) + .collect::, _>>() + .and_then(|vec| Ok(vec.into())) + } else { + ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into())) + } + } +} + +#[cfg(test)] +mod ssz_tests { + use super::*; + use ssz::*; + use typenum::*; + + #[test] + fn encode() { + let vec: FixedVector = vec![0; 2].into(); + assert_eq!(vec.as_ssz_bytes(), vec![0, 0, 0, 0]); + assert_eq!( as Encode>::ssz_fixed_len(), 4); + } + + fn round_trip(item: T) { + let encoded = &item.as_ssz_bytes(); + assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); + } + + #[test] + fn u16_len_8() { + round_trip::>(vec![42; 8].into()); + round_trip::>(vec![0; 8].into()); + } +} diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs new file mode 100644 index 000000000..37aa24875 --- /dev/null +++ b/eth2/utils/ssz_types/src/lib.rs @@ -0,0 +1,23 @@ +mod bit_vector; +mod bitfield; +mod fixed_vector; +mod variable_list; + +pub use bit_vector::{BitList, BitVector}; +pub use fixed_vector::FixedVector; +pub use typenum; +pub use variable_list::VariableList; + +/// Returned when a variable-length item encounters an error. +#[derive(PartialEq, Debug)] +pub enum VariableSizedError { + /// The operation would cause the maximum length to be exceeded. + ExceedsMaxLength { len: usize, max_len: usize }, +} + +/// Returned when a fixed-length item encounters an error. +#[derive(PartialEq, Debug)] +pub enum FixedSizedError { + /// The operation would create an item of an invalid size. + InvalidLength { len: usize, fixed_len: usize }, +} diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs new file mode 100644 index 000000000..3d0bf31c9 --- /dev/null +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -0,0 +1,321 @@ +use crate::VariableSizedError as Error; +use serde_derive::{Deserialize, Serialize}; +use std::marker::PhantomData; +use std::ops::{Deref, Index, IndexMut}; +use std::slice::SliceIndex; +use typenum::Unsigned; + +pub use typenum; + +/// Emulates a SSZ `List`. +/// +/// An ordered, heap-allocated, variable-length, homogeneous collection of `T`, with no more than +/// `N` values. +/// +/// This struct is backed by a Rust `Vec` but constrained such that it must be instantiated with a +/// fixed number of elements and you may not add or remove elements, only modify. +/// +/// The length of this struct is fixed at the type-level using +/// [typenum](https://crates.io/crates/typenum). +/// +/// ## Example +/// +/// ``` +/// use ssz_types::{VariableList, typenum}; +/// +/// let base: Vec = vec![1, 2, 3, 4]; +/// +/// // Create a `VariableList` from a `Vec` that has the expected length. +/// let exact: VariableList<_, typenum::U4> = VariableList::from(base.clone()); +/// assert_eq!(&exact[..], &[1, 2, 3, 4]); +/// +/// // Create a `VariableList` from a `Vec` that is too long and the `Vec` is truncated. +/// let short: VariableList<_, typenum::U3> = VariableList::from(base.clone()); +/// assert_eq!(&short[..], &[1, 2, 3]); +/// +/// // Create a `VariableList` from a `Vec` that is shorter than the maximum. +/// let mut long: VariableList<_, typenum::U5> = VariableList::from(base); +/// assert_eq!(&long[..], &[1, 2, 3, 4]); +/// +/// // Push a value to if it does not exceed the maximum +/// long.push(5).unwrap(); +/// assert_eq!(&long[..], &[1, 2, 3, 4, 5]); +/// +/// // Push a value to if it _does_ exceed the maximum. +/// assert!(long.push(6).is_err()); +/// ``` +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct VariableList { + vec: Vec, + _phantom: PhantomData, +} + +impl VariableList { + /// Returns `Some` if the given `vec` equals the fixed length of `Self`. Otherwise returns + /// `None`. + pub fn new(vec: Vec) -> Result { + if vec.len() <= N::to_usize() { + Ok(Self { + vec, + _phantom: PhantomData, + }) + } else { + Err(Error::ExceedsMaxLength { + len: vec.len(), + max_len: Self::max_len(), + }) + } + } + + /// Returns the number of values presently in `self`. + pub fn len(&self) -> usize { + self.vec.len() + } + + /// True if `self` does not contain any values. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the type-level maximum length. + pub fn max_len() -> usize { + N::to_usize() + } + + /// Appends `value` to the back of `self`. + /// + /// Returns `Err(())` when appending `value` would exceed the maximum length. + pub fn push(&mut self, value: T) -> Result<(), Error> { + if self.vec.len() < Self::max_len() { + Ok(self.vec.push(value)) + } else { + Err(Error::ExceedsMaxLength { + len: self.vec.len() + 1, + max_len: Self::max_len(), + }) + } + } +} + +impl From> for VariableList { + fn from(mut vec: Vec) -> Self { + vec.truncate(N::to_usize()); + + Self { + vec, + _phantom: PhantomData, + } + } +} + +impl Into> for VariableList { + fn into(self) -> Vec { + self.vec + } +} + +impl Default for VariableList { + fn default() -> Self { + Self { + vec: Vec::default(), + _phantom: PhantomData, + } + } +} + +impl> Index for VariableList { + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&self.vec, index) + } +} + +impl> IndexMut for VariableList { + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut self.vec, index) + } +} + +impl Deref for VariableList { + type Target = [T]; + + fn deref(&self) -> &[T] { + &self.vec[..] + } +} + +#[cfg(test)] +mod test { + use super::*; + use typenum::*; + + #[test] + fn new() { + let vec = vec![42; 5]; + let fixed: Result, _> = VariableList::new(vec.clone()); + assert!(fixed.is_err()); + + let vec = vec![42; 3]; + let fixed: Result, _> = VariableList::new(vec.clone()); + assert!(fixed.is_ok()); + + let vec = vec![42; 4]; + let fixed: Result, _> = VariableList::new(vec.clone()); + assert!(fixed.is_ok()); + } + + #[test] + fn indexing() { + let vec = vec![1, 2]; + + let mut fixed: VariableList = vec.clone().into(); + + assert_eq!(fixed[0], 1); + assert_eq!(&fixed[0..1], &vec[0..1]); + assert_eq!((&fixed[..]).len(), 2); + + fixed[1] = 3; + assert_eq!(fixed[1], 3); + } + + #[test] + fn length() { + let vec = vec![42; 5]; + let fixed: VariableList = VariableList::from(vec.clone()); + assert_eq!(&fixed[..], &vec[0..4]); + + let vec = vec![42; 3]; + let fixed: VariableList = VariableList::from(vec.clone()); + assert_eq!(&fixed[0..3], &vec[..]); + assert_eq!(&fixed[..], &vec![42, 42, 42][..]); + + let vec = vec![]; + let fixed: VariableList = VariableList::from(vec.clone()); + assert_eq!(&fixed[..], &vec![][..]); + } + + #[test] + fn deref() { + let vec = vec![0, 2, 4, 6]; + let fixed: VariableList = VariableList::from(vec); + + assert_eq!(fixed.get(0), Some(&0)); + assert_eq!(fixed.get(3), Some(&6)); + assert_eq!(fixed.get(4), None); + } +} + +/* +impl tree_hash::TreeHash for VariableList +where + T: tree_hash::TreeHash, +{ + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::Vector + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + tree_hash::impls::vec_tree_hash_root(&self.vec) + } +} + +impl cached_tree_hash::CachedTreeHash for VariableList +where + T: cached_tree_hash::CachedTreeHash + tree_hash::TreeHash, +{ + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let (cache, _overlay) = cached_tree_hash::vec::new_tree_hash_cache(&self.vec, depth)?; + + Ok(cache) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + cached_tree_hash::vec::produce_schema(&self.vec, depth) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + cached_tree_hash::vec::update_tree_hash_cache(&self.vec, cache)?; + + Ok(()) + } +} +*/ + +impl ssz::Encode for VariableList +where + T: ssz::Encode, +{ + fn is_ssz_fixed_len() -> bool { + >::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + >::ssz_fixed_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.vec.ssz_append(buf) + } +} + +impl ssz::Decode for VariableList +where + T: ssz::Decode + Default, +{ + fn is_ssz_fixed_len() -> bool { + >::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + >::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let vec = >::from_ssz_bytes(bytes)?; + + Self::new(vec).map_err(|e| ssz::DecodeError::BytesInvalid(format!("VariableList {:?}", e))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ssz::*; + use typenum::*; + + #[test] + fn encode() { + let vec: VariableList = vec![0; 2].into(); + assert_eq!(vec.as_ssz_bytes(), vec![0, 0, 0, 0]); + assert_eq!( as Encode>::ssz_fixed_len(), 4); + } + + fn round_trip(item: T) { + let encoded = &item.as_ssz_bytes(); + assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); + } + + #[test] + fn u16_len_8() { + round_trip::>(vec![42; 8].into()); + round_trip::>(vec![0; 8].into()); + } +} From 2b7d5560ad5265e54a32382c739d125979ef632e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 6 Jul 2019 15:51:15 +1000 Subject: [PATCH 46/72] Refactor SSZ types bitfield Removes superfulous `Bitfield` struct --- eth2/utils/ssz_types/Cargo.toml | 2 +- eth2/utils/ssz_types/src/bit_vector.rs | 229 --------- eth2/utils/ssz_types/src/bitfield.rs | 603 +++++++++++++++-------- eth2/utils/ssz_types/src/fixed_vector.rs | 4 +- eth2/utils/ssz_types/src/lib.rs | 21 +- 5 files changed, 410 insertions(+), 449 deletions(-) delete mode 100644 eth2/utils/ssz_types/src/bit_vector.rs diff --git a/eth2/utils/ssz_types/Cargo.toml b/eth2/utils/ssz_types/Cargo.toml index 31d567d49..f5680ecb1 100644 --- a/eth2/utils/ssz_types/Cargo.toml +++ b/eth2/utils/ssz_types/Cargo.toml @@ -12,7 +12,7 @@ tree_hash = { path = "../tree_hash" } serde = "1.0" serde_derive = "1.0" serde_hex = { path = "../serde_hex" } -ssz = { path = "../ssz" } +eth2_ssz = { path = "../ssz" } typenum = "1.10" [dev-dependencies] diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs deleted file mode 100644 index 48c91f94d..000000000 --- a/eth2/utils/ssz_types/src/bit_vector.rs +++ /dev/null @@ -1,229 +0,0 @@ -use crate::bitfield::{Bitfield, Error}; -use crate::{FixedSizedError, VariableSizedError}; -use std::marker::PhantomData; -use typenum::Unsigned; - -/// Provides a common `impl` for structs that wrap a `Bitfield`. -macro_rules! common_impl { - ($name: ident, $error: ident) => { - impl $name { - /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. - /// - /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` - /// regardless of `bit`. - pub fn from_elem(initial_len: usize, bit: bool) -> Result { - let bitfield = Bitfield::from_elem(initial_len, bit); - Self::from_bitfield(bitfield) - } - - /// Create a new BitList using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Result { - let bitfield = Bitfield::from_bytes(bytes); - Self::from_bitfield(bitfield) - } - - /// Returns a vector of bytes representing the bitfield - pub fn to_bytes(&self) -> Vec { - self.bitfield.to_bytes() - } - - /// Read the value of a bit. - /// - /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0. - /// If the index is out of bounds, we return an error to that extent. - pub fn get(&self, i: usize) -> Result { - self.bitfield.get(i) - } - - fn capacity() -> usize { - N::to_usize() - } - - /// Set the value of a bit. - /// - /// Returns an `Err` if `i` is outside of the maximum permitted length. - pub fn set(&mut self, i: usize, value: bool) -> Result<(), VariableSizedError> { - if i < Self::capacity() { - self.bitfield.set(i, value); - Ok(()) - } else { - Err(VariableSizedError::ExceedsMaxLength { - len: Self::capacity() + 1, - max_len: Self::capacity(), - }) - } - } - - /// Returns the number of bits in this bitfield. - pub fn len(&self) -> usize { - self.bitfield.len() - } - - /// Returns true if `self.len() == 0` - pub fn is_empty(&self) -> bool { - self.bitfield.is_empty() - } - - /// Returns true if all bits are set to 0. - pub fn is_zero(&self) -> bool { - self.bitfield.is_zero() - } - - /// Returns the number of bytes required to represent this bitfield. - pub fn num_bytes(&self) -> usize { - self.bitfield.num_bytes() - } - - /// Returns the number of `1` bits in the bitfield - pub fn num_set_bits(&self) -> usize { - self.bitfield.num_set_bits() - } - } - }; -} - -/// Emulates a SSZ `Bitvector`. -/// -/// An ordered, heap-allocated, fixed-length, collection of `bool` values, with `N` values. -pub struct BitVector { - bitfield: Bitfield, - _phantom: PhantomData, -} - -common_impl!(BitVector, FixedSizedError); - -impl BitVector { - /// Create a new bitfield. - pub fn new() -> Self { - Self { - bitfield: Bitfield::with_capacity(N::to_usize()), - _phantom: PhantomData, - } - } - - fn from_bitfield(bitfield: Bitfield) -> Result { - if bitfield.len() != Self::capacity() { - Err(FixedSizedError::InvalidLength { - len: bitfield.len(), - fixed_len: Self::capacity(), - }) - } else { - Ok(Self { - bitfield, - _phantom: PhantomData, - }) - } - } -} - -/// Emulates a SSZ `Bitlist`. -/// -/// An ordered, heap-allocated, variable-length, collection of `bool` values, limited to `N` -/// values. -pub struct BitList { - bitfield: Bitfield, - _phantom: PhantomData, -} - -common_impl!(BitList, VariableSizedError); - -impl BitList { - /// Create a new, empty BitList. - pub fn new() -> Self { - Self { - bitfield: Bitfield::default(), - _phantom: PhantomData, - } - } - - /// Create a new BitList list with `initial_len` bits all set to `false`. - pub fn with_capacity(initial_len: usize) -> Result { - Self::from_elem(initial_len, false) - } - - /// The maximum possible number of bits. - pub fn max_len() -> usize { - N::to_usize() - } - - fn from_bitfield(bitfield: Bitfield) -> Result { - if bitfield.len() > Self::max_len() { - Err(VariableSizedError::ExceedsMaxLength { - len: bitfield.len(), - max_len: Self::max_len(), - }) - } else { - Ok(Self { - bitfield, - _phantom: PhantomData, - }) - } - } - - /// Compute the intersection (binary-and) of this bitfield with another - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn intersection(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let bitfield = self.bitfield.intersection(&other.bitfield); - Self::from_bitfield(bitfield).expect( - "An intersection of two same-sized sets cannot be larger than one of the initial sets", - ) - } - - /// Like `intersection` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn intersection_inplace(&mut self, other: &Self) { - self.bitfield.intersection_inplace(&other.bitfield); - } - - /// Compute the union (binary-or) of this bitfield with another. Lengths must match. - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn union(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let bitfield = self.bitfield.union(&other.bitfield); - Self::from_bitfield(bitfield) - .expect("A union of two same-sized sets cannot be larger than one of the initial sets") - } - - /// Like `union` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn union_inplace(&mut self, other: &Self) { - self.bitfield.union_inplace(&other.bitfield) - } - - /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. - /// - /// Computes `self - other`. - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn difference(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let bitfield = self.bitfield.difference(&other.bitfield); - Self::from_bitfield(bitfield).expect( - "A difference of two same-sized sets cannot be larger than one of the initial sets", - ) - } - - /// Like `difference` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn difference_inplace(&mut self, other: &Self) { - self.bitfield.difference_inplace(&other.bitfield) - } -} diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index d77f63cf7..8462b53df 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -1,209 +1,227 @@ +use crate::FixedSizedError; use bit_reverse::LookupReverse; -use bit_vec::BitVec; -use cached_tree_hash::cached_tree_hash_bytes_as_list; +use bit_vec::BitVec as Bitfield; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode, PrefixedHexVisitor}; use ssz::{Decode, Encode}; use std::cmp; use std::default; +use std::marker::PhantomData; +use typenum::Unsigned; -/// A Bitfield represents a set of booleans compactly stored as a vector of bits. -/// The Bitfield is given a fixed size during construction. Reads outside of the current size return an out-of-bounds error. Writes outside of the current size expand the size of the set. -#[derive(Debug, Clone)] -pub struct Bitfield(BitVec); +/// Provides a common `impl` for structs that wrap a `$name`. +macro_rules! common_impl { + ($name: ident, $error: ident) => { + impl $name { + /// Create a new BitList list with `initial_len` bits all set to `false`. + pub fn with_capacity(initial_len: usize) -> Result { + Self::from_elem(initial_len, false) + } -/// Error represents some reason a request against a bitfield was not satisfied -#[derive(Debug, PartialEq)] -pub enum Error { - /// OutOfBounds refers to indexing into a bitfield where no bits exist; returns the illegal index and the current size of the bitfield, respectively - OutOfBounds(usize, usize), -} + /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. + /// + /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` + /// regardless of `bit`. + pub fn from_elem(initial_len: usize, bit: bool) -> Result { + // BitVec can panic if we don't set the len to be a multiple of 8. + let full_len = ((initial_len + 7) / 8) * 8; -impl Bitfield { - pub fn with_capacity(initial_len: usize) -> Self { - Self::from_elem(initial_len, false) - } + Self::validate_length(full_len)?; - /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. - /// - /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` - /// regardless of `bit`. - pub fn from_elem(initial_len: usize, bit: bool) -> Self { - // BitVec can panic if we don't set the len to be a multiple of 8. - let full_len = ((initial_len + 7) / 8) * 8; - let mut bitfield = BitVec::from_elem(full_len, false); + let mut bitfield = Bitfield::from_elem(full_len, false); - if bit { - for i in 0..initial_len { - bitfield.set(i, true); + if bit { + for i in 0..initial_len { + bitfield.set(i, true); + } + } + + Ok(Self { + bitfield, + _phantom: PhantomData, + }) + } + + /// Create a new bitfield using the supplied `bytes` as input + pub fn from_bytes(bytes: &[u8]) -> Result { + Self::validate_length(bytes.len().saturating_mul(8))?; + + Ok(Self { + bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), + _phantom: PhantomData, + }) + } + /// Returns a vector of bytes representing the bitfield + pub fn to_bytes(&self) -> Vec { + reverse_bit_order(self.bitfield.to_bytes().to_vec()) + } + + /// Read the value of a bit. + /// + /// If the index is in bounds, then result is Ok(value) where value is `true` if the + /// bit is 1 and `false` if the bit is 0. If the index is out of bounds, we return an + /// error to that extent. + pub fn get(&self, i: usize) -> Result { + if i < N::to_usize() { + match self.bitfield.get(i) { + Some(value) => Ok(value), + None => Err($error::OutOfBounds { + i, + len: self.bitfield.len(), + }), + } + } else { + Err($error::InvalidLength { + i, + len: N::to_usize(), + }) + } + } + + /// Set the value of a bit. + /// + /// If the index is out of bounds, we expand the size of the underlying set to include + /// the new index. Returns the previous value if there was one. + pub fn set(&mut self, i: usize, value: bool) -> Result<(), $error> { + match self.get(i) { + Ok(previous) => Some(previous), + Err($error::OutOfBounds { len, .. }) => { + let new_len = i - len + 1; + self.bitfield.grow(new_len, false); + None + } + Err(e) => return Err(e), + }; + + self.bitfield.set(i, value); + + Ok(()) + } + + /// Returns the number of bits in this bitfield. + pub fn len(&self) -> usize { + self.bitfield.len() + } + + /// Returns true if `self.len() == 0` + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns true if all bits are set to 0. + pub fn is_zero(&self) -> bool { + self.bitfield.none() + } + + /// Returns the number of bytes required to represent this bitfield. + pub fn num_bytes(&self) -> usize { + self.to_bytes().len() + } + + /// Returns the number of `1` bits in the bitfield + pub fn num_set_bits(&self) -> usize { + self.bitfield.iter().filter(|&bit| bit).count() } } - Self { 0: bitfield } - } - - /// Create a new bitfield using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Self { - Self { - 0: BitVec::from_bytes(&reverse_bit_order(bytes.to_vec())), - } - } - - /// Returns a vector of bytes representing the bitfield - pub fn to_bytes(&self) -> Vec { - reverse_bit_order(self.0.to_bytes().to_vec()) - } - - /// Read the value of a bit. - /// - /// If the index is in bounds, then result is Ok(value) where value is `true` if the bit is 1 and `false` if the bit is 0. - /// If the index is out of bounds, we return an error to that extent. - pub fn get(&self, i: usize) -> Result { - match self.0.get(i) { - Some(value) => Ok(value), - None => Err(Error::OutOfBounds(i, self.0.len())), - } - } - - /// Set the value of a bit. - /// - /// If the index is out of bounds, we expand the size of the underlying set to include the new index. - /// Returns the previous value if there was one. - pub fn set(&mut self, i: usize, value: bool) -> Option { - let previous = match self.get(i) { - Ok(previous) => Some(previous), - Err(Error::OutOfBounds(_, len)) => { - let new_len = i - len + 1; - self.0.grow(new_len, false); - None - } - }; - self.0.set(i, value); - previous - } - - /// Returns the number of bits in this bitfield. - pub fn len(&self) -> usize { - self.0.len() - } - - /// Returns true if `self.len() == 0` - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns true if all bits are set to 0. - pub fn is_zero(&self) -> bool { - self.0.none() - } - - /// Returns the number of bytes required to represent this bitfield. - pub fn num_bytes(&self) -> usize { - self.to_bytes().len() - } - - /// Returns the number of `1` bits in the bitfield - pub fn num_set_bits(&self) -> usize { - self.0.iter().filter(|&bit| bit).count() - } - - /// Compute the intersection (binary-and) of this bitfield with another. Lengths must match. - pub fn intersection(&self, other: &Self) -> Self { - let mut res = self.clone(); - res.intersection_inplace(other); - res - } - - /// Like `intersection` but in-place (updates `self`). - pub fn intersection_inplace(&mut self, other: &Self) { - self.0.intersect(&other.0); - } - - /// Compute the union (binary-or) of this bitfield with another. Lengths must match. - pub fn union(&self, other: &Self) -> Self { - let mut res = self.clone(); - res.union_inplace(other); - res - } - - /// Like `union` but in-place (updates `self`). - pub fn union_inplace(&mut self, other: &Self) { - self.0.union(&other.0); - } - - /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. - /// - /// Computes `self - other`. - pub fn difference(&self, other: &Self) -> Self { - let mut res = self.clone(); - res.difference_inplace(other); - res - } - - /// Like `difference` but in-place (updates `self`). - pub fn difference_inplace(&mut self, other: &Self) { - self.0.difference(&other.0); - } -} - -impl default::Default for Bitfield { - /// default provides the "empty" bitfield - /// Note: the empty bitfield is set to the `0` byte. - fn default() -> Self { - Self::from_elem(8, false) - } -} - -impl cmp::PartialEq for Bitfield { - /// Determines equality by comparing the `ssz` encoding of the two candidates. - /// This method ensures that the presence of high-order (empty) bits in the highest byte do not exclude equality when they are in fact representing the same information. - fn eq(&self, other: &Self) -> bool { - ssz::ssz_encode(self) == ssz::ssz_encode(other) - } -} - -/// Create a new bitfield that is a union of two other bitfields. -/// -/// For example `union(0101, 1000) == 1101` -// TODO: length-independent intersection for BitAnd -impl std::ops::BitOr for Bitfield { - type Output = Self; - - fn bitor(self, other: Self) -> Self { - let (biggest, smallest) = if self.len() > other.len() { - (&self, &other) - } else { - (&other, &self) - }; - let mut new = biggest.clone(); - for i in 0..smallest.len() { - if let Ok(true) = smallest.get(i) { - new.set(i, true); + impl cmp::PartialEq for $name { + /// Determines equality by comparing the `ssz` encoding of the two candidates. This + /// method ensures that the presence of high-order (empty) bits in the highest byte do + /// not exclude equality when they are in fact representing the same information. + fn eq(&self, other: &Self) -> bool { + ssz::ssz_encode(self) == ssz::ssz_encode(other) } } - new - } -} -impl Encode for Bitfield { - fn is_ssz_fixed_len() -> bool { - false - } + /// Create a new bitfield that is a union of two other bitfields. + /// + /// For example `union(0101, 1000) == 1101` + // TODO: length-independent intersection for BitAnd + impl std::ops::BitOr for $name { + type Output = Self; - fn ssz_append(&self, buf: &mut Vec) { - buf.append(&mut self.to_bytes()) - } -} + fn bitor(self, other: Self) -> Self { + let (biggest, smallest) = if self.len() > other.len() { + (&self, &other) + } else { + (&other, &self) + }; + let mut new = (*biggest).clone(); + for i in 0..smallest.len() { + if let Ok(true) = smallest.get(i) { + new.set(i, true) + .expect("Cannot produce bitfield larger than smallest of two given"); + } + } + new + } + } -impl Decode for Bitfield { - fn is_ssz_fixed_len() -> bool { - false - } + impl Encode for $name { + fn is_ssz_fixed_len() -> bool { + false + } - fn from_ssz_bytes(bytes: &[u8]) -> Result { - Ok(Bitfield::from_bytes(bytes)) - } + fn ssz_append(&self, buf: &mut Vec) { + buf.append(&mut self.to_bytes()) + } + } + + impl Decode for $name { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + $name::from_bytes(bytes) + .map_err(|e| ssz::DecodeError::BytesInvalid(format!("Bitlist {:?}", e))) + } + } + + impl Serialize for $name { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&encode(self.to_bytes())) + } + } + + impl<'de, N: Unsigned> Deserialize<'de> for $name { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + $name::from_bytes(&bytes) + .map_err(|e| serde::de::Error::custom(format!("Bitlist {:?}", e))) + } + } + + impl tree_hash::TreeHash for $name { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.to_bytes().tree_hash_root() + } + } + }; } // Reverse the bit order of a whole byte vec, so that the ith bit @@ -215,50 +233,208 @@ fn reverse_bit_order(mut bytes: Vec) -> Vec { bytes.into_iter().map(LookupReverse::swap_bits).collect() } -impl Serialize for Bitfield { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&encode(self.to_bytes())) +/// Emulates a SSZ `Bitvector`. +/// +/// An ordered, heap-allocated, fixed-length, collection of `bool` values, with `N` values. +/// +/// ## Notes +/// +/// Considering this struct is backed by bytes, errors may be raised when attempting to decode +/// bytes into a `BitVector` where `N` is not a multiple of 8. It is advised to always set `N` to +/// a multiple of 8. +/// +/// ## Example +/// ``` +/// use ssz_types::{BitVector, typenum}; +/// +/// let mut bitvec: BitVector = BitVector::new(); +/// +/// assert_eq!(bitvec.len(), 8); +/// +/// for i in 0..8 { +/// assert_eq!(bitvec.get(i).unwrap(), false); // Defaults to false. +/// } +/// +/// assert!(bitvec.get(8).is_err()); // Cannot get out-of-bounds. +/// +/// assert!(bitvec.set(7, true).is_ok()); +/// assert!(bitvec.set(8, true).is_err()); // Cannot set out-of-bounds. +/// ``` +#[derive(Debug, Clone)] +pub struct BitVector { + bitfield: Bitfield, + _phantom: PhantomData, +} + +common_impl!(BitVector, FixedSizedError); + +impl BitVector { + /// Create a new bitfield. + pub fn new() -> Self { + Self::with_capacity(Self::capacity()).expect("Capacity must be correct") + } + + fn capacity() -> usize { + N::to_usize() + } + + fn validate_length(len: usize) -> Result<(), FixedSizedError> { + let fixed_len = N::to_usize(); + + if len > fixed_len { + Err(FixedSizedError::InvalidLength { + i: len, + len: fixed_len, + }) + } else { + Ok(()) + } } } -impl<'de> Deserialize<'de> for Bitfield { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - // We reverse the bit-order so that the BitVec library can read its 0th - // bit from the end of the hex string, e.g. - // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] - let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; - Ok(Bitfield::from_bytes(&bytes)) +/// Emulates a SSZ `Bitlist`. +/// +/// An ordered, heap-allocated, variable-length, collection of `bool` values, limited to `N` +/// values. +/// +/// ## Notes +/// +/// Considering this struct is backed by bytes, errors may be raised when attempting to decode +/// bytes into a `BitList` where `N` is not a multiple of 8. It is advised to always set `N` to +/// a multiple of 8. +/// +/// ## Example +/// ``` +/// use ssz_types::{BitList, typenum}; +/// +/// let mut bitlist: BitList = BitList::new(); +/// +/// assert_eq!(bitlist.len(), 0); +/// +/// assert!(bitlist.get(0).is_err()); // Cannot get at or below the length. +/// +/// for i in 0..8 { +/// assert!(bitlist.set(i, true).is_ok()); +/// } +/// +/// assert!(bitlist.set(8, true).is_err()); // Cannot set out-of-bounds. +/// +/// // Cannot create with an excessive capacity. +/// let result: Result, _> = BitList::with_capacity(9); +/// assert!(result.is_err()); +/// ``` +#[derive(Debug, Clone)] +pub struct BitList { + bitfield: Bitfield, + _phantom: PhantomData, +} + +common_impl!(BitList, FixedSizedError); + +impl BitList { + /// Create a new, empty BitList. + pub fn new() -> Self { + Self { + bitfield: Bitfield::default(), + _phantom: PhantomData, + } + } + + fn validate_length(len: usize) -> Result<(), FixedSizedError> { + let max_len = Self::max_len(); + + if len > max_len { + Err(FixedSizedError::InvalidLength { + i: len, + len: max_len, + }) + } else { + Ok(()) + } + } + + /// The maximum possible number of bits. + pub fn max_len() -> usize { + N::to_usize() } } -impl tree_hash::TreeHash for Bitfield { - fn tree_hash_type() -> tree_hash::TreeHashType { - tree_hash::TreeHashType::List +impl BitList { + /// Compute the intersection (binary-and) of this bitfield with another + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn intersection(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let mut res: Self = self.to_owned(); + res.intersection_inplace(other); + res } - fn tree_hash_packed_encoding(&self) -> Vec { - unreachable!("List should never be packed.") + /// Like `intersection` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn intersection_inplace(&mut self, other: &Self) { + self.bitfield.intersect(&other.bitfield); } - fn tree_hash_packing_factor() -> usize { - unreachable!("List should never be packed.") + /// Compute the union (binary-or) of this bitfield with another. Lengths must match. + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn union(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let mut res = self.clone(); + res.union_inplace(other); + res } - fn tree_hash_root(&self) -> Vec { - self.to_bytes().tree_hash_root() + /// Like `union` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn union_inplace(&mut self, other: &Self) { + self.bitfield.union(&other.bitfield); + } + + /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. + /// + /// Computes `self - other`. + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn difference(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let mut res = self.clone(); + res.difference_inplace(other); + res + } + + /// Like `difference` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn difference_inplace(&mut self, other: &Self) { + self.bitfield.difference(&other.bitfield); } } -cached_tree_hash_bytes_as_list!(Bitfield); +impl default::Default for BitList { + /// Default provides the "empty" bitfield + /// Note: the empty bitfield is set to the `0` byte. + fn default() -> Self { + Self::from_elem(0, false).expect("Zero cannot be larger than the maximum length") + } +} +/* #[cfg(test)] mod tests { use super::*; @@ -568,3 +744,4 @@ mod tests { assert!(a.difference(&a).is_zero()); } } +*/ diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index 071532e22..c4e101dc9 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -60,8 +60,8 @@ impl FixedVector { }) } else { Err(Error::InvalidLength { - len: vec.len(), - fixed_len: Self::capacity(), + i: vec.len(), + len: Self::capacity(), }) } } diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 37aa24875..3bcf8ed1f 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -1,9 +1,8 @@ -mod bit_vector; mod bitfield; mod fixed_vector; mod variable_list; -pub use bit_vector::{BitList, BitVector}; +pub use bitfield::{BitList, BitVector}; pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; @@ -12,12 +11,26 @@ pub use variable_list::VariableList; #[derive(PartialEq, Debug)] pub enum VariableSizedError { /// The operation would cause the maximum length to be exceeded. - ExceedsMaxLength { len: usize, max_len: usize }, + ExceedsMaxLength { + len: usize, + max_len: usize, + }, + OutOfBounds { + i: usize, + len: usize, + }, } /// Returned when a fixed-length item encounters an error. #[derive(PartialEq, Debug)] pub enum FixedSizedError { /// The operation would create an item of an invalid size. - InvalidLength { len: usize, fixed_len: usize }, + InvalidLength { + i: usize, + len: usize, + }, + OutOfBounds { + i: usize, + len: usize, + }, } From c8c5c8ff16b3bade38779b1ef89e34d459de19f9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 6 Jul 2019 15:57:11 +1000 Subject: [PATCH 47/72] Collect errors into a single error --- eth2/utils/ssz_types/src/bitfield.rs | 32 +++++++++++------------ eth2/utils/ssz_types/src/fixed_vector.rs | 2 +- eth2/utils/ssz_types/src/lib.rs | 29 +++----------------- eth2/utils/ssz_types/src/variable_list.rs | 14 +++++----- 4 files changed, 28 insertions(+), 49 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 8462b53df..9befbb809 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -1,4 +1,4 @@ -use crate::FixedSizedError; +use crate::Error; use bit_reverse::LookupReverse; use bit_vec::BitVec as Bitfield; use serde::de::{Deserialize, Deserializer}; @@ -12,10 +12,10 @@ use typenum::Unsigned; /// Provides a common `impl` for structs that wrap a `$name`. macro_rules! common_impl { - ($name: ident, $error: ident) => { + ($name: ident) => { impl $name { /// Create a new BitList list with `initial_len` bits all set to `false`. - pub fn with_capacity(initial_len: usize) -> Result { + pub fn with_capacity(initial_len: usize) -> Result { Self::from_elem(initial_len, false) } @@ -23,7 +23,7 @@ macro_rules! common_impl { /// /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` /// regardless of `bit`. - pub fn from_elem(initial_len: usize, bit: bool) -> Result { + pub fn from_elem(initial_len: usize, bit: bool) -> Result { // BitVec can panic if we don't set the len to be a multiple of 8. let full_len = ((initial_len + 7) / 8) * 8; @@ -44,7 +44,7 @@ macro_rules! common_impl { } /// Create a new bitfield using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { Self::validate_length(bytes.len().saturating_mul(8))?; Ok(Self { @@ -62,17 +62,17 @@ macro_rules! common_impl { /// If the index is in bounds, then result is Ok(value) where value is `true` if the /// bit is 1 and `false` if the bit is 0. If the index is out of bounds, we return an /// error to that extent. - pub fn get(&self, i: usize) -> Result { + pub fn get(&self, i: usize) -> Result { if i < N::to_usize() { match self.bitfield.get(i) { Some(value) => Ok(value), - None => Err($error::OutOfBounds { + None => Err(Error::OutOfBounds { i, len: self.bitfield.len(), }), } } else { - Err($error::InvalidLength { + Err(Error::InvalidLength { i, len: N::to_usize(), }) @@ -83,10 +83,10 @@ macro_rules! common_impl { /// /// If the index is out of bounds, we expand the size of the underlying set to include /// the new index. Returns the previous value if there was one. - pub fn set(&mut self, i: usize, value: bool) -> Result<(), $error> { + pub fn set(&mut self, i: usize, value: bool) -> Result<(), Error> { match self.get(i) { Ok(previous) => Some(previous), - Err($error::OutOfBounds { len, .. }) => { + Err(Error::OutOfBounds { len, .. }) => { let new_len = i - len + 1; self.bitfield.grow(new_len, false); None @@ -266,7 +266,7 @@ pub struct BitVector { _phantom: PhantomData, } -common_impl!(BitVector, FixedSizedError); +common_impl!(BitVector); impl BitVector { /// Create a new bitfield. @@ -278,11 +278,11 @@ impl BitVector { N::to_usize() } - fn validate_length(len: usize) -> Result<(), FixedSizedError> { + fn validate_length(len: usize) -> Result<(), Error> { let fixed_len = N::to_usize(); if len > fixed_len { - Err(FixedSizedError::InvalidLength { + Err(Error::InvalidLength { i: len, len: fixed_len, }) @@ -329,7 +329,7 @@ pub struct BitList { _phantom: PhantomData, } -common_impl!(BitList, FixedSizedError); +common_impl!(BitList); impl BitList { /// Create a new, empty BitList. @@ -340,11 +340,11 @@ impl BitList { } } - fn validate_length(len: usize) -> Result<(), FixedSizedError> { + fn validate_length(len: usize) -> Result<(), Error> { let max_len = Self::max_len(); if len > max_len { - Err(FixedSizedError::InvalidLength { + Err(Error::InvalidLength { i: len, len: max_len, }) diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index c4e101dc9..b5d422760 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -1,4 +1,4 @@ -use crate::FixedSizedError as Error; +use crate::Error; use serde_derive::{Deserialize, Serialize}; use std::marker::PhantomData; use std::ops::{Deref, Index, IndexMut}; diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 3bcf8ed1f..9fba87bc4 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -7,30 +7,9 @@ pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; -/// Returned when a variable-length item encounters an error. +/// Returned when an item encounters an error. #[derive(PartialEq, Debug)] -pub enum VariableSizedError { - /// The operation would cause the maximum length to be exceeded. - ExceedsMaxLength { - len: usize, - max_len: usize, - }, - OutOfBounds { - i: usize, - len: usize, - }, -} - -/// Returned when a fixed-length item encounters an error. -#[derive(PartialEq, Debug)] -pub enum FixedSizedError { - /// The operation would create an item of an invalid size. - InvalidLength { - i: usize, - len: usize, - }, - OutOfBounds { - i: usize, - len: usize, - }, +pub enum Error { + InvalidLength { i: usize, len: usize }, + OutOfBounds { i: usize, len: usize }, } diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs index 3d0bf31c9..34450be8a 100644 --- a/eth2/utils/ssz_types/src/variable_list.rs +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -1,4 +1,4 @@ -use crate::VariableSizedError as Error; +use crate::Error; use serde_derive::{Deserialize, Serialize}; use std::marker::PhantomData; use std::ops::{Deref, Index, IndexMut}; @@ -61,9 +61,9 @@ impl VariableList { _phantom: PhantomData, }) } else { - Err(Error::ExceedsMaxLength { - len: vec.len(), - max_len: Self::max_len(), + Err(Error::InvalidLength { + i: vec.len(), + len: Self::max_len(), }) } } @@ -90,9 +90,9 @@ impl VariableList { if self.vec.len() < Self::max_len() { Ok(self.vec.push(value)) } else { - Err(Error::ExceedsMaxLength { - len: self.vec.len() + 1, - max_len: Self::max_len(), + Err(Error::InvalidLength { + i: self.vec.len() + 1, + len: Self::max_len(), }) } } From ecb0bf11c72d4dcc4c442f82a29739fdcf852790 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 8 Jul 2019 09:36:52 +1000 Subject: [PATCH 48/72] Move bitlist and bitvector into own files --- eth2/utils/ssz_types/src/bit_list.rs | 443 +++++++++++ eth2/utils/ssz_types/src/bit_vector.rs | 69 ++ eth2/utils/ssz_types/src/bitfield.rs | 747 ------------------ eth2/utils/ssz_types/src/impl_bitfield_fns.rs | 229 ++++++ eth2/utils/ssz_types/src/lib.rs | 10 +- 5 files changed, 749 insertions(+), 749 deletions(-) create mode 100644 eth2/utils/ssz_types/src/bit_list.rs create mode 100644 eth2/utils/ssz_types/src/bit_vector.rs delete mode 100644 eth2/utils/ssz_types/src/bitfield.rs create mode 100644 eth2/utils/ssz_types/src/impl_bitfield_fns.rs diff --git a/eth2/utils/ssz_types/src/bit_list.rs b/eth2/utils/ssz_types/src/bit_list.rs new file mode 100644 index 000000000..07324148b --- /dev/null +++ b/eth2/utils/ssz_types/src/bit_list.rs @@ -0,0 +1,443 @@ +use super::*; +use crate::{impl_bitfield_fns, reverse_bit_order, Error}; +use bit_vec::BitVec as Bitfield; +use serde::de::{Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode, PrefixedHexVisitor}; +use ssz::{Decode, Encode}; +use std::cmp; +use std::default; +use std::marker::PhantomData; +use typenum::Unsigned; + +/// Emulates a SSZ `Bitlist`. +/// +/// An ordered, heap-allocated, variable-length, collection of `bool` values, limited to `N` +/// values. +/// +/// ## Notes +/// +/// Considering this struct is backed by bytes, errors may be raised when attempting to decode +/// bytes into a `BitList` where `N` is not a multiple of 8. It is advised to always set `N` to +/// a multiple of 8. +/// +/// ## Example +/// ``` +/// use ssz_types::{BitList, typenum}; +/// +/// let mut bitlist: BitList = BitList::new(); +/// +/// assert_eq!(bitlist.len(), 0); +/// +/// assert!(bitlist.get(0).is_err()); // Cannot get at or below the length. +/// +/// for i in 0..8 { +/// assert!(bitlist.set(i, true).is_ok()); +/// } +/// +/// assert!(bitlist.set(8, true).is_err()); // Cannot set out-of-bounds. +/// +/// // Cannot create with an excessive capacity. +/// let result: Result, _> = BitList::with_capacity(9); +/// assert!(result.is_err()); +/// ``` +#[derive(Debug, Clone)] +pub struct BitList { + bitfield: Bitfield, + _phantom: PhantomData, +} + +impl_bitfield_fns!(BitList); + +impl BitList { + /// Create a new, empty BitList. + pub fn new() -> Self { + Self { + bitfield: Bitfield::default(), + _phantom: PhantomData, + } + } + + fn validate_length(len: usize) -> Result<(), Error> { + let max_len = Self::max_len(); + + if len > max_len { + Err(Error::InvalidLength { + i: len, + len: max_len, + }) + } else { + Ok(()) + } + } + + /// The maximum possible number of bits. + pub fn max_len() -> usize { + N::to_usize() + } +} + +impl BitList { + /// Compute the intersection (binary-and) of this bitfield with another + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn intersection(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let mut res: Self = self.to_owned(); + res.intersection_inplace(other); + res + } + + /// Like `intersection` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn intersection_inplace(&mut self, other: &Self) { + self.bitfield.intersect(&other.bitfield); + } + + /// Compute the union (binary-or) of this bitfield with another. Lengths must match. + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn union(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let mut res = self.clone(); + res.union_inplace(other); + res + } + + /// Like `union` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn union_inplace(&mut self, other: &Self) { + self.bitfield.union(&other.bitfield); + } + + /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. + /// + /// Computes `self - other`. + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn difference(&self, other: &Self) -> Self { + assert_eq!(self.len(), other.len()); + let mut res = self.clone(); + res.difference_inplace(other); + res + } + + /// Like `difference` but in-place (updates `self`). + /// + /// ## Panics + /// + /// If `self` and `other` have different lengths. + pub fn difference_inplace(&mut self, other: &Self) { + self.bitfield.difference(&other.bitfield); + } +} + +impl default::Default for BitList { + /// Default provides the "empty" bitfield + /// Note: the empty bitfield is set to the `0` byte. + fn default() -> Self { + Self::from_elem(0, false).expect("Zero cannot be larger than the maximum length") + } +} + +#[cfg(test)] +mod test_bitlist { + use super::*; + use serde_yaml; + use ssz::ssz_encode; + // use tree_hash::TreeHash; + + pub type BitList1024 = BitList; + + /* + #[test] + pub fn cached_tree_hash() { + let original = BitList1024::from_bytes(&vec![18; 12][..]); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let modified = BitList1024::from_bytes(&vec![2; 1][..]); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } + */ + + #[test] + fn new_bitfield() { + let mut field = BitList1024::new(); + let original_len = field.len(); + + for i in 0..100 { + if i < original_len { + assert!(!field.get(i).unwrap()); + } else { + assert!(field.get(i).is_err()); + } + field.set(i, true).unwrap(); + } + } + + #[test] + fn empty_bitfield() { + let mut field = BitList1024::from_elem(0, false).unwrap(); + let original_len = field.len(); + + assert_eq!(original_len, 0); + + for i in 0..100 { + if i < original_len { + assert!(!field.get(i).unwrap()); + } else { + assert!(field.get(i).is_err()); + } + field.set(i, true).unwrap(); + } + + assert_eq!(field.len(), 100); + assert_eq!(field.num_set_bits(), 100); + } + + const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; + + #[test] + fn get_from_bitfield() { + let field = BitList1024::from_bytes(INPUT).unwrap(); + field.get(0).unwrap(); + field.get(6).unwrap(); + field.get(14).unwrap(); + } + + #[test] + fn set_for_bitfield() { + let mut field = BitList1024::from_bytes(INPUT).unwrap(); + field.set(10, true).unwrap(); + field.get(10).unwrap(); + field.set(6, false).unwrap(); + field.get(6).unwrap(); + } + + #[test] + fn len() { + let field = BitList1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.len(), 16); + + let field = BitList1024::new(); + assert_eq!(field.len(), 0); + } + + #[test] + fn num_set_bits() { + let field = BitList1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.num_set_bits(), 2); + + let field = BitList1024::new(); + assert_eq!(field.num_set_bits(), 0); + } + + #[test] + fn to_bytes() { + let field = BitList1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.to_bytes(), INPUT); + + let field = BitList1024::new(); + assert_eq!(field.to_bytes(), vec![0]); + } + + #[test] + fn out_of_bounds() { + let mut field = BitList1024::from_bytes(INPUT).unwrap(); + + let out_of_bounds_index = field.len(); + assert!(field.set(out_of_bounds_index, true).is_ok()); + assert!(field.len() == out_of_bounds_index + 1); + assert!(field.get(out_of_bounds_index).unwrap()); + + for i in 0..100 { + if i <= out_of_bounds_index { + assert!(field.set(i, true).is_ok()); + } else { + assert!(field.set(i, true).is_ok()); + } + } + } + + #[test] + fn grows_with_false() { + let input_all_set: &[u8] = &[0b1111_1111, 0b1111_1111]; + let mut field = BitList1024::from_bytes(input_all_set).unwrap(); + + // Define `a` and `b`, where both are out of bounds and `b` is greater than `a`. + let a = field.len(); + let b = a + 1; + + // Ensure `a` is out-of-bounds for test integrity. + assert!(field.get(a).is_err()); + + // Set `b` to `true`.. + assert!(field.set(b, true).is_ok()); + + // Ensure that `a` wasn't also set to `true` during the grow. + assert_eq!(field.get(a), Ok(false)); + assert_eq!(field.get(b), Ok(true)); + } + + #[test] + fn num_bytes() { + let field = BitList1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.num_bytes(), 2); + + let field = BitList1024::from_elem(2, true).unwrap(); + assert_eq!(field.num_bytes(), 1); + + let field = BitList1024::from_elem(13, true).unwrap(); + assert_eq!(field.num_bytes(), 2); + } + + #[test] + fn ssz_encoding() { + let field = create_bitfield(); + assert_eq!(field.as_ssz_bytes(), vec![0b0000_0011, 0b1000_0111]); + + let field = BitList1024::from_elem(18, true).unwrap(); + assert_eq!( + field.as_ssz_bytes(), + vec![0b0000_0011, 0b1111_1111, 0b1111_1111] + ); + + let mut b = BitList1024::new(); + b.set(1, true).unwrap(); + assert_eq!(ssz_encode(&b), vec![0b0000_0010]); + } + + fn create_bitfield() -> BitList1024 { + let count = 2 * 8; + let mut field = BitList1024::with_capacity(count).unwrap(); + + let indices = &[0, 1, 2, 7, 8, 9]; + for &i in indices { + field.set(i, true).unwrap(); + } + field + } + + #[test] + fn ssz_decode() { + let encoded = vec![0b0000_0011, 0b1000_0111]; + let field = BitList1024::from_ssz_bytes(&encoded).unwrap(); + let expected = create_bitfield(); + assert_eq!(field, expected); + + let encoded = vec![255, 255, 3]; + let field = BitList1024::from_ssz_bytes(&encoded).unwrap(); + let expected = BitList1024::from_bytes(&[255, 255, 3]).unwrap(); + assert_eq!(field, expected); + } + + #[test] + fn serialize_deserialize() { + use serde_yaml::Value; + + let data: &[(_, &[_])] = &[ + ("0x01", &[0b00000001]), + ("0xf301", &[0b11110011, 0b00000001]), + ]; + for (hex_data, bytes) in data { + let bitfield = BitList1024::from_bytes(bytes).unwrap(); + assert_eq!( + serde_yaml::from_str::(hex_data).unwrap(), + bitfield + ); + assert_eq!( + serde_yaml::to_value(&bitfield).unwrap(), + Value::String(hex_data.to_string()) + ); + } + } + + #[test] + fn ssz_round_trip() { + let original = BitList1024::from_bytes(&vec![18; 12][..]).unwrap(); + let ssz = ssz_encode(&original); + let decoded = BitList1024::from_ssz_bytes(&ssz).unwrap(); + assert_eq!(original, decoded); + } + + #[test] + fn bitor() { + let a = BitList1024::from_bytes(&vec![2, 8, 1][..]).unwrap(); + let b = BitList1024::from_bytes(&vec![4, 8, 16][..]).unwrap(); + let c = BitList1024::from_bytes(&vec![6, 8, 17][..]).unwrap(); + assert_eq!(c, a | b); + } + + #[test] + fn is_zero() { + let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]]; + for bytes in yes_data { + assert!(BitList1024::from_bytes(bytes).unwrap().is_zero()); + } + let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]]; + for bytes in no_data { + assert!(!BitList1024::from_bytes(bytes).unwrap().is_zero()); + } + } + + #[test] + fn intersection() { + let a = BitList1024::from_bytes(&[0b1100, 0b0001]).unwrap(); + let b = BitList1024::from_bytes(&[0b1011, 0b1001]).unwrap(); + let c = BitList1024::from_bytes(&[0b1000, 0b0001]).unwrap(); + assert_eq!(a.intersection(&b), c); + assert_eq!(b.intersection(&a), c); + assert_eq!(a.intersection(&c), c); + assert_eq!(b.intersection(&c), c); + assert_eq!(a.intersection(&a), a); + assert_eq!(b.intersection(&b), b); + assert_eq!(c.intersection(&c), c); + } + + #[test] + fn union() { + let a = BitList1024::from_bytes(&[0b1100, 0b0001]).unwrap(); + let b = BitList1024::from_bytes(&[0b1011, 0b1001]).unwrap(); + let c = BitList1024::from_bytes(&[0b1111, 0b1001]).unwrap(); + assert_eq!(a.union(&b), c); + assert_eq!(b.union(&a), c); + assert_eq!(a.union(&a), a); + assert_eq!(b.union(&b), b); + assert_eq!(c.union(&c), c); + } + + #[test] + fn difference() { + let a = BitList1024::from_bytes(&[0b1100, 0b0001]).unwrap(); + let b = BitList1024::from_bytes(&[0b1011, 0b1001]).unwrap(); + let a_b = BitList1024::from_bytes(&[0b0100, 0b0000]).unwrap(); + let b_a = BitList1024::from_bytes(&[0b0011, 0b1000]).unwrap(); + assert_eq!(a.difference(&b), a_b); + assert_eq!(b.difference(&a), b_a); + assert!(a.difference(&a).is_zero()); + } +} diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs new file mode 100644 index 000000000..830412757 --- /dev/null +++ b/eth2/utils/ssz_types/src/bit_vector.rs @@ -0,0 +1,69 @@ +use super::*; +use crate::{impl_bitfield_fns, reverse_bit_order, Error}; +use bit_vec::BitVec as Bitfield; +use serde::de::{Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode, PrefixedHexVisitor}; +use ssz::{Decode, Encode}; +use std::cmp; +use std::marker::PhantomData; +use typenum::Unsigned; + +/// Emulates a SSZ `Bitvector`. +/// +/// An ordered, heap-allocated, fixed-length, collection of `bool` values, with `N` values. +/// +/// ## Notes +/// +/// Considering this struct is backed by bytes, errors may be raised when attempting to decode +/// bytes into a `BitVector` where `N` is not a multiple of 8. It is advised to always set `N` to +/// a multiple of 8. +/// +/// ## Example +/// ``` +/// use ssz_types::{BitVector, typenum}; +/// +/// let mut bitvec: BitVector = BitVector::new(); +/// +/// assert_eq!(bitvec.len(), 8); +/// +/// for i in 0..8 { +/// assert_eq!(bitvec.get(i).unwrap(), false); // Defaults to false. +/// } +/// +/// assert!(bitvec.get(8).is_err()); // Cannot get out-of-bounds. +/// +/// assert!(bitvec.set(7, true).is_ok()); +/// assert!(bitvec.set(8, true).is_err()); // Cannot set out-of-bounds. +/// ``` +#[derive(Debug, Clone)] +pub struct BitVector { + bitfield: Bitfield, + _phantom: PhantomData, +} + +impl_bitfield_fns!(BitVector); + +impl BitVector { + /// Create a new bitfield. + pub fn new() -> Self { + Self::with_capacity(Self::capacity()).expect("Capacity must be correct") + } + + fn capacity() -> usize { + N::to_usize() + } + + fn validate_length(len: usize) -> Result<(), Error> { + let fixed_len = N::to_usize(); + + if len > fixed_len { + Err(Error::InvalidLength { + i: len, + len: fixed_len, + }) + } else { + Ok(()) + } + } +} diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs deleted file mode 100644 index 9befbb809..000000000 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ /dev/null @@ -1,747 +0,0 @@ -use crate::Error; -use bit_reverse::LookupReverse; -use bit_vec::BitVec as Bitfield; -use serde::de::{Deserialize, Deserializer}; -use serde::ser::{Serialize, Serializer}; -use serde_hex::{encode, PrefixedHexVisitor}; -use ssz::{Decode, Encode}; -use std::cmp; -use std::default; -use std::marker::PhantomData; -use typenum::Unsigned; - -/// Provides a common `impl` for structs that wrap a `$name`. -macro_rules! common_impl { - ($name: ident) => { - impl $name { - /// Create a new BitList list with `initial_len` bits all set to `false`. - pub fn with_capacity(initial_len: usize) -> Result { - Self::from_elem(initial_len, false) - } - - /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. - /// - /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` - /// regardless of `bit`. - pub fn from_elem(initial_len: usize, bit: bool) -> Result { - // BitVec can panic if we don't set the len to be a multiple of 8. - let full_len = ((initial_len + 7) / 8) * 8; - - Self::validate_length(full_len)?; - - let mut bitfield = Bitfield::from_elem(full_len, false); - - if bit { - for i in 0..initial_len { - bitfield.set(i, true); - } - } - - Ok(Self { - bitfield, - _phantom: PhantomData, - }) - } - - /// Create a new bitfield using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Result { - Self::validate_length(bytes.len().saturating_mul(8))?; - - Ok(Self { - bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), - _phantom: PhantomData, - }) - } - /// Returns a vector of bytes representing the bitfield - pub fn to_bytes(&self) -> Vec { - reverse_bit_order(self.bitfield.to_bytes().to_vec()) - } - - /// Read the value of a bit. - /// - /// If the index is in bounds, then result is Ok(value) where value is `true` if the - /// bit is 1 and `false` if the bit is 0. If the index is out of bounds, we return an - /// error to that extent. - pub fn get(&self, i: usize) -> Result { - if i < N::to_usize() { - match self.bitfield.get(i) { - Some(value) => Ok(value), - None => Err(Error::OutOfBounds { - i, - len: self.bitfield.len(), - }), - } - } else { - Err(Error::InvalidLength { - i, - len: N::to_usize(), - }) - } - } - - /// Set the value of a bit. - /// - /// If the index is out of bounds, we expand the size of the underlying set to include - /// the new index. Returns the previous value if there was one. - pub fn set(&mut self, i: usize, value: bool) -> Result<(), Error> { - match self.get(i) { - Ok(previous) => Some(previous), - Err(Error::OutOfBounds { len, .. }) => { - let new_len = i - len + 1; - self.bitfield.grow(new_len, false); - None - } - Err(e) => return Err(e), - }; - - self.bitfield.set(i, value); - - Ok(()) - } - - /// Returns the number of bits in this bitfield. - pub fn len(&self) -> usize { - self.bitfield.len() - } - - /// Returns true if `self.len() == 0` - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns true if all bits are set to 0. - pub fn is_zero(&self) -> bool { - self.bitfield.none() - } - - /// Returns the number of bytes required to represent this bitfield. - pub fn num_bytes(&self) -> usize { - self.to_bytes().len() - } - - /// Returns the number of `1` bits in the bitfield - pub fn num_set_bits(&self) -> usize { - self.bitfield.iter().filter(|&bit| bit).count() - } - } - - impl cmp::PartialEq for $name { - /// Determines equality by comparing the `ssz` encoding of the two candidates. This - /// method ensures that the presence of high-order (empty) bits in the highest byte do - /// not exclude equality when they are in fact representing the same information. - fn eq(&self, other: &Self) -> bool { - ssz::ssz_encode(self) == ssz::ssz_encode(other) - } - } - - /// Create a new bitfield that is a union of two other bitfields. - /// - /// For example `union(0101, 1000) == 1101` - // TODO: length-independent intersection for BitAnd - impl std::ops::BitOr for $name { - type Output = Self; - - fn bitor(self, other: Self) -> Self { - let (biggest, smallest) = if self.len() > other.len() { - (&self, &other) - } else { - (&other, &self) - }; - let mut new = (*biggest).clone(); - for i in 0..smallest.len() { - if let Ok(true) = smallest.get(i) { - new.set(i, true) - .expect("Cannot produce bitfield larger than smallest of two given"); - } - } - new - } - } - - impl Encode for $name { - fn is_ssz_fixed_len() -> bool { - false - } - - fn ssz_append(&self, buf: &mut Vec) { - buf.append(&mut self.to_bytes()) - } - } - - impl Decode for $name { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - $name::from_bytes(bytes) - .map_err(|e| ssz::DecodeError::BytesInvalid(format!("Bitlist {:?}", e))) - } - } - - impl Serialize for $name { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&encode(self.to_bytes())) - } - } - - impl<'de, N: Unsigned> Deserialize<'de> for $name { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - // We reverse the bit-order so that the BitVec library can read its 0th - // bit from the end of the hex string, e.g. - // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] - let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; - $name::from_bytes(&bytes) - .map_err(|e| serde::de::Error::custom(format!("Bitlist {:?}", e))) - } - } - - impl tree_hash::TreeHash for $name { - fn tree_hash_type() -> tree_hash::TreeHashType { - tree_hash::TreeHashType::List - } - - fn tree_hash_packed_encoding(&self) -> Vec { - unreachable!("List should never be packed.") - } - - fn tree_hash_packing_factor() -> usize { - unreachable!("List should never be packed.") - } - - fn tree_hash_root(&self) -> Vec { - self.to_bytes().tree_hash_root() - } - } - }; -} - -// Reverse the bit order of a whole byte vec, so that the ith bit -// of the input vec is placed in the (N - i)th bit of the output vec. -// This function is necessary for converting bitfields to and from YAML, -// as the BitVec library and the hex-parser use opposing bit orders. -fn reverse_bit_order(mut bytes: Vec) -> Vec { - bytes.reverse(); - bytes.into_iter().map(LookupReverse::swap_bits).collect() -} - -/// Emulates a SSZ `Bitvector`. -/// -/// An ordered, heap-allocated, fixed-length, collection of `bool` values, with `N` values. -/// -/// ## Notes -/// -/// Considering this struct is backed by bytes, errors may be raised when attempting to decode -/// bytes into a `BitVector` where `N` is not a multiple of 8. It is advised to always set `N` to -/// a multiple of 8. -/// -/// ## Example -/// ``` -/// use ssz_types::{BitVector, typenum}; -/// -/// let mut bitvec: BitVector = BitVector::new(); -/// -/// assert_eq!(bitvec.len(), 8); -/// -/// for i in 0..8 { -/// assert_eq!(bitvec.get(i).unwrap(), false); // Defaults to false. -/// } -/// -/// assert!(bitvec.get(8).is_err()); // Cannot get out-of-bounds. -/// -/// assert!(bitvec.set(7, true).is_ok()); -/// assert!(bitvec.set(8, true).is_err()); // Cannot set out-of-bounds. -/// ``` -#[derive(Debug, Clone)] -pub struct BitVector { - bitfield: Bitfield, - _phantom: PhantomData, -} - -common_impl!(BitVector); - -impl BitVector { - /// Create a new bitfield. - pub fn new() -> Self { - Self::with_capacity(Self::capacity()).expect("Capacity must be correct") - } - - fn capacity() -> usize { - N::to_usize() - } - - fn validate_length(len: usize) -> Result<(), Error> { - let fixed_len = N::to_usize(); - - if len > fixed_len { - Err(Error::InvalidLength { - i: len, - len: fixed_len, - }) - } else { - Ok(()) - } - } -} - -/// Emulates a SSZ `Bitlist`. -/// -/// An ordered, heap-allocated, variable-length, collection of `bool` values, limited to `N` -/// values. -/// -/// ## Notes -/// -/// Considering this struct is backed by bytes, errors may be raised when attempting to decode -/// bytes into a `BitList` where `N` is not a multiple of 8. It is advised to always set `N` to -/// a multiple of 8. -/// -/// ## Example -/// ``` -/// use ssz_types::{BitList, typenum}; -/// -/// let mut bitlist: BitList = BitList::new(); -/// -/// assert_eq!(bitlist.len(), 0); -/// -/// assert!(bitlist.get(0).is_err()); // Cannot get at or below the length. -/// -/// for i in 0..8 { -/// assert!(bitlist.set(i, true).is_ok()); -/// } -/// -/// assert!(bitlist.set(8, true).is_err()); // Cannot set out-of-bounds. -/// -/// // Cannot create with an excessive capacity. -/// let result: Result, _> = BitList::with_capacity(9); -/// assert!(result.is_err()); -/// ``` -#[derive(Debug, Clone)] -pub struct BitList { - bitfield: Bitfield, - _phantom: PhantomData, -} - -common_impl!(BitList); - -impl BitList { - /// Create a new, empty BitList. - pub fn new() -> Self { - Self { - bitfield: Bitfield::default(), - _phantom: PhantomData, - } - } - - fn validate_length(len: usize) -> Result<(), Error> { - let max_len = Self::max_len(); - - if len > max_len { - Err(Error::InvalidLength { - i: len, - len: max_len, - }) - } else { - Ok(()) - } - } - - /// The maximum possible number of bits. - pub fn max_len() -> usize { - N::to_usize() - } -} - -impl BitList { - /// Compute the intersection (binary-and) of this bitfield with another - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn intersection(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let mut res: Self = self.to_owned(); - res.intersection_inplace(other); - res - } - - /// Like `intersection` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn intersection_inplace(&mut self, other: &Self) { - self.bitfield.intersect(&other.bitfield); - } - - /// Compute the union (binary-or) of this bitfield with another. Lengths must match. - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn union(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let mut res = self.clone(); - res.union_inplace(other); - res - } - - /// Like `union` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn union_inplace(&mut self, other: &Self) { - self.bitfield.union(&other.bitfield); - } - - /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. - /// - /// Computes `self - other`. - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn difference(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let mut res = self.clone(); - res.difference_inplace(other); - res - } - - /// Like `difference` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn difference_inplace(&mut self, other: &Self) { - self.bitfield.difference(&other.bitfield); - } -} - -impl default::Default for BitList { - /// Default provides the "empty" bitfield - /// Note: the empty bitfield is set to the `0` byte. - fn default() -> Self { - Self::from_elem(0, false).expect("Zero cannot be larger than the maximum length") - } -} - -/* -#[cfg(test)] -mod tests { - use super::*; - use serde_yaml; - use ssz::ssz_encode; - use tree_hash::TreeHash; - - impl Bitfield { - /// Create a new bitfield. - pub fn new() -> Self { - Default::default() - } - } - - #[test] - pub fn test_cached_tree_hash() { - let original = Bitfield::from_bytes(&vec![18; 12][..]); - - let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); - - assert_eq!( - cache.tree_hash_root().unwrap().to_vec(), - original.tree_hash_root() - ); - - let modified = Bitfield::from_bytes(&vec![2; 1][..]); - - cache.update(&modified).unwrap(); - - assert_eq!( - cache.tree_hash_root().unwrap().to_vec(), - modified.tree_hash_root() - ); - } - - #[test] - fn test_new_bitfield() { - let mut field = Bitfield::new(); - let original_len = field.len(); - - for i in 0..100 { - if i < original_len { - assert!(!field.get(i).unwrap()); - } else { - assert!(field.get(i).is_err()); - } - let previous = field.set(i, true); - if i < original_len { - assert!(!previous.unwrap()); - } else { - assert!(previous.is_none()); - } - } - } - - #[test] - fn test_empty_bitfield() { - let mut field = Bitfield::from_elem(0, false); - let original_len = field.len(); - - assert_eq!(original_len, 0); - - for i in 0..100 { - if i < original_len { - assert!(!field.get(i).unwrap()); - } else { - assert!(field.get(i).is_err()); - } - let previous = field.set(i, true); - if i < original_len { - assert!(!previous.unwrap()); - } else { - assert!(previous.is_none()); - } - } - - assert_eq!(field.len(), 100); - assert_eq!(field.num_set_bits(), 100); - } - - const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; - - #[test] - fn test_get_from_bitfield() { - let field = Bitfield::from_bytes(INPUT); - let unset = field.get(0).unwrap(); - assert!(!unset); - let set = field.get(6).unwrap(); - assert!(set); - let set = field.get(14).unwrap(); - assert!(set); - } - - #[test] - fn test_set_for_bitfield() { - let mut field = Bitfield::from_bytes(INPUT); - let previous = field.set(10, true).unwrap(); - assert!(!previous); - let previous = field.get(10).unwrap(); - assert!(previous); - let previous = field.set(6, false).unwrap(); - assert!(previous); - let previous = field.get(6).unwrap(); - assert!(!previous); - } - - #[test] - fn test_len() { - let field = Bitfield::from_bytes(INPUT); - assert_eq!(field.len(), 16); - - let field = Bitfield::new(); - assert_eq!(field.len(), 8); - } - - #[test] - fn test_num_set_bits() { - let field = Bitfield::from_bytes(INPUT); - assert_eq!(field.num_set_bits(), 2); - - let field = Bitfield::new(); - assert_eq!(field.num_set_bits(), 0); - } - - #[test] - fn test_to_bytes() { - let field = Bitfield::from_bytes(INPUT); - assert_eq!(field.to_bytes(), INPUT); - - let field = Bitfield::new(); - assert_eq!(field.to_bytes(), vec![0]); - } - - #[test] - fn test_out_of_bounds() { - let mut field = Bitfield::from_bytes(INPUT); - - let out_of_bounds_index = field.len(); - assert!(field.set(out_of_bounds_index, true).is_none()); - assert!(field.len() == out_of_bounds_index + 1); - assert!(field.get(out_of_bounds_index).unwrap()); - - for i in 0..100 { - if i <= out_of_bounds_index { - assert!(field.set(i, true).is_some()); - } else { - assert!(field.set(i, true).is_none()); - } - } - } - - #[test] - fn test_grows_with_false() { - let input_all_set: &[u8] = &[0b1111_1111, 0b1111_1111]; - let mut field = Bitfield::from_bytes(input_all_set); - - // Define `a` and `b`, where both are out of bounds and `b` is greater than `a`. - let a = field.len(); - let b = a + 1; - - // Ensure `a` is out-of-bounds for test integrity. - assert!(field.get(a).is_err()); - - // Set `b` to `true`. Also, for test integrity, ensure it was previously out-of-bounds. - assert!(field.set(b, true).is_none()); - - // Ensure that `a` wasn't also set to `true` during the grow. - assert_eq!(field.get(a), Ok(false)); - assert_eq!(field.get(b), Ok(true)); - } - - #[test] - fn test_num_bytes() { - let field = Bitfield::from_bytes(INPUT); - assert_eq!(field.num_bytes(), 2); - - let field = Bitfield::from_elem(2, true); - assert_eq!(field.num_bytes(), 1); - - let field = Bitfield::from_elem(13, true); - assert_eq!(field.num_bytes(), 2); - } - - #[test] - fn test_ssz_encode() { - let field = create_test_bitfield(); - assert_eq!(field.as_ssz_bytes(), vec![0b0000_0011, 0b1000_0111]); - - let field = Bitfield::from_elem(18, true); - assert_eq!( - field.as_ssz_bytes(), - vec![0b0000_0011, 0b1111_1111, 0b1111_1111] - ); - - let mut b = Bitfield::new(); - b.set(1, true); - assert_eq!(ssz_encode(&b), vec![0b0000_0010]); - } - - fn create_test_bitfield() -> Bitfield { - let count = 2 * 8; - let mut field = Bitfield::with_capacity(count); - - let indices = &[0, 1, 2, 7, 8, 9]; - for &i in indices { - field.set(i, true); - } - field - } - - #[test] - fn test_ssz_decode() { - let encoded = vec![0b0000_0011, 0b1000_0111]; - let field = Bitfield::from_ssz_bytes(&encoded).unwrap(); - let expected = create_test_bitfield(); - assert_eq!(field, expected); - - let encoded = vec![255, 255, 3]; - let field = Bitfield::from_ssz_bytes(&encoded).unwrap(); - let expected = Bitfield::from_bytes(&[255, 255, 3]); - assert_eq!(field, expected); - } - - #[test] - fn test_serialize_deserialize() { - use serde_yaml::Value; - - let data: &[(_, &[_])] = &[ - ("0x01", &[0b00000001]), - ("0xf301", &[0b11110011, 0b00000001]), - ]; - for (hex_data, bytes) in data { - let bitfield = Bitfield::from_bytes(bytes); - assert_eq!( - serde_yaml::from_str::(hex_data).unwrap(), - bitfield - ); - assert_eq!( - serde_yaml::to_value(&bitfield).unwrap(), - Value::String(hex_data.to_string()) - ); - } - } - - #[test] - fn test_ssz_round_trip() { - let original = Bitfield::from_bytes(&vec![18; 12][..]); - let ssz = ssz_encode(&original); - let decoded = Bitfield::from_ssz_bytes(&ssz).unwrap(); - assert_eq!(original, decoded); - } - - #[test] - fn test_bitor() { - let a = Bitfield::from_bytes(&vec![2, 8, 1][..]); - let b = Bitfield::from_bytes(&vec![4, 8, 16][..]); - let c = Bitfield::from_bytes(&vec![6, 8, 17][..]); - assert_eq!(c, a | b); - } - - #[test] - fn test_is_zero() { - let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]]; - for bytes in yes_data { - assert!(Bitfield::from_bytes(bytes).is_zero()); - } - let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]]; - for bytes in no_data { - assert!(!Bitfield::from_bytes(bytes).is_zero()); - } - } - - #[test] - fn test_intersection() { - let a = Bitfield::from_bytes(&[0b1100, 0b0001]); - let b = Bitfield::from_bytes(&[0b1011, 0b1001]); - let c = Bitfield::from_bytes(&[0b1000, 0b0001]); - assert_eq!(a.intersection(&b), c); - assert_eq!(b.intersection(&a), c); - assert_eq!(a.intersection(&c), c); - assert_eq!(b.intersection(&c), c); - assert_eq!(a.intersection(&a), a); - assert_eq!(b.intersection(&b), b); - assert_eq!(c.intersection(&c), c); - } - - #[test] - fn test_union() { - let a = Bitfield::from_bytes(&[0b1100, 0b0001]); - let b = Bitfield::from_bytes(&[0b1011, 0b1001]); - let c = Bitfield::from_bytes(&[0b1111, 0b1001]); - assert_eq!(a.union(&b), c); - assert_eq!(b.union(&a), c); - assert_eq!(a.union(&a), a); - assert_eq!(b.union(&b), b); - assert_eq!(c.union(&c), c); - } - - #[test] - fn test_difference() { - let a = Bitfield::from_bytes(&[0b1100, 0b0001]); - let b = Bitfield::from_bytes(&[0b1011, 0b1001]); - let a_b = Bitfield::from_bytes(&[0b0100, 0b0000]); - let b_a = Bitfield::from_bytes(&[0b0011, 0b1000]); - assert_eq!(a.difference(&b), a_b); - assert_eq!(b.difference(&a), b_a); - assert!(a.difference(&a).is_zero()); - } -} -*/ diff --git a/eth2/utils/ssz_types/src/impl_bitfield_fns.rs b/eth2/utils/ssz_types/src/impl_bitfield_fns.rs new file mode 100644 index 000000000..febe386fb --- /dev/null +++ b/eth2/utils/ssz_types/src/impl_bitfield_fns.rs @@ -0,0 +1,229 @@ +use bit_reverse::LookupReverse; + +/// Provides a common `impl` for structs that wrap a `$name`. +#[macro_export] +macro_rules! impl_bitfield_fns { + ($name: ident) => { + impl $name { + /// Create a new BitList list with `initial_len` bits all set to `false`. + pub fn with_capacity(initial_len: usize) -> Result { + Self::from_elem(initial_len, false) + } + + /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. + /// + /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` + /// regardless of `bit`. + pub fn from_elem(initial_len: usize, bit: bool) -> Result { + // BitVec can panic if we don't set the len to be a multiple of 8. + let full_len = ((initial_len + 7) / 8) * 8; + + Self::validate_length(full_len)?; + + let mut bitfield = Bitfield::from_elem(full_len, false); + + if bit { + for i in 0..initial_len { + bitfield.set(i, true); + } + } + + Ok(Self { + bitfield, + _phantom: PhantomData, + }) + } + + /// Create a new bitfield using the supplied `bytes` as input + pub fn from_bytes(bytes: &[u8]) -> Result { + Self::validate_length(bytes.len().saturating_mul(8))?; + + Ok(Self { + bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), + _phantom: PhantomData, + }) + } + /// Returns a vector of bytes representing the bitfield + pub fn to_bytes(&self) -> Vec { + if self.bitfield.is_empty() { + vec![0] // Empty bitfield should be represented as a zero byte. + } else { + reverse_bit_order(self.bitfield.to_bytes().to_vec()) + } + } + + /// Read the value of a bit. + /// + /// If the index is in bounds, then result is Ok(value) where value is `true` if the + /// bit is 1 and `false` if the bit is 0. If the index is out of bounds, we return an + /// error to that extent. + pub fn get(&self, i: usize) -> Result { + if i < N::to_usize() { + match self.bitfield.get(i) { + Some(value) => Ok(value), + None => Err(Error::OutOfBounds { + i, + len: self.bitfield.len(), + }), + } + } else { + Err(Error::InvalidLength { + i, + len: N::to_usize(), + }) + } + } + + /// Set the value of a bit. + /// + /// If the index is out of bounds, we expand the size of the underlying set to include + /// the new index. Returns the previous value if there was one. + pub fn set(&mut self, i: usize, value: bool) -> Result<(), Error> { + match self.get(i) { + Ok(previous) => Some(previous), + Err(Error::OutOfBounds { len, .. }) => { + let new_len = i - len + 1; + self.bitfield.grow(new_len, false); + None + } + Err(e) => return Err(e), + }; + + self.bitfield.set(i, value); + + Ok(()) + } + + /// Returns the number of bits in this bitfield. + pub fn len(&self) -> usize { + self.bitfield.len() + } + + /// Returns true if `self.len() == 0` + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns true if all bits are set to 0. + pub fn is_zero(&self) -> bool { + self.bitfield.none() + } + + /// Returns the number of bytes required to represent this bitfield. + pub fn num_bytes(&self) -> usize { + self.to_bytes().len() + } + + /// Returns the number of `1` bits in the bitfield + pub fn num_set_bits(&self) -> usize { + self.bitfield.iter().filter(|&bit| bit).count() + } + } + + impl cmp::PartialEq for $name { + /// Determines equality by comparing the `ssz` encoding of the two candidates. This + /// method ensures that the presence of high-order (empty) bits in the highest byte do + /// not exclude equality when they are in fact representing the same information. + fn eq(&self, other: &Self) -> bool { + ssz::ssz_encode(self) == ssz::ssz_encode(other) + } + } + + /// Create a new bitfield that is a union of two other bitfields. + /// + /// For example `union(0101, 1000) == 1101` + // TODO: length-independent intersection for BitAnd + impl std::ops::BitOr for $name { + type Output = Self; + + fn bitor(self, other: Self) -> Self { + let (biggest, smallest) = if self.len() > other.len() { + (&self, &other) + } else { + (&other, &self) + }; + let mut new = (*biggest).clone(); + for i in 0..smallest.len() { + if let Ok(true) = smallest.get(i) { + new.set(i, true) + .expect("Cannot produce bitfield larger than smallest of two given"); + } + } + new + } + } + + impl Encode for $name { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_append(&self, buf: &mut Vec) { + buf.append(&mut self.to_bytes()) + } + } + + impl Decode for $name { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + $name::from_bytes(bytes) + .map_err(|e| ssz::DecodeError::BytesInvalid(format!("Bitfield {:?}", e))) + } + } + + impl Serialize for $name { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&encode(self.to_bytes())) + } + } + + impl<'de, N: Unsigned> Deserialize<'de> for $name { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + $name::from_bytes(&bytes) + .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) + } + } + + impl tree_hash::TreeHash for $name { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.to_bytes().tree_hash_root() + } + } + }; +} + +// Reverse the bit order of a whole byte vec, so that the ith bit +// of the input vec is placed in the (N - i)th bit of the output vec. +// This function is necessary for converting bitfields to and from YAML, +// as the BitVec library and the hex-parser use opposing bit orders. +pub fn reverse_bit_order(mut bytes: Vec) -> Vec { + bytes.reverse(); + bytes.into_iter().map(LookupReverse::swap_bits).collect() +} diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 9fba87bc4..1e49c6edd 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -1,8 +1,14 @@ -mod bitfield; +#[macro_use] +mod impl_bitfield_fns; +mod bit_list; +mod bit_vector; mod fixed_vector; mod variable_list; -pub use bitfield::{BitList, BitVector}; +use impl_bitfield_fns::reverse_bit_order; + +pub use bit_list::BitList; +pub use bit_vector::BitVector; pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; From 636ebb0d4e798438d0227406dede03e77b4fd551 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 8 Jul 2019 11:54:47 +1000 Subject: [PATCH 49/72] Add progress on bitfields --- eth2/utils/ssz_types/src/bit_list.rs | 12 +- eth2/utils/ssz_types/src/bit_vector.rs | 275 ++++++++++++++++++ eth2/utils/ssz_types/src/impl_bitfield_fns.rs | 80 ++++- 3 files changed, 357 insertions(+), 10 deletions(-) diff --git a/eth2/utils/ssz_types/src/bit_list.rs b/eth2/utils/ssz_types/src/bit_list.rs index 07324148b..bbe4a653b 100644 --- a/eth2/utils/ssz_types/src/bit_list.rs +++ b/eth2/utils/ssz_types/src/bit_list.rs @@ -75,6 +75,16 @@ impl BitList { pub fn max_len() -> usize { N::to_usize() } + + /// Create a new bitfield using the supplied `bytes` as input + pub fn from_bytes(bytes: &[u8]) -> Result { + Self::validate_length(bytes.len().saturating_mul(8))?; + + Ok(Self { + bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), + _phantom: PhantomData, + }) + } } impl BitList { @@ -153,7 +163,7 @@ impl default::Default for BitList { } #[cfg(test)] -mod test_bitlist { +mod test { use super::*; use serde_yaml; use ssz::ssz_encode; diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs index 830412757..ea5fcf978 100644 --- a/eth2/utils/ssz_types/src/bit_vector.rs +++ b/eth2/utils/ssz_types/src/bit_vector.rs @@ -54,6 +54,18 @@ impl BitVector { N::to_usize() } + /// Create a new bitfield using the supplied `bytes` as input + pub fn from_bytes(bytes: &[u8]) -> Result { + if Self::capacity() >= 8 && bytes.len() != 1 { + Self::validate_length(bytes.len().saturating_mul(8))?; + } + + Ok(Self { + bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), + _phantom: PhantomData, + }) + } + fn validate_length(len: usize) -> Result<(), Error> { let fixed_len = N::to_usize(); @@ -67,3 +79,266 @@ impl BitVector { } } } + +#[cfg(test)] +mod test { + use super::*; + use serde_yaml; + use ssz::ssz_encode; + // use tree_hash::TreeHash; + + pub type BitVector4 = BitVector; + pub type BitVector1024 = BitVector; + + /* + #[test] + pub fn cached_tree_hash() { + let original = BitVector1024::from_bytes(&vec![18; 12][..]); + + let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + original.tree_hash_root() + ); + + let modified = BitVector1024::from_bytes(&vec![2; 1][..]); + + cache.update(&modified).unwrap(); + + assert_eq!( + cache.tree_hash_root().unwrap().to_vec(), + modified.tree_hash_root() + ); + } + */ + + #[test] + fn new_bitfield() { + let mut field = BitVector1024::new(); + let original_len = field.len(); + + assert_eq!(original_len, 1024); + + for i in 0..1028 { + if i < original_len { + assert!(!field.get(i).unwrap()); + assert!(field.set(i, true).is_ok()); + } else { + assert!(field.get(i).is_err()); + assert!(field.set(i, true).is_err()); + } + } + } + + #[test] + fn from_bytes_bitvec4() { + let bytes = &[3]; + + let bitvec = BitVector4::from_bytes(bytes).unwrap(); + + assert_eq!(bitvec.get(0), Ok(true)); + assert_eq!(bitvec.get(1), Ok(true)); + assert_eq!(bitvec.get(2), Ok(false)); + assert_eq!(bitvec.get(3), Ok(false)); + + assert!(bitvec.get(4).is_err()); + } + + /* + #[test] + fn from_bytes_bytes_too_long() { + let bytes = &[0, 0]; + + assert_eq!( + BitVector4::from_bytes(bytes), + Err(Error::InvalidLength { i: 16, len: 4 }) + ); + } + + const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; + + #[test] + fn get_from_bitfield() { + let field = BitVector1024::from_bytes(INPUT).unwrap(); + field.get(0).unwrap(); + field.get(6).unwrap(); + field.get(14).unwrap(); + } + + #[test] + fn set_for_bitfield() { + let mut field = BitVector1024::from_bytes(INPUT).unwrap(); + field.set(10, true).unwrap(); + field.get(10).unwrap(); + field.set(6, false).unwrap(); + field.get(6).unwrap(); + } + + #[test] + fn len() { + let field = BitVector1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.len(), 16); + + let field = BitVector1024::new(); + assert_eq!(field.len(), 0); + } + + #[test] + fn num_set_bits() { + let field = BitVector1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.num_set_bits(), 2); + + let field = BitVector1024::new(); + assert_eq!(field.num_set_bits(), 0); + } + + #[test] + fn to_bytes() { + let field = BitVector1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.to_bytes(), INPUT); + + let field = BitVector1024::new(); + assert_eq!(field.to_bytes(), vec![0]); + } + + #[test] + fn out_of_bounds() { + let mut field = BitVector1024::from_bytes(INPUT).unwrap(); + + let out_of_bounds_index = field.len(); + assert!(field.set(out_of_bounds_index, true).is_ok()); + assert!(field.len() == out_of_bounds_index + 1); + assert!(field.get(out_of_bounds_index).unwrap()); + + for i in 0..100 { + if i <= out_of_bounds_index { + assert!(field.set(i, true).is_ok()); + } else { + assert!(field.set(i, true).is_ok()); + } + } + } + + #[test] + fn grows_with_false() { + let input_all_set: &[u8] = &[0b1111_1111, 0b1111_1111]; + let mut field = BitVector1024::from_bytes(input_all_set).unwrap(); + + // Define `a` and `b`, where both are out of bounds and `b` is greater than `a`. + let a = field.len(); + let b = a + 1; + + // Ensure `a` is out-of-bounds for test integrity. + assert!(field.get(a).is_err()); + + // Set `b` to `true`.. + assert!(field.set(b, true).is_ok()); + + // Ensure that `a` wasn't also set to `true` during the grow. + assert_eq!(field.get(a), Ok(false)); + assert_eq!(field.get(b), Ok(true)); + } + + #[test] + fn num_bytes() { + let field = BitVector1024::from_bytes(INPUT).unwrap(); + assert_eq!(field.num_bytes(), 2); + + let field = BitVector1024::from_elem(2, true).unwrap(); + assert_eq!(field.num_bytes(), 1); + + let field = BitVector1024::from_elem(13, true).unwrap(); + assert_eq!(field.num_bytes(), 2); + } + + #[test] + fn ssz_encoding() { + let field = create_bitfield(); + assert_eq!(field.as_ssz_bytes(), vec![0b0000_0011, 0b1000_0111]); + + let field = BitVector1024::from_elem(18, true).unwrap(); + assert_eq!( + field.as_ssz_bytes(), + vec![0b0000_0011, 0b1111_1111, 0b1111_1111] + ); + + let mut b = BitVector1024::new(); + b.set(1, true).unwrap(); + assert_eq!(ssz_encode(&b), vec![0b0000_0010]); + } + + fn create_bitfield() -> BitVector1024 { + let count = 2 * 8; + let mut field = BitVector1024::with_capacity(count).unwrap(); + + let indices = &[0, 1, 2, 7, 8, 9]; + for &i in indices { + field.set(i, true).unwrap(); + } + field + } + + #[test] + fn ssz_decode() { + let encoded = vec![0b0000_0011, 0b1000_0111]; + let field = BitVector1024::from_ssz_bytes(&encoded).unwrap(); + let expected = create_bitfield(); + assert_eq!(field, expected); + + let encoded = vec![255, 255, 3]; + let field = BitVector1024::from_ssz_bytes(&encoded).unwrap(); + let expected = BitVector1024::from_bytes(&[255, 255, 3]).unwrap(); + assert_eq!(field, expected); + } + + #[test] + fn serialize_deserialize() { + use serde_yaml::Value; + + let data: &[(_, &[_])] = &[ + ("0x01", &[0b00000001]), + ("0xf301", &[0b11110011, 0b00000001]), + ]; + for (hex_data, bytes) in data { + let bitfield = BitVector1024::from_bytes(bytes).unwrap(); + assert_eq!( + serde_yaml::from_str::(hex_data).unwrap(), + bitfield + ); + assert_eq!( + serde_yaml::to_value(&bitfield).unwrap(), + Value::String(hex_data.to_string()) + ); + } + } + + #[test] + fn ssz_round_trip() { + let original = BitVector1024::from_bytes(&vec![18; 12][..]).unwrap(); + let ssz = ssz_encode(&original); + let decoded = BitVector1024::from_ssz_bytes(&ssz).unwrap(); + assert_eq!(original, decoded); + } + + #[test] + fn bitor() { + let a = BitVector1024::from_bytes(&vec![2, 8, 1][..]).unwrap(); + let b = BitVector1024::from_bytes(&vec![4, 8, 16][..]).unwrap(); + let c = BitVector1024::from_bytes(&vec![6, 8, 17][..]).unwrap(); + assert_eq!(c, a | b); + } + + #[test] + fn is_zero() { + let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]]; + for bytes in yes_data { + assert!(BitVector1024::from_bytes(bytes).unwrap().is_zero()); + } + let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]]; + for bytes in no_data { + assert!(!BitVector1024::from_bytes(bytes).unwrap().is_zero()); + } + } + */ +} diff --git a/eth2/utils/ssz_types/src/impl_bitfield_fns.rs b/eth2/utils/ssz_types/src/impl_bitfield_fns.rs index febe386fb..dd11f4f30 100644 --- a/eth2/utils/ssz_types/src/impl_bitfield_fns.rs +++ b/eth2/utils/ssz_types/src/impl_bitfield_fns.rs @@ -34,15 +34,6 @@ macro_rules! impl_bitfield_fns { }) } - /// Create a new bitfield using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Result { - Self::validate_length(bytes.len().saturating_mul(8))?; - - Ok(Self { - bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), - _phantom: PhantomData, - }) - } /// Returns a vector of bytes representing the bitfield pub fn to_bytes(&self) -> Vec { if self.bitfield.is_empty() { @@ -227,3 +218,74 @@ pub fn reverse_bit_order(mut bytes: Vec) -> Vec { bytes.reverse(); bytes.into_iter().map(LookupReverse::swap_bits).collect() } + +/* +/// Verify that the given `bytes` faithfully represent a bitfield of length `bit_len`. +/// +/// The only valid `bytes` for `bit_len == 0` is `&[0]`. +pub fn verify_bitfield_bytes(bytes: &[u8], bit_len: usize) -> bool { + if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { + true // A bitfield with `bit_len` 0 can only be represented by a single zero byte. + } else if bytes.len() != ((bit_len + 7) / 8) || bytes.is_empty() { + false // The number of bytes must be the minimum required to represent `bit_len`. + } else { + // Ensure there are no bits higher than `bit_len` that are set to true. + let (mask, _) = u8::max_value().overflowing_shl(8 - (bit_len as u32 % 8)); + (bytes.last().expect("Bytes cannot be empty") & !mask) == 0 + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn bitfield_bytes_length() { + assert!(verify_bitfield_bytes(&[0b0000_0000], 0)); + assert!(verify_bitfield_bytes(&[0b1000_0000], 1)); + assert!(verify_bitfield_bytes(&[0b1100_0000], 2)); + assert!(verify_bitfield_bytes(&[0b1110_0000], 3)); + assert!(verify_bitfield_bytes(&[0b1111_0000], 4)); + assert!(verify_bitfield_bytes(&[0b1111_1000], 5)); + assert!(verify_bitfield_bytes(&[0b1111_1100], 6)); + assert!(verify_bitfield_bytes(&[0b1111_1110], 7)); + assert!(verify_bitfield_bytes(&[0b1111_1111], 8)); + + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b0000_0000], 9)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1000_0000], 9)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1100_0000], 10)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1110_0000], 11)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_0000], 12)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_1000], 13)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_1100], 14)); + assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_1110], 15)); + + for i in 0..8 { + assert!(!verify_bitfield_bytes(&[], i)); + assert!(!verify_bitfield_bytes(&[0b1111_1111], i)); + assert!(!verify_bitfield_bytes(&[0b1111_1110, 0b0000_0000], i)); + } + + assert!(!verify_bitfield_bytes(&[0b1000_0000], 0)); + + assert!(!verify_bitfield_bytes(&[0b1000_0000], 0)); + assert!(!verify_bitfield_bytes(&[0b1100_0000], 1)); + assert!(!verify_bitfield_bytes(&[0b1110_0000], 2)); + assert!(!verify_bitfield_bytes(&[0b1111_0000], 3)); + assert!(!verify_bitfield_bytes(&[0b1111_1000], 4)); + assert!(!verify_bitfield_bytes(&[0b1111_1100], 5)); + assert!(!verify_bitfield_bytes(&[0b1111_1110], 6)); + assert!(!verify_bitfield_bytes(&[0b1111_1111], 7)); + + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1000_0000], 8)); + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1100_0000], 9)); + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1110_0000], 10)); + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_0000], 11)); + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_1000], 12)); + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_1100], 13)); + assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_1110], 14)); + + assert!(!verify_bitfield_bytes(&[0b1111_1110], 6)); + } +} +*/ From bbcc58dca3094ecb838075f3302a170758a45558 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 8 Jul 2019 16:07:40 +1000 Subject: [PATCH 50/72] Start building new bitfield struct --- eth2/utils/ssz_types/Cargo.toml | 2 - eth2/utils/ssz_types/src/bit_list.rs | 39 +- eth2/utils/ssz_types/src/bit_vector.rs | 17 +- eth2/utils/ssz_types/src/bitfield.rs | 520 ++++++++++++++++++ eth2/utils/ssz_types/src/impl_bitfield_fns.rs | 291 ---------- eth2/utils/ssz_types/src/lib.rs | 4 +- 6 files changed, 540 insertions(+), 333 deletions(-) create mode 100644 eth2/utils/ssz_types/src/bitfield.rs delete mode 100644 eth2/utils/ssz_types/src/impl_bitfield_fns.rs diff --git a/eth2/utils/ssz_types/Cargo.toml b/eth2/utils/ssz_types/Cargo.toml index f5680ecb1..2e4cbc899 100644 --- a/eth2/utils/ssz_types/Cargo.toml +++ b/eth2/utils/ssz_types/Cargo.toml @@ -5,8 +5,6 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -bit_reverse = "0.1" -bit-vec = "0.5.0" cached_tree_hash = { path = "../cached_tree_hash" } tree_hash = { path = "../tree_hash" } serde = "1.0" diff --git a/eth2/utils/ssz_types/src/bit_list.rs b/eth2/utils/ssz_types/src/bit_list.rs index bbe4a653b..83fa6a0bd 100644 --- a/eth2/utils/ssz_types/src/bit_list.rs +++ b/eth2/utils/ssz_types/src/bit_list.rs @@ -1,6 +1,5 @@ use super::*; -use crate::{impl_bitfield_fns, reverse_bit_order, Error}; -use bit_vec::BitVec as Bitfield; +use crate::{bitfield::Bitfield, impl_bitfield_fns, Error}; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode, PrefixedHexVisitor}; @@ -53,7 +52,7 @@ impl BitList { /// Create a new, empty BitList. pub fn new() -> Self { Self { - bitfield: Bitfield::default(), + bitfield: Bitfield::with_capacity(Self::max_len()), _phantom: PhantomData, } } @@ -75,18 +74,20 @@ impl BitList { pub fn max_len() -> usize { N::to_usize() } - - /// Create a new bitfield using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Result { - Self::validate_length(bytes.len().saturating_mul(8))?; - - Ok(Self { - bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), - _phantom: PhantomData, - }) - } } +/* +fn encode_bitfield(bitfield: Bitfield) -> Vec { + // Set the next bit of the bitfield to true. + // + // SSZ spec: + // + // An additional leading 1 bit is added so that the length in bits will also be known. + bitfield.set(bitfield.len(), true); + let bytes = bitfield.to_bytes(); +} +*/ + impl BitList { /// Compute the intersection (binary-and) of this bitfield with another /// @@ -106,7 +107,7 @@ impl BitList { /// /// If `self` and `other` have different lengths. pub fn intersection_inplace(&mut self, other: &Self) { - self.bitfield.intersect(&other.bitfield); + self.bitfield.intersection(&other.bitfield); } /// Compute the union (binary-or) of this bitfield with another. Lengths must match. @@ -154,14 +155,7 @@ impl BitList { } } -impl default::Default for BitList { - /// Default provides the "empty" bitfield - /// Note: the empty bitfield is set to the `0` byte. - fn default() -> Self { - Self::from_elem(0, false).expect("Zero cannot be larger than the maximum length") - } -} - +/* #[cfg(test)] mod test { use super::*; @@ -451,3 +445,4 @@ mod test { assert!(a.difference(&a).is_zero()); } } +*/ diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs index ea5fcf978..ba63281cf 100644 --- a/eth2/utils/ssz_types/src/bit_vector.rs +++ b/eth2/utils/ssz_types/src/bit_vector.rs @@ -1,6 +1,5 @@ use super::*; -use crate::{impl_bitfield_fns, reverse_bit_order, Error}; -use bit_vec::BitVec as Bitfield; +use crate::{bitfield::Bitfield, impl_bitfield_fns, Error}; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use serde_hex::{encode, PrefixedHexVisitor}; @@ -54,18 +53,6 @@ impl BitVector { N::to_usize() } - /// Create a new bitfield using the supplied `bytes` as input - pub fn from_bytes(bytes: &[u8]) -> Result { - if Self::capacity() >= 8 && bytes.len() != 1 { - Self::validate_length(bytes.len().saturating_mul(8))?; - } - - Ok(Self { - bitfield: Bitfield::from_bytes(&reverse_bit_order(bytes.to_vec())), - _phantom: PhantomData, - }) - } - fn validate_length(len: usize) -> Result<(), Error> { let fixed_len = N::to_usize(); @@ -113,6 +100,7 @@ mod test { } */ + /* #[test] fn new_bitfield() { let mut field = BitVector1024::new(); @@ -145,7 +133,6 @@ mod test { assert!(bitvec.get(4).is_err()); } - /* #[test] fn from_bytes_bytes_too_long() { let bytes = &[0, 0]; diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs new file mode 100644 index 000000000..a64615bd7 --- /dev/null +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -0,0 +1,520 @@ +/// A heap-allocated, ordered, fixed-length, collection of `bool` values. +/// +/// The length of the Bitfield is set at instantiation (i.e., runtime, not compile time). +/// +/// The internal representation of the bitfield is the same as that required by SSZ - the highest +/// byte (by `Vec` index) stores the lowest bit-indices and the right-most bit stores the lowest +/// bit-index. E.g., `vec![0b0000_0010, 0b0000_0001]` has bits `1, 9` set. +#[derive(Clone, Debug, PartialEq)] +pub struct Bitfield { + bytes: Vec, + len: usize, +} + +impl Bitfield { + pub fn with_capacity(num_bits: usize) -> Self { + Self { + bytes: vec![0; Self::bytes_for_bit_len(num_bits)], + len: num_bits, + } + } + + pub fn set(&mut self, i: usize, value: bool) -> Option<()> { + if i < self.len { + let byte = { + let num_bytes = self.bytes.len(); + let offset = i / 8; + self.bytes + .get_mut(num_bytes - offset - 1) + .expect("Cannot be OOB if less than self.len") + }; + + if value { + *byte |= 1 << (i % 8) + } else { + *byte &= !(1 << (i % 8)) + } + + Some(()) + } else { + None + } + } + + pub fn get(&self, i: usize) -> Option { + if i < self.len { + let byte = { + let num_bytes = self.bytes.len(); + let offset = i / 8; + self.bytes + .get(num_bytes - offset - 1) + .expect("Cannot be OOB if less than self.len") + }; + + Some(*byte & 1 << (i % 8) > 0) + } else { + None + } + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + fn bytes_for_bit_len(bit_len: usize) -> usize { + (bit_len + 7) / 8 + } + + /// Verify that the given `bytes` faithfully represent a bitfield of length `bit_len`. + /// + /// The only valid `bytes` for `bit_len == 0` is `&[0]`. + fn verify_bitfield_bytes(bytes: &[u8], bit_len: usize) -> bool { + if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { + true // A bitfield with `bit_len` 0 can only be represented by a single zero byte. + } else if bytes.len() != Bitfield::bytes_for_bit_len(bit_len) || bytes.is_empty() { + false // The number of bytes must be the minimum required to represent `bit_len`. + } else { + // Ensure there are no bits higher than `bit_len` that are set to true. + let (mask, _) = u8::max_value().overflowing_shr(bit_len as u32 % 8); + (bytes.last().expect("Bytes cannot be empty") & !mask) == 0 + } + } + + pub fn to_bytes(self) -> Vec { + self.bytes + } + + pub fn as_slice(&self) -> &[u8] { + &self.bytes + } + + pub fn from_bytes(bytes: Vec, bit_len: usize) -> Option { + if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { + // A bitfield with `bit_len` 0 can only be represented by a single zero byte. + Some(Self { bytes, len: 0 }) + } else if bytes.len() != Bitfield::bytes_for_bit_len(bit_len) || bytes.is_empty() { + // The number of bytes must be the minimum required to represent `bit_len`. + None + } else { + // Ensure there are no bits higher than `bit_len` that are set to true. + let (mask, _) = u8::max_value().overflowing_shr(8 - (bit_len as u32 % 8)); + + if (bytes.first().expect("Bytes cannot be empty") & !mask) == 0 { + Some(Self { + bytes, + len: bit_len, + }) + } else { + None + } + } + } + + pub fn iter(&self) -> BitIter<'_> { + BitIter { + bitfield: self, + i: 0, + } + } + + pub fn is_zero(&self) -> bool { + !self.bytes.iter().any(|byte| (*byte & u8::max_value()) > 0) + } + + pub fn intersection(&self, other: &Self) -> Option { + if self.is_comparable(other) { + let mut res = self.clone(); + res.intersection_inplace(other); + Some(res) + } else { + None + } + } + + pub fn intersection_inplace(&mut self, other: &Self) -> Option<()> { + if self.is_comparable(other) { + for i in 0..self.bytes.len() { + self.bytes[i] = self.bytes[i] & other.bytes[i]; + } + Some(()) + } else { + None + } + } + + pub fn union(&self, other: &Self) -> Option { + if self.is_comparable(other) { + let mut res = self.clone(); + res.union_inplace(other); + Some(res) + } else { + None + } + } + + pub fn union_inplace(&mut self, other: &Self) -> Option<()> { + if self.is_comparable(other) { + for i in 0..self.bytes.len() { + self.bytes[i] = self.bytes[i] | other.bytes[i]; + } + Some(()) + } else { + None + } + } + + pub fn difference(&self, other: &Self) -> Option { + if self.is_comparable(other) { + let mut res = self.clone(); + res.difference_inplace(other); + Some(res) + } else { + None + } + } + + pub fn difference_inplace(&mut self, other: &Self) -> Option<()> { + if self.is_comparable(other) { + for i in 0..self.bytes.len() { + self.bytes[i] = self.bytes[i] & !other.bytes[i]; + } + Some(()) + } else { + None + } + } + + pub fn is_comparable(&self, other: &Self) -> bool { + (self.len() == other.len()) && (self.bytes.len() == other.bytes.len()) + } +} + +pub struct BitIter<'a> { + bitfield: &'a Bitfield, + i: usize, +} + +impl<'a> Iterator for BitIter<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + let res = self.bitfield.get(self.i); + self.i += 1; + res + } +} + +/// Provides a common `impl` for structs that wrap a `$name`. +#[macro_export] +macro_rules! impl_bitfield_fns { + ($name: ident) => { + impl $name { + pub fn with_capacity(initial_len: usize) -> Result { + Self::validate_length(initial_len)?; + + Self::with_capacity(initial_len) + } + + pub fn get(&self, i: usize) -> Result { + if i < N::to_usize() { + match self.bitfield.get(i) { + Some(value) => Ok(value), + None => Err(Error::OutOfBounds { + i, + len: self.bitfield.len(), + }), + } + } else { + Err(Error::InvalidLength { + i, + len: N::to_usize(), + }) + } + } + + pub fn set(&mut self, i: usize, value: bool) -> Option<()> { + self.bitfield.set(i, value) + } + + /// Returns the number of bits in this bitfield. + pub fn len(&self) -> usize { + self.bitfield.len() + } + + /// Returns true if `self.len() == 0` + pub fn is_empty(&self) -> bool { + self.bitfield.is_empty() + } + + /// Returns true if all bits are set to 0. + pub fn is_zero(&self) -> bool { + self.bitfield.is_zero() + } + + /// Returns the number of bytes presently used to store the bitfield. + pub fn num_bytes(&self) -> usize { + self.bitfield.as_slice().len() + } + + /// Returns the number of `1` bits in the bitfield + pub fn num_set_bits(&self) -> usize { + self.bitfield.iter().filter(|&bit| bit).count() + } + } + + /* + impl Encode for $name { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_append(&self, buf: &mut Vec) { + buf.append(&mut self.bitfield.to_bytes()) + } + } + + impl Decode for $name { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let bitfield = + Bitfield::from_bytes(bytes.to_vec(), bytes.len() * 8).expect("Cannot fail"); + Ok(Self { + bitfield, + _phantom: PhantomData, + }) + /* + $name::from_bytes(bytes) + .map_err(|e| ssz::DecodeError::BytesInvalid(format!("Bitfield {:?}", e))) + */ + } + } + + impl Serialize for $name { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&encode(self.bitfield.to_bytes())) + } + } + + impl<'de, N: Unsigned> Deserialize<'de> for $name { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + $name::from_bytes(&bytes) + .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) + } + } + + impl tree_hash::TreeHash for $name { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.to_bytes().tree_hash_root() + } + } + */ + }; +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn from_bytes() { + assert!(Bitfield::from_bytes(vec![0b0000_0000], 0).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_0001], 1).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_0011], 2).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_0111], 3).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_1111], 4).is_some()); + assert!(Bitfield::from_bytes(vec![0b0001_1111], 5).is_some()); + assert!(Bitfield::from_bytes(vec![0b0011_1111], 6).is_some()); + assert!(Bitfield::from_bytes(vec![0b0111_1111], 7).is_some()); + assert!(Bitfield::from_bytes(vec![0b1111_1111], 8).is_some()); + + assert!(Bitfield::from_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_some()); + assert!(Bitfield::from_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_some()); + assert!(Bitfield::from_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_some()); + assert!(Bitfield::from_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_some()); + assert!(Bitfield::from_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_some()); + assert!(Bitfield::from_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_some()); + + for i in 0..8 { + assert!(Bitfield::from_bytes(vec![], i).is_none()); + assert!(Bitfield::from_bytes(vec![0b1111_1111], i).is_none()); + assert!(Bitfield::from_bytes(vec![0b1111_1110, 0b0000_0000], i).is_none()); + } + + assert!(Bitfield::from_bytes(vec![0b0000_0001], 0).is_none()); + + assert!(Bitfield::from_bytes(vec![0b0000_0001], 0).is_none()); + assert!(Bitfield::from_bytes(vec![0b0000_0011], 1).is_none()); + assert!(Bitfield::from_bytes(vec![0b0000_0111], 2).is_none()); + assert!(Bitfield::from_bytes(vec![0b0000_1111], 3).is_none()); + assert!(Bitfield::from_bytes(vec![0b0001_1111], 4).is_none()); + assert!(Bitfield::from_bytes(vec![0b0011_1111], 5).is_none()); + assert!(Bitfield::from_bytes(vec![0b0111_1111], 6).is_none()); + assert!(Bitfield::from_bytes(vec![0b1111_1111], 7).is_none()); + + assert!(Bitfield::from_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_none()); + assert!(Bitfield::from_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_none()); + assert!(Bitfield::from_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_none()); + assert!(Bitfield::from_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_none()); + assert!(Bitfield::from_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_none()); + assert!(Bitfield::from_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_none()); + assert!(Bitfield::from_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_none()); + assert!(Bitfield::from_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_none()); + } + + fn test_set_unset(num_bits: usize) { + let mut bitfield = Bitfield::with_capacity(num_bits); + + for i in 0..num_bits + 1 { + dbg!(i); + if i < num_bits { + // Starts as false + assert_eq!(bitfield.get(i), Some(false)); + // Can be set true. + assert!(bitfield.set(i, true).is_some()); + assert_eq!(bitfield.get(i), Some(true)); + // Can be set false + assert!(bitfield.set(i, false).is_some()); + assert_eq!(bitfield.get(i), Some(false)); + } else { + assert_eq!(bitfield.get(i), None); + assert!(bitfield.set(i, true).is_none()); + assert_eq!(bitfield.get(i), None); + } + } + } + + fn test_bytes_round_trip(num_bits: usize) { + dbg!(num_bits); + for i in 0..num_bits { + dbg!(i); + let mut bitfield = Bitfield::with_capacity(num_bits); + bitfield.set(i, true).unwrap(); + + let bytes = bitfield.clone().to_bytes(); + dbg!(&bytes); + assert_eq!(bitfield, Bitfield::from_bytes(bytes, num_bits).unwrap()); + } + } + + #[test] + fn set_unset() { + for i in 0..8 * 5 { + test_set_unset(i) + } + } + + #[test] + fn bytes_round_trip() { + for i in 0..8 * 5 { + test_bytes_round_trip(i) + } + } + + #[test] + fn to_bytes() { + let mut bitfield = Bitfield::with_capacity(9); + bitfield.set(0, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_0001]); + bitfield.set(1, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_0011]); + bitfield.set(2, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_0111]); + bitfield.set(3, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_1111]); + bitfield.set(4, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0001_1111]); + bitfield.set(5, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0011_1111]); + bitfield.set(6, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0111_1111]); + bitfield.set(7, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b1111_1111]); + bitfield.set(8, true); + assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0001, 0b1111_1111]); + } + + #[test] + fn intersection() { + let a = Bitfield::from_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = Bitfield::from_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let c = Bitfield::from_bytes(vec![0b1000, 0b0001], 16).unwrap(); + + assert_eq!(a.intersection(&b).unwrap(), c); + assert_eq!(b.intersection(&a).unwrap(), c); + assert_eq!(a.intersection(&c).unwrap(), c); + assert_eq!(b.intersection(&c).unwrap(), c); + assert_eq!(a.intersection(&a).unwrap(), a); + assert_eq!(b.intersection(&b).unwrap(), b); + assert_eq!(c.intersection(&c).unwrap(), c); + } + + #[test] + fn union() { + let a = Bitfield::from_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = Bitfield::from_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let c = Bitfield::from_bytes(vec![0b1111, 0b1001], 16).unwrap(); + + assert_eq!(a.union(&b).unwrap(), c); + assert_eq!(b.union(&a).unwrap(), c); + assert_eq!(a.union(&a).unwrap(), a); + assert_eq!(b.union(&b).unwrap(), b); + assert_eq!(c.union(&c).unwrap(), c); + } + + #[test] + fn difference() { + let a = Bitfield::from_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = Bitfield::from_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let a_b = Bitfield::from_bytes(vec![0b0100, 0b0000], 16).unwrap(); + let b_a = Bitfield::from_bytes(vec![0b0011, 0b1000], 16).unwrap(); + + assert_eq!(a.difference(&b).unwrap(), a_b); + assert_eq!(b.difference(&a).unwrap(), b_a); + assert!(a.difference(&a).unwrap().is_zero()); + } + + #[test] + fn iter() { + let mut bitfield = Bitfield::with_capacity(9); + bitfield.set(2, true); + bitfield.set(8, true); + + assert_eq!( + bitfield.iter().collect::>(), + vec![false, false, true, false, false, false, false, false, true] + ); + } +} diff --git a/eth2/utils/ssz_types/src/impl_bitfield_fns.rs b/eth2/utils/ssz_types/src/impl_bitfield_fns.rs deleted file mode 100644 index dd11f4f30..000000000 --- a/eth2/utils/ssz_types/src/impl_bitfield_fns.rs +++ /dev/null @@ -1,291 +0,0 @@ -use bit_reverse::LookupReverse; - -/// Provides a common `impl` for structs that wrap a `$name`. -#[macro_export] -macro_rules! impl_bitfield_fns { - ($name: ident) => { - impl $name { - /// Create a new BitList list with `initial_len` bits all set to `false`. - pub fn with_capacity(initial_len: usize) -> Result { - Self::from_elem(initial_len, false) - } - - /// Create a new bitfield with the given length `initial_len` and all values set to `bit`. - /// - /// Note: if `initial_len` is not a multiple of 8, the remaining bits will be set to `false` - /// regardless of `bit`. - pub fn from_elem(initial_len: usize, bit: bool) -> Result { - // BitVec can panic if we don't set the len to be a multiple of 8. - let full_len = ((initial_len + 7) / 8) * 8; - - Self::validate_length(full_len)?; - - let mut bitfield = Bitfield::from_elem(full_len, false); - - if bit { - for i in 0..initial_len { - bitfield.set(i, true); - } - } - - Ok(Self { - bitfield, - _phantom: PhantomData, - }) - } - - /// Returns a vector of bytes representing the bitfield - pub fn to_bytes(&self) -> Vec { - if self.bitfield.is_empty() { - vec![0] // Empty bitfield should be represented as a zero byte. - } else { - reverse_bit_order(self.bitfield.to_bytes().to_vec()) - } - } - - /// Read the value of a bit. - /// - /// If the index is in bounds, then result is Ok(value) where value is `true` if the - /// bit is 1 and `false` if the bit is 0. If the index is out of bounds, we return an - /// error to that extent. - pub fn get(&self, i: usize) -> Result { - if i < N::to_usize() { - match self.bitfield.get(i) { - Some(value) => Ok(value), - None => Err(Error::OutOfBounds { - i, - len: self.bitfield.len(), - }), - } - } else { - Err(Error::InvalidLength { - i, - len: N::to_usize(), - }) - } - } - - /// Set the value of a bit. - /// - /// If the index is out of bounds, we expand the size of the underlying set to include - /// the new index. Returns the previous value if there was one. - pub fn set(&mut self, i: usize, value: bool) -> Result<(), Error> { - match self.get(i) { - Ok(previous) => Some(previous), - Err(Error::OutOfBounds { len, .. }) => { - let new_len = i - len + 1; - self.bitfield.grow(new_len, false); - None - } - Err(e) => return Err(e), - }; - - self.bitfield.set(i, value); - - Ok(()) - } - - /// Returns the number of bits in this bitfield. - pub fn len(&self) -> usize { - self.bitfield.len() - } - - /// Returns true if `self.len() == 0` - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns true if all bits are set to 0. - pub fn is_zero(&self) -> bool { - self.bitfield.none() - } - - /// Returns the number of bytes required to represent this bitfield. - pub fn num_bytes(&self) -> usize { - self.to_bytes().len() - } - - /// Returns the number of `1` bits in the bitfield - pub fn num_set_bits(&self) -> usize { - self.bitfield.iter().filter(|&bit| bit).count() - } - } - - impl cmp::PartialEq for $name { - /// Determines equality by comparing the `ssz` encoding of the two candidates. This - /// method ensures that the presence of high-order (empty) bits in the highest byte do - /// not exclude equality when they are in fact representing the same information. - fn eq(&self, other: &Self) -> bool { - ssz::ssz_encode(self) == ssz::ssz_encode(other) - } - } - - /// Create a new bitfield that is a union of two other bitfields. - /// - /// For example `union(0101, 1000) == 1101` - // TODO: length-independent intersection for BitAnd - impl std::ops::BitOr for $name { - type Output = Self; - - fn bitor(self, other: Self) -> Self { - let (biggest, smallest) = if self.len() > other.len() { - (&self, &other) - } else { - (&other, &self) - }; - let mut new = (*biggest).clone(); - for i in 0..smallest.len() { - if let Ok(true) = smallest.get(i) { - new.set(i, true) - .expect("Cannot produce bitfield larger than smallest of two given"); - } - } - new - } - } - - impl Encode for $name { - fn is_ssz_fixed_len() -> bool { - false - } - - fn ssz_append(&self, buf: &mut Vec) { - buf.append(&mut self.to_bytes()) - } - } - - impl Decode for $name { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - $name::from_bytes(bytes) - .map_err(|e| ssz::DecodeError::BytesInvalid(format!("Bitfield {:?}", e))) - } - } - - impl Serialize for $name { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&encode(self.to_bytes())) - } - } - - impl<'de, N: Unsigned> Deserialize<'de> for $name { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - // We reverse the bit-order so that the BitVec library can read its 0th - // bit from the end of the hex string, e.g. - // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] - let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; - $name::from_bytes(&bytes) - .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) - } - } - - impl tree_hash::TreeHash for $name { - fn tree_hash_type() -> tree_hash::TreeHashType { - tree_hash::TreeHashType::List - } - - fn tree_hash_packed_encoding(&self) -> Vec { - unreachable!("List should never be packed.") - } - - fn tree_hash_packing_factor() -> usize { - unreachable!("List should never be packed.") - } - - fn tree_hash_root(&self) -> Vec { - self.to_bytes().tree_hash_root() - } - } - }; -} - -// Reverse the bit order of a whole byte vec, so that the ith bit -// of the input vec is placed in the (N - i)th bit of the output vec. -// This function is necessary for converting bitfields to and from YAML, -// as the BitVec library and the hex-parser use opposing bit orders. -pub fn reverse_bit_order(mut bytes: Vec) -> Vec { - bytes.reverse(); - bytes.into_iter().map(LookupReverse::swap_bits).collect() -} - -/* -/// Verify that the given `bytes` faithfully represent a bitfield of length `bit_len`. -/// -/// The only valid `bytes` for `bit_len == 0` is `&[0]`. -pub fn verify_bitfield_bytes(bytes: &[u8], bit_len: usize) -> bool { - if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { - true // A bitfield with `bit_len` 0 can only be represented by a single zero byte. - } else if bytes.len() != ((bit_len + 7) / 8) || bytes.is_empty() { - false // The number of bytes must be the minimum required to represent `bit_len`. - } else { - // Ensure there are no bits higher than `bit_len` that are set to true. - let (mask, _) = u8::max_value().overflowing_shl(8 - (bit_len as u32 % 8)); - (bytes.last().expect("Bytes cannot be empty") & !mask) == 0 - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn bitfield_bytes_length() { - assert!(verify_bitfield_bytes(&[0b0000_0000], 0)); - assert!(verify_bitfield_bytes(&[0b1000_0000], 1)); - assert!(verify_bitfield_bytes(&[0b1100_0000], 2)); - assert!(verify_bitfield_bytes(&[0b1110_0000], 3)); - assert!(verify_bitfield_bytes(&[0b1111_0000], 4)); - assert!(verify_bitfield_bytes(&[0b1111_1000], 5)); - assert!(verify_bitfield_bytes(&[0b1111_1100], 6)); - assert!(verify_bitfield_bytes(&[0b1111_1110], 7)); - assert!(verify_bitfield_bytes(&[0b1111_1111], 8)); - - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b0000_0000], 9)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1000_0000], 9)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1100_0000], 10)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1110_0000], 11)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_0000], 12)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_1000], 13)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_1100], 14)); - assert!(verify_bitfield_bytes(&[0b1111_1111, 0b1111_1110], 15)); - - for i in 0..8 { - assert!(!verify_bitfield_bytes(&[], i)); - assert!(!verify_bitfield_bytes(&[0b1111_1111], i)); - assert!(!verify_bitfield_bytes(&[0b1111_1110, 0b0000_0000], i)); - } - - assert!(!verify_bitfield_bytes(&[0b1000_0000], 0)); - - assert!(!verify_bitfield_bytes(&[0b1000_0000], 0)); - assert!(!verify_bitfield_bytes(&[0b1100_0000], 1)); - assert!(!verify_bitfield_bytes(&[0b1110_0000], 2)); - assert!(!verify_bitfield_bytes(&[0b1111_0000], 3)); - assert!(!verify_bitfield_bytes(&[0b1111_1000], 4)); - assert!(!verify_bitfield_bytes(&[0b1111_1100], 5)); - assert!(!verify_bitfield_bytes(&[0b1111_1110], 6)); - assert!(!verify_bitfield_bytes(&[0b1111_1111], 7)); - - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1000_0000], 8)); - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1100_0000], 9)); - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1110_0000], 10)); - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_0000], 11)); - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_1000], 12)); - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_1100], 13)); - assert!(!verify_bitfield_bytes(&[0b1111_1111, 0b1111_1110], 14)); - - assert!(!verify_bitfield_bytes(&[0b1111_1110], 6)); - } -} -*/ diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 1e49c6edd..9d76b2280 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -1,12 +1,10 @@ #[macro_use] -mod impl_bitfield_fns; +mod bitfield; mod bit_list; mod bit_vector; mod fixed_vector; mod variable_list; -use impl_bitfield_fns::reverse_bit_order; - pub use bit_list::BitList; pub use bit_vector::BitVector; pub use fixed_vector::FixedVector; From 93cd38da55fb9d677b738a9f9640339ed6543b5e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 8 Jul 2019 16:27:08 +1000 Subject: [PATCH 51/72] Progress further on Bitfield struct --- eth2/utils/ssz_types/src/bit_vector.rs | 18 ++++-------------- eth2/utils/ssz_types/src/bitfield.rs | 23 +---------------------- 2 files changed, 5 insertions(+), 36 deletions(-) diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs index ba63281cf..cc4b1bdd5 100644 --- a/eth2/utils/ssz_types/src/bit_vector.rs +++ b/eth2/utils/ssz_types/src/bit_vector.rs @@ -46,25 +46,15 @@ impl_bitfield_fns!(BitVector); impl BitVector { /// Create a new bitfield. pub fn new() -> Self { - Self::with_capacity(Self::capacity()).expect("Capacity must be correct") + Self { + bitfield: Bitfield::with_capacity(Self::capacity()), + _phantom: PhantomData, + } } fn capacity() -> usize { N::to_usize() } - - fn validate_length(len: usize) -> Result<(), Error> { - let fixed_len = N::to_usize(); - - if len > fixed_len { - Err(Error::InvalidLength { - i: len, - len: fixed_len, - }) - } else { - Ok(()) - } - } } #[cfg(test)] diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index a64615bd7..380b56f16 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -4,7 +4,7 @@ /// /// The internal representation of the bitfield is the same as that required by SSZ - the highest /// byte (by `Vec` index) stores the lowest bit-indices and the right-most bit stores the lowest -/// bit-index. E.g., `vec![0b0000_0010, 0b0000_0001]` has bits `1, 9` set. +/// bit-index. E.g., `vec![0b0000_0010, 0b0000_0001]` has bits `0, 9` set. #[derive(Clone, Debug, PartialEq)] pub struct Bitfield { bytes: Vec, @@ -69,21 +69,6 @@ impl Bitfield { (bit_len + 7) / 8 } - /// Verify that the given `bytes` faithfully represent a bitfield of length `bit_len`. - /// - /// The only valid `bytes` for `bit_len == 0` is `&[0]`. - fn verify_bitfield_bytes(bytes: &[u8], bit_len: usize) -> bool { - if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { - true // A bitfield with `bit_len` 0 can only be represented by a single zero byte. - } else if bytes.len() != Bitfield::bytes_for_bit_len(bit_len) || bytes.is_empty() { - false // The number of bytes must be the minimum required to represent `bit_len`. - } else { - // Ensure there are no bits higher than `bit_len` that are set to true. - let (mask, _) = u8::max_value().overflowing_shr(bit_len as u32 % 8); - (bytes.last().expect("Bytes cannot be empty") & !mask) == 0 - } - } - pub fn to_bytes(self) -> Vec { self.bytes } @@ -213,12 +198,6 @@ impl<'a> Iterator for BitIter<'a> { macro_rules! impl_bitfield_fns { ($name: ident) => { impl $name { - pub fn with_capacity(initial_len: usize) -> Result { - Self::validate_length(initial_len)?; - - Self::with_capacity(initial_len) - } - pub fn get(&self, i: usize) -> Result { if i < N::to_usize() { match self.bitfield.get(i) { From 1484773cd11b1da7c211471b9faa98cf5f5a59e9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 8 Jul 2019 18:41:43 +1000 Subject: [PATCH 52/72] Refactor to use Bitfield struct with type variants --- eth2/utils/ssz_types/src/bit_list.rs | 448 ------------------ eth2/utils/ssz_types/src/bit_vector.rs | 321 ------------- eth2/utils/ssz_types/src/bitfield.rs | 607 ++++++++++++++++--------- eth2/utils/ssz_types/src/lib.rs | 9 +- 4 files changed, 402 insertions(+), 983 deletions(-) delete mode 100644 eth2/utils/ssz_types/src/bit_list.rs delete mode 100644 eth2/utils/ssz_types/src/bit_vector.rs diff --git a/eth2/utils/ssz_types/src/bit_list.rs b/eth2/utils/ssz_types/src/bit_list.rs deleted file mode 100644 index 83fa6a0bd..000000000 --- a/eth2/utils/ssz_types/src/bit_list.rs +++ /dev/null @@ -1,448 +0,0 @@ -use super::*; -use crate::{bitfield::Bitfield, impl_bitfield_fns, Error}; -use serde::de::{Deserialize, Deserializer}; -use serde::ser::{Serialize, Serializer}; -use serde_hex::{encode, PrefixedHexVisitor}; -use ssz::{Decode, Encode}; -use std::cmp; -use std::default; -use std::marker::PhantomData; -use typenum::Unsigned; - -/// Emulates a SSZ `Bitlist`. -/// -/// An ordered, heap-allocated, variable-length, collection of `bool` values, limited to `N` -/// values. -/// -/// ## Notes -/// -/// Considering this struct is backed by bytes, errors may be raised when attempting to decode -/// bytes into a `BitList` where `N` is not a multiple of 8. It is advised to always set `N` to -/// a multiple of 8. -/// -/// ## Example -/// ``` -/// use ssz_types::{BitList, typenum}; -/// -/// let mut bitlist: BitList = BitList::new(); -/// -/// assert_eq!(bitlist.len(), 0); -/// -/// assert!(bitlist.get(0).is_err()); // Cannot get at or below the length. -/// -/// for i in 0..8 { -/// assert!(bitlist.set(i, true).is_ok()); -/// } -/// -/// assert!(bitlist.set(8, true).is_err()); // Cannot set out-of-bounds. -/// -/// // Cannot create with an excessive capacity. -/// let result: Result, _> = BitList::with_capacity(9); -/// assert!(result.is_err()); -/// ``` -#[derive(Debug, Clone)] -pub struct BitList { - bitfield: Bitfield, - _phantom: PhantomData, -} - -impl_bitfield_fns!(BitList); - -impl BitList { - /// Create a new, empty BitList. - pub fn new() -> Self { - Self { - bitfield: Bitfield::with_capacity(Self::max_len()), - _phantom: PhantomData, - } - } - - fn validate_length(len: usize) -> Result<(), Error> { - let max_len = Self::max_len(); - - if len > max_len { - Err(Error::InvalidLength { - i: len, - len: max_len, - }) - } else { - Ok(()) - } - } - - /// The maximum possible number of bits. - pub fn max_len() -> usize { - N::to_usize() - } -} - -/* -fn encode_bitfield(bitfield: Bitfield) -> Vec { - // Set the next bit of the bitfield to true. - // - // SSZ spec: - // - // An additional leading 1 bit is added so that the length in bits will also be known. - bitfield.set(bitfield.len(), true); - let bytes = bitfield.to_bytes(); -} -*/ - -impl BitList { - /// Compute the intersection (binary-and) of this bitfield with another - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn intersection(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let mut res: Self = self.to_owned(); - res.intersection_inplace(other); - res - } - - /// Like `intersection` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn intersection_inplace(&mut self, other: &Self) { - self.bitfield.intersection(&other.bitfield); - } - - /// Compute the union (binary-or) of this bitfield with another. Lengths must match. - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn union(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let mut res = self.clone(); - res.union_inplace(other); - res - } - - /// Like `union` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn union_inplace(&mut self, other: &Self) { - self.bitfield.union(&other.bitfield); - } - - /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. - /// - /// Computes `self - other`. - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn difference(&self, other: &Self) -> Self { - assert_eq!(self.len(), other.len()); - let mut res = self.clone(); - res.difference_inplace(other); - res - } - - /// Like `difference` but in-place (updates `self`). - /// - /// ## Panics - /// - /// If `self` and `other` have different lengths. - pub fn difference_inplace(&mut self, other: &Self) { - self.bitfield.difference(&other.bitfield); - } -} - -/* -#[cfg(test)] -mod test { - use super::*; - use serde_yaml; - use ssz::ssz_encode; - // use tree_hash::TreeHash; - - pub type BitList1024 = BitList; - - /* - #[test] - pub fn cached_tree_hash() { - let original = BitList1024::from_bytes(&vec![18; 12][..]); - - let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); - - assert_eq!( - cache.tree_hash_root().unwrap().to_vec(), - original.tree_hash_root() - ); - - let modified = BitList1024::from_bytes(&vec![2; 1][..]); - - cache.update(&modified).unwrap(); - - assert_eq!( - cache.tree_hash_root().unwrap().to_vec(), - modified.tree_hash_root() - ); - } - */ - - #[test] - fn new_bitfield() { - let mut field = BitList1024::new(); - let original_len = field.len(); - - for i in 0..100 { - if i < original_len { - assert!(!field.get(i).unwrap()); - } else { - assert!(field.get(i).is_err()); - } - field.set(i, true).unwrap(); - } - } - - #[test] - fn empty_bitfield() { - let mut field = BitList1024::from_elem(0, false).unwrap(); - let original_len = field.len(); - - assert_eq!(original_len, 0); - - for i in 0..100 { - if i < original_len { - assert!(!field.get(i).unwrap()); - } else { - assert!(field.get(i).is_err()); - } - field.set(i, true).unwrap(); - } - - assert_eq!(field.len(), 100); - assert_eq!(field.num_set_bits(), 100); - } - - const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; - - #[test] - fn get_from_bitfield() { - let field = BitList1024::from_bytes(INPUT).unwrap(); - field.get(0).unwrap(); - field.get(6).unwrap(); - field.get(14).unwrap(); - } - - #[test] - fn set_for_bitfield() { - let mut field = BitList1024::from_bytes(INPUT).unwrap(); - field.set(10, true).unwrap(); - field.get(10).unwrap(); - field.set(6, false).unwrap(); - field.get(6).unwrap(); - } - - #[test] - fn len() { - let field = BitList1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.len(), 16); - - let field = BitList1024::new(); - assert_eq!(field.len(), 0); - } - - #[test] - fn num_set_bits() { - let field = BitList1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.num_set_bits(), 2); - - let field = BitList1024::new(); - assert_eq!(field.num_set_bits(), 0); - } - - #[test] - fn to_bytes() { - let field = BitList1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.to_bytes(), INPUT); - - let field = BitList1024::new(); - assert_eq!(field.to_bytes(), vec![0]); - } - - #[test] - fn out_of_bounds() { - let mut field = BitList1024::from_bytes(INPUT).unwrap(); - - let out_of_bounds_index = field.len(); - assert!(field.set(out_of_bounds_index, true).is_ok()); - assert!(field.len() == out_of_bounds_index + 1); - assert!(field.get(out_of_bounds_index).unwrap()); - - for i in 0..100 { - if i <= out_of_bounds_index { - assert!(field.set(i, true).is_ok()); - } else { - assert!(field.set(i, true).is_ok()); - } - } - } - - #[test] - fn grows_with_false() { - let input_all_set: &[u8] = &[0b1111_1111, 0b1111_1111]; - let mut field = BitList1024::from_bytes(input_all_set).unwrap(); - - // Define `a` and `b`, where both are out of bounds and `b` is greater than `a`. - let a = field.len(); - let b = a + 1; - - // Ensure `a` is out-of-bounds for test integrity. - assert!(field.get(a).is_err()); - - // Set `b` to `true`.. - assert!(field.set(b, true).is_ok()); - - // Ensure that `a` wasn't also set to `true` during the grow. - assert_eq!(field.get(a), Ok(false)); - assert_eq!(field.get(b), Ok(true)); - } - - #[test] - fn num_bytes() { - let field = BitList1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.num_bytes(), 2); - - let field = BitList1024::from_elem(2, true).unwrap(); - assert_eq!(field.num_bytes(), 1); - - let field = BitList1024::from_elem(13, true).unwrap(); - assert_eq!(field.num_bytes(), 2); - } - - #[test] - fn ssz_encoding() { - let field = create_bitfield(); - assert_eq!(field.as_ssz_bytes(), vec![0b0000_0011, 0b1000_0111]); - - let field = BitList1024::from_elem(18, true).unwrap(); - assert_eq!( - field.as_ssz_bytes(), - vec![0b0000_0011, 0b1111_1111, 0b1111_1111] - ); - - let mut b = BitList1024::new(); - b.set(1, true).unwrap(); - assert_eq!(ssz_encode(&b), vec![0b0000_0010]); - } - - fn create_bitfield() -> BitList1024 { - let count = 2 * 8; - let mut field = BitList1024::with_capacity(count).unwrap(); - - let indices = &[0, 1, 2, 7, 8, 9]; - for &i in indices { - field.set(i, true).unwrap(); - } - field - } - - #[test] - fn ssz_decode() { - let encoded = vec![0b0000_0011, 0b1000_0111]; - let field = BitList1024::from_ssz_bytes(&encoded).unwrap(); - let expected = create_bitfield(); - assert_eq!(field, expected); - - let encoded = vec![255, 255, 3]; - let field = BitList1024::from_ssz_bytes(&encoded).unwrap(); - let expected = BitList1024::from_bytes(&[255, 255, 3]).unwrap(); - assert_eq!(field, expected); - } - - #[test] - fn serialize_deserialize() { - use serde_yaml::Value; - - let data: &[(_, &[_])] = &[ - ("0x01", &[0b00000001]), - ("0xf301", &[0b11110011, 0b00000001]), - ]; - for (hex_data, bytes) in data { - let bitfield = BitList1024::from_bytes(bytes).unwrap(); - assert_eq!( - serde_yaml::from_str::(hex_data).unwrap(), - bitfield - ); - assert_eq!( - serde_yaml::to_value(&bitfield).unwrap(), - Value::String(hex_data.to_string()) - ); - } - } - - #[test] - fn ssz_round_trip() { - let original = BitList1024::from_bytes(&vec![18; 12][..]).unwrap(); - let ssz = ssz_encode(&original); - let decoded = BitList1024::from_ssz_bytes(&ssz).unwrap(); - assert_eq!(original, decoded); - } - - #[test] - fn bitor() { - let a = BitList1024::from_bytes(&vec![2, 8, 1][..]).unwrap(); - let b = BitList1024::from_bytes(&vec![4, 8, 16][..]).unwrap(); - let c = BitList1024::from_bytes(&vec![6, 8, 17][..]).unwrap(); - assert_eq!(c, a | b); - } - - #[test] - fn is_zero() { - let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]]; - for bytes in yes_data { - assert!(BitList1024::from_bytes(bytes).unwrap().is_zero()); - } - let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]]; - for bytes in no_data { - assert!(!BitList1024::from_bytes(bytes).unwrap().is_zero()); - } - } - - #[test] - fn intersection() { - let a = BitList1024::from_bytes(&[0b1100, 0b0001]).unwrap(); - let b = BitList1024::from_bytes(&[0b1011, 0b1001]).unwrap(); - let c = BitList1024::from_bytes(&[0b1000, 0b0001]).unwrap(); - assert_eq!(a.intersection(&b), c); - assert_eq!(b.intersection(&a), c); - assert_eq!(a.intersection(&c), c); - assert_eq!(b.intersection(&c), c); - assert_eq!(a.intersection(&a), a); - assert_eq!(b.intersection(&b), b); - assert_eq!(c.intersection(&c), c); - } - - #[test] - fn union() { - let a = BitList1024::from_bytes(&[0b1100, 0b0001]).unwrap(); - let b = BitList1024::from_bytes(&[0b1011, 0b1001]).unwrap(); - let c = BitList1024::from_bytes(&[0b1111, 0b1001]).unwrap(); - assert_eq!(a.union(&b), c); - assert_eq!(b.union(&a), c); - assert_eq!(a.union(&a), a); - assert_eq!(b.union(&b), b); - assert_eq!(c.union(&c), c); - } - - #[test] - fn difference() { - let a = BitList1024::from_bytes(&[0b1100, 0b0001]).unwrap(); - let b = BitList1024::from_bytes(&[0b1011, 0b1001]).unwrap(); - let a_b = BitList1024::from_bytes(&[0b0100, 0b0000]).unwrap(); - let b_a = BitList1024::from_bytes(&[0b0011, 0b1000]).unwrap(); - assert_eq!(a.difference(&b), a_b); - assert_eq!(b.difference(&a), b_a); - assert!(a.difference(&a).is_zero()); - } -} -*/ diff --git a/eth2/utils/ssz_types/src/bit_vector.rs b/eth2/utils/ssz_types/src/bit_vector.rs deleted file mode 100644 index cc4b1bdd5..000000000 --- a/eth2/utils/ssz_types/src/bit_vector.rs +++ /dev/null @@ -1,321 +0,0 @@ -use super::*; -use crate::{bitfield::Bitfield, impl_bitfield_fns, Error}; -use serde::de::{Deserialize, Deserializer}; -use serde::ser::{Serialize, Serializer}; -use serde_hex::{encode, PrefixedHexVisitor}; -use ssz::{Decode, Encode}; -use std::cmp; -use std::marker::PhantomData; -use typenum::Unsigned; - -/// Emulates a SSZ `Bitvector`. -/// -/// An ordered, heap-allocated, fixed-length, collection of `bool` values, with `N` values. -/// -/// ## Notes -/// -/// Considering this struct is backed by bytes, errors may be raised when attempting to decode -/// bytes into a `BitVector` where `N` is not a multiple of 8. It is advised to always set `N` to -/// a multiple of 8. -/// -/// ## Example -/// ``` -/// use ssz_types::{BitVector, typenum}; -/// -/// let mut bitvec: BitVector = BitVector::new(); -/// -/// assert_eq!(bitvec.len(), 8); -/// -/// for i in 0..8 { -/// assert_eq!(bitvec.get(i).unwrap(), false); // Defaults to false. -/// } -/// -/// assert!(bitvec.get(8).is_err()); // Cannot get out-of-bounds. -/// -/// assert!(bitvec.set(7, true).is_ok()); -/// assert!(bitvec.set(8, true).is_err()); // Cannot set out-of-bounds. -/// ``` -#[derive(Debug, Clone)] -pub struct BitVector { - bitfield: Bitfield, - _phantom: PhantomData, -} - -impl_bitfield_fns!(BitVector); - -impl BitVector { - /// Create a new bitfield. - pub fn new() -> Self { - Self { - bitfield: Bitfield::with_capacity(Self::capacity()), - _phantom: PhantomData, - } - } - - fn capacity() -> usize { - N::to_usize() - } -} - -#[cfg(test)] -mod test { - use super::*; - use serde_yaml; - use ssz::ssz_encode; - // use tree_hash::TreeHash; - - pub type BitVector4 = BitVector; - pub type BitVector1024 = BitVector; - - /* - #[test] - pub fn cached_tree_hash() { - let original = BitVector1024::from_bytes(&vec![18; 12][..]); - - let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap(); - - assert_eq!( - cache.tree_hash_root().unwrap().to_vec(), - original.tree_hash_root() - ); - - let modified = BitVector1024::from_bytes(&vec![2; 1][..]); - - cache.update(&modified).unwrap(); - - assert_eq!( - cache.tree_hash_root().unwrap().to_vec(), - modified.tree_hash_root() - ); - } - */ - - /* - #[test] - fn new_bitfield() { - let mut field = BitVector1024::new(); - let original_len = field.len(); - - assert_eq!(original_len, 1024); - - for i in 0..1028 { - if i < original_len { - assert!(!field.get(i).unwrap()); - assert!(field.set(i, true).is_ok()); - } else { - assert!(field.get(i).is_err()); - assert!(field.set(i, true).is_err()); - } - } - } - - #[test] - fn from_bytes_bitvec4() { - let bytes = &[3]; - - let bitvec = BitVector4::from_bytes(bytes).unwrap(); - - assert_eq!(bitvec.get(0), Ok(true)); - assert_eq!(bitvec.get(1), Ok(true)); - assert_eq!(bitvec.get(2), Ok(false)); - assert_eq!(bitvec.get(3), Ok(false)); - - assert!(bitvec.get(4).is_err()); - } - - #[test] - fn from_bytes_bytes_too_long() { - let bytes = &[0, 0]; - - assert_eq!( - BitVector4::from_bytes(bytes), - Err(Error::InvalidLength { i: 16, len: 4 }) - ); - } - - const INPUT: &[u8] = &[0b0100_0000, 0b0100_0000]; - - #[test] - fn get_from_bitfield() { - let field = BitVector1024::from_bytes(INPUT).unwrap(); - field.get(0).unwrap(); - field.get(6).unwrap(); - field.get(14).unwrap(); - } - - #[test] - fn set_for_bitfield() { - let mut field = BitVector1024::from_bytes(INPUT).unwrap(); - field.set(10, true).unwrap(); - field.get(10).unwrap(); - field.set(6, false).unwrap(); - field.get(6).unwrap(); - } - - #[test] - fn len() { - let field = BitVector1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.len(), 16); - - let field = BitVector1024::new(); - assert_eq!(field.len(), 0); - } - - #[test] - fn num_set_bits() { - let field = BitVector1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.num_set_bits(), 2); - - let field = BitVector1024::new(); - assert_eq!(field.num_set_bits(), 0); - } - - #[test] - fn to_bytes() { - let field = BitVector1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.to_bytes(), INPUT); - - let field = BitVector1024::new(); - assert_eq!(field.to_bytes(), vec![0]); - } - - #[test] - fn out_of_bounds() { - let mut field = BitVector1024::from_bytes(INPUT).unwrap(); - - let out_of_bounds_index = field.len(); - assert!(field.set(out_of_bounds_index, true).is_ok()); - assert!(field.len() == out_of_bounds_index + 1); - assert!(field.get(out_of_bounds_index).unwrap()); - - for i in 0..100 { - if i <= out_of_bounds_index { - assert!(field.set(i, true).is_ok()); - } else { - assert!(field.set(i, true).is_ok()); - } - } - } - - #[test] - fn grows_with_false() { - let input_all_set: &[u8] = &[0b1111_1111, 0b1111_1111]; - let mut field = BitVector1024::from_bytes(input_all_set).unwrap(); - - // Define `a` and `b`, where both are out of bounds and `b` is greater than `a`. - let a = field.len(); - let b = a + 1; - - // Ensure `a` is out-of-bounds for test integrity. - assert!(field.get(a).is_err()); - - // Set `b` to `true`.. - assert!(field.set(b, true).is_ok()); - - // Ensure that `a` wasn't also set to `true` during the grow. - assert_eq!(field.get(a), Ok(false)); - assert_eq!(field.get(b), Ok(true)); - } - - #[test] - fn num_bytes() { - let field = BitVector1024::from_bytes(INPUT).unwrap(); - assert_eq!(field.num_bytes(), 2); - - let field = BitVector1024::from_elem(2, true).unwrap(); - assert_eq!(field.num_bytes(), 1); - - let field = BitVector1024::from_elem(13, true).unwrap(); - assert_eq!(field.num_bytes(), 2); - } - - #[test] - fn ssz_encoding() { - let field = create_bitfield(); - assert_eq!(field.as_ssz_bytes(), vec![0b0000_0011, 0b1000_0111]); - - let field = BitVector1024::from_elem(18, true).unwrap(); - assert_eq!( - field.as_ssz_bytes(), - vec![0b0000_0011, 0b1111_1111, 0b1111_1111] - ); - - let mut b = BitVector1024::new(); - b.set(1, true).unwrap(); - assert_eq!(ssz_encode(&b), vec![0b0000_0010]); - } - - fn create_bitfield() -> BitVector1024 { - let count = 2 * 8; - let mut field = BitVector1024::with_capacity(count).unwrap(); - - let indices = &[0, 1, 2, 7, 8, 9]; - for &i in indices { - field.set(i, true).unwrap(); - } - field - } - - #[test] - fn ssz_decode() { - let encoded = vec![0b0000_0011, 0b1000_0111]; - let field = BitVector1024::from_ssz_bytes(&encoded).unwrap(); - let expected = create_bitfield(); - assert_eq!(field, expected); - - let encoded = vec![255, 255, 3]; - let field = BitVector1024::from_ssz_bytes(&encoded).unwrap(); - let expected = BitVector1024::from_bytes(&[255, 255, 3]).unwrap(); - assert_eq!(field, expected); - } - - #[test] - fn serialize_deserialize() { - use serde_yaml::Value; - - let data: &[(_, &[_])] = &[ - ("0x01", &[0b00000001]), - ("0xf301", &[0b11110011, 0b00000001]), - ]; - for (hex_data, bytes) in data { - let bitfield = BitVector1024::from_bytes(bytes).unwrap(); - assert_eq!( - serde_yaml::from_str::(hex_data).unwrap(), - bitfield - ); - assert_eq!( - serde_yaml::to_value(&bitfield).unwrap(), - Value::String(hex_data.to_string()) - ); - } - } - - #[test] - fn ssz_round_trip() { - let original = BitVector1024::from_bytes(&vec![18; 12][..]).unwrap(); - let ssz = ssz_encode(&original); - let decoded = BitVector1024::from_ssz_bytes(&ssz).unwrap(); - assert_eq!(original, decoded); - } - - #[test] - fn bitor() { - let a = BitVector1024::from_bytes(&vec![2, 8, 1][..]).unwrap(); - let b = BitVector1024::from_bytes(&vec![4, 8, 16][..]).unwrap(); - let c = BitVector1024::from_bytes(&vec![6, 8, 17][..]).unwrap(); - assert_eq!(c, a | b); - } - - #[test] - fn is_zero() { - let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]]; - for bytes in yes_data { - assert!(BitVector1024::from_bytes(bytes).unwrap().is_zero()); - } - let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]]; - for bytes in no_data { - assert!(!BitVector1024::from_bytes(bytes).unwrap().is_zero()); - } - } - */ -} diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 380b56f16..683fc67e1 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -1,24 +1,127 @@ -/// A heap-allocated, ordered, fixed-length, collection of `bool` values. +use core::marker::PhantomData; +use serde::de::{Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; +use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; +use ssz::{Decode, Encode}; +use typenum::Unsigned; + +pub trait BitfieldBehaviour: Clone {} + +/// A marker struct used to define SSZ `BitList` functionality on a `Bitfield`. +#[derive(Clone, PartialEq, Debug)] +pub struct BitList { + _phantom: PhantomData, +} + +/// A marker struct used to define SSZ `BitVector` functionality on a `Bitfield`. +#[derive(Clone, PartialEq, Debug)] +pub struct BitVector { + _phantom: PhantomData, +} + +impl BitfieldBehaviour for BitList {} +impl BitfieldBehaviour for BitVector {} + +/// A heap-allocated, ordered, fixed-length, collection of `bool` values. Must be used with the `BitList` or +/// `BitVector` marker structs. /// -/// The length of the Bitfield is set at instantiation (i.e., runtime, not compile time). +/// The length of the Bitfield is set at instantiation (i.e., runtime, not compile time). However, +/// use with a `BitList` sets a type-level (i.e., compile-time) maximum length and `BitVector` +/// provides a type-level fixed length. +/// +/// ## Note /// /// The internal representation of the bitfield is the same as that required by SSZ - the highest /// byte (by `Vec` index) stores the lowest bit-indices and the right-most bit stores the lowest /// bit-index. E.g., `vec![0b0000_0010, 0b0000_0001]` has bits `0, 9` set. #[derive(Clone, Debug, PartialEq)] -pub struct Bitfield { +pub struct Bitfield { bytes: Vec, len: usize, + _phantom: PhantomData, } -impl Bitfield { - pub fn with_capacity(num_bits: usize) -> Self { - Self { - bytes: vec![0; Self::bytes_for_bit_len(num_bits)], - len: num_bits, +impl Bitfield> { + pub fn with_capacity(num_bits: usize) -> Option { + if num_bits <= N::to_usize() { + Some(Self { + bytes: vec![0; bytes_for_bit_len(num_bits)], + len: num_bits, + _phantom: PhantomData, + }) + } else { + None } } + pub fn capacity() -> usize { + N::to_usize() + } + + pub fn to_bytes(&self) -> Vec { + let len = self.len(); + let mut bytes = self.as_slice().to_vec(); + + if bytes_for_bit_len(len + 1) == bytes.len() + 1 { + bytes.insert(0, 0); + } + + let mut bitfield: Bitfield> = Bitfield::from_raw_bytes(bytes, len + 1) + .expect("Bitfield capacity has been confirmed earlier."); + bitfield + .set(len, true) + .expect("Bitfield capacity has been confirmed earlier."); + + bitfield.bytes + } + + pub fn from_bytes(bytes: Vec) -> Option { + let mut initial_bitfield: Bitfield> = { + let num_bits = bytes.len() * 8; + Bitfield::from_raw_bytes(bytes, num_bits) + .expect("Must have adequate bytes for bit count.") + }; + + let len = initial_bitfield.highest_set_bit()?; + initial_bitfield + .set(len, false) + .expect("Bit has been confirmed to exist"); + + let mut bytes = initial_bitfield.to_raw_bytes(); + + if bytes_for_bit_len(len) < bytes.len() { + bytes.remove(0); + } + + Self::from_raw_bytes(bytes, len) + } +} + +impl Bitfield> { + pub fn new() -> Self { + let num_bits = N::to_usize(); + + Self { + bytes: vec![0; num_bits], + len: num_bits, + _phantom: PhantomData, + } + } + + pub fn capacity() -> usize { + N::to_usize() + } + + pub fn to_bytes(self) -> Vec { + self.to_raw_bytes() + } + + pub fn from_bytes(bytes: Vec) -> Option { + Self::from_raw_bytes(bytes, Self::capacity()) + } +} + +impl Bitfield { pub fn set(&mut self, i: usize, value: bool) -> Option<()> { if i < self.len { let byte = { @@ -65,11 +168,7 @@ impl Bitfield { self.len == 0 } - fn bytes_for_bit_len(bit_len: usize) -> usize { - (bit_len + 7) / 8 - } - - pub fn to_bytes(self) -> Vec { + pub fn to_raw_bytes(self) -> Vec { self.bytes } @@ -77,11 +176,15 @@ impl Bitfield { &self.bytes } - pub fn from_bytes(bytes: Vec, bit_len: usize) -> Option { + pub fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { // A bitfield with `bit_len` 0 can only be represented by a single zero byte. - Some(Self { bytes, len: 0 }) - } else if bytes.len() != Bitfield::bytes_for_bit_len(bit_len) || bytes.is_empty() { + Some(Self { + bytes, + len: 0, + _phantom: PhantomData, + }) + } else if bytes.len() != bytes_for_bit_len(bit_len) || bytes.is_empty() { // The number of bytes must be the minimum required to represent `bit_len`. None } else { @@ -92,6 +195,7 @@ impl Bitfield { Some(Self { bytes, len: bit_len, + _phantom: PhantomData, }) } else { None @@ -99,7 +203,14 @@ impl Bitfield { } } - pub fn iter(&self) -> BitIter<'_> { + pub fn highest_set_bit(&self) -> Option { + let byte_i = self.bytes.iter().position(|byte| *byte > 0)?; + let bit_i = 7 - self.bytes[byte_i].leading_zeros() as usize; + + Some((self.bytes.len().saturating_sub(1) - byte_i) * 8 + bit_i) + } + + pub fn iter(&self) -> BitIter<'_, T> { BitIter { bitfield: self, i: 0, @@ -178,12 +289,16 @@ impl Bitfield { } } -pub struct BitIter<'a> { - bitfield: &'a Bitfield, +fn bytes_for_bit_len(bit_len: usize) -> usize { + (bit_len + 7) / 8 +} + +pub struct BitIter<'a, T> { + bitfield: &'a Bitfield, i: usize, } -impl<'a> Iterator for BitIter<'a> { +impl<'a, T: BitfieldBehaviour> Iterator for BitIter<'a, T> { type Item = bool; fn next(&mut self) -> Option { @@ -193,188 +308,200 @@ impl<'a> Iterator for BitIter<'a> { } } -/// Provides a common `impl` for structs that wrap a `$name`. -#[macro_export] -macro_rules! impl_bitfield_fns { - ($name: ident) => { - impl $name { - pub fn get(&self, i: usize) -> Result { - if i < N::to_usize() { - match self.bitfield.get(i) { - Some(value) => Ok(value), - None => Err(Error::OutOfBounds { - i, - len: self.bitfield.len(), - }), - } - } else { - Err(Error::InvalidLength { - i, - len: N::to_usize(), - }) - } - } +impl Encode for Bitfield> { + fn is_ssz_fixed_len() -> bool { + false + } - pub fn set(&mut self, i: usize, value: bool) -> Option<()> { - self.bitfield.set(i, value) - } - - /// Returns the number of bits in this bitfield. - pub fn len(&self) -> usize { - self.bitfield.len() - } - - /// Returns true if `self.len() == 0` - pub fn is_empty(&self) -> bool { - self.bitfield.is_empty() - } - - /// Returns true if all bits are set to 0. - pub fn is_zero(&self) -> bool { - self.bitfield.is_zero() - } - - /// Returns the number of bytes presently used to store the bitfield. - pub fn num_bytes(&self) -> usize { - self.bitfield.as_slice().len() - } - - /// Returns the number of `1` bits in the bitfield - pub fn num_set_bits(&self) -> usize { - self.bitfield.iter().filter(|&bit| bit).count() - } - } - - /* - impl Encode for $name { - fn is_ssz_fixed_len() -> bool { - false - } - - fn ssz_append(&self, buf: &mut Vec) { - buf.append(&mut self.bitfield.to_bytes()) - } - } - - impl Decode for $name { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - let bitfield = - Bitfield::from_bytes(bytes.to_vec(), bytes.len() * 8).expect("Cannot fail"); - Ok(Self { - bitfield, - _phantom: PhantomData, - }) - /* - $name::from_bytes(bytes) - .map_err(|e| ssz::DecodeError::BytesInvalid(format!("Bitfield {:?}", e))) - */ - } - } - - impl Serialize for $name { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&encode(self.bitfield.to_bytes())) - } - } - - impl<'de, N: Unsigned> Deserialize<'de> for $name { - /// Serde serialization is compliant with the Ethereum YAML test format. - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - // We reverse the bit-order so that the BitVec library can read its 0th - // bit from the end of the hex string, e.g. - // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] - let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; - $name::from_bytes(&bytes) - .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) - } - } - - impl tree_hash::TreeHash for $name { - fn tree_hash_type() -> tree_hash::TreeHashType { - tree_hash::TreeHashType::List - } - - fn tree_hash_packed_encoding(&self) -> Vec { - unreachable!("List should never be packed.") - } - - fn tree_hash_packing_factor() -> usize { - unreachable!("List should never be packed.") - } - - fn tree_hash_root(&self) -> Vec { - self.to_bytes().tree_hash_root() - } - } - */ - }; + fn ssz_append(&self, buf: &mut Vec) { + buf.append(&mut self.clone().to_bytes()) + } } +impl Decode for Bitfield> { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + Self::from_bytes(bytes.to_vec()) + .ok_or_else(|| ssz::DecodeError::BytesInvalid("BitList failed to decode".to_string())) + } +} + +impl Encode for Bitfield> { + fn is_ssz_fixed_len() -> bool { + true + } + + fn ssz_fixed_len() -> usize { + bytes_for_bit_len(N::to_usize()) + } + + fn ssz_append(&self, buf: &mut Vec) { + buf.append(&mut self.clone().to_bytes()) + } +} + +impl Decode for Bitfield> { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + Self::from_bytes(bytes.to_vec()) + .ok_or_else(|| ssz::DecodeError::BytesInvalid("BitVector failed to decode".to_string())) + } +} + +impl Serialize for Bitfield> { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&hex_encode(self.as_ssz_bytes())) + } +} + +impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + Self::from_ssz_bytes(&bytes) + .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) + } +} + +impl Serialize for Bitfield> { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&hex_encode(self.as_ssz_bytes())) + } +} + +impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { + /// Serde serialization is compliant with the Ethereum YAML test format. + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // We reverse the bit-order so that the BitVec library can read its 0th + // bit from the end of the hex string, e.g. + // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] + let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; + Self::from_ssz_bytes(&bytes) + .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) + } +} + +impl tree_hash::TreeHash for Bitfield> { + fn tree_hash_type() -> tree_hash::TreeHashType { + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + unreachable!("List should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + unreachable!("List should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + // TODO: pad this out to max length. + self.as_ssz_bytes().tree_hash_root() + } +} + +impl tree_hash::TreeHash for Bitfield> { + fn tree_hash_type() -> tree_hash::TreeHashType { + // TODO: move this to be a vector. + tree_hash::TreeHashType::List + } + + fn tree_hash_packed_encoding(&self) -> Vec { + // TODO: move this to be a vector. + unreachable!("Vector should never be packed.") + } + + fn tree_hash_packing_factor() -> usize { + // TODO: move this to be a vector. + unreachable!("Vector should never be packed.") + } + + fn tree_hash_root(&self) -> Vec { + self.as_ssz_bytes().tree_hash_root() + } +} + +// TODO: test ssz decode a zero-length bitlist. + #[cfg(test)] mod test { use super::*; - #[test] - fn from_bytes() { - assert!(Bitfield::from_bytes(vec![0b0000_0000], 0).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_0001], 1).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_0011], 2).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_0111], 3).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_1111], 4).is_some()); - assert!(Bitfield::from_bytes(vec![0b0001_1111], 5).is_some()); - assert!(Bitfield::from_bytes(vec![0b0011_1111], 6).is_some()); - assert!(Bitfield::from_bytes(vec![0b0111_1111], 7).is_some()); - assert!(Bitfield::from_bytes(vec![0b1111_1111], 8).is_some()); + type Bitfield = super::Bitfield>; - assert!(Bitfield::from_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_some()); - assert!(Bitfield::from_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_some()); - assert!(Bitfield::from_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_some()); - assert!(Bitfield::from_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_some()); - assert!(Bitfield::from_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_some()); - assert!(Bitfield::from_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_some()); + #[test] + fn from_raw_bytes() { + assert!(Bitfield::from_raw_bytes(vec![0b0000_0000], 0).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0001], 1).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0011], 2).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0111], 3).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_1111], 4).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0001_1111], 5).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0011_1111], 6).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0111_1111], 7).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b1111_1111], 8).is_some()); + + assert!(Bitfield::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_some()); + assert!(Bitfield::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_some()); for i in 0..8 { - assert!(Bitfield::from_bytes(vec![], i).is_none()); - assert!(Bitfield::from_bytes(vec![0b1111_1111], i).is_none()); - assert!(Bitfield::from_bytes(vec![0b1111_1110, 0b0000_0000], i).is_none()); + assert!(Bitfield::from_raw_bytes(vec![], i).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b1111_1111], i).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b1111_1110, 0b0000_0000], i).is_none()); } - assert!(Bitfield::from_bytes(vec![0b0000_0001], 0).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0001], 0).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_0001], 0).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_0011], 1).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_0111], 2).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_1111], 3).is_none()); - assert!(Bitfield::from_bytes(vec![0b0001_1111], 4).is_none()); - assert!(Bitfield::from_bytes(vec![0b0011_1111], 5).is_none()); - assert!(Bitfield::from_bytes(vec![0b0111_1111], 6).is_none()); - assert!(Bitfield::from_bytes(vec![0b1111_1111], 7).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0001], 0).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0011], 1).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0111], 2).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_1111], 3).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0001_1111], 4).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0011_1111], 5).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0111_1111], 6).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b1111_1111], 7).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_none()); - assert!(Bitfield::from_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_none()); - assert!(Bitfield::from_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_none()); - assert!(Bitfield::from_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_none()); - assert!(Bitfield::from_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_none()); - assert!(Bitfield::from_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_none()); + assert!(Bitfield::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_none()); } fn test_set_unset(num_bits: usize) { - let mut bitfield = Bitfield::with_capacity(num_bits); + let mut bitfield = Bitfield::with_capacity(num_bits).unwrap(); for i in 0..num_bits + 1 { dbg!(i); @@ -399,12 +526,12 @@ mod test { dbg!(num_bits); for i in 0..num_bits { dbg!(i); - let mut bitfield = Bitfield::with_capacity(num_bits); + let mut bitfield = Bitfield::with_capacity(num_bits).unwrap(); bitfield.set(i, true).unwrap(); - let bytes = bitfield.clone().to_bytes(); + let bytes = bitfield.clone().to_raw_bytes(); dbg!(&bytes); - assert_eq!(bitfield, Bitfield::from_bytes(bytes, num_bits).unwrap()); + assert_eq!(bitfield, Bitfield::from_raw_bytes(bytes, num_bits).unwrap()); } } @@ -423,33 +550,93 @@ mod test { } #[test] - fn to_bytes() { - let mut bitfield = Bitfield::with_capacity(9); + fn to_raw_bytes() { + let mut bitfield = Bitfield::with_capacity(9).unwrap(); bitfield.set(0, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_0001]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0000_0001] + ); bitfield.set(1, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_0011]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0000_0011] + ); bitfield.set(2, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_0111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0000_0111] + ); bitfield.set(3, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0000_1111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0000_1111] + ); bitfield.set(4, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0001_1111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0001_1111] + ); bitfield.set(5, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0011_1111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0011_1111] + ); bitfield.set(6, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b0111_1111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b0111_1111] + ); bitfield.set(7, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0000, 0b1111_1111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0000, 0b1111_1111] + ); bitfield.set(8, true); - assert_eq!(bitfield.clone().to_bytes(), vec![0b0000_0001, 0b1111_1111]); + assert_eq!( + bitfield.clone().to_raw_bytes(), + vec![0b0000_0001, 0b1111_1111] + ); + } + + #[test] + fn highest_set_bit() { + assert_eq!(Bitfield::with_capacity(16).unwrap().highest_set_bit(), None); + + assert_eq!( + Bitfield::from_raw_bytes(vec![0b0000_000, 0b0000_0001], 16) + .unwrap() + .highest_set_bit(), + Some(0) + ); + + assert_eq!( + Bitfield::from_raw_bytes(vec![0b0000_000, 0b0000_0010], 16) + .unwrap() + .highest_set_bit(), + Some(1) + ); + + assert_eq!( + Bitfield::from_raw_bytes(vec![0b0000_1000], 8) + .unwrap() + .highest_set_bit(), + Some(3) + ); + + assert_eq!( + Bitfield::from_raw_bytes(vec![0b1000_0000, 0b0000_0000], 16) + .unwrap() + .highest_set_bit(), + Some(15) + ); } #[test] fn intersection() { - let a = Bitfield::from_bytes(vec![0b1100, 0b0001], 16).unwrap(); - let b = Bitfield::from_bytes(vec![0b1011, 0b1001], 16).unwrap(); - let c = Bitfield::from_bytes(vec![0b1000, 0b0001], 16).unwrap(); + let a = Bitfield::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = Bitfield::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let c = Bitfield::from_raw_bytes(vec![0b1000, 0b0001], 16).unwrap(); assert_eq!(a.intersection(&b).unwrap(), c); assert_eq!(b.intersection(&a).unwrap(), c); @@ -462,9 +649,9 @@ mod test { #[test] fn union() { - let a = Bitfield::from_bytes(vec![0b1100, 0b0001], 16).unwrap(); - let b = Bitfield::from_bytes(vec![0b1011, 0b1001], 16).unwrap(); - let c = Bitfield::from_bytes(vec![0b1111, 0b1001], 16).unwrap(); + let a = Bitfield::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = Bitfield::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let c = Bitfield::from_raw_bytes(vec![0b1111, 0b1001], 16).unwrap(); assert_eq!(a.union(&b).unwrap(), c); assert_eq!(b.union(&a).unwrap(), c); @@ -475,10 +662,10 @@ mod test { #[test] fn difference() { - let a = Bitfield::from_bytes(vec![0b1100, 0b0001], 16).unwrap(); - let b = Bitfield::from_bytes(vec![0b1011, 0b1001], 16).unwrap(); - let a_b = Bitfield::from_bytes(vec![0b0100, 0b0000], 16).unwrap(); - let b_a = Bitfield::from_bytes(vec![0b0011, 0b1000], 16).unwrap(); + let a = Bitfield::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = Bitfield::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let a_b = Bitfield::from_raw_bytes(vec![0b0100, 0b0000], 16).unwrap(); + let b_a = Bitfield::from_raw_bytes(vec![0b0011, 0b1000], 16).unwrap(); assert_eq!(a.difference(&b).unwrap(), a_b); assert_eq!(b.difference(&a).unwrap(), b_a); @@ -487,7 +674,7 @@ mod test { #[test] fn iter() { - let mut bitfield = Bitfield::with_capacity(9); + let mut bitfield = Bitfield::with_capacity(9).unwrap(); bitfield.set(2, true); bitfield.set(8, true); diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 9d76b2280..03e1bf20f 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -1,12 +1,13 @@ #[macro_use] mod bitfield; -mod bit_list; -mod bit_vector; +// mod bit_list; +// mod bit_vector; mod fixed_vector; mod variable_list; -pub use bit_list::BitList; -pub use bit_vector::BitVector; +// pub use bit_list::BitList; +// pub use bit_vector::BitVector; +pub use bitfield::{BitList, BitVector, Bitfield}; pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; From 5a7c44ed372e2754a8714f41c9d02303924a4b7b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 8 Jul 2019 18:53:25 +1000 Subject: [PATCH 53/72] Add failing doc tests --- eth2/utils/ssz_types/src/bitfield.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 683fc67e1..e64f790dd 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -29,6 +29,22 @@ impl BitfieldBehaviour for BitVector {} /// use with a `BitList` sets a type-level (i.e., compile-time) maximum length and `BitVector` /// provides a type-level fixed length. /// +/// ## Example +/// ``` +/// use ssz_types::{Bitfield, BitVector, BitList, typenum}; +/// +/// type BitList8 = Bitfield>; +/// +/// // The `N` type parameter specifies a maximum length. Creating a `BitList` with a larger +/// // capacity returns `None`. +/// assert!(BitList8::with_capacity(9).is_none()); +/// +/// let mut bitlist = BitList8::with_capacity(4); // `BitList` permits a capacity of less than the maximum. +/// assert!(bitlist.set(3, true).is_some()); // Setting inside the instantiation capacity is permitted. +/// assert!(bitlist.set(4, true).is_none()); // Setting outside that capacity is not. +/// +/// ``` +/// /// ## Note /// /// The internal representation of the bitfield is the same as that required by SSZ - the highest From 94265272b484c2ad68c5a98ade5bed52b1d57292 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 09:28:22 +1000 Subject: [PATCH 54/72] Tidy bitfield docs --- eth2/utils/ssz_types/src/bitfield.rs | 25 ++++++++++++++++--------- eth2/utils/ssz_types/src/lib.rs | 6 +----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index e64f790dd..a929405ed 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -33,21 +33,32 @@ impl BitfieldBehaviour for BitVector {} /// ``` /// use ssz_types::{Bitfield, BitVector, BitList, typenum}; /// +/// // `BitList` has a type-level maximum length. The length of the list is specified at runtime +/// // and it must be less than or equal to `N`. After instantiation, `BitList` cannot grow or +/// // shrink. /// type BitList8 = Bitfield>; /// -/// // The `N` type parameter specifies a maximum length. Creating a `BitList` with a larger -/// // capacity returns `None`. +/// // Creating a `BitList` with a larger-than-`N` capacity returns `None`. /// assert!(BitList8::with_capacity(9).is_none()); /// -/// let mut bitlist = BitList8::with_capacity(4); // `BitList` permits a capacity of less than the maximum. +/// let mut bitlist = BitList8::with_capacity(4).unwrap(); // `BitList` permits a capacity of less than the maximum. /// assert!(bitlist.set(3, true).is_some()); // Setting inside the instantiation capacity is permitted. -/// assert!(bitlist.set(4, true).is_none()); // Setting outside that capacity is not. +/// assert!(bitlist.set(5, true).is_none()); // Setting outside that capacity is not. +/// +/// // `BitVector` has a type-level fixed length. Unlike `BitList`, it cannot be instantiated with a custom length +/// // or grow/shrink. +/// type BitVector8 = Bitfield>; +/// +/// let mut bitvector = BitVector8::new(); +/// assert_eq!(bitvector.len(), 8); // `BitVector` length is fixed at the type-level. +/// assert!(bitvector.set(7, true).is_some()); // Setting inside the capacity is permitted. +/// assert!(bitvector.set(9, true).is_none()); // Setting outside the capacity is not. /// /// ``` /// /// ## Note /// -/// The internal representation of the bitfield is the same as that required by SSZ - the highest +/// The internal representation of the bitfield is the same as that required by SSZ. The highest /// byte (by `Vec` index) stores the lowest bit-indices and the right-most bit stores the lowest /// bit-index. E.g., `vec![0b0000_0010, 0b0000_0001]` has bits `0, 9` set. #[derive(Clone, Debug, PartialEq)] @@ -520,7 +531,6 @@ mod test { let mut bitfield = Bitfield::with_capacity(num_bits).unwrap(); for i in 0..num_bits + 1 { - dbg!(i); if i < num_bits { // Starts as false assert_eq!(bitfield.get(i), Some(false)); @@ -539,14 +549,11 @@ mod test { } fn test_bytes_round_trip(num_bits: usize) { - dbg!(num_bits); for i in 0..num_bits { - dbg!(i); let mut bitfield = Bitfield::with_capacity(num_bits).unwrap(); bitfield.set(i, true).unwrap(); let bytes = bitfield.clone().to_raw_bytes(); - dbg!(&bytes); assert_eq!(bitfield, Bitfield::from_raw_bytes(bytes, num_bits).unwrap()); } } diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 03e1bf20f..8eff665de 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -1,13 +1,9 @@ #[macro_use] mod bitfield; -// mod bit_list; -// mod bit_vector; mod fixed_vector; mod variable_list; -// pub use bit_list::BitList; -// pub use bit_vector::BitVector; -pub use bitfield::{BitList, BitVector, Bitfield}; +pub use bitfield::{BitList, BitVector, Bitfield, BitfieldBehaviour}; pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; From e4ef0fc9d4fa322d4b3356d025f5526aa2805a52 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 11:00:40 +1000 Subject: [PATCH 55/72] Add crate-level docs to ssz_types --- eth2/utils/ssz_types/src/bitfield.rs | 77 +++++++++++++++++++++++++++- eth2/utils/ssz_types/src/lib.rs | 35 +++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index a929405ed..b7cdb2876 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -471,7 +471,82 @@ impl tree_hash::TreeHash for Bitfield> { } } -// TODO: test ssz decode a zero-length bitlist. +impl cached_tree_hash::CachedTreeHash for Bitfield> { + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let bytes = self.to_bytes(); + + let (mut cache, schema) = cached_tree_hash::vec::new_tree_hash_cache(&bytes, depth)?; + + cache.add_length_nodes(schema.into_overlay(0).chunk_range(), bytes.len())?; + + Ok(cache) + } + + fn num_tree_hash_cache_chunks(&self) -> usize { + // Add two extra nodes to cater for the node before and after to allow mixing-in length. + cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks() + 2 + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + let bytes = self.to_bytes(); + cached_tree_hash::vec::produce_schema(&bytes, depth) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + let bytes = self.to_bytes(); + + // Skip the length-mixed-in root node. + cache.chunk_index += 1; + + // Update the cache, returning the new overlay. + let new_overlay = cached_tree_hash::vec::update_tree_hash_cache(&bytes, cache)?; + + // Mix in length + cache.mix_in_length(new_overlay.chunk_range(), bytes.len())?; + + // Skip an extra node to clear the length node. + cache.chunk_index += 1; + + Ok(()) + } +} + +impl cached_tree_hash::CachedTreeHash for Bitfield> { + fn new_tree_hash_cache( + &self, + depth: usize, + ) -> Result { + let (cache, _schema) = + cached_tree_hash::vec::new_tree_hash_cache(&ssz::ssz_encode(self), depth)?; + + Ok(cache) + } + + fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema { + let lengths = vec![ + 1; + cached_tree_hash::merkleize::num_unsanitized_leaves(bytes_for_bit_len( + N::to_usize() + )) + ]; + cached_tree_hash::BTreeSchema::from_lengths(depth, lengths) + } + + fn update_tree_hash_cache( + &self, + cache: &mut cached_tree_hash::TreeHashCache, + ) -> Result<(), cached_tree_hash::Error> { + cached_tree_hash::vec::update_tree_hash_cache(&ssz::ssz_encode(self), cache)?; + + Ok(()) + } +} #[cfg(test)] mod test { diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 8eff665de..610888229 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -1,3 +1,38 @@ +//! Provides types with unique properties required for SSZ serialization and Merklization: +//! +//! - `FixedVector`: A heap-allocated list with a size that is fixed at compile time. +//! - `VariableList`: A heap-allocated list that cannot grow past a type-level maximum length. +//! - `Bitfield`: A heap-allocated bitfield that with a type-level _maximum_ length. +//! - `Bitfield`: A heap-allocated bitfield that with a type-level _fixed__ length. +//! +//! These structs are required as SSZ serialization and Merklization rely upon type-level lengths +//! for padding and verification. +//! +//! ## Example +//! ``` +//! use ssz_types::*; +//! +//! pub struct Example { +//! bit_vector: Bitfield>, +//! bit_list: Bitfield>, +//! variable_list: VariableList, +//! fixed_vector: FixedVector, +//! } +//! +//! let mut example = Example { +//! bit_vector: Bitfield::new(), +//! bit_list: Bitfield::with_capacity(4).unwrap(), +//! variable_list: <_>::from(vec![0, 1]), +//! fixed_vector: <_>::from(vec![2, 3]), +//! }; +//! +//! assert_eq!(example.bit_vector.len(), 8); +//! assert_eq!(example.bit_list.len(), 4); +//! assert_eq!(&example.variable_list[..], &[0, 1]); +//! assert_eq!(&example.fixed_vector[..], &[2, 3, 0, 0, 0, 0, 0, 0]); +//! +//! ``` + #[macro_use] mod bitfield; mod fixed_vector; From efcbb1649140d73c771b2846fb38b7ce68d62d99 Mon Sep 17 00:00:00 2001 From: jzaki Date: Tue, 9 Jul 2019 14:52:05 +1000 Subject: [PATCH 56/72] Update installation notes to include git-lfs step. --- docs/installation.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 135304890..9c317fab9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -13,14 +13,17 @@ installed): - `clang`: required by RocksDB. - `protobuf`: required for protobuf serialization (gRPC). - `cmake`: required for building protobuf + - `git-lfs`: The Git extension for [Large File Support](https://git-lfs.github.com/) (required for EF tests submodule). 6. Navigate to the working directory. - 7. Run the test by using command `cargo test --all`. By running, it will pass all the required test cases. + 7. If you haven't already, clone the repository with submodules: `git clone --recursive https://github.com/sigp/lighthouse`. + Alternatively, run `git submodule init` in a repository which was cloned without submodules. + 8. Run the test by using command `cargo test --all --release`. By running, it will pass all the required test cases. If you are doing it for the first time, then you can grab a coffee in the meantime. Usually, it takes time to build, compile and pass all test cases. If there is no error then it means everything is working properly and it's time to get your hands dirty. In case, if there is an error, then please raise the [issue](https://github.com/sigp/lighthouse/issues). We will help you. - 8. As an alternative to, or instead of the above step, you may also run benchmarks by using + 9. As an alternative to, or instead of the above step, you may also run benchmarks by using the command `cargo bench --all` ## Notes: From 000d941e2e67fb24ed975cda3dcd0c22b2e98dbf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 16:03:02 +1000 Subject: [PATCH 57/72] Add tests for BitList --- eth2/utils/ssz_types/src/bitfield.rs | 167 ++++++++++++++++++++++++--- 1 file changed, 151 insertions(+), 16 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index b7cdb2876..1cb53ab1e 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -71,8 +71,10 @@ pub struct Bitfield { impl Bitfield> { pub fn with_capacity(num_bits: usize) -> Option { if num_bits <= N::to_usize() { + let num_bytes = std::cmp::max(bytes_for_bit_len(num_bits), 1); + Some(Self { - bytes: vec![0; bytes_for_bit_len(num_bits)], + bytes: vec![0; num_bytes], len: num_bits, _phantom: PhantomData, }) @@ -81,7 +83,7 @@ impl Bitfield> { } } - pub fn capacity() -> usize { + pub fn max_len() -> usize { N::to_usize() } @@ -89,15 +91,13 @@ impl Bitfield> { let len = self.len(); let mut bytes = self.as_slice().to_vec(); - if bytes_for_bit_len(len + 1) == bytes.len() + 1 { + while bytes_for_bit_len(len + 1) > bytes.len() { bytes.insert(0, 0); } let mut bitfield: Bitfield> = Bitfield::from_raw_bytes(bytes, len + 1) .expect("Bitfield capacity has been confirmed earlier."); - bitfield - .set(len, true) - .expect("Bitfield capacity has been confirmed earlier."); + bitfield.set(len, true).expect("Bitfield index must exist."); bitfield.bytes } @@ -110,17 +110,22 @@ impl Bitfield> { }; let len = initial_bitfield.highest_set_bit()?; - initial_bitfield - .set(len, false) - .expect("Bit has been confirmed to exist"); - let mut bytes = initial_bitfield.to_raw_bytes(); + if len <= Self::max_len() { + initial_bitfield + .set(len, false) + .expect("Bit has been confirmed to exist"); - if bytes_for_bit_len(len) < bytes.len() { - bytes.remove(0); + let mut bytes = initial_bitfield.to_raw_bytes(); + + if bytes_for_bit_len(len) < bytes.len() && bytes != &[0] { + bytes.remove(0); + } + + Self::from_raw_bytes(bytes, len) + } else { + None } - - Self::from_raw_bytes(bytes, len) } } @@ -203,7 +208,7 @@ impl Bitfield { &self.bytes } - pub fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { + fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { // A bitfield with `bit_len` 0 can only be represented by a single zero byte. Some(Self { @@ -552,7 +557,137 @@ impl cached_tree_hash::CachedTreeHash for Bitfield>; + mod bitlist { + use super::*; + + pub type BitList = crate::Bitfield>; + pub type BitList0 = BitList; + pub type BitList1 = BitList; + pub type BitList8 = BitList; + pub type BitList16 = BitList; + + #[test] + fn ssz_encode() { + assert_eq!( + BitList0::with_capacity(0).unwrap().as_ssz_bytes(), + vec![0b0000_00001], + ); + + assert_eq!( + BitList1::with_capacity(0).unwrap().as_ssz_bytes(), + vec![0b0000_00001], + ); + + assert_eq!( + BitList1::with_capacity(1).unwrap().as_ssz_bytes(), + vec![0b0000_00010], + ); + + assert_eq!( + BitList8::with_capacity(8).unwrap().as_ssz_bytes(), + vec![0b0000_0001, 0b0000_0000], + ); + + assert_eq!( + BitList8::with_capacity(7).unwrap().as_ssz_bytes(), + vec![0b1000_0000] + ); + + let mut b = BitList8::with_capacity(8).unwrap(); + for i in 0..8 { + b.set(i, true).unwrap(); + } + assert_eq!(b.as_ssz_bytes(), vec![0b0000_0001, 255]); + + let mut b = BitList8::with_capacity(8).unwrap(); + for i in 0..4 { + b.set(i, true).unwrap(); + } + assert_eq!(b.as_ssz_bytes(), vec![0b0000_0001, 0b0000_1111]); + + assert_eq!( + BitList16::with_capacity(16).unwrap().as_ssz_bytes(), + vec![0b0000_0001, 0b0000_0000, 0b0000_0000] + ); + } + + #[test] + fn ssz_decode() { + assert!(BitList0::from_ssz_bytes(&[0b0000_0000]).is_err()); + assert!(BitList1::from_ssz_bytes(&[0b0000_0000, 0b0000_0000]).is_err()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0000]).is_err()); + assert!(BitList16::from_ssz_bytes(&[0b0000_0000]).is_err()); + + assert!(BitList0::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitList0::from_ssz_bytes(&[0b0000_0010]).is_err()); + + assert!(BitList1::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitList1::from_ssz_bytes(&[0b0000_0010]).is_ok()); + assert!(BitList1::from_ssz_bytes(&[0b0000_0100]).is_err()); + + assert!(BitList8::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0010]).is_ok()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0001, 0b0000_0100]).is_ok()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0010, 0b0000_0100]).is_err()); + } + + #[test] + fn ssz_round_trip() { + assert_round_trip(BitList0::with_capacity(0).unwrap()); + + for i in 0..2 { + assert_round_trip(BitList1::with_capacity(i).unwrap()); + } + for i in 0..9 { + assert_round_trip(BitList8::with_capacity(i).unwrap()); + } + for i in 0..17 { + assert_round_trip(BitList16::with_capacity(i).unwrap()); + } + + let mut b = BitList1::with_capacity(1).unwrap(); + b.set(0, true); + assert_round_trip(b); + + for i in 0..8 { + let mut b = BitList8::with_capacity(i).unwrap(); + for j in 0..i { + if j % 2 == 0 { + b.set(j, true); + } + } + assert_round_trip(b); + + let mut b = BitList8::with_capacity(i).unwrap(); + for j in 0..i { + b.set(j, true); + } + assert_round_trip(b); + } + + for i in 0..16 { + let mut b = BitList16::with_capacity(i).unwrap(); + for j in 0..i { + if j % 2 == 0 { + b.set(j, true); + } + } + assert_round_trip(b); + + let mut b = BitList16::with_capacity(i).unwrap(); + for j in 0..i { + b.set(j, true); + } + assert_round_trip(b); + } + } + + fn assert_round_trip(t: T) { + assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t); + } + } + + type Bitfield = crate::Bitfield>; #[test] fn from_raw_bytes() { From 57cc9460180d7b3ceaabf68aef4ad04db0b86ade Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 16:37:09 +1000 Subject: [PATCH 58/72] Add bitvector tests --- eth2/utils/ssz_types/src/bitfield.rs | 431 ++++++++++++++++----------- 1 file changed, 265 insertions(+), 166 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 1cb53ab1e..b88c271c8 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -132,9 +132,10 @@ impl Bitfield> { impl Bitfield> { pub fn new() -> Self { let num_bits = N::to_usize(); + let num_bytes = std::cmp::max(bytes_for_bit_len(num_bits), 1); Self { - bytes: vec![0; num_bits], + bytes: vec![0; num_bytes], len: num_bits, _phantom: PhantomData, } @@ -554,191 +555,286 @@ impl cached_tree_hash::CachedTreeHash for Bitfield = crate::Bitfield>; + pub type BitVector0 = BitVector; + pub type BitVector1 = BitVector; + pub type BitVector4 = BitVector; + pub type BitVector8 = BitVector; + pub type BitVector16 = BitVector; - pub type BitList = crate::Bitfield>; - pub type BitList0 = BitList; - pub type BitList1 = BitList; - pub type BitList8 = BitList; - pub type BitList16 = BitList; + #[test] + fn ssz_encode() { + assert_eq!(BitVector0::new().as_ssz_bytes(), vec![0b0000_0000]); + assert_eq!(BitVector1::new().as_ssz_bytes(), vec![0b0000_0000]); + assert_eq!(BitVector4::new().as_ssz_bytes(), vec![0b0000_0000]); + assert_eq!(BitVector8::new().as_ssz_bytes(), vec![0b0000_0000]); + assert_eq!( + BitVector16::new().as_ssz_bytes(), + vec![0b0000_0000, 0b0000_0000] + ); - #[test] - fn ssz_encode() { - assert_eq!( - BitList0::with_capacity(0).unwrap().as_ssz_bytes(), - vec![0b0000_00001], - ); + let mut b = BitVector8::new(); + for i in 0..8 { + b.set(i, true).unwrap(); + } + assert_eq!(b.as_ssz_bytes(), vec![255]); - assert_eq!( - BitList1::with_capacity(0).unwrap().as_ssz_bytes(), - vec![0b0000_00001], - ); + let mut b = BitVector4::new(); + for i in 0..4 { + b.set(i, true).unwrap(); + } + assert_eq!(b.as_ssz_bytes(), vec![0b0000_1111]); + } - assert_eq!( - BitList1::with_capacity(1).unwrap().as_ssz_bytes(), - vec![0b0000_00010], - ); + #[test] + fn ssz_decode() { + assert!(BitVector0::from_ssz_bytes(&[0b0000_0000]).is_ok()); + assert!(BitVector0::from_ssz_bytes(&[0b0000_0001]).is_err()); + assert!(BitVector0::from_ssz_bytes(&[0b0000_0010]).is_err()); - assert_eq!( - BitList8::with_capacity(8).unwrap().as_ssz_bytes(), - vec![0b0000_0001, 0b0000_0000], - ); + assert!(BitVector1::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitVector1::from_ssz_bytes(&[0b0000_0010]).is_err()); + assert!(BitVector1::from_ssz_bytes(&[0b0000_0100]).is_err()); + assert!(BitVector1::from_ssz_bytes(&[0b0000_0000, 0b0000_0000]).is_err()); - assert_eq!( - BitList8::with_capacity(7).unwrap().as_ssz_bytes(), - vec![0b1000_0000] - ); + assert!(BitVector8::from_ssz_bytes(&[0b0000_0000]).is_ok()); + assert!(BitVector8::from_ssz_bytes(&[1, 0b0000_0000]).is_err()); + assert!(BitVector8::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitVector8::from_ssz_bytes(&[0b0000_0010]).is_ok()); + assert!(BitVector8::from_ssz_bytes(&[0b0000_0001, 0b0000_0100]).is_err()); + assert!(BitVector8::from_ssz_bytes(&[0b0000_0010, 0b0000_0100]).is_err()); - let mut b = BitList8::with_capacity(8).unwrap(); - for i in 0..8 { - b.set(i, true).unwrap(); + assert!(BitVector16::from_ssz_bytes(&[0b0000_0000]).is_err()); + assert!(BitVector16::from_ssz_bytes(&[0b0000_0000, 0b0000_0000]).is_ok()); + assert!(BitVector16::from_ssz_bytes(&[1, 0b0000_0000, 0b0000_0000]).is_err()); + } + + #[test] + fn ssz_round_trip() { + assert_round_trip(BitVector0::new()); + + let mut b = BitVector1::new(); + b.set(0, true); + assert_round_trip(b); + + let mut b = BitVector8::new(); + for j in 0..8 { + if j % 2 == 0 { + b.set(j, true); } - assert_eq!(b.as_ssz_bytes(), vec![0b0000_0001, 255]); + } + assert_round_trip(b); - let mut b = BitList8::with_capacity(8).unwrap(); - for i in 0..4 { - b.set(i, true).unwrap(); + let mut b = BitVector8::new(); + for j in 0..8 { + b.set(j, true); + } + assert_round_trip(b); + + let mut b = BitVector16::new(); + for j in 0..16 { + if j % 2 == 0 { + b.set(j, true); } - assert_eq!(b.as_ssz_bytes(), vec![0b0000_0001, 0b0000_1111]); + } + assert_round_trip(b); - assert_eq!( - BitList16::with_capacity(16).unwrap().as_ssz_bytes(), - vec![0b0000_0001, 0b0000_0000, 0b0000_0000] - ); + let mut b = BitVector16::new(); + for j in 0..16 { + b.set(j, true); + } + assert_round_trip(b); + } + + fn assert_round_trip(t: T) { + assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t); + } +} + +#[cfg(test)] +mod bitlist { + use super::*; + + pub type BitList = super::Bitfield>; + pub type BitList0 = BitList; + pub type BitList1 = BitList; + pub type BitList8 = BitList; + pub type BitList16 = BitList; + pub type BitList1024 = BitList; + + #[test] + fn ssz_encode() { + assert_eq!( + BitList0::with_capacity(0).unwrap().as_ssz_bytes(), + vec![0b0000_00001], + ); + + assert_eq!( + BitList1::with_capacity(0).unwrap().as_ssz_bytes(), + vec![0b0000_00001], + ); + + assert_eq!( + BitList1::with_capacity(1).unwrap().as_ssz_bytes(), + vec![0b0000_00010], + ); + + assert_eq!( + BitList8::with_capacity(8).unwrap().as_ssz_bytes(), + vec![0b0000_0001, 0b0000_0000], + ); + + assert_eq!( + BitList8::with_capacity(7).unwrap().as_ssz_bytes(), + vec![0b1000_0000] + ); + + let mut b = BitList8::with_capacity(8).unwrap(); + for i in 0..8 { + b.set(i, true).unwrap(); + } + assert_eq!(b.as_ssz_bytes(), vec![0b0000_0001, 255]); + + let mut b = BitList8::with_capacity(8).unwrap(); + for i in 0..4 { + b.set(i, true).unwrap(); + } + assert_eq!(b.as_ssz_bytes(), vec![0b0000_0001, 0b0000_1111]); + + assert_eq!( + BitList16::with_capacity(16).unwrap().as_ssz_bytes(), + vec![0b0000_0001, 0b0000_0000, 0b0000_0000] + ); + } + + #[test] + fn ssz_decode() { + assert!(BitList0::from_ssz_bytes(&[0b0000_0000]).is_err()); + assert!(BitList1::from_ssz_bytes(&[0b0000_0000, 0b0000_0000]).is_err()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0000]).is_err()); + assert!(BitList16::from_ssz_bytes(&[0b0000_0000]).is_err()); + + assert!(BitList0::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitList0::from_ssz_bytes(&[0b0000_0010]).is_err()); + + assert!(BitList1::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitList1::from_ssz_bytes(&[0b0000_0010]).is_ok()); + assert!(BitList1::from_ssz_bytes(&[0b0000_0100]).is_err()); + + assert!(BitList8::from_ssz_bytes(&[0b0000_0001]).is_ok()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0010]).is_ok()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0001, 0b0000_0100]).is_ok()); + assert!(BitList8::from_ssz_bytes(&[0b0000_0010, 0b0000_0100]).is_err()); + } + + #[test] + fn ssz_round_trip() { + assert_round_trip(BitList0::with_capacity(0).unwrap()); + + for i in 0..2 { + assert_round_trip(BitList1::with_capacity(i).unwrap()); + } + for i in 0..9 { + assert_round_trip(BitList8::with_capacity(i).unwrap()); + } + for i in 0..17 { + assert_round_trip(BitList16::with_capacity(i).unwrap()); } - #[test] - fn ssz_decode() { - assert!(BitList0::from_ssz_bytes(&[0b0000_0000]).is_err()); - assert!(BitList1::from_ssz_bytes(&[0b0000_0000, 0b0000_0000]).is_err()); - assert!(BitList8::from_ssz_bytes(&[0b0000_0000]).is_err()); - assert!(BitList16::from_ssz_bytes(&[0b0000_0000]).is_err()); + let mut b = BitList1::with_capacity(1).unwrap(); + b.set(0, true); + assert_round_trip(b); - assert!(BitList0::from_ssz_bytes(&[0b0000_0001]).is_ok()); - assert!(BitList0::from_ssz_bytes(&[0b0000_0010]).is_err()); - - assert!(BitList1::from_ssz_bytes(&[0b0000_0001]).is_ok()); - assert!(BitList1::from_ssz_bytes(&[0b0000_0010]).is_ok()); - assert!(BitList1::from_ssz_bytes(&[0b0000_0100]).is_err()); - - assert!(BitList8::from_ssz_bytes(&[0b0000_0001]).is_ok()); - assert!(BitList8::from_ssz_bytes(&[0b0000_0010]).is_ok()); - assert!(BitList8::from_ssz_bytes(&[0b0000_0001, 0b0000_0100]).is_ok()); - assert!(BitList8::from_ssz_bytes(&[0b0000_0010, 0b0000_0100]).is_err()); - } - - #[test] - fn ssz_round_trip() { - assert_round_trip(BitList0::with_capacity(0).unwrap()); - - for i in 0..2 { - assert_round_trip(BitList1::with_capacity(i).unwrap()); + for i in 0..8 { + let mut b = BitList8::with_capacity(i).unwrap(); + for j in 0..i { + if j % 2 == 0 { + b.set(j, true); + } } - for i in 0..9 { - assert_round_trip(BitList8::with_capacity(i).unwrap()); - } - for i in 0..17 { - assert_round_trip(BitList16::with_capacity(i).unwrap()); - } - - let mut b = BitList1::with_capacity(1).unwrap(); - b.set(0, true); assert_round_trip(b); - for i in 0..8 { - let mut b = BitList8::with_capacity(i).unwrap(); - for j in 0..i { - if j % 2 == 0 { - b.set(j, true); - } - } - assert_round_trip(b); - - let mut b = BitList8::with_capacity(i).unwrap(); - for j in 0..i { - b.set(j, true); - } - assert_round_trip(b); - } - - for i in 0..16 { - let mut b = BitList16::with_capacity(i).unwrap(); - for j in 0..i { - if j % 2 == 0 { - b.set(j, true); - } - } - assert_round_trip(b); - - let mut b = BitList16::with_capacity(i).unwrap(); - for j in 0..i { - b.set(j, true); - } - assert_round_trip(b); + let mut b = BitList8::with_capacity(i).unwrap(); + for j in 0..i { + b.set(j, true); } + assert_round_trip(b); } - fn assert_round_trip(t: T) { - assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t); + for i in 0..16 { + let mut b = BitList16::with_capacity(i).unwrap(); + for j in 0..i { + if j % 2 == 0 { + b.set(j, true); + } + } + assert_round_trip(b); + + let mut b = BitList16::with_capacity(i).unwrap(); + for j in 0..i { + b.set(j, true); + } + assert_round_trip(b); } } - type Bitfield = crate::Bitfield>; + fn assert_round_trip(t: T) { + assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t); + } #[test] fn from_raw_bytes() { - assert!(Bitfield::from_raw_bytes(vec![0b0000_0000], 0).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0001], 1).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0011], 2).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0111], 3).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_1111], 4).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0001_1111], 5).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0011_1111], 6).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0111_1111], 7).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b1111_1111], 8).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0000], 0).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 1).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011], 2).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111], 3).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111], 4).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111], 5).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111], 6).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111], 7).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], 8).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_some()); - assert!(Bitfield::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_some()); for i in 0..8 { - assert!(Bitfield::from_raw_bytes(vec![], i).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b1111_1111], i).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b1111_1110, 0b0000_0000], i).is_none()); + assert!(BitList1024::from_raw_bytes(vec![], i).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], i).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1110, 0b0000_0000], i).is_none()); } - assert!(Bitfield::from_raw_bytes(vec![0b0000_0001], 0).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 0).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0001], 0).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0011], 1).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0111], 2).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_1111], 3).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0001_1111], 4).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0011_1111], 5).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0111_1111], 6).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b1111_1111], 7).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 0).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011], 1).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111], 2).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111], 3).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111], 4).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111], 5).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111], 6).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], 7).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_none()); - assert!(Bitfield::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_none()); } fn test_set_unset(num_bits: usize) { - let mut bitfield = Bitfield::with_capacity(num_bits).unwrap(); + let mut bitfield = BitList1024::with_capacity(num_bits).unwrap(); for i in 0..num_bits + 1 { if i < num_bits { @@ -760,7 +856,7 @@ mod test { fn test_bytes_round_trip(num_bits: usize) { for i in 0..num_bits { - let mut bitfield = Bitfield::with_capacity(num_bits).unwrap(); + let mut bitfield = BitList1024::with_capacity(num_bits).unwrap(); bitfield.set(i, true).unwrap(); let bytes = bitfield.clone().to_raw_bytes(); @@ -784,7 +880,7 @@ mod test { #[test] fn to_raw_bytes() { - let mut bitfield = Bitfield::with_capacity(9).unwrap(); + let mut bitfield = BitList1024::with_capacity(9).unwrap(); bitfield.set(0, true); assert_eq!( bitfield.clone().to_raw_bytes(), @@ -834,31 +930,34 @@ mod test { #[test] fn highest_set_bit() { - assert_eq!(Bitfield::with_capacity(16).unwrap().highest_set_bit(), None); + assert_eq!( + BitList1024::with_capacity(16).unwrap().highest_set_bit(), + None + ); assert_eq!( - Bitfield::from_raw_bytes(vec![0b0000_000, 0b0000_0001], 16) + BitList1024::from_raw_bytes(vec![0b0000_000, 0b0000_0001], 16) .unwrap() .highest_set_bit(), Some(0) ); assert_eq!( - Bitfield::from_raw_bytes(vec![0b0000_000, 0b0000_0010], 16) + BitList1024::from_raw_bytes(vec![0b0000_000, 0b0000_0010], 16) .unwrap() .highest_set_bit(), Some(1) ); assert_eq!( - Bitfield::from_raw_bytes(vec![0b0000_1000], 8) + BitList1024::from_raw_bytes(vec![0b0000_1000], 8) .unwrap() .highest_set_bit(), Some(3) ); assert_eq!( - Bitfield::from_raw_bytes(vec![0b1000_0000, 0b0000_0000], 16) + BitList1024::from_raw_bytes(vec![0b1000_0000, 0b0000_0000], 16) .unwrap() .highest_set_bit(), Some(15) @@ -867,9 +966,9 @@ mod test { #[test] fn intersection() { - let a = Bitfield::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); - let b = Bitfield::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); - let c = Bitfield::from_raw_bytes(vec![0b1000, 0b0001], 16).unwrap(); + let a = BitList1024::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = BitList1024::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let c = BitList1024::from_raw_bytes(vec![0b1000, 0b0001], 16).unwrap(); assert_eq!(a.intersection(&b).unwrap(), c); assert_eq!(b.intersection(&a).unwrap(), c); @@ -882,9 +981,9 @@ mod test { #[test] fn union() { - let a = Bitfield::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); - let b = Bitfield::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); - let c = Bitfield::from_raw_bytes(vec![0b1111, 0b1001], 16).unwrap(); + let a = BitList1024::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = BitList1024::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let c = BitList1024::from_raw_bytes(vec![0b1111, 0b1001], 16).unwrap(); assert_eq!(a.union(&b).unwrap(), c); assert_eq!(b.union(&a).unwrap(), c); @@ -895,10 +994,10 @@ mod test { #[test] fn difference() { - let a = Bitfield::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); - let b = Bitfield::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); - let a_b = Bitfield::from_raw_bytes(vec![0b0100, 0b0000], 16).unwrap(); - let b_a = Bitfield::from_raw_bytes(vec![0b0011, 0b1000], 16).unwrap(); + let a = BitList1024::from_raw_bytes(vec![0b1100, 0b0001], 16).unwrap(); + let b = BitList1024::from_raw_bytes(vec![0b1011, 0b1001], 16).unwrap(); + let a_b = BitList1024::from_raw_bytes(vec![0b0100, 0b0000], 16).unwrap(); + let b_a = BitList1024::from_raw_bytes(vec![0b0011, 0b1000], 16).unwrap(); assert_eq!(a.difference(&b).unwrap(), a_b); assert_eq!(b.difference(&a).unwrap(), b_a); @@ -907,7 +1006,7 @@ mod test { #[test] fn iter() { - let mut bitfield = Bitfield::with_capacity(9).unwrap(); + let mut bitfield = BitList1024::with_capacity(9).unwrap(); bitfield.set(2, true); bitfield.set(8, true); From 7283fdff1525ea1a7d476c7703c91a631015b414 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 16:58:53 +1000 Subject: [PATCH 59/72] Fix clippy lints in `ssz_types` --- eth2/utils/ssz_types/src/bitfield.rs | 58 +++++++++++++---------- eth2/utils/ssz_types/src/variable_list.rs | 3 +- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index b88c271c8..0638699cf 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -87,7 +87,7 @@ impl Bitfield> { N::to_usize() } - pub fn to_bytes(&self) -> Vec { + pub fn into_bytes(self) -> Vec { let len = self.len(); let mut bytes = self.as_slice().to_vec(); @@ -116,9 +116,9 @@ impl Bitfield> { .set(len, false) .expect("Bit has been confirmed to exist"); - let mut bytes = initial_bitfield.to_raw_bytes(); + let mut bytes = initial_bitfield.into_raw_bytes(); - if bytes_for_bit_len(len) < bytes.len() && bytes != &[0] { + if bytes_for_bit_len(len) < bytes.len() && bytes != [0] { bytes.remove(0); } @@ -145,8 +145,8 @@ impl Bitfield> { N::to_usize() } - pub fn to_bytes(self) -> Vec { - self.to_raw_bytes() + pub fn into_bytes(self) -> Vec { + self.into_raw_bytes() } pub fn from_bytes(bytes: Vec) -> Option { @@ -154,6 +154,12 @@ impl Bitfield> { } } +impl Default for Bitfield> { + fn default() -> Self { + Self::new() + } +} + impl Bitfield { pub fn set(&mut self, i: usize, value: bool) -> Option<()> { if i < self.len { @@ -201,7 +207,7 @@ impl Bitfield { self.len == 0 } - pub fn to_raw_bytes(self) -> Vec { + pub fn into_raw_bytes(self) -> Vec { self.bytes } @@ -210,7 +216,7 @@ impl Bitfield { } fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { - if bytes.len() == 1 && bit_len == 0 && bytes == &[0] { + if bytes.len() == 1 && bit_len == 0 && bytes == [0] { // A bitfield with `bit_len` 0 can only be represented by a single zero byte. Some(Self { bytes, @@ -267,7 +273,7 @@ impl Bitfield { pub fn intersection_inplace(&mut self, other: &Self) -> Option<()> { if self.is_comparable(other) { for i in 0..self.bytes.len() { - self.bytes[i] = self.bytes[i] & other.bytes[i]; + self.bytes[i] &= other.bytes[i]; } Some(()) } else { @@ -288,7 +294,7 @@ impl Bitfield { pub fn union_inplace(&mut self, other: &Self) -> Option<()> { if self.is_comparable(other) { for i in 0..self.bytes.len() { - self.bytes[i] = self.bytes[i] | other.bytes[i]; + self.bytes[i] |= other.bytes[i]; } Some(()) } else { @@ -309,7 +315,7 @@ impl Bitfield { pub fn difference_inplace(&mut self, other: &Self) -> Option<()> { if self.is_comparable(other) { for i in 0..self.bytes.len() { - self.bytes[i] = self.bytes[i] & !other.bytes[i]; + self.bytes[i] &= !other.bytes[i]; } Some(()) } else { @@ -347,7 +353,7 @@ impl Encode for Bitfield> { } fn ssz_append(&self, buf: &mut Vec) { - buf.append(&mut self.clone().to_bytes()) + buf.append(&mut self.clone().into_bytes()) } } @@ -372,7 +378,7 @@ impl Encode for Bitfield> { } fn ssz_append(&self, buf: &mut Vec) { - buf.append(&mut self.clone().to_bytes()) + buf.append(&mut self.clone().into_bytes()) } } @@ -482,7 +488,7 @@ impl cached_tree_hash::CachedTreeHash for Bitfield Result { - let bytes = self.to_bytes(); + let bytes = self.clone().into_bytes(); let (mut cache, schema) = cached_tree_hash::vec::new_tree_hash_cache(&bytes, depth)?; @@ -497,7 +503,7 @@ impl cached_tree_hash::CachedTreeHash for Bitfield cached_tree_hash::BTreeSchema { - let bytes = self.to_bytes(); + let bytes = self.clone().into_bytes(); cached_tree_hash::vec::produce_schema(&bytes, depth) } @@ -505,7 +511,7 @@ impl cached_tree_hash::CachedTreeHash for Bitfield Result<(), cached_tree_hash::Error> { - let bytes = self.to_bytes(); + let bytes = self.clone().into_bytes(); // Skip the length-mixed-in root node. cache.chunk_index += 1; @@ -859,7 +865,7 @@ mod bitlist { let mut bitfield = BitList1024::with_capacity(num_bits).unwrap(); bitfield.set(i, true).unwrap(); - let bytes = bitfield.clone().to_raw_bytes(); + let bytes = bitfield.clone().into_raw_bytes(); assert_eq!(bitfield, Bitfield::from_raw_bytes(bytes, num_bits).unwrap()); } } @@ -879,51 +885,51 @@ mod bitlist { } #[test] - fn to_raw_bytes() { + fn into_raw_bytes() { let mut bitfield = BitList1024::with_capacity(9).unwrap(); bitfield.set(0, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_0001] ); bitfield.set(1, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_0011] ); bitfield.set(2, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_0111] ); bitfield.set(3, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_1111] ); bitfield.set(4, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0001_1111] ); bitfield.set(5, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0011_1111] ); bitfield.set(6, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0111_1111] ); bitfield.set(7, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b1111_1111] ); bitfield.set(8, true); assert_eq!( - bitfield.clone().to_raw_bytes(), + bitfield.clone().into_raw_bytes(), vec![0b0000_0001, 0b1111_1111] ); } diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs index 34450be8a..1d9a3349e 100644 --- a/eth2/utils/ssz_types/src/variable_list.rs +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -88,7 +88,8 @@ impl VariableList { /// Returns `Err(())` when appending `value` would exceed the maximum length. pub fn push(&mut self, value: T) -> Result<(), Error> { if self.vec.len() < Self::max_len() { - Ok(self.vec.push(value)) + self.vec.push(value); + Ok(()) } else { Err(Error::InvalidLength { i: self.vec.len() + 1, From 734aa3b8bd81d794d33928a9df79384c9ee319aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 17:01:37 +1000 Subject: [PATCH 60/72] Satisfy clippy lint in SSZ --- eth2/utils/ssz/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index bcb9f525c..886433f14 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -47,9 +47,9 @@ pub use encode::{Encode, SszEncoder}; pub const BYTES_PER_LENGTH_OFFSET: usize = 4; /// The maximum value that can be represented using `BYTES_PER_LENGTH_OFFSET`. #[cfg(target_pointer_width = "32")] -pub const MAX_LENGTH_VALUE: usize = (std::u32::MAX >> 8 * (4 - BYTES_PER_LENGTH_OFFSET)) as usize; +pub const MAX_LENGTH_VALUE: usize = (std::u32::MAX >> (8 * (4 - BYTES_PER_LENGTH_OFFSET))) as usize; #[cfg(target_pointer_width = "64")] -pub const MAX_LENGTH_VALUE: usize = (std::u64::MAX >> 8 * (8 - BYTES_PER_LENGTH_OFFSET)) as usize; +pub const MAX_LENGTH_VALUE: usize = (std::u64::MAX >> (8 * (8 - BYTES_PER_LENGTH_OFFSET))) as usize; /// Convenience function to SSZ encode an object supporting ssz::Encode. /// From 090133b088c58fcf329147589dacab51b1bf42fe Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 17:31:34 +1000 Subject: [PATCH 61/72] Add more comments to bitfield --- eth2/utils/ssz_types/src/bitfield.rs | 87 ++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 0638699cf..06d6b54b9 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -5,15 +5,20 @@ use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; use ssz::{Decode, Encode}; use typenum::Unsigned; +/// A marker trait applied to `BitList` and `BitVector` that defines the behaviour of a `Bitfield`. pub trait BitfieldBehaviour: Clone {} /// A marker struct used to define SSZ `BitList` functionality on a `Bitfield`. +/// +/// See the [`Bitfield`](struct.Bitfield.html) docs for usage. #[derive(Clone, PartialEq, Debug)] pub struct BitList { _phantom: PhantomData, } /// A marker struct used to define SSZ `BitVector` functionality on a `Bitfield`. +/// +/// See the [`Bitfield`](struct.Bitfield.html) docs for usage. #[derive(Clone, PartialEq, Debug)] pub struct BitVector { _phantom: PhantomData, @@ -69,6 +74,12 @@ pub struct Bitfield { } impl Bitfield> { + /// Instantiate with capacity for `num_bits` boolean values. The length cannot be grown or + /// shrunk after instantiation. + /// + /// All bits are initialized to `false`. + /// + /// Returns `None` if `num_bits > N`. pub fn with_capacity(num_bits: usize) -> Option { if num_bits <= N::to_usize() { let num_bytes = std::cmp::max(bytes_for_bit_len(num_bits), 1); @@ -83,10 +94,26 @@ impl Bitfield> { } } + /// Equal to `N` regardless of the value supplied to `with_capacity`. pub fn max_len() -> usize { N::to_usize() } + /// Consumes `self`, returning a serialized representation. + /// + /// The output is faithful to the SSZ encoding of `self`, such that a leading `true` bit is + /// used to indicate the length of the bitfield. + /// + /// ## Example + /// ``` + /// use ssz_types::{Bitfield, typenum}; + /// + /// type BitList = Bitfield>; + /// + /// let b = BitList::with_capacity(4).unwrap(); + /// + /// assert_eq!(b.into_bytes(), vec![0b0001_0000]); + /// ``` pub fn into_bytes(self) -> Vec { let len = self.len(); let mut bytes = self.as_slice().to_vec(); @@ -102,6 +129,10 @@ impl Bitfield> { bitfield.bytes } + /// Instantiates a new instance from `bytes`. Consumes the same format that `self.into_bytes()` + /// produces (SSZ). + /// + /// Returns `None` if `bytes` are not a valid encoding. pub fn from_bytes(bytes: Vec) -> Option { let mut initial_bitfield: Bitfield> = { let num_bits = bytes.len() * 8; @@ -130,6 +161,9 @@ impl Bitfield> { } impl Bitfield> { + /// Instantiate a new `Bitfield` with a fixed-length of `N` bits. + /// + /// All bits are initialized to `false`. pub fn new() -> Self { let num_bits = N::to_usize(); let num_bytes = std::cmp::max(bytes_for_bit_len(num_bits), 1); @@ -141,14 +175,31 @@ impl Bitfield> { } } + /// Returns `N`, the number of bits in `Self`. pub fn capacity() -> usize { N::to_usize() } + /// Consumes `self`, returning a serialized representation. + /// + /// The output is faithful to the SSZ encoding of `self`. + /// + /// ## Example + /// ``` + /// use ssz_types::{Bitfield, typenum}; + /// + /// type BitVector = Bitfield>; + /// + /// assert_eq!(BitVector::new().into_bytes(), vec![0b0000_0000]); + /// ``` pub fn into_bytes(self) -> Vec { self.into_raw_bytes() } + /// Instantiates a new instance from `bytes`. Consumes the same format that `self.into_bytes()` + /// produces (SSZ). + /// + /// Returns `None` if `bytes` are not a valid encoding. pub fn from_bytes(bytes: Vec) -> Option { Self::from_raw_bytes(bytes, Self::capacity()) } @@ -161,6 +212,9 @@ impl Default for Bitfield> { } impl Bitfield { + /// Sets the `i`'th bit to `value`. + /// + /// Returns `None` if `i` is out-of-bounds of `self`. pub fn set(&mut self, i: usize, value: bool) -> Option<()> { if i < self.len { let byte = { @@ -183,6 +237,9 @@ impl Bitfield { } } + /// Returns the value of the `i`'th bit. + /// + /// Returns `None` if `i` is out-of-bounds of `self`. pub fn get(&self, i: usize) -> Option { if i < self.len { let byte = { @@ -199,22 +256,34 @@ impl Bitfield { } } + /// Returns the number of bits stored in `self`. pub fn len(&self) -> usize { self.len } + /// Returns `true` if `self.len() == 0`. pub fn is_empty(&self) -> bool { self.len == 0 } + /// Returns the underlying bytes representation of the bitfield. pub fn into_raw_bytes(self) -> Vec { self.bytes } + /// Returns a view into the underlying bytes representation of the bitfield. pub fn as_slice(&self) -> &[u8] { &self.bytes } + /// Instantiates from the given `bytes`, which are the same format as output from + /// `self.into_raw_bytes()`. + /// + /// Returns `None` if: + /// + /// - `bytes` is not the minimal required bytes to represent a bitfield of `bit_len` bits. + /// - `bit_len` is not a multiple of 8 and `bytes` contains set bits that are higher than, or + /// equal to `bit_len`. fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { if bytes.len() == 1 && bit_len == 0 && bytes == [0] { // A bitfield with `bit_len` 0 can only be represented by a single zero byte. @@ -242,6 +311,8 @@ impl Bitfield { } } + /// Returns the `Some(i)` where `i` is the highest index with a set bit. Returns `None` if + /// there are no set bits. pub fn highest_set_bit(&self) -> Option { let byte_i = self.bytes.iter().position(|byte| *byte > 0)?; let bit_i = 7 - self.bytes[byte_i].leading_zeros() as usize; @@ -249,6 +320,7 @@ impl Bitfield { Some((self.bytes.len().saturating_sub(1) - byte_i) * 8 + bit_i) } + /// Returns an iterator across bitfield `bool` values, starting at the lowest index. pub fn iter(&self) -> BitIter<'_, T> { BitIter { bitfield: self, @@ -256,10 +328,14 @@ impl Bitfield { } } + /// Returns true if no bits are set. pub fn is_zero(&self) -> bool { !self.bytes.iter().any(|byte| (*byte & u8::max_value()) > 0) } + /// Compute the intersection (binary-and) of this bitfield with another. + /// + /// Returns `None` if `self.is_comparable(other) == false`. pub fn intersection(&self, other: &Self) -> Option { if self.is_comparable(other) { let mut res = self.clone(); @@ -270,6 +346,7 @@ impl Bitfield { } } + /// Like `intersection` but in-place (updates `self`). pub fn intersection_inplace(&mut self, other: &Self) -> Option<()> { if self.is_comparable(other) { for i in 0..self.bytes.len() { @@ -281,6 +358,9 @@ impl Bitfield { } } + /// Compute the union (binary-or) of this bitfield with another. + /// + /// Returns `None` if `self.is_comparable(other) == false`. pub fn union(&self, other: &Self) -> Option { if self.is_comparable(other) { let mut res = self.clone(); @@ -291,6 +371,7 @@ impl Bitfield { } } + /// Like `union` but in-place (updates `self`). pub fn union_inplace(&mut self, other: &Self) -> Option<()> { if self.is_comparable(other) { for i in 0..self.bytes.len() { @@ -302,6 +383,9 @@ impl Bitfield { } } + /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match. + /// + /// Returns `None` if `self.is_comparable(other) == false`. pub fn difference(&self, other: &Self) -> Option { if self.is_comparable(other) { let mut res = self.clone(); @@ -312,6 +396,7 @@ impl Bitfield { } } + /// Like `difference` but in-place (updates `self`). pub fn difference_inplace(&mut self, other: &Self) -> Option<()> { if self.is_comparable(other) { for i in 0..self.bytes.len() { @@ -323,6 +408,8 @@ impl Bitfield { } } + /// Returns true if `self` and `other` have the same lengths and can be used in binary + /// comparison operations. pub fn is_comparable(&self, other: &Self) -> bool { (self.len() == other.len()) && (self.bytes.len() == other.bytes.len()) } From daa8916e6b0ac62ccb5c52db1e917a18f6956f55 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 20:28:19 +1000 Subject: [PATCH 62/72] Add comments after self-review --- eth2/utils/ssz_types/src/bitfield.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 06d6b54b9..3d2303777 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -8,7 +8,7 @@ use typenum::Unsigned; /// A marker trait applied to `BitList` and `BitVector` that defines the behaviour of a `Bitfield`. pub trait BitfieldBehaviour: Clone {} -/// A marker struct used to define SSZ `BitList` functionality on a `Bitfield`. +/// A marker struct used to declare SSZ `BitList` behaviour on a `Bitfield`. /// /// See the [`Bitfield`](struct.Bitfield.html) docs for usage. #[derive(Clone, PartialEq, Debug)] @@ -16,7 +16,7 @@ pub struct BitList { _phantom: PhantomData, } -/// A marker struct used to define SSZ `BitVector` functionality on a `Bitfield`. +/// A marker struct used to declare SSZ `BitVector` behaviour on a `Bitfield`. /// /// See the [`Bitfield`](struct.Bitfield.html) docs for usage. #[derive(Clone, PartialEq, Debug)] @@ -415,10 +415,14 @@ impl Bitfield { } } +/// Returns the minimum required bytes to represent a given number of bits. +/// +/// `bit_len == 0` requires a single byte. fn bytes_for_bit_len(bit_len: usize) -> usize { (bit_len + 7) / 8 } +/// An iterator over the bits in a `Bitfield`. pub struct BitIter<'a, T> { bitfield: &'a Bitfield, i: usize, From bb3b3fd8b9f3f37de91d5c331b4f779f7bc6213a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 20:29:10 +1000 Subject: [PATCH 63/72] Fix bug around single-byte for 0-bits --- eth2/utils/ssz_types/src/bitfield.rs | 33 ++++++++++++++-------------- eth2/utils/ssz_types/src/lib.rs | 2 +- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 3d2303777..465f26f67 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -82,10 +82,8 @@ impl Bitfield> { /// Returns `None` if `num_bits > N`. pub fn with_capacity(num_bits: usize) -> Option { if num_bits <= N::to_usize() { - let num_bytes = std::cmp::max(bytes_for_bit_len(num_bits), 1); - Some(Self { - bytes: vec![0; num_bytes], + bytes: vec![0; bytes_for_bit_len(num_bits)], len: num_bits, _phantom: PhantomData, }) @@ -165,12 +163,9 @@ impl Bitfield> { /// /// All bits are initialized to `false`. pub fn new() -> Self { - let num_bits = N::to_usize(); - let num_bytes = std::cmp::max(bytes_for_bit_len(num_bits), 1); - Self { - bytes: vec![0; num_bytes], - len: num_bits, + bytes: vec![0; bytes_for_bit_len(Self::capacity())], + len: Self::capacity(), _phantom: PhantomData, } } @@ -285,13 +280,17 @@ impl Bitfield { /// - `bit_len` is not a multiple of 8 and `bytes` contains set bits that are higher than, or /// equal to `bit_len`. fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { - if bytes.len() == 1 && bit_len == 0 && bytes == [0] { - // A bitfield with `bit_len` 0 can only be represented by a single zero byte. - Some(Self { - bytes, - len: 0, - _phantom: PhantomData, - }) + if bit_len == 0 { + if bytes.len() == 1 && bytes == [0] { + // A bitfield with `bit_len` 0 can only be represented by a single zero byte. + Some(Self { + bytes, + len: 0, + _phantom: PhantomData, + }) + } else { + None + } } else if bytes.len() != bytes_for_bit_len(bit_len) || bytes.is_empty() { // The number of bytes must be the minimum required to represent `bit_len`. None @@ -299,7 +298,7 @@ impl Bitfield { // Ensure there are no bits higher than `bit_len` that are set to true. let (mask, _) = u8::max_value().overflowing_shr(8 - (bit_len as u32 % 8)); - if (bytes.first().expect("Bytes cannot be empty") & !mask) == 0 { + if (bytes.first().expect("Guarded against empty bytes") & !mask) == 0 { Some(Self { bytes, len: bit_len, @@ -419,7 +418,7 @@ impl Bitfield { /// /// `bit_len == 0` requires a single byte. fn bytes_for_bit_len(bit_len: usize) -> usize { - (bit_len + 7) / 8 + std::cmp::max(1, (bit_len + 7) / 8) } /// An iterator over the bits in a `Bitfield`. diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 610888229..42989f173 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -38,7 +38,7 @@ mod bitfield; mod fixed_vector; mod variable_list; -pub use bitfield::{BitList, BitVector, Bitfield, BitfieldBehaviour}; +pub use bitfield::{BitList, BitVector, Bitfield}; pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; From 5e1a2ebf25649a21823b1584fab1c7983f0d49a2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Jul 2019 20:30:29 +1000 Subject: [PATCH 64/72] Uncomment broken tree hash impl for variable list --- eth2/utils/ssz_types/src/variable_list.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs index 1d9a3349e..68f03deed 100644 --- a/eth2/utils/ssz_types/src/variable_list.rs +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -210,7 +210,6 @@ mod test { } } -/* impl tree_hash::TreeHash for VariableList where T: tree_hash::TreeHash, @@ -258,7 +257,6 @@ where Ok(()) } } -*/ impl ssz::Encode for VariableList where From 08069704c1373d91bc28fe79ff23e86a7d632f6b Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Wed, 10 Jul 2019 10:27:44 +1000 Subject: [PATCH 65/72] Add cli flag for logging to JSON file --- account_manager/src/main.rs | 24 +++++++-------- beacon_node/client/Cargo.toml | 3 +- beacon_node/client/src/config.rs | 51 +++++++++++++++++++++++++++++-- beacon_node/src/main.rs | 11 +++++-- validator_client/Cargo.toml | 3 +- validator_client/src/config.rs | 52 +++++++++++++++++++++++++++++--- validator_client/src/main.rs | 11 +++++-- 7 files changed, 129 insertions(+), 26 deletions(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index 68c304ec5..3c55c39e2 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -14,13 +14,20 @@ fn main() { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::CompactFormat::new(decorator).build().fuse(); let drain = slog_async::Async::new(drain).build().fuse(); - let log = slog::Logger::root(drain, o!()); + let mut log = slog::Logger::root(drain, o!()); // CLI let matches = App::new("Lighthouse Accounts Manager") .version("0.0.1") .author("Sigma Prime ") .about("Eth 2.0 Accounts Manager") + .arg( + Arg::with_name("logfile") + .long("logfile") + .value_name("logfile") + .help("File path where output will be written.") + .takes_value(true), + ) .arg( Arg::with_name("datadir") .long("datadir") @@ -91,21 +98,12 @@ fn main() { let mut client_config = ValidatorClientConfig::default(); - if let Err(e) = client_config.apply_cli_args(&matches) { - crit!(log, "Failed to apply CLI args"; "error" => format!("{:?}", e)); - return; - }; - // Ensure the `data_dir` in the config matches that supplied to the CLI. client_config.data_dir = data_dir.clone(); - // Update the client config with any CLI args. - match client_config.apply_cli_args(&matches) { - Ok(()) => (), - Err(s) => { - crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); - return; - } + if let Err(e) = client_config.apply_cli_args(&matches, &mut log) { + crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => format!("{:?}", e)); + return; }; // Log configuration diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 94a529ea7..d3b1e6294 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -21,8 +21,9 @@ serde_derive = "1.0" error-chain = "0.12.0" eth2_ssz = { path = "../../eth2/utils/ssz" } slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } -slog-term = "^2.4.0" slog-async = "^2.3.0" +slog-json = "^2.3" +slog-term = "^2.4.0" tokio = "0.1.15" clap = "2.32.0" dirs = "1.0.3" diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 415ef0ec9..864577559 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -2,8 +2,10 @@ use clap::ArgMatches; use http_server::HttpServerConfig; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; -use std::fs; +use slog::{info, o, Drain}; +use std::fs::{self, OpenOptions}; use std::path::PathBuf; +use std::sync::Mutex; /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -11,6 +13,7 @@ pub struct Config { pub data_dir: PathBuf, pub db_type: String, db_name: String, + pub log_file: PathBuf, pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, pub http: HttpServerConfig, @@ -20,6 +23,7 @@ impl Default for Config { fn default() -> Self { Self { data_dir: PathBuf::from(".lighthouse"), + log_file: PathBuf::from(""), db_type: "disk".to_string(), db_name: "chain_db".to_string(), // Note: there are no default bootnodes specified. @@ -45,23 +49,64 @@ impl Config { Some(path) } + // Update the logger to output in JSON to specified file + fn update_logger(&mut self, log: &mut slog::Logger) -> Result<(), &'static str> { + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&self.log_file); + + if file.is_err() { + return Err("Cannot open log file"); + } + let file = file.unwrap(); + + if let Some(file) = self.log_file.to_str() { + info!( + *log, + "Log file specified, output will now be written to {} in json.", file + ); + } else { + info!( + *log, + "Log file specified output will now be written in json" + ); + } + + let drain = Mutex::new(slog_json::Json::default(file)).fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + *log = slog::Logger::root(drain, o!()); + + Ok(()) + } + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. /// /// Returns an error if arguments are obviously invalid. May succeed even if some values are /// invalid. - pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), String> { + pub fn apply_cli_args( + &mut self, + args: &ArgMatches, + log: &mut slog::Logger, + ) -> Result<(), String> { if let Some(dir) = args.value_of("datadir") { self.data_dir = PathBuf::from(dir); }; if let Some(dir) = args.value_of("db") { self.db_type = dir.to_string(); - } + }; self.network.apply_cli_args(args)?; self.rpc.apply_cli_args(args)?; self.http.apply_cli_args(args)?; + if let Some(log_file) = args.value_of("logfile") { + self.log_file = PathBuf::from(log_file); + self.update_logger(log)?; + }; + Ok(()) } } diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 791feae54..6beb0fd64 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -29,6 +29,13 @@ fn main() { .help("Data directory for keys and databases.") .takes_value(true) ) + .arg( + Arg::with_name("logfile") + .long("logfile") + .value_name("logfile") + .help("File path where output will be written.") + .takes_value(true), + ) // network related arguments .arg( Arg::with_name("listen-address") @@ -159,7 +166,7 @@ fn main() { _ => drain.filter_level(Level::Info), }; - let log = slog::Logger::root(drain.fuse(), o!()); + let mut log = slog::Logger::root(drain.fuse(), o!()); let data_dir = match matches .value_of("datadir") @@ -214,7 +221,7 @@ fn main() { client_config.data_dir = data_dir.clone(); // Update the client config with any CLI args. - match client_config.apply_cli_args(&matches) { + match client_config.apply_cli_args(&matches, &mut log) { Ok(()) => (), Err(s) => { crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 1972f870c..d2824bc2f 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -26,8 +26,9 @@ types = { path = "../eth2/types" } serde = "1.0" serde_derive = "1.0" slog = "^2.2.3" -slog-term = "^2.4.0" slog-async = "^2.3.0" +slog-json = "^2.3" +slog-term = "^2.4.0" tokio = "0.1.18" tokio-timer = "0.2.10" toml = "^0.5" diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index d7664c161..7bc504b23 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -2,11 +2,11 @@ use bincode; use bls::Keypair; use clap::ArgMatches; use serde_derive::{Deserialize, Serialize}; -use slog::{debug, error, info}; -use std::fs; -use std::fs::File; +use slog::{debug, error, info, o, Drain}; +use std::fs::{self, File, OpenOptions}; use std::io::{Error, ErrorKind}; use std::path::PathBuf; +use std::sync::Mutex; use types::{EthSpec, MainnetEthSpec}; /// Stores the core configuration for this validator instance. @@ -14,6 +14,8 @@ use types::{EthSpec, MainnetEthSpec}; pub struct Config { /// The data directory, which stores all validator databases pub data_dir: PathBuf, + /// The path where the logs will be outputted + pub log_file: PathBuf, /// The server at which the Beacon Node can be contacted pub server: String, /// The number of slots per epoch. @@ -27,6 +29,7 @@ impl Default for Config { fn default() -> Self { Self { data_dir: PathBuf::from(".lighthouse-validator"), + log_file: PathBuf::from(""), server: "localhost:5051".to_string(), slots_per_epoch: MainnetEthSpec::slots_per_epoch(), } @@ -38,11 +41,20 @@ impl Config { /// /// Returns an error if arguments are obviously invalid. May succeed even if some values are /// invalid. - pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + pub fn apply_cli_args( + &mut self, + args: &ArgMatches, + log: &mut slog::Logger, + ) -> Result<(), &'static str> { if let Some(datadir) = args.value_of("datadir") { self.data_dir = PathBuf::from(datadir); }; + if let Some(log_file) = args.value_of("logfile") { + self.log_file = PathBuf::from(log_file); + self.update_logger(log)?; + }; + if let Some(srv) = args.value_of("server") { self.server = srv.to_string(); }; @@ -50,6 +62,38 @@ impl Config { Ok(()) } + // Update the logger to output in JSON to specified file + fn update_logger(&mut self, log: &mut slog::Logger) -> Result<(), &'static str> { + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&self.log_file); + + if file.is_err() { + return Err("Cannot open log file"); + } + let file = file.unwrap(); + + if let Some(file) = self.log_file.to_str() { + info!( + *log, + "Log file specified, output will now be written to {} in json.", file + ); + } else { + info!( + *log, + "Log file specified output will now be written in json" + ); + } + + let drain = Mutex::new(slog_json::Json::default(file)).fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + *log = slog::Logger::root(drain, o!()); + + Ok(()) + } + /// Try to load keys from validator_dir, returning None if none are found or an error. #[allow(dead_code)] pub fn fetch_keys(&self, log: &slog::Logger) -> Option> { diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 5beea4c38..c12cae6a2 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -26,7 +26,7 @@ fn main() { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::CompactFormat::new(decorator).build().fuse(); let drain = slog_async::Async::new(drain).build().fuse(); - let log = slog::Logger::root(drain, o!()); + let mut log = slog::Logger::root(drain, o!()); // CLI let matches = App::new("Lighthouse Validator Client") @@ -41,6 +41,13 @@ fn main() { .help("Data directory for keys and databases.") .takes_value(true), ) + .arg( + Arg::with_name("logfile") + .long("logfile") + .value_name("logfile") + .help("File path where output will be written.") + .takes_value(true), + ) .arg( Arg::with_name("eth2-spec") .long("eth2-spec") @@ -123,7 +130,7 @@ fn main() { client_config.data_dir = data_dir.clone(); // Update the client config with any CLI args. - match client_config.apply_cli_args(&matches) { + match client_config.apply_cli_args(&matches, &mut log) { Ok(()) => (), Err(s) => { crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); From 1d1ae5c0752bf13de5176adb31bacbc5f01f920e Mon Sep 17 00:00:00 2001 From: jzaki Date: Wed, 10 Jul 2019 13:15:38 +1000 Subject: [PATCH 66/72] Update instructions --- docs/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 9c317fab9..1b0ced7bd 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -14,9 +14,9 @@ installed): - `protobuf`: required for protobuf serialization (gRPC). - `cmake`: required for building protobuf - `git-lfs`: The Git extension for [Large File Support](https://git-lfs.github.com/) (required for EF tests submodule). - 6. Navigate to the working directory. - 7. If you haven't already, clone the repository with submodules: `git clone --recursive https://github.com/sigp/lighthouse`. + 6. If you haven't already, clone the repository with submodules: `git clone --recursive https://github.com/sigp/lighthouse`. Alternatively, run `git submodule init` in a repository which was cloned without submodules. + 7. Change directory to the root of the repository. 8. Run the test by using command `cargo test --all --release`. By running, it will pass all the required test cases. If you are doing it for the first time, then you can grab a coffee in the meantime. Usually, it takes time to build, compile and pass all test cases. If there is no error then it means everything is working properly From 2c1afcc2d6bfdbd052944e9a7645fecce994b369 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 11 Jul 2019 12:40:37 +1000 Subject: [PATCH 67/72] Rename marker structs for Bitfield --- eth2/utils/ssz_types/src/bitfield.rs | 98 ++++++++++++++++------------ eth2/utils/ssz_types/src/lib.rs | 12 ++-- 2 files changed, 66 insertions(+), 44 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 465f26f67..23231e09d 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -5,43 +5,61 @@ use serde_hex::{encode as hex_encode, PrefixedHexVisitor}; use ssz::{Decode, Encode}; use typenum::Unsigned; -/// A marker trait applied to `BitList` and `BitVector` that defines the behaviour of a `Bitfield`. +/// A marker trait applied to `Variable` and `Fixed` that defines the behaviour of a `Bitfield`. pub trait BitfieldBehaviour: Clone {} -/// A marker struct used to declare SSZ `BitList` behaviour on a `Bitfield`. +/// A marker struct used to declare SSZ `Variable` behaviour on a `Bitfield`. /// /// See the [`Bitfield`](struct.Bitfield.html) docs for usage. #[derive(Clone, PartialEq, Debug)] -pub struct BitList { +pub struct Variable { _phantom: PhantomData, } -/// A marker struct used to declare SSZ `BitVector` behaviour on a `Bitfield`. +/// A marker struct used to declare SSZ `Fixed` behaviour on a `Bitfield`. /// /// See the [`Bitfield`](struct.Bitfield.html) docs for usage. #[derive(Clone, PartialEq, Debug)] -pub struct BitVector { +pub struct Fixed { _phantom: PhantomData, } -impl BitfieldBehaviour for BitList {} -impl BitfieldBehaviour for BitVector {} +impl BitfieldBehaviour for Variable {} +impl BitfieldBehaviour for Fixed {} -/// A heap-allocated, ordered, fixed-length, collection of `bool` values. Must be used with the `BitList` or -/// `BitVector` marker structs. +/// A heap-allocated, ordered, variable-length collection of `bool` values, limited to `N` bits. +pub type BitList = Bitfield>; + +/// A heap-allocated, ordered, fixed-length collection of `bool` values, with `N` bits. +/// +/// See [Bitfield](struct.Bitfield.html) documentation. +pub type BitVector = Bitfield>; + +/// A heap-allocated, ordered, fixed-length, collection of `bool` values. Use of +/// [`BitList`](type.BitList.html) or [`BitVector`](type.BitVector.html) type aliases is preferred +/// over direct use of this struct. +/// +/// The `T` type parameter is used to define length behaviour with the `Variable` or `Fixed` marker +/// structs. /// /// The length of the Bitfield is set at instantiation (i.e., runtime, not compile time). However, -/// use with a `BitList` sets a type-level (i.e., compile-time) maximum length and `BitVector` +/// use with a `Variable` sets a type-level (i.e., compile-time) maximum length and `Fixed` /// provides a type-level fixed length. /// /// ## Example +/// +/// The example uses the following crate-level type aliases: +/// +/// - `BitList` is an alias for `Bitfield>` +/// - `BitVector` is an alias for `Bitfield>` +/// /// ``` -/// use ssz_types::{Bitfield, BitVector, BitList, typenum}; +/// use ssz_types::{BitVector, BitList, typenum}; /// /// // `BitList` has a type-level maximum length. The length of the list is specified at runtime /// // and it must be less than or equal to `N`. After instantiation, `BitList` cannot grow or /// // shrink. -/// type BitList8 = Bitfield>; +/// type BitList8 = BitList; /// /// // Creating a `BitList` with a larger-than-`N` capacity returns `None`. /// assert!(BitList8::with_capacity(9).is_none()); @@ -52,7 +70,7 @@ impl BitfieldBehaviour for BitVector {} /// /// // `BitVector` has a type-level fixed length. Unlike `BitList`, it cannot be instantiated with a custom length /// // or grow/shrink. -/// type BitVector8 = Bitfield>; +/// type BitVector8 = BitVector; /// /// let mut bitvector = BitVector8::new(); /// assert_eq!(bitvector.len(), 8); // `BitVector` length is fixed at the type-level. @@ -73,7 +91,7 @@ pub struct Bitfield { _phantom: PhantomData, } -impl Bitfield> { +impl Bitfield> { /// Instantiate with capacity for `num_bits` boolean values. The length cannot be grown or /// shrunk after instantiation. /// @@ -104,11 +122,11 @@ impl Bitfield> { /// /// ## Example /// ``` - /// use ssz_types::{Bitfield, typenum}; + /// use ssz_types::{BitList, typenum}; /// - /// type BitList = Bitfield>; + /// type BitList8 = BitList; /// - /// let b = BitList::with_capacity(4).unwrap(); + /// let b = BitList8::with_capacity(4).unwrap(); /// /// assert_eq!(b.into_bytes(), vec![0b0001_0000]); /// ``` @@ -120,7 +138,7 @@ impl Bitfield> { bytes.insert(0, 0); } - let mut bitfield: Bitfield> = Bitfield::from_raw_bytes(bytes, len + 1) + let mut bitfield: Bitfield> = Bitfield::from_raw_bytes(bytes, len + 1) .expect("Bitfield capacity has been confirmed earlier."); bitfield.set(len, true).expect("Bitfield index must exist."); @@ -132,7 +150,7 @@ impl Bitfield> { /// /// Returns `None` if `bytes` are not a valid encoding. pub fn from_bytes(bytes: Vec) -> Option { - let mut initial_bitfield: Bitfield> = { + let mut initial_bitfield: Bitfield> = { let num_bits = bytes.len() * 8; Bitfield::from_raw_bytes(bytes, num_bits) .expect("Must have adequate bytes for bit count.") @@ -158,7 +176,7 @@ impl Bitfield> { } } -impl Bitfield> { +impl Bitfield> { /// Instantiate a new `Bitfield` with a fixed-length of `N` bits. /// /// All bits are initialized to `false`. @@ -181,11 +199,11 @@ impl Bitfield> { /// /// ## Example /// ``` - /// use ssz_types::{Bitfield, typenum}; + /// use ssz_types::{BitVector, typenum}; /// - /// type BitVector = Bitfield>; + /// type BitVector4 = BitVector; /// - /// assert_eq!(BitVector::new().into_bytes(), vec![0b0000_0000]); + /// assert_eq!(BitVector4::new().into_bytes(), vec![0b0000_0000]); /// ``` pub fn into_bytes(self) -> Vec { self.into_raw_bytes() @@ -200,7 +218,7 @@ impl Bitfield> { } } -impl Default for Bitfield> { +impl Default for Bitfield> { fn default() -> Self { Self::new() } @@ -437,7 +455,7 @@ impl<'a, T: BitfieldBehaviour> Iterator for BitIter<'a, T> { } } -impl Encode for Bitfield> { +impl Encode for Bitfield> { fn is_ssz_fixed_len() -> bool { false } @@ -447,18 +465,18 @@ impl Encode for Bitfield> { } } -impl Decode for Bitfield> { +impl Decode for Bitfield> { fn is_ssz_fixed_len() -> bool { false } fn from_ssz_bytes(bytes: &[u8]) -> Result { Self::from_bytes(bytes.to_vec()) - .ok_or_else(|| ssz::DecodeError::BytesInvalid("BitList failed to decode".to_string())) + .ok_or_else(|| ssz::DecodeError::BytesInvalid("Variable failed to decode".to_string())) } } -impl Encode for Bitfield> { +impl Encode for Bitfield> { fn is_ssz_fixed_len() -> bool { true } @@ -472,18 +490,18 @@ impl Encode for Bitfield> { } } -impl Decode for Bitfield> { +impl Decode for Bitfield> { fn is_ssz_fixed_len() -> bool { false } fn from_ssz_bytes(bytes: &[u8]) -> Result { Self::from_bytes(bytes.to_vec()) - .ok_or_else(|| ssz::DecodeError::BytesInvalid("BitVector failed to decode".to_string())) + .ok_or_else(|| ssz::DecodeError::BytesInvalid("Fixed failed to decode".to_string())) } } -impl Serialize for Bitfield> { +impl Serialize for Bitfield> { /// Serde serialization is compliant with the Ethereum YAML test format. fn serialize(&self, serializer: S) -> Result where @@ -493,7 +511,7 @@ impl Serialize for Bitfield> { } } -impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { +impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { /// Serde serialization is compliant with the Ethereum YAML test format. fn deserialize(deserializer: D) -> Result where @@ -508,7 +526,7 @@ impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { } } -impl Serialize for Bitfield> { +impl Serialize for Bitfield> { /// Serde serialization is compliant with the Ethereum YAML test format. fn serialize(&self, serializer: S) -> Result where @@ -518,7 +536,7 @@ impl Serialize for Bitfield> { } } -impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { +impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { /// Serde serialization is compliant with the Ethereum YAML test format. fn deserialize(deserializer: D) -> Result where @@ -533,7 +551,7 @@ impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { } } -impl tree_hash::TreeHash for Bitfield> { +impl tree_hash::TreeHash for Bitfield> { fn tree_hash_type() -> tree_hash::TreeHashType { tree_hash::TreeHashType::List } @@ -552,7 +570,7 @@ impl tree_hash::TreeHash for Bitfield> { } } -impl tree_hash::TreeHash for Bitfield> { +impl tree_hash::TreeHash for Bitfield> { fn tree_hash_type() -> tree_hash::TreeHashType { // TODO: move this to be a vector. tree_hash::TreeHashType::List @@ -573,7 +591,7 @@ impl tree_hash::TreeHash for Bitfield> { } } -impl cached_tree_hash::CachedTreeHash for Bitfield> { +impl cached_tree_hash::CachedTreeHash for Bitfield> { fn new_tree_hash_cache( &self, depth: usize, @@ -619,7 +637,7 @@ impl cached_tree_hash::CachedTreeHash for Bitfield cached_tree_hash::CachedTreeHash for Bitfield> { +impl cached_tree_hash::CachedTreeHash for Bitfield> { fn new_tree_hash_cache( &self, depth: usize, @@ -653,8 +671,8 @@ impl cached_tree_hash::CachedTreeHash for Bitfield = crate::Bitfield>; pub type BitVector0 = BitVector; pub type BitVector1 = BitVector; pub type BitVector4 = BitVector; @@ -753,8 +771,8 @@ mod bitvector { #[cfg(test)] mod bitlist { use super::*; + use crate::BitList; - pub type BitList = super::Bitfield>; pub type BitList0 = BitList; pub type BitList1 = BitList; pub type BitList8 = BitList; diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 42989f173..8e7595d48 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -2,8 +2,8 @@ //! //! - `FixedVector`: A heap-allocated list with a size that is fixed at compile time. //! - `VariableList`: A heap-allocated list that cannot grow past a type-level maximum length. -//! - `Bitfield`: A heap-allocated bitfield that with a type-level _maximum_ length. -//! - `Bitfield`: A heap-allocated bitfield that with a type-level _fixed__ length. +//! - `BitList`: A heap-allocated bitfield that with a type-level _maximum_ length. +//! - `BitVector`: A heap-allocated bitfield that with a type-level _fixed__ length. //! //! These structs are required as SSZ serialization and Merklization rely upon type-level lengths //! for padding and verification. @@ -13,8 +13,8 @@ //! use ssz_types::*; //! //! pub struct Example { -//! bit_vector: Bitfield>, -//! bit_list: Bitfield>, +//! bit_vector: BitVector, +//! bit_list: BitList, //! variable_list: VariableList, //! fixed_vector: FixedVector, //! } @@ -43,6 +43,10 @@ pub use fixed_vector::FixedVector; pub use typenum; pub use variable_list::VariableList; +pub mod length { + pub use crate::bitfield::{Fixed, Variable}; +} + /// Returned when an item encounters an error. #[derive(PartialEq, Debug)] pub enum Error { From 61406b34bc8a7fc259f6b360452a82efe34c760e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 11 Jul 2019 12:45:34 +1000 Subject: [PATCH 68/72] Resolve issues raised from @michaelsproul review --- eth2/utils/ssz_types/src/bitfield.rs | 8 +------- eth2/utils/ssz_types/src/fixed_vector.rs | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 23231e09d..6af82b48d 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -347,7 +347,7 @@ impl Bitfield { /// Returns true if no bits are set. pub fn is_zero(&self) -> bool { - !self.bytes.iter().any(|byte| (*byte & u8::max_value()) > 0) + self.bytes.iter().all(|byte| *byte == 0) } /// Compute the intersection (binary-and) of this bitfield with another. @@ -517,9 +517,6 @@ impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { where D: Deserializer<'de>, { - // We reverse the bit-order so that the BitVec library can read its 0th - // bit from the end of the hex string, e.g. - // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; Self::from_ssz_bytes(&bytes) .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) @@ -542,9 +539,6 @@ impl<'de, N: Unsigned + Clone> Deserialize<'de> for Bitfield> { where D: Deserializer<'de>, { - // We reverse the bit-order so that the BitVec library can read its 0th - // bit from the end of the hex string, e.g. - // "0xef01" => [0xef, 0x01] => [0b1000_0000, 0b1111_1110] let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?; Self::from_ssz_bytes(&bytes) .map_err(|e| serde::de::Error::custom(format!("Bitfield {:?}", e))) diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index b5d422760..8d7c2264e 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -50,8 +50,8 @@ pub struct FixedVector { } impl FixedVector { - /// Returns `Some` if the given `vec` equals the fixed length of `Self`. Otherwise returns - /// `None`. + /// Returns `Ok` if the given `vec` equals the fixed length of `Self`. Otherwise returns + /// `Err`. pub fn new(vec: Vec) -> Result { if vec.len() == Self::capacity() { Ok(Self { From 561cec0bf69f9f7914b2a3435d3da062916b1165 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 11 Jul 2019 13:19:38 +1000 Subject: [PATCH 69/72] Move many bitfield Options to Results --- eth2/utils/ssz_types/src/bitfield.rs | 208 ++++++++++++---------- eth2/utils/ssz_types/src/fixed_vector.rs | 2 +- eth2/utils/ssz_types/src/lib.rs | 15 +- eth2/utils/ssz_types/src/variable_list.rs | 4 +- 4 files changed, 127 insertions(+), 102 deletions(-) diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 6af82b48d..de9a198f3 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -1,3 +1,4 @@ +use crate::Error; use core::marker::PhantomData; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; @@ -62,11 +63,11 @@ pub type BitVector = Bitfield>; /// type BitList8 = BitList; /// /// // Creating a `BitList` with a larger-than-`N` capacity returns `None`. -/// assert!(BitList8::with_capacity(9).is_none()); +/// assert!(BitList8::with_capacity(9).is_err()); /// /// let mut bitlist = BitList8::with_capacity(4).unwrap(); // `BitList` permits a capacity of less than the maximum. -/// assert!(bitlist.set(3, true).is_some()); // Setting inside the instantiation capacity is permitted. -/// assert!(bitlist.set(5, true).is_none()); // Setting outside that capacity is not. +/// assert!(bitlist.set(3, true).is_ok()); // Setting inside the instantiation capacity is permitted. +/// assert!(bitlist.set(5, true).is_err()); // Setting outside that capacity is not. /// /// // `BitVector` has a type-level fixed length. Unlike `BitList`, it cannot be instantiated with a custom length /// // or grow/shrink. @@ -74,8 +75,8 @@ pub type BitVector = Bitfield>; /// /// let mut bitvector = BitVector8::new(); /// assert_eq!(bitvector.len(), 8); // `BitVector` length is fixed at the type-level. -/// assert!(bitvector.set(7, true).is_some()); // Setting inside the capacity is permitted. -/// assert!(bitvector.set(9, true).is_none()); // Setting outside the capacity is not. +/// assert!(bitvector.set(7, true).is_ok()); // Setting inside the capacity is permitted. +/// assert!(bitvector.set(9, true).is_err()); // Setting outside the capacity is not. /// /// ``` /// @@ -98,15 +99,18 @@ impl Bitfield> { /// All bits are initialized to `false`. /// /// Returns `None` if `num_bits > N`. - pub fn with_capacity(num_bits: usize) -> Option { + pub fn with_capacity(num_bits: usize) -> Result { if num_bits <= N::to_usize() { - Some(Self { + Ok(Self { bytes: vec![0; bytes_for_bit_len(num_bits)], len: num_bits, _phantom: PhantomData, }) } else { - None + Err(Error::OutOfBounds { + i: Self::max_len(), + len: Self::max_len(), + }) } } @@ -149,14 +153,16 @@ impl Bitfield> { /// produces (SSZ). /// /// Returns `None` if `bytes` are not a valid encoding. - pub fn from_bytes(bytes: Vec) -> Option { + pub fn from_bytes(bytes: Vec) -> Result { let mut initial_bitfield: Bitfield> = { let num_bits = bytes.len() * 8; Bitfield::from_raw_bytes(bytes, num_bits) .expect("Must have adequate bytes for bit count.") }; - let len = initial_bitfield.highest_set_bit()?; + let len = initial_bitfield + .highest_set_bit() + .ok_or_else(|| Error::MissingLengthInformation)?; if len <= Self::max_len() { initial_bitfield @@ -171,7 +177,10 @@ impl Bitfield> { Self::from_raw_bytes(bytes, len) } else { - None + Err(Error::OutOfBounds { + i: Self::max_len(), + len: Self::max_len(), + }) } } } @@ -213,7 +222,7 @@ impl Bitfield> { /// produces (SSZ). /// /// Returns `None` if `bytes` are not a valid encoding. - pub fn from_bytes(bytes: Vec) -> Option { + pub fn from_bytes(bytes: Vec) -> Result { Self::from_raw_bytes(bytes, Self::capacity()) } } @@ -228,7 +237,7 @@ impl Bitfield { /// Sets the `i`'th bit to `value`. /// /// Returns `None` if `i` is out-of-bounds of `self`. - pub fn set(&mut self, i: usize, value: bool) -> Option<()> { + pub fn set(&mut self, i: usize, value: bool) -> Result<(), Error> { if i < self.len { let byte = { let num_bytes = self.bytes.len(); @@ -244,16 +253,16 @@ impl Bitfield { *byte &= !(1 << (i % 8)) } - Some(()) + Ok(()) } else { - None + Err(Error::OutOfBounds { i, len: self.len }) } } /// Returns the value of the `i`'th bit. /// /// Returns `None` if `i` is out-of-bounds of `self`. - pub fn get(&self, i: usize) -> Option { + pub fn get(&self, i: usize) -> Result { if i < self.len { let byte = { let num_bytes = self.bytes.len(); @@ -263,9 +272,9 @@ impl Bitfield { .expect("Cannot be OOB if less than self.len") }; - Some(*byte & 1 << (i % 8) > 0) + Ok(*byte & 1 << (i % 8) > 0) } else { - None + Err(Error::OutOfBounds { i, len: self.len }) } } @@ -297,33 +306,36 @@ impl Bitfield { /// - `bytes` is not the minimal required bytes to represent a bitfield of `bit_len` bits. /// - `bit_len` is not a multiple of 8 and `bytes` contains set bits that are higher than, or /// equal to `bit_len`. - fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Option { + fn from_raw_bytes(bytes: Vec, bit_len: usize) -> Result { if bit_len == 0 { if bytes.len() == 1 && bytes == [0] { // A bitfield with `bit_len` 0 can only be represented by a single zero byte. - Some(Self { + Ok(Self { bytes, len: 0, _phantom: PhantomData, }) } else { - None + Err(Error::ExcessBits) } - } else if bytes.len() != bytes_for_bit_len(bit_len) || bytes.is_empty() { + } else if bytes.len() != bytes_for_bit_len(bit_len) { // The number of bytes must be the minimum required to represent `bit_len`. - None + Err(Error::InvalidByteCount { + given: bytes.len(), + expected: bytes_for_bit_len(bit_len), + }) } else { // Ensure there are no bits higher than `bit_len` that are set to true. let (mask, _) = u8::max_value().overflowing_shr(8 - (bit_len as u32 % 8)); if (bytes.first().expect("Guarded against empty bytes") & !mask) == 0 { - Some(Self { + Ok(Self { bytes, len: bit_len, _phantom: PhantomData, }) } else { - None + Err(Error::ExcessBits) } } } @@ -449,9 +461,9 @@ impl<'a, T: BitfieldBehaviour> Iterator for BitIter<'a, T> { type Item = bool; fn next(&mut self) -> Option { - let res = self.bitfield.get(self.i); + let res = self.bitfield.get(self.i).ok()?; self.i += 1; - res + Some(res) } } @@ -471,8 +483,9 @@ impl Decode for Bitfield> { } fn from_ssz_bytes(bytes: &[u8]) -> Result { - Self::from_bytes(bytes.to_vec()) - .ok_or_else(|| ssz::DecodeError::BytesInvalid("Variable failed to decode".to_string())) + Self::from_bytes(bytes.to_vec()).map_err(|e| { + ssz::DecodeError::BytesInvalid(format!("BitList failed to decode: {:?}", e)) + }) } } @@ -496,8 +509,9 @@ impl Decode for Bitfield> { } fn from_ssz_bytes(bytes: &[u8]) -> Result { - Self::from_bytes(bytes.to_vec()) - .ok_or_else(|| ssz::DecodeError::BytesInvalid("Fixed failed to decode".to_string())) + Self::from_bytes(bytes.to_vec()).map_err(|e| { + ssz::DecodeError::BytesInvalid(format!("BitVector failed to decode: {:?}", e)) + }) } } @@ -725,34 +739,34 @@ mod bitvector { assert_round_trip(BitVector0::new()); let mut b = BitVector1::new(); - b.set(0, true); + b.set(0, true).unwrap(); assert_round_trip(b); let mut b = BitVector8::new(); for j in 0..8 { if j % 2 == 0 { - b.set(j, true); + b.set(j, true).unwrap(); } } assert_round_trip(b); let mut b = BitVector8::new(); for j in 0..8 { - b.set(j, true); + b.set(j, true).unwrap(); } assert_round_trip(b); let mut b = BitVector16::new(); for j in 0..16 { if j % 2 == 0 { - b.set(j, true); + b.set(j, true).unwrap(); } } assert_round_trip(b); let mut b = BitVector16::new(); for j in 0..16 { - b.set(j, true); + b.set(j, true).unwrap(); } assert_round_trip(b); } @@ -853,21 +867,21 @@ mod bitlist { } let mut b = BitList1::with_capacity(1).unwrap(); - b.set(0, true); + b.set(0, true).unwrap(); assert_round_trip(b); for i in 0..8 { let mut b = BitList8::with_capacity(i).unwrap(); for j in 0..i { if j % 2 == 0 { - b.set(j, true); + b.set(j, true).unwrap(); } } assert_round_trip(b); let mut b = BitList8::with_capacity(i).unwrap(); for j in 0..i { - b.set(j, true); + b.set(j, true).unwrap(); } assert_round_trip(b); } @@ -876,14 +890,14 @@ mod bitlist { let mut b = BitList16::with_capacity(i).unwrap(); for j in 0..i { if j % 2 == 0 { - b.set(j, true); + b.set(j, true).unwrap(); } } assert_round_trip(b); let mut b = BitList16::with_capacity(i).unwrap(); for j in 0..i { - b.set(j, true); + b.set(j, true).unwrap(); } assert_round_trip(b); } @@ -895,50 +909,50 @@ mod bitlist { #[test] fn from_raw_bytes() { - assert!(BitList1024::from_raw_bytes(vec![0b0000_0000], 0).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 1).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0011], 2).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0111], 3).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_1111], 4).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0001_1111], 5).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0011_1111], 6).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0111_1111], 7).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], 8).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0000], 0).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 1).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011], 2).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111], 3).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111], 4).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111], 5).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111], 6).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111], 7).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], 8).is_ok()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_some()); - assert!(BitList1024::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_some()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 9).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 10).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 11).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 12).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 13).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 14).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 15).is_ok()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 16).is_ok()); for i in 0..8 { - assert!(BitList1024::from_raw_bytes(vec![], i).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], i).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b1111_1110, 0b0000_0000], i).is_none()); + assert!(BitList1024::from_raw_bytes(vec![], i).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], i).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1110, 0b0000_0000], i).is_err()); } - assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 0).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 0).is_err()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 0).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0011], 1).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0111], 2).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_1111], 3).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0001_1111], 4).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0011_1111], 5).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0111_1111], 6).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], 7).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001], 0).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011], 1).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111], 2).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111], 3).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111], 4).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111], 5).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111], 6).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111], 7).is_err()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_none()); - assert!(BitList1024::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_none()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0001, 0b1111_1111], 8).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0011, 0b1111_1111], 9).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_0111, 0b1111_1111], 10).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0000_1111, 0b1111_1111], 11).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0001_1111, 0b1111_1111], 12).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0011_1111, 0b1111_1111], 13).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b0111_1111, 0b1111_1111], 14).is_err()); + assert!(BitList1024::from_raw_bytes(vec![0b1111_1111, 0b1111_1111], 15).is_err()); } fn test_set_unset(num_bits: usize) { @@ -947,17 +961,17 @@ mod bitlist { for i in 0..num_bits + 1 { if i < num_bits { // Starts as false - assert_eq!(bitfield.get(i), Some(false)); + assert_eq!(bitfield.get(i), Ok(false)); // Can be set true. - assert!(bitfield.set(i, true).is_some()); - assert_eq!(bitfield.get(i), Some(true)); + assert!(bitfield.set(i, true).is_ok()); + assert_eq!(bitfield.get(i), Ok(true)); // Can be set false - assert!(bitfield.set(i, false).is_some()); - assert_eq!(bitfield.get(i), Some(false)); + assert!(bitfield.set(i, false).is_ok()); + assert_eq!(bitfield.get(i), Ok(false)); } else { - assert_eq!(bitfield.get(i), None); - assert!(bitfield.set(i, true).is_none()); - assert_eq!(bitfield.get(i), None); + assert!(bitfield.get(i).is_err()); + assert!(bitfield.set(i, true).is_err()); + assert!(bitfield.get(i).is_err()); } } } @@ -989,47 +1003,47 @@ mod bitlist { #[test] fn into_raw_bytes() { let mut bitfield = BitList1024::with_capacity(9).unwrap(); - bitfield.set(0, true); + bitfield.set(0, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_0001] ); - bitfield.set(1, true); + bitfield.set(1, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_0011] ); - bitfield.set(2, true); + bitfield.set(2, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_0111] ); - bitfield.set(3, true); + bitfield.set(3, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0000_1111] ); - bitfield.set(4, true); + bitfield.set(4, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0001_1111] ); - bitfield.set(5, true); + bitfield.set(5, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0011_1111] ); - bitfield.set(6, true); + bitfield.set(6, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b0111_1111] ); - bitfield.set(7, true); + bitfield.set(7, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0000, 0b1111_1111] ); - bitfield.set(8, true); + bitfield.set(8, true).unwrap(); assert_eq!( bitfield.clone().into_raw_bytes(), vec![0b0000_0001, 0b1111_1111] @@ -1115,8 +1129,8 @@ mod bitlist { #[test] fn iter() { let mut bitfield = BitList1024::with_capacity(9).unwrap(); - bitfield.set(2, true); - bitfield.set(8, true); + bitfield.set(2, true).unwrap(); + bitfield.set(8, true).unwrap(); assert_eq!( bitfield.iter().collect::>(), diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index 8d7c2264e..687d7d738 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -59,7 +59,7 @@ impl FixedVector { _phantom: PhantomData, }) } else { - Err(Error::InvalidLength { + Err(Error::OutOfBounds { i: vec.len(), len: Self::capacity(), }) diff --git a/eth2/utils/ssz_types/src/lib.rs b/eth2/utils/ssz_types/src/lib.rs index 8e7595d48..59869b7c0 100644 --- a/eth2/utils/ssz_types/src/lib.rs +++ b/eth2/utils/ssz_types/src/lib.rs @@ -50,6 +50,17 @@ pub mod length { /// Returned when an item encounters an error. #[derive(PartialEq, Debug)] pub enum Error { - InvalidLength { i: usize, len: usize }, - OutOfBounds { i: usize, len: usize }, + OutOfBounds { + i: usize, + len: usize, + }, + /// A `BitList` does not have a set bit, therefore it's length is unknowable. + MissingLengthInformation, + /// A `BitList` has excess bits set to true. + ExcessBits, + /// A `BitList` has an invalid number of bytes for a given bit length. + InvalidByteCount { + given: usize, + expected: usize, + }, } diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs index 68f03deed..52872ada6 100644 --- a/eth2/utils/ssz_types/src/variable_list.rs +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -61,7 +61,7 @@ impl VariableList { _phantom: PhantomData, }) } else { - Err(Error::InvalidLength { + Err(Error::OutOfBounds { i: vec.len(), len: Self::max_len(), }) @@ -91,7 +91,7 @@ impl VariableList { self.vec.push(value); Ok(()) } else { - Err(Error::InvalidLength { + Err(Error::OutOfBounds { i: self.vec.len() + 1, len: Self::max_len(), }) From 88c6d15c32a83e029362daec6920ff4b82d33b51 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 16 Jul 2019 14:40:56 +1000 Subject: [PATCH 70/72] Padding efficent merkle root algo (#436) * Add initial work on padding efficent merkle roots * Improve merklize_padded * Improve tree_hash crate -- fix bugs, docs * Update codebase for tree_hash API change * Remove dbg statements, fix import error * Fix clippy lints, doc error * Tidy tree hash comments * Increase tree_hash max tree height * Fix PR review comments * Fix typos * Fix cache access off-by-one in tree hash * Set max tree depth to 48 (from 64) --- eth2/utils/tree_hash/Cargo.toml | 2 + eth2/utils/tree_hash/src/impls.rs | 2 +- eth2/utils/tree_hash/src/lib.rs | 16 +- eth2/utils/tree_hash/src/merkleize_padded.rs | 366 ++++++++++++++++++ .../{merkleize.rs => merkleize_standard.rs} | 23 +- eth2/utils/tree_hash_derive/src/lib.rs | 4 +- eth2/utils/tree_hash_derive/tests/tests.rs | 2 +- 7 files changed, 403 insertions(+), 12 deletions(-) create mode 100644 eth2/utils/tree_hash/src/merkleize_padded.rs rename eth2/utils/tree_hash/src/{merkleize.rs => merkleize_standard.rs} (72%) diff --git a/eth2/utils/tree_hash/Cargo.toml b/eth2/utils/tree_hash/Cargo.toml index 7e23d2165..948e0fe4f 100644 --- a/eth2/utils/tree_hash/Cargo.toml +++ b/eth2/utils/tree_hash/Cargo.toml @@ -5,9 +5,11 @@ authors = ["Paul Hauner "] edition = "2018" [dev-dependencies] +rand = "0.7" tree_hash_derive = { path = "../tree_hash_derive" } [dependencies] ethereum-types = "0.5" hashing = { path = "../hashing" } int_to_bytes = { path = "../int_to_bytes" } +lazy_static = "0.1" diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs index 212aba3c8..b91147830 100644 --- a/eth2/utils/tree_hash/src/impls.rs +++ b/eth2/utils/tree_hash/src/impls.rs @@ -1,5 +1,5 @@ use super::*; -use crate::merkleize::merkle_root; +use crate::merkle_root; use ethereum_types::H256; use hashing::hash; use int_to_bytes::int_to_bytes32; diff --git a/eth2/utils/tree_hash/src/lib.rs b/eth2/utils/tree_hash/src/lib.rs index 2554e70c3..a1d7a048e 100644 --- a/eth2/utils/tree_hash/src/lib.rs +++ b/eth2/utils/tree_hash/src/lib.rs @@ -1,5 +1,17 @@ +#[macro_use] +extern crate lazy_static; + pub mod impls; -pub mod merkleize; +mod merkleize_padded; +mod merkleize_standard; + +pub use merkleize_padded::merkleize_padded; +pub use merkleize_standard::merkleize_standard; + +/// Alias to `merkleize_padded(&bytes, 0)` +pub fn merkle_root(bytes: &[u8]) -> Vec { + merkleize_padded(&bytes, 0) +} pub const BYTES_PER_CHUNK: usize = 32; pub const HASHSIZE: usize = 32; @@ -44,7 +56,7 @@ macro_rules! tree_hash_ssz_encoding_as_vector { } fn tree_hash_root(&self) -> Vec { - tree_hash::merkleize::merkle_root(&ssz::ssz_encode(self)) + tree_hash::merkle_root(&ssz::ssz_encode(self)) } } }; diff --git a/eth2/utils/tree_hash/src/merkleize_padded.rs b/eth2/utils/tree_hash/src/merkleize_padded.rs new file mode 100644 index 000000000..43bd247d8 --- /dev/null +++ b/eth2/utils/tree_hash/src/merkleize_padded.rs @@ -0,0 +1,366 @@ +use super::BYTES_PER_CHUNK; +use hashing::hash; + +/// The size of the cache that stores padding nodes for a given height. +/// +/// Currently, we panic if we encounter a tree with a height larger than `MAX_TREE_DEPTH`. +/// +/// It is set to 48 as we expect it to be sufficiently high that we won't exceed it. +pub const MAX_TREE_DEPTH: usize = 48; + +lazy_static! { + /// Cached zero hashes where `ZERO_HASHES[i]` is the hash of a Merkle tree with 2^i zero leaves. + static ref ZERO_HASHES: Vec> = { + let mut hashes = vec![vec![0; 32]; MAX_TREE_DEPTH + 1]; + + for i in 0..MAX_TREE_DEPTH { + hashes[i + 1] = hash_concat(&hashes[i], &hashes[i]); + } + + hashes + }; +} + +/// Merkleize `bytes` and return the root, optionally padding the tree out to `min_leaves` number of +/// leaves. +/// +/// First all nodes are extracted from `bytes` and then a padding node is added until the number of +/// leaf chunks is greater than or equal to `min_leaves`. Callers may set `min_leaves` to `0` if no +/// adding additional chunks should be added to the given `bytes`. +/// +/// If `bytes.len() <= BYTES_PER_CHUNK`, no hashing is done and `bytes` is returned, potentially +/// padded out to `BYTES_PER_CHUNK` length with `0`. +/// +/// ## CPU Performance +/// +/// A cache of `MAX_TREE_DEPTH` hashes are stored to avoid re-computing the hashes of padding nodes +/// (or their parents). Therefore, adding padding nodes only incurs one more hash per additional +/// height of the tree. +/// +/// ## Memory Performance +/// +/// This algorithm has two interesting memory usage properties: +/// +/// 1. The maximum memory footprint is roughly `O(V / 2)` memory, where `V` is the number of leaf +/// chunks with values (i.e., leaves that are not padding). The means adding padding nodes to +/// the tree does not increase the memory footprint. +/// 2. At each height of the tree half of the memory is freed until only a single chunk is stored. +/// 3. The input `bytes` are not copied into another list before processing. +/// +/// _Note: there are some minor memory overheads, including a handful of usizes and a list of +/// `MAX_TREE_DEPTH` hashes as `lazy_static` constants._ +pub fn merkleize_padded(bytes: &[u8], min_leaves: usize) -> Vec { + // If the bytes are just one chunk or less, pad to one chunk and return without hashing. + if bytes.len() <= BYTES_PER_CHUNK && min_leaves <= 1 { + let mut o = bytes.to_vec(); + o.resize(BYTES_PER_CHUNK, 0); + return o; + } + + assert!( + bytes.len() > BYTES_PER_CHUNK || min_leaves > 1, + "Merkle hashing only needs to happen if there is more than one chunk" + ); + + // The number of leaves that can be made directly from `bytes`. + let leaves_with_values = (bytes.len() + (BYTES_PER_CHUNK - 1)) / BYTES_PER_CHUNK; + + // The number of parents that have at least one non-padding leaf. + // + // Since there is more than one node in this tree (see prior assertion), there should always be + // one or more initial parent nodes. + let initial_parents_with_values = std::cmp::max(1, next_even_number(leaves_with_values) / 2); + + // The number of leaves in the full tree (including padding nodes). + let num_leaves = std::cmp::max(leaves_with_values, min_leaves).next_power_of_two(); + + // The number of levels in the tree. + // + // A tree with a single node has `height == 1`. + let height = num_leaves.trailing_zeros() as usize + 1; + + assert!(height >= 2, "The tree should have two or more heights"); + + // A buffer/scratch-space used for storing each round of hashes at each height. + // + // This buffer is kept as small as possible; it will shrink so it never stores a padding node. + let mut chunks = ChunkStore::with_capacity(initial_parents_with_values); + + // Create a parent in the `chunks` buffer for every two chunks in `bytes`. + // + // I.e., do the first round of hashing, hashing from the `bytes` slice and filling the `chunks` + // struct. + for i in 0..initial_parents_with_values { + let start = i * BYTES_PER_CHUNK * 2; + + // Hash two chunks, creating a parent chunk. + let hash = match bytes.get(start..start + BYTES_PER_CHUNK * 2) { + // All bytes are available, hash as usual. + Some(slice) => hash(slice), + // Unable to get all the bytes, get a small slice and pad it out. + None => { + let mut preimage = bytes + .get(start..) + .expect("`i` can only be larger than zero if there are bytes to read") + .to_vec(); + preimage.resize(BYTES_PER_CHUNK * 2, 0); + hash(&preimage) + } + }; + + assert_eq!( + hash.len(), + BYTES_PER_CHUNK, + "Hashes should be exactly one chunk" + ); + + // Store the parent node. + chunks + .set(i, &hash) + .expect("Buffer should always have capacity for parent nodes") + } + + // Iterate through all heights above the leaf nodes and either (a) hash two children or, (b) + // hash a left child and a right padding node. + // + // Skip the 0'th height because the leaves have already been processed. Skip the highest-height + // in the tree as it is the root does not require hashing. + // + // The padding nodes for each height are cached via `lazy static` to simulate non-adjacent + // padding nodes (i.e., avoid doing unnecessary hashing). + for height in 1..height - 1 { + let child_nodes = chunks.len(); + let parent_nodes = next_even_number(child_nodes) / 2; + + // For each pair of nodes stored in `chunks`: + // + // - If two nodes are available, hash them to form a parent. + // - If one node is available, hash it and a cached padding node to form a parent. + for i in 0..parent_nodes { + let (left, right) = match (chunks.get(i * 2), chunks.get(i * 2 + 1)) { + (Ok(left), Ok(right)) => (left, right), + (Ok(left), Err(_)) => (left, get_zero_hash(height)), + // Deriving `parent_nodes` from `chunks.len()` has ensured that we never encounter the + // scenario where we expect two nodes but there are none. + (Err(_), Err(_)) => unreachable!("Parent must have one child"), + // `chunks` is a contiguous array so it is impossible for an index to be missing + // when a higher index is present. + (Err(_), Ok(_)) => unreachable!("Parent must have a left child"), + }; + + assert!( + left.len() == right.len() && right.len() == BYTES_PER_CHUNK, + "Both children should be `BYTES_PER_CHUNK` bytes." + ); + + let hash = hash_concat(left, right); + + // Store a parent node. + chunks + .set(i, &hash) + .expect("Buf is adequate size for parent"); + } + + // Shrink the buffer so it neatly fits the number of new nodes created in this round. + // + // The number of `parent_nodes` is either decreasing or stable. It never increases. + chunks.truncate(parent_nodes); + } + + // There should be a single chunk left in the buffer and it is the Merkle root. + let root = chunks.into_vec(); + + assert_eq!(root.len(), BYTES_PER_CHUNK, "Only one chunk should remain"); + + root +} + +/// A helper struct for storing words of `BYTES_PER_CHUNK` size in a flat byte array. +#[derive(Debug)] +struct ChunkStore(Vec); + +impl ChunkStore { + /// Creates a new instance with `chunks` padding nodes. + fn with_capacity(chunks: usize) -> Self { + Self(vec![0; chunks * BYTES_PER_CHUNK]) + } + + /// Set the `i`th chunk to `value`. + /// + /// Returns `Err` if `value.len() != BYTES_PER_CHUNK` or `i` is out-of-bounds. + fn set(&mut self, i: usize, value: &[u8]) -> Result<(), ()> { + if i < self.len() && value.len() == BYTES_PER_CHUNK { + let slice = &mut self.0[i * BYTES_PER_CHUNK..i * BYTES_PER_CHUNK + BYTES_PER_CHUNK]; + slice.copy_from_slice(value); + Ok(()) + } else { + Err(()) + } + } + + /// Gets the `i`th chunk. + /// + /// Returns `Err` if `i` is out-of-bounds. + fn get(&self, i: usize) -> Result<&[u8], ()> { + if i < self.len() { + Ok(&self.0[i * BYTES_PER_CHUNK..i * BYTES_PER_CHUNK + BYTES_PER_CHUNK]) + } else { + Err(()) + } + } + + /// Returns the number of chunks presently stored in `self`. + fn len(&self) -> usize { + self.0.len() / BYTES_PER_CHUNK + } + + /// Truncates 'self' to `num_chunks` chunks. + /// + /// Functionally identical to `Vec::truncate`. + fn truncate(&mut self, num_chunks: usize) { + self.0.truncate(num_chunks * BYTES_PER_CHUNK) + } + + /// Consumes `self`, returning the underlying byte array. + fn into_vec(self) -> Vec { + self.0 + } +} + +/// Returns a cached padding node for a given height. +fn get_zero_hash(height: usize) -> &'static [u8] { + if height <= MAX_TREE_DEPTH { + &ZERO_HASHES[height] + } else { + panic!("Tree exceeds MAX_TREE_DEPTH of {}", MAX_TREE_DEPTH) + } +} + +/// Concatenate two vectors. +fn concat(mut vec1: Vec, mut vec2: Vec) -> Vec { + vec1.append(&mut vec2); + vec1 +} + +/// Compute the hash of two other hashes concatenated. +fn hash_concat(h1: &[u8], h2: &[u8]) -> Vec { + hash(&concat(h1.to_vec(), h2.to_vec())) +} + +/// Returns the next even number following `n`. If `n` is even, `n` is returned. +fn next_even_number(n: usize) -> usize { + n + n % 2 +} + +#[cfg(test)] +mod test { + use super::*; + + pub fn reference_root(bytes: &[u8]) -> Vec { + crate::merkleize_standard(&bytes)[0..32].to_vec() + } + + macro_rules! common_tests { + ($get_bytes: ident) => { + #[test] + fn zero_value_0_nodes() { + test_against_reference(&$get_bytes(0 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_1_nodes() { + test_against_reference(&$get_bytes(1 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_2_nodes() { + test_against_reference(&$get_bytes(2 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_3_nodes() { + test_against_reference(&$get_bytes(3 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_4_nodes() { + test_against_reference(&$get_bytes(4 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_8_nodes() { + test_against_reference(&$get_bytes(8 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_9_nodes() { + test_against_reference(&$get_bytes(9 * BYTES_PER_CHUNK), 0); + } + + #[test] + fn zero_value_8_nodes_varying_min_length() { + for i in 0..64 { + test_against_reference(&$get_bytes(8 * BYTES_PER_CHUNK), i); + } + } + + #[test] + fn zero_value_range_of_nodes() { + for i in 0..32 * BYTES_PER_CHUNK { + test_against_reference(&$get_bytes(i), 0); + } + } + + #[test] + fn max_tree_depth_min_nodes() { + let input = vec![0; 10 * BYTES_PER_CHUNK]; + let min_nodes = 2usize.pow(MAX_TREE_DEPTH as u32); + assert_eq!( + merkleize_padded(&input, min_nodes), + get_zero_hash(MAX_TREE_DEPTH) + ); + } + }; + } + + mod zero_value { + use super::*; + + fn zero_bytes(bytes: usize) -> Vec { + vec![0; bytes] + } + + common_tests!(zero_bytes); + } + + mod random_value { + use super::*; + use rand::RngCore; + + fn random_bytes(bytes: usize) -> Vec { + let mut bytes = Vec::with_capacity(bytes); + rand::thread_rng().fill_bytes(&mut bytes); + bytes + } + + common_tests!(random_bytes); + } + + fn test_against_reference(input: &[u8], min_nodes: usize) { + let mut reference_input = input.to_vec(); + reference_input.resize( + std::cmp::max( + reference_input.len(), + min_nodes.next_power_of_two() * BYTES_PER_CHUNK, + ), + 0, + ); + + assert_eq!( + reference_root(&reference_input), + merkleize_padded(&input, min_nodes), + "input.len(): {:?}", + input.len() + ); + } +} diff --git a/eth2/utils/tree_hash/src/merkleize.rs b/eth2/utils/tree_hash/src/merkleize_standard.rs similarity index 72% rename from eth2/utils/tree_hash/src/merkleize.rs rename to eth2/utils/tree_hash/src/merkleize_standard.rs index 9482895ec..c47d342dd 100644 --- a/eth2/utils/tree_hash/src/merkleize.rs +++ b/eth2/utils/tree_hash/src/merkleize_standard.rs @@ -1,12 +1,23 @@ use super::*; use hashing::hash; -pub fn merkle_root(bytes: &[u8]) -> Vec { - // TODO: replace this with a more memory efficient method. - efficient_merkleize(&bytes)[0..32].to_vec() -} - -pub fn efficient_merkleize(bytes: &[u8]) -> Vec { +/// Merkleizes bytes and returns the root, using a simple algorithm that does not optimize to avoid +/// processing or storing padding bytes. +/// +/// The input `bytes` will be padded to ensure that the number of leaves is a power-of-two. +/// +/// It is likely a better choice to use [merkleize_padded](fn.merkleize_padded.html) instead. +/// +/// ## CPU Performance +/// +/// Will hash all nodes in the tree, even if they are padding and pre-determined. +/// +/// ## Memory Performance +/// +/// - Duplicates the input `bytes`. +/// - Stores all internal nodes, even if they are padding. +/// - Does not free up unused memory during operation. +pub fn merkleize_standard(bytes: &[u8]) -> Vec { // If the bytes are just one chunk (or less than one chunk) just return them. if bytes.len() <= HASHSIZE { let mut o = bytes.to_vec(); diff --git a/eth2/utils/tree_hash_derive/src/lib.rs b/eth2/utils/tree_hash_derive/src/lib.rs index fe94af181..f815dd529 100644 --- a/eth2/utils/tree_hash_derive/src/lib.rs +++ b/eth2/utils/tree_hash_derive/src/lib.rs @@ -150,7 +150,7 @@ pub fn tree_hash_derive(input: TokenStream) -> TokenStream { leaves.append(&mut self.#idents.tree_hash_root()); )* - tree_hash::merkleize::merkle_root(&leaves) + tree_hash::merkle_root(&leaves) } } }; @@ -180,7 +180,7 @@ pub fn tree_hash_signed_root_derive(input: TokenStream) -> TokenStream { leaves.append(&mut self.#idents.tree_hash_root()); )* - tree_hash::merkleize::merkle_root(&leaves) + tree_hash::merkle_root(&leaves) } } }; diff --git a/eth2/utils/tree_hash_derive/tests/tests.rs b/eth2/utils/tree_hash_derive/tests/tests.rs index d4fd55165..ab11730ff 100644 --- a/eth2/utils/tree_hash_derive/tests/tests.rs +++ b/eth2/utils/tree_hash_derive/tests/tests.rs @@ -1,5 +1,5 @@ use cached_tree_hash::{CachedTreeHash, TreeHashCache}; -use tree_hash::{merkleize::merkle_root, SignedRoot, TreeHash}; +use tree_hash::{merkle_root, SignedRoot, TreeHash}; use tree_hash_derive::{CachedTreeHash, SignedRoot, TreeHash}; #[derive(Clone, Debug, TreeHash, CachedTreeHash)] From 051355925299862ea2298c6f5d24fdf1dfc6fbd0 Mon Sep 17 00:00:00 2001 From: Kirk Baird Date: Tue, 16 Jul 2019 17:28:15 +1000 Subject: [PATCH 71/72] =?UTF-8?q?Fix=20syncing=20bugs=20by=20recursively?= =?UTF-8?q?=20attempting=20to=20process=20parents=20in=20the=20=E2=80=A6?= =?UTF-8?q?=20(#429)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix syncing bugs by recursively attempting to process parents in the import queue, change BlockRootsIterator * Swap from crossbeam channel to tokio mpsc * Recursion fix * Remove exess block processing * Fix network lag, correct attestation topic * Correct network poll logic * Overhaul of SimpleSync and modify BlockRootsIterator to return start_slot * Fix bug in tests relating to StateRootsIterator * Remove old, commented-out heartbeat code. * Tidy docs on import queue enum * Change source logging msg in simple sync * Rename function parameter in simple sync * Use `BestBlockRootsIterator` in `reduced_tree` * Update comments for `BestBlockRootsIterator` * Fix duplicate dep in cargo.toml --- beacon_node/Cargo.toml | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 15 +- beacon_node/beacon_chain/src/test_utils.rs | 2 +- beacon_node/client/Cargo.toml | 2 +- beacon_node/client/src/notifier.rs | 33 ++- beacon_node/eth2-libp2p/Cargo.toml | 2 +- beacon_node/eth2-libp2p/src/service.rs | 2 - beacon_node/http_server/Cargo.toml | 3 +- beacon_node/http_server/src/lib.rs | 3 +- beacon_node/network/Cargo.toml | 3 +- beacon_node/network/src/message_handler.rs | 33 +-- beacon_node/network/src/service.rs | 105 ++++---- beacon_node/network/src/sync/import_queue.rs | 127 ++++++---- beacon_node/network/src/sync/simple_sync.rs | 237 ++++++++++++------- beacon_node/rpc/Cargo.toml | 3 +- beacon_node/rpc/src/attestation.rs | 9 +- beacon_node/rpc/src/beacon_block.rs | 6 +- beacon_node/rpc/src/lib.rs | 3 +- beacon_node/src/main.rs | 2 +- beacon_node/store/src/iter.rs | 167 ++++++++++++- eth2/lmd_ghost/src/reduced_tree.rs | 8 +- 21 files changed, 515 insertions(+), 252 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 9e96f8484..5c8786c70 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -13,7 +13,7 @@ client = { path = "client" } version = { path = "version" } clap = "2.32.0" serde = "1.0" -slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } +slog = { version = "^2.2.3" , features = ["max_level_trace"] } slog-term = "^2.4.0" slog-async = "^2.3.0" ctrlc = { version = "3.1.1", features = ["termination"] } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 2d8282270..96ebe4b41 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -18,7 +18,7 @@ use state_processing::{ per_slot_processing, BlockProcessingError, }; use std::sync::Arc; -use store::iter::{BlockIterator, BlockRootsIterator, StateRootsIterator}; +use store::iter::{BestBlockRootsIterator, BlockIterator, BlockRootsIterator, StateRootsIterator}; use store::{Error as DBError, Store}; use tree_hash::TreeHash; use types::*; @@ -226,6 +226,19 @@ impl BeaconChain { BlockRootsIterator::owned(self.store.clone(), self.state.read().clone(), slot) } + /// Iterates in reverse (highest to lowest slot) through all block roots from largest + /// `slot <= beacon_state.slot` through to genesis. + /// + /// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. + /// + /// Contains duplicate roots when skip slots are encountered. + pub fn rev_iter_best_block_roots( + &self, + slot: Slot, + ) -> BestBlockRootsIterator { + BestBlockRootsIterator::owned(self.store.clone(), self.state.read().clone(), slot) + } + /// Iterates in reverse (highest to lowest slot) through all state roots from `slot` through to /// genesis. /// diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 9b3f7c1cb..991d29418 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -191,7 +191,7 @@ where fn get_state_at_slot(&self, state_slot: Slot) -> BeaconState { let state_root = self .chain - .rev_iter_state_roots(self.chain.current_state().slot) + .rev_iter_state_roots(self.chain.current_state().slot - 1) .find(|(_hash, slot)| *slot == state_slot) .map(|(hash, _slot)| hash) .expect("could not find state root"); diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index d3b1e6294..7afaf92ac 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -20,7 +20,7 @@ serde = "1.0.93" serde_derive = "1.0" error-chain = "0.12.0" eth2_ssz = { path = "../../eth2/utils/ssz" } -slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } +slog = { version = "^2.2.3" , features = ["max_level_trace"] } slog-async = "^2.3.0" slog-json = "^2.3" slog-term = "^2.4.0" diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 977342b1a..987f064a4 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -3,39 +3,34 @@ use beacon_chain::BeaconChainTypes; use exit_future::Exit; use futures::{Future, Stream}; use slog::{debug, o}; -use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; -/// Thread that monitors the client and reports useful statistics to the user. +/// The interval between heartbeat events. +pub const HEARTBEAT_INTERVAL_SECONDS: u64 = 5; +/// Spawns a thread that can be used to run code periodically, on `HEARTBEAT_INTERVAL_SECONDS` +/// durations. +/// +/// Presently unused, but remains for future use. pub fn run( client: &Client, executor: TaskExecutor, exit: Exit, ) { // notification heartbeat - let interval = Interval::new(Instant::now(), Duration::from_secs(5)); + let interval = Interval::new( + Instant::now(), + Duration::from_secs(HEARTBEAT_INTERVAL_SECONDS), + ); let _log = client.log.new(o!("Service" => "Notifier")); - // TODO: Debugging only - let counter = Arc::new(Mutex::new(0)); - let network = client.network.clone(); - - // build heartbeat logic here - let heartbeat = move |_| { - //debug!(log, "Temp heartbeat output"); - //TODO: Remove this logic. Testing only - let mut count = counter.lock().unwrap(); - *count += 1; - - if *count % 5 == 0 { - // debug!(log, "Sending Message"); - network.send_message(); - } - + let heartbeat = |_| { + // There is not presently any heartbeat logic. + // + // We leave this function empty for future use. Ok(()) }; diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 1fbd30872..2fbedf780 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -15,7 +15,7 @@ serde = "1.0" serde_derive = "1.0" eth2_ssz = { path = "../../eth2/utils/ssz" } eth2_ssz_derive = { path = "../../eth2/utils/ssz_derive" } -slog = { version = "^2.4.1" , features = ["max_level_trace", "release_max_level_trace"] } +slog = { version = "^2.4.1" , features = ["max_level_trace"] } version = { path = "../version" } tokio = "0.1.16" futures = "0.1.25" diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 69f8a1ca5..2eecfac97 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -109,8 +109,6 @@ impl Stream for Service { fn poll(&mut self) -> Poll, Self::Error> { loop { - // TODO: Currently only gossipsub events passed here. - // Build a type for more generic events match self.swarm.poll() { //Behaviour events Ok(Async::Ready(Some(event))) => match event { diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml index 9a7a4b0a5..3e428357d 100644 --- a/beacon_node/http_server/Cargo.toml +++ b/beacon_node/http_server/Cargo.toml @@ -27,9 +27,8 @@ futures = "0.1.23" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" -slog = "^2.2.3" +slog = { version = "^2.2.3" , features = ["max_level_trace"] } slog-term = "^2.4.0" slog-async = "^2.3.0" tokio = "0.1.17" exit-future = "0.1.4" -crossbeam-channel = "0.3.8" diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index ab1176d61..f1d006a5b 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -14,6 +14,7 @@ use slog::{info, o, warn}; use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::TaskExecutor; +use tokio::sync::mpsc; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct HttpServerConfig { @@ -75,7 +76,7 @@ pub fn create_iron_http_server( pub fn start_service( config: &HttpServerConfig, executor: &TaskExecutor, - _network_chan: crossbeam_channel::Sender, + _network_chan: mpsc::UnboundedSender, beacon_chain: Arc>, db_path: PathBuf, metrics_registry: Registry, diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 23fbdd7d9..1499ac580 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -13,10 +13,9 @@ store = { path = "../store" } eth2-libp2p = { path = "../eth2-libp2p" } version = { path = "../version" } types = { path = "../../eth2/types" } -slog = { version = "^2.2.3" } +slog = { version = "^2.2.3" , features = ["max_level_trace"] } eth2_ssz = { path = "../../eth2/utils/ssz" } tree_hash = { path = "../../eth2/utils/tree_hash" } futures = "0.1.25" error-chain = "0.12.0" -crossbeam-channel = "0.3.8" tokio = "0.1.16" diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index 40a396c3b..40538798a 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -2,17 +2,18 @@ use crate::error; use crate::service::{NetworkMessage, OutgoingMessage}; use crate::sync::SimpleSync; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use crossbeam_channel::{unbounded as channel, Sender}; use eth2_libp2p::{ behaviour::PubsubMessage, rpc::{methods::GoodbyeReason, RPCRequest, RPCResponse, RequestId}, PeerId, RPCEvent, }; -use futures::future; +use futures::future::Future; +use futures::stream::Stream; use slog::{debug, warn}; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; +use tokio::sync::mpsc; /// Timeout for RPC requests. // const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); @@ -48,13 +49,13 @@ impl MessageHandler { /// Initializes and runs the MessageHandler. pub fn spawn( beacon_chain: Arc>, - network_send: crossbeam_channel::Sender, + network_send: mpsc::UnboundedSender, executor: &tokio::runtime::TaskExecutor, log: slog::Logger, - ) -> error::Result> { + ) -> error::Result> { debug!(log, "Service starting"); - let (handler_send, handler_recv) = channel(); + let (handler_send, handler_recv) = mpsc::unbounded_channel(); // Initialise sync and begin processing in thread // generate the Message handler @@ -69,13 +70,13 @@ impl MessageHandler { // spawn handler task // TODO: Handle manual termination of thread - executor.spawn(future::poll_fn(move || -> Result<_, _> { - loop { - handler.handle_message(handler_recv.recv().map_err(|_| { + executor.spawn( + handler_recv + .for_each(move |msg| Ok(handler.handle_message(msg))) + .map_err(move |_| { debug!(log, "Network message handler terminated."); - })?); - } - })); + }), + ); Ok(handler_send) } @@ -222,7 +223,7 @@ impl MessageHandler { pub struct NetworkContext { /// The network channel to relay messages to the Network service. - network_send: crossbeam_channel::Sender, + network_send: mpsc::UnboundedSender, /// A mapping of peers and the RPC id we have sent an RPC request to. outstanding_outgoing_request_ids: HashMap<(PeerId, RequestId), Instant>, /// Stores the next `RequestId` we should include on an outgoing `RPCRequest` to a `PeerId`. @@ -232,7 +233,7 @@ pub struct NetworkContext { } impl NetworkContext { - pub fn new(network_send: crossbeam_channel::Sender, log: slog::Logger) -> Self { + pub fn new(network_send: mpsc::UnboundedSender, log: slog::Logger) -> Self { Self { network_send, outstanding_outgoing_request_ids: HashMap::new(), @@ -278,13 +279,13 @@ impl NetworkContext { ); } - fn send_rpc_event(&self, peer_id: PeerId, rpc_event: RPCEvent) { + fn send_rpc_event(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { self.send(peer_id, OutgoingMessage::RPC(rpc_event)) } - fn send(&self, peer_id: PeerId, outgoing_message: OutgoingMessage) { + fn send(&mut self, peer_id: PeerId, outgoing_message: OutgoingMessage) { self.network_send - .send(NetworkMessage::Send(peer_id, outgoing_message)) + .try_send(NetworkMessage::Send(peer_id, outgoing_message)) .unwrap_or_else(|_| { warn!( self.log, diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index b2ecc1a0b..a2265bb8e 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -2,24 +2,23 @@ use crate::error; use crate::message_handler::{HandlerMessage, MessageHandler}; use crate::NetworkConfig; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use crossbeam_channel::{unbounded as channel, Sender, TryRecvError}; use eth2_libp2p::Service as LibP2PService; use eth2_libp2p::Topic; use eth2_libp2p::{Libp2pEvent, PeerId}; use eth2_libp2p::{PubsubMessage, RPCEvent}; use futures::prelude::*; -use futures::sync::oneshot; use futures::Stream; use slog::{debug, info, o, trace}; use std::marker::PhantomData; use std::sync::Arc; use tokio::runtime::TaskExecutor; +use tokio::sync::{mpsc, oneshot}; /// Service that handles communication between internal services and the eth2_libp2p network service. pub struct Service { //libp2p_service: Arc>, _libp2p_exit: oneshot::Sender<()>, - network_send: crossbeam_channel::Sender, + network_send: mpsc::UnboundedSender, _phantom: PhantomData, //message_handler: MessageHandler, //message_handler_send: Sender } @@ -30,9 +29,9 @@ impl Service { config: &NetworkConfig, executor: &TaskExecutor, log: slog::Logger, - ) -> error::Result<(Arc, Sender)> { + ) -> error::Result<(Arc, mpsc::UnboundedSender)> { // build the network channel - let (network_send, network_recv) = channel::(); + let (network_send, network_recv) = mpsc::unbounded_channel::(); // launch message handler thread let message_handler_log = log.new(o!("Service" => "MessageHandler")); let message_handler_send = MessageHandler::spawn( @@ -64,9 +63,9 @@ impl Service { } // TODO: Testing only - pub fn send_message(&self) { + pub fn send_message(&mut self) { self.network_send - .send(NetworkMessage::Send( + .try_send(NetworkMessage::Send( PeerId::random(), OutgoingMessage::NotifierTest, )) @@ -76,12 +75,12 @@ impl Service { fn spawn_service( libp2p_service: LibP2PService, - network_recv: crossbeam_channel::Receiver, - message_handler_send: crossbeam_channel::Sender, + network_recv: mpsc::UnboundedReceiver, + message_handler_send: mpsc::UnboundedSender, executor: &TaskExecutor, log: slog::Logger, -) -> error::Result> { - let (network_exit, exit_rx) = oneshot::channel(); +) -> error::Result> { + let (network_exit, exit_rx) = tokio::sync::oneshot::channel(); // spawn on the current executor executor.spawn( @@ -105,25 +104,61 @@ fn spawn_service( //TODO: Potentially handle channel errors fn network_service( mut libp2p_service: LibP2PService, - network_recv: crossbeam_channel::Receiver, - message_handler_send: crossbeam_channel::Sender, + mut network_recv: mpsc::UnboundedReceiver, + mut message_handler_send: mpsc::UnboundedSender, log: slog::Logger, ) -> impl futures::Future { futures::future::poll_fn(move || -> Result<_, eth2_libp2p::error::Error> { - // poll the swarm - loop { + // only end the loop once both major polls are not ready. + let mut not_ready_count = 0; + while not_ready_count < 2 { + not_ready_count = 0; + // poll the network channel + match network_recv.poll() { + Ok(Async::Ready(Some(message))) => { + match message { + // TODO: Testing message - remove + NetworkMessage::Send(peer_id, outgoing_message) => { + match outgoing_message { + OutgoingMessage::RPC(rpc_event) => { + trace!(log, "Sending RPC Event: {:?}", rpc_event); + //TODO: Make swarm private + //TODO: Implement correct peer id topic message handling + libp2p_service.swarm.send_rpc(peer_id, rpc_event); + } + OutgoingMessage::NotifierTest => { + // debug!(log, "Received message from notifier"); + } + }; + } + NetworkMessage::Publish { topics, message } => { + debug!(log, "Sending pubsub message"; "topics" => format!("{:?}",topics)); + libp2p_service.swarm.publish(topics, *message); + } + } + } + Ok(Async::NotReady) => not_ready_count += 1, + Ok(Async::Ready(None)) => { + return Err(eth2_libp2p::error::Error::from("Network channel closed")); + } + Err(_) => { + return Err(eth2_libp2p::error::Error::from("Network channel error")); + } + } + + // poll the swarm match libp2p_service.poll() { Ok(Async::Ready(Some(event))) => match event { Libp2pEvent::RPC(peer_id, rpc_event) => { trace!(log, "RPC Event: RPC message received: {:?}", rpc_event); message_handler_send - .send(HandlerMessage::RPC(peer_id, rpc_event)) + .try_send(HandlerMessage::RPC(peer_id, rpc_event)) .map_err(|_| "failed to send rpc to handler")?; } Libp2pEvent::PeerDialed(peer_id) => { debug!(log, "Peer Dialed: {:?}", peer_id); message_handler_send - .send(HandlerMessage::PeerDialed(peer_id)) + .try_send(HandlerMessage::PeerDialed(peer_id)) .map_err(|_| "failed to send rpc to handler")?; } Libp2pEvent::PubsubMessage { @@ -132,43 +167,13 @@ fn network_service( //TODO: Decide if we need to propagate the topic upwards. (Potentially for //attestations) message_handler_send - .send(HandlerMessage::PubsubMessage(source, message)) + .try_send(HandlerMessage::PubsubMessage(source, message)) .map_err(|_| " failed to send pubsub message to handler")?; } }, Ok(Async::Ready(None)) => unreachable!("Stream never ends"), - Ok(Async::NotReady) => break, - Err(_) => break, - } - } - // poll the network channel - // TODO: refactor - combine poll_fn's? - loop { - match network_recv.try_recv() { - // TODO: Testing message - remove - Ok(NetworkMessage::Send(peer_id, outgoing_message)) => { - match outgoing_message { - OutgoingMessage::RPC(rpc_event) => { - trace!(log, "Sending RPC Event: {:?}", rpc_event); - //TODO: Make swarm private - //TODO: Implement correct peer id topic message handling - libp2p_service.swarm.send_rpc(peer_id, rpc_event); - } - OutgoingMessage::NotifierTest => { - // debug!(log, "Received message from notifier"); - } - }; - } - Ok(NetworkMessage::Publish { topics, message }) => { - debug!(log, "Sending pubsub message on topics {:?}", topics); - libp2p_service.swarm.publish(topics, *message); - } - Err(TryRecvError::Empty) => break, - Err(TryRecvError::Disconnected) => { - return Err(eth2_libp2p::error::Error::from( - "Network channel disconnected", - )); - } + Ok(Async::NotReady) => not_ready_count += 1, + Err(_) => not_ready_count += 1, } } Ok(Async::NotReady) diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 8cc3dd65d..fe640aaa0 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -41,31 +41,23 @@ impl ImportQueue { } } - /// Completes all possible partials into `BeaconBlock` and returns them, sorted by increasing - /// slot number. Does not delete the partials from the queue, this must be done manually. - /// - /// Returns `(queue_index, block, sender)`: - /// - /// - `block_root`: may be used to remove the entry if it is successfully processed. - /// - `block`: the completed block. - /// - `sender`: the `PeerId` the provided the `BeaconBlockBody` which completed the partial. - pub fn complete_blocks(&self) -> Vec<(Hash256, BeaconBlock, PeerId)> { - let mut complete: Vec<(Hash256, BeaconBlock, PeerId)> = self - .partials - .iter() - .filter_map(|(_, partial)| partial.clone().complete()) - .collect(); - - // Sort the completable partials to be in ascending slot order. - complete.sort_unstable_by(|a, b| a.1.slot.partial_cmp(&b.1.slot).unwrap()); - - complete - } - + /// Returns true of the if the `BlockRoot` is found in the `import_queue`. pub fn contains_block_root(&self, block_root: Hash256) -> bool { self.partials.contains_key(&block_root) } + /// Attempts to complete the `BlockRoot` if it is found in the `import_queue`. + /// + /// Returns an Enum with a `PartialBeaconBlockCompletion`. + /// Does not remove the `block_root` from the `import_queue`. + pub fn attempt_complete_block(&self, block_root: Hash256) -> PartialBeaconBlockCompletion { + if let Some(partial) = self.partials.get(&block_root) { + partial.attempt_complete() + } else { + PartialBeaconBlockCompletion::MissingRoot + } + } + /// Removes the first `PartialBeaconBlock` with a matching `block_root`, returning the partial /// if it exists. pub fn remove(&mut self, block_root: Hash256) -> Option { @@ -102,6 +94,8 @@ impl ImportQueue { block_roots: &[BlockRootSlot], sender: PeerId, ) -> Vec { + // TODO: This will currently not return a `BlockRootSlot` if this root exists but there is no header. + // It would be more robust if it did. let new_block_root_slots: Vec = block_roots .iter() // Ignore any roots already stored in the queue. @@ -135,12 +129,8 @@ impl ImportQueue { /// the queue and it's block root is included in the output. /// /// If a `header` is already in the queue, but not yet processed by the chain the block root is - /// included in the output and the `inserted` time for the partial record is set to + /// not included in the output and the `inserted` time for the partial record is set to /// `Instant::now()`. Updating the `inserted` time stops the partial from becoming stale. - /// - /// Presently the queue enforces that a `BeaconBlockHeader` _must_ be received before its - /// `BeaconBlockBody`. This is not a natural requirement and we could enhance the queue to lift - /// this restraint. pub fn enqueue_headers( &mut self, headers: Vec, @@ -152,8 +142,10 @@ impl ImportQueue { let block_root = Hash256::from_slice(&header.canonical_root()[..]); if self.chain_has_not_seen_block(&block_root) { - self.insert_header(block_root, header, sender.clone()); - required_bodies.push(block_root); + if !self.insert_header(block_root, header, sender.clone()) { + // If a body is empty + required_bodies.push(block_root); + } } } @@ -163,10 +155,17 @@ impl ImportQueue { /// If there is a matching `header` for this `body`, adds it to the queue. /// /// If there is no `header` for the `body`, the body is simply discarded. - pub fn enqueue_bodies(&mut self, bodies: Vec, sender: PeerId) { + pub fn enqueue_bodies( + &mut self, + bodies: Vec, + sender: PeerId, + ) -> Option { + let mut last_block_hash = None; for body in bodies { - self.insert_body(body, sender.clone()); + last_block_hash = self.insert_body(body, sender.clone()); } + + last_block_hash } pub fn enqueue_full_blocks(&mut self, blocks: Vec, sender: PeerId) { @@ -179,12 +178,22 @@ impl ImportQueue { /// /// If the header already exists, the `inserted` time is set to `now` and not other /// modifications are made. - fn insert_header(&mut self, block_root: Hash256, header: BeaconBlockHeader, sender: PeerId) { + /// Returns true is `body` exists. + fn insert_header( + &mut self, + block_root: Hash256, + header: BeaconBlockHeader, + sender: PeerId, + ) -> bool { + let mut exists = false; self.partials .entry(block_root) .and_modify(|partial| { partial.header = Some(header.clone()); partial.inserted = Instant::now(); + if partial.body.is_some() { + exists = true; + } }) .or_insert_with(|| PartialBeaconBlock { slot: header.slot, @@ -194,28 +203,30 @@ impl ImportQueue { inserted: Instant::now(), sender, }); + exists } /// Updates an existing partial with the `body`. /// - /// If there is no header for the `body`, the body is simply discarded. - /// /// If the body already existed, the `inserted` time is set to `now`. - fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) { + /// + /// Returns the block hash of the inserted body + fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) -> Option { let body_root = Hash256::from_slice(&body.tree_hash_root()[..]); + let mut last_root = None; - self.partials.iter_mut().for_each(|(_, mut p)| { + self.partials.iter_mut().for_each(|(root, mut p)| { if let Some(header) = &mut p.header { if body_root == header.block_body_root { p.inserted = Instant::now(); - - if p.body.is_none() { - p.body = Some(body.clone()); - p.sender = sender.clone(); - } + p.body = Some(body.clone()); + p.sender = sender.clone(); + last_root = Some(*root); } } }); + + last_root } /// Updates an existing `partial` with the completed block, or adds a new (complete) partial. @@ -257,13 +268,33 @@ pub struct PartialBeaconBlock { } impl PartialBeaconBlock { - /// Consumes `self` and returns a full built `BeaconBlock`, it's root and the `sender` - /// `PeerId`, if enough information exists to complete the block. Otherwise, returns `None`. - pub fn complete(self) -> Option<(Hash256, BeaconBlock, PeerId)> { - Some(( - self.block_root, - self.header?.into_block(self.body?), - self.sender, - )) + /// Attempts to build a block. + /// + /// Does not comsume the `PartialBeaconBlock`. + pub fn attempt_complete(&self) -> PartialBeaconBlockCompletion { + if self.header.is_none() { + PartialBeaconBlockCompletion::MissingHeader(self.slot) + } else if self.body.is_none() { + PartialBeaconBlockCompletion::MissingBody + } else { + PartialBeaconBlockCompletion::Complete( + self.header + .clone() + .unwrap() + .into_block(self.body.clone().unwrap()), + ) + } } } + +/// The result of trying to convert a `BeaconBlock` into a `PartialBeaconBlock`. +pub enum PartialBeaconBlockCompletion { + /// The partial contains a valid BeaconBlock. + Complete(BeaconBlock), + /// The partial does not exist. + MissingRoot, + /// The partial contains a `BeaconBlockRoot` but no `BeaconBlockHeader`. + MissingHeader(Slot), + /// The partial contains a `BeaconBlockRoot` and `BeaconBlockHeader` but no `BeaconBlockBody`. + MissingBody, +} diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 5899e5aea..5ce921057 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,4 +1,4 @@ -use super::import_queue::ImportQueue; +use super::import_queue::{ImportQueue, PartialBeaconBlockCompletion}; use crate::message_handler::NetworkContext; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; @@ -17,7 +17,7 @@ use types::{ const SLOT_IMPORT_TOLERANCE: u64 = 100; /// The amount of seconds a block (or partial block) may exist in the import queue. -const QUEUE_STALE_SECS: u64 = 6; +const QUEUE_STALE_SECS: u64 = 100; /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it. /// Otherwise we queue it. @@ -227,7 +227,12 @@ impl SimpleSync { // // Therefore, there are some blocks between the local finalized epoch and the remote // head that are worth downloading. - debug!(self.log, "UsefulPeer"; "peer" => format!("{:?}", peer_id)); + debug!( + self.log, "UsefulPeer"; + "peer" => format!("{:?}", peer_id), + "local_finalized_epoch" => local.latest_finalized_epoch, + "remote_latest_finalized_epoch" => remote.latest_finalized_epoch, + ); let start_slot = local .latest_finalized_epoch @@ -238,7 +243,7 @@ impl SimpleSync { peer_id, BeaconBlockRootsRequest { start_slot, - count: required_slots.into(), + count: required_slots.as_u64(), }, network, ); @@ -247,7 +252,7 @@ impl SimpleSync { fn root_at_slot(&self, target_slot: Slot) -> Option { self.chain - .rev_iter_block_roots(target_slot) + .rev_iter_best_block_roots(target_slot) .take(1) .find(|(_root, slot)| *slot == target_slot) .map(|(root, _slot)| root) @@ -271,8 +276,7 @@ impl SimpleSync { let mut roots: Vec = self .chain - .rev_iter_block_roots(req.start_slot + req.count) - .skip(1) + .rev_iter_best_block_roots(req.start_slot + req.count) .take(req.count as usize) .map(|(block_root, slot)| BlockRootSlot { slot, block_root }) .collect(); @@ -356,7 +360,7 @@ impl SimpleSync { BeaconBlockHeadersRequest { start_root: first.block_root, start_slot: first.slot, - max_headers: (last.slot - first.slot).as_u64(), + max_headers: (last.slot - first.slot + 1).as_u64(), skip_slots: 0, }, network, @@ -386,7 +390,7 @@ impl SimpleSync { // unnecessary block deserialization when `req.skip_slots > 0`. let mut roots: Vec = self .chain - .rev_iter_block_roots(req.start_slot + (count - 1)) + .rev_iter_best_block_roots(req.start_slot + count) .take(count as usize) .map(|(root, _slot)| root) .collect(); @@ -499,14 +503,26 @@ impl SimpleSync { "count" => res.block_bodies.len(), ); - self.import_queue - .enqueue_bodies(res.block_bodies, peer_id.clone()); + if !res.block_bodies.is_empty() { + // Import all blocks to queue + let last_root = self + .import_queue + .enqueue_bodies(res.block_bodies, peer_id.clone()); + + // Attempt to process all recieved bodies by recursively processing the latest block + if let Some(root) = last_root { + match self.attempt_process_partial_block(peer_id, root, network, &"rpc") { + Some(BlockProcessingOutcome::Processed { block_root: _ }) => { + // If processing is successful remove from `import_queue` + self.import_queue.remove(root); + } + _ => {} + } + } + } // Clear out old entries self.import_queue.remove_stale(); - - // Import blocks, if possible. - self.process_import_queue(network); } /// Process a gossip message declaring a new block. @@ -526,31 +542,35 @@ impl SimpleSync { match outcome { BlockProcessingOutcome::Processed { .. } => SHOULD_FORWARD_GOSSIP_BLOCK, BlockProcessingOutcome::ParentUnknown { parent } => { - // Clean the stale entries from the queue. - self.import_queue.remove_stale(); - // Add this block to the queue self.import_queue - .enqueue_full_blocks(vec![block], peer_id.clone()); - trace!( - self.log, - "NewGossipBlock"; + .enqueue_full_blocks(vec![block.clone()], peer_id.clone()); + debug!( + self.log, "RequestParentBlock"; + "parent_root" => format!("{}", parent), + "parent_slot" => block.slot - 1, "peer" => format!("{:?}", peer_id), ); - // Unless the parent is in the queue, request the parent block from the peer. - // - // It is likely that this is duplicate work, given we already send a hello - // request. However, I believe there are some edge-cases where the hello - // message doesn't suffice, so we perform this request as well. - if !self.import_queue.contains_block_root(parent) { - // Send a hello to learn of the clients best slot so we can then sync the required - // parent(s). - network.send_rpc_request( - peer_id.clone(), - RPCRequest::Hello(hello_message(&self.chain)), - ); - } + // Request roots between parent and start of finality from peer. + let start_slot = self + .chain + .head() + .beacon_state + .finalized_epoch + .start_slot(T::EthSpec::slots_per_epoch()); + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + // Request blocks between `latest_finalized_slot` and the `block` + start_slot, + count: block.slot.as_u64() - start_slot.as_u64(), + }, + network, + ); + + // Clean the stale entries from the queue. + self.import_queue.remove_stale(); SHOULD_FORWARD_GOSSIP_BLOCK } @@ -592,40 +612,6 @@ impl SimpleSync { } } - /// Iterate through the `import_queue` and process any complete blocks. - /// - /// If a block is successfully processed it is removed from the queue, otherwise it remains in - /// the queue. - pub fn process_import_queue(&mut self, network: &mut NetworkContext) { - let mut successful = 0; - - // Loop through all of the complete blocks in the queue. - for (block_root, block, sender) in self.import_queue.complete_blocks() { - let processing_result = self.process_block(sender, block.clone(), network, &"gossip"); - - let should_dequeue = match processing_result { - Some(BlockProcessingOutcome::ParentUnknown { .. }) => false, - Some(BlockProcessingOutcome::FutureSlot { - present_slot, - block_slot, - }) if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot => false, - _ => true, - }; - - if processing_result == Some(BlockProcessingOutcome::Processed { block_root }) { - successful += 1; - } - - if should_dequeue { - self.import_queue.remove(block_root); - } - } - - if successful > 0 { - info!(self.log, "Imported {} blocks", successful) - } - } - /// Request some `BeaconBlockRoots` from the remote peer. fn request_block_roots( &mut self, @@ -700,6 +686,89 @@ impl SimpleSync { hello_message(&self.chain) } + /// Helper function to attempt to process a partial block. + /// + /// If the block can be completed recursively call `process_block` + /// else request missing parts. + fn attempt_process_partial_block( + &mut self, + peer_id: PeerId, + block_root: Hash256, + network: &mut NetworkContext, + source: &str, + ) -> Option { + match self.import_queue.attempt_complete_block(block_root) { + PartialBeaconBlockCompletion::MissingBody => { + // Unable to complete the block because the block body is missing. + debug!( + self.log, "RequestParentBody"; + "source" => source, + "block_root" => format!("{}", block_root), + "peer" => format!("{:?}", peer_id), + ); + + // Request the block body from the peer. + self.request_block_bodies( + peer_id, + BeaconBlockBodiesRequest { + block_roots: vec![block_root], + }, + network, + ); + + None + } + PartialBeaconBlockCompletion::MissingHeader(slot) => { + // Unable to complete the block because the block header is missing. + debug!( + self.log, "RequestParentHeader"; + "source" => source, + "block_root" => format!("{}", block_root), + "peer" => format!("{:?}", peer_id), + ); + + // Request the block header from the peer. + self.request_block_headers( + peer_id, + BeaconBlockHeadersRequest { + start_root: block_root, + start_slot: slot, + max_headers: 1, + skip_slots: 0, + }, + network, + ); + + None + } + PartialBeaconBlockCompletion::MissingRoot => { + // The `block_root` is not known to the queue. + debug!( + self.log, "MissingParentRoot"; + "source" => source, + "block_root" => format!("{}", block_root), + "peer" => format!("{:?}", peer_id), + ); + + // Do nothing. + + None + } + PartialBeaconBlockCompletion::Complete(block) => { + // The block exists in the queue, attempt to process it + trace!( + self.log, "AttemptProcessParent"; + "source" => source, + "block_root" => format!("{}", block_root), + "parent_slot" => block.slot, + "peer" => format!("{:?}", peer_id), + ); + + self.process_block(peer_id.clone(), block, network, source) + } + } + } + /// Processes the `block` that was received from `peer_id`. /// /// If the block was submitted to the beacon chain without internal error, `Some(outcome)` is @@ -726,6 +795,7 @@ impl SimpleSync { if let Ok(outcome) = processing_result { match outcome { BlockProcessingOutcome::Processed { block_root } => { + // The block was valid and we processed it successfully. debug!( self.log, "Imported block from network"; "source" => source, @@ -735,26 +805,29 @@ impl SimpleSync { ); } BlockProcessingOutcome::ParentUnknown { parent } => { - // The block was valid and we processed it successfully. - debug!( + // The parent has not been processed + trace!( self.log, "ParentBlockUnknown"; "source" => source, "parent_root" => format!("{}", parent), + "baby_block_slot" => block.slot, "peer" => format!("{:?}", peer_id), ); - // Unless the parent is in the queue, request the parent block from the peer. - // - // It is likely that this is duplicate work, given we already send a hello - // request. However, I believe there are some edge-cases where the hello - // message doesn't suffice, so we perform this request as well. - if !self.import_queue.contains_block_root(parent) { - // Send a hello to learn of the clients best slot so we can then sync the require - // parent(s). - network.send_rpc_request( - peer_id.clone(), - RPCRequest::Hello(hello_message(&self.chain)), - ); + // If the parent is in the `import_queue` attempt to complete it then process it. + match self.attempt_process_partial_block(peer_id, parent, network, source) { + // If processing parent is sucessful, re-process block and remove parent from queue + Some(BlockProcessingOutcome::Processed { block_root: _ }) => { + self.import_queue.remove(parent); + + // Attempt to process `block` again + match self.chain.process_block(block) { + Ok(outcome) => return Some(outcome), + Err(_) => return None, + } + } + // All other cases leave `parent` in `import_queue` and return original outcome. + _ => {} } } BlockProcessingOutcome::FutureSlot { diff --git a/beacon_node/rpc/Cargo.toml b/beacon_node/rpc/Cargo.toml index a37492796..99cd78e6a 100644 --- a/beacon_node/rpc/Cargo.toml +++ b/beacon_node/rpc/Cargo.toml @@ -22,9 +22,8 @@ dirs = "1.0.3" futures = "0.1.23" serde = "1.0" serde_derive = "1.0" -slog = "^2.2.3" +slog = { version = "^2.2.3" , features = ["max_level_trace"] } slog-term = "^2.4.0" slog-async = "^2.3.0" tokio = "0.1.17" exit-future = "0.1.4" -crossbeam-channel = "0.3.8" diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index 86f4331f1..b85d4e947 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,7 +1,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::PubsubMessage; use eth2_libp2p::TopicBuilder; -use eth2_libp2p::SHARD_TOPIC_PREFIX; +use eth2_libp2p::BEACON_ATTESTATION_TOPIC; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use network::NetworkMessage; @@ -13,12 +13,13 @@ use protos::services_grpc::AttestationService; use slog::{error, info, trace, warn}; use ssz::{ssz_encode, Decode}; use std::sync::Arc; +use tokio::sync::mpsc; use types::Attestation; #[derive(Clone)] pub struct AttestationServiceInstance { pub chain: Arc>, - pub network_chan: crossbeam_channel::Sender, + pub network_chan: mpsc::UnboundedSender, pub log: slog::Logger, } @@ -139,11 +140,11 @@ impl AttestationService for AttestationServiceInstance { ); // valid attestation, propagate to the network - let topic = TopicBuilder::new(SHARD_TOPIC_PREFIX).build(); + let topic = TopicBuilder::new(BEACON_ATTESTATION_TOPIC).build(); let message = PubsubMessage::Attestation(attestation); self.network_chan - .send(NetworkMessage::Publish { + .try_send(NetworkMessage::Publish { topics: vec![topic], message: Box::new(message), }) diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index cdf46a1ab..faaf2232a 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -1,5 +1,4 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; -use crossbeam_channel; use eth2_libp2p::BEACON_PUBSUB_TOPIC; use eth2_libp2p::{PubsubMessage, TopicBuilder}; use futures::Future; @@ -14,12 +13,13 @@ use slog::Logger; use slog::{error, info, trace, warn}; use ssz::{ssz_encode, Decode}; use std::sync::Arc; +use tokio::sync::mpsc; use types::{BeaconBlock, Signature, Slot}; #[derive(Clone)] pub struct BeaconBlockServiceInstance { pub chain: Arc>, - pub network_chan: crossbeam_channel::Sender, + pub network_chan: mpsc::UnboundedSender, pub log: Logger, } @@ -111,7 +111,7 @@ impl BeaconBlockService for BeaconBlockServiceInstance { // Publish the block to the p2p network via gossipsub. self.network_chan - .send(NetworkMessage::Publish { + .try_send(NetworkMessage::Publish { topics: vec![topic], message: Box::new(message), }) diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index 3e6fd3e73..eef009292 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -20,11 +20,12 @@ use protos::services_grpc::{ use slog::{info, o, warn}; use std::sync::Arc; use tokio::runtime::TaskExecutor; +use tokio::sync::mpsc; pub fn start_server( config: &RPCConfig, executor: &TaskExecutor, - network_chan: crossbeam_channel::Sender, + network_chan: mpsc::UnboundedSender, beacon_chain: Arc>, log: &slog::Logger, ) -> exit_future::Signal { diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 6beb0fd64..55c86672a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -163,7 +163,7 @@ fn main() { 0 => drain.filter_level(Level::Info), 1 => drain.filter_level(Level::Debug), 2 => drain.filter_level(Level::Trace), - _ => drain.filter_level(Level::Info), + _ => drain.filter_level(Level::Trace), }; let mut log = slog::Logger::root(drain.fuse(), o!()); diff --git a/beacon_node/store/src/iter.rs b/beacon_node/store/src/iter.rs index 76807ce8f..d545cf2fe 100644 --- a/beacon_node/store/src/iter.rs +++ b/beacon_node/store/src/iter.rs @@ -15,15 +15,15 @@ impl<'a, T: EthSpec, U: Store> StateRootsIterator<'a, T, U> { Self { store, beacon_state: Cow::Borrowed(beacon_state), - slot: start_slot, + slot: start_slot + 1, } } pub fn owned(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { Self { - slot: start_slot, - beacon_state: Cow::Owned(beacon_state), store, + beacon_state: Cow::Owned(beacon_state), + slot: start_slot + 1, } } } @@ -90,13 +90,19 @@ impl<'a, T: EthSpec, U: Store> Iterator for BlockIterator<'a, T, U> { } } -/// Iterates backwards through block roots. +/// Iterates backwards through block roots. If any specified slot is unable to be retrieved, the +/// iterator returns `None` indefinitely. /// /// Uses the `latest_block_roots` field of `BeaconState` to as the source of block roots and will /// perform a lookup on the `Store` for a prior `BeaconState` if `latest_block_roots` has been /// exhausted. /// /// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. +/// +/// ## Notes +/// +/// See [`BestBlockRootsIterator`](struct.BestBlockRootsIterator.html), which has different +/// `start_slot` logic. #[derive(Clone)] pub struct BlockRootsIterator<'a, T: EthSpec, U> { store: Arc, @@ -108,18 +114,18 @@ impl<'a, T: EthSpec, U: Store> BlockRootsIterator<'a, T, U> { /// Create a new iterator over all block roots in the given `beacon_state` and prior states. pub fn new(store: Arc, beacon_state: &'a BeaconState, start_slot: Slot) -> Self { Self { - slot: start_slot, - beacon_state: Cow::Borrowed(beacon_state), store, + beacon_state: Cow::Borrowed(beacon_state), + slot: start_slot + 1, } } /// Create a new iterator over all block roots in the given `beacon_state` and prior states. pub fn owned(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { Self { - slot: start_slot, - beacon_state: Cow::Owned(beacon_state), store, + beacon_state: Cow::Owned(beacon_state), + slot: start_slot + 1, } } } @@ -156,6 +162,104 @@ impl<'a, T: EthSpec, U: Store> Iterator for BlockRootsIterator<'a, T, U> { } } +/// Iterates backwards through block roots with `start_slot` highest possible value +/// `<= beacon_state.slot`. +/// +/// The distinction between `BestBlockRootsIterator` and `BlockRootsIterator` is: +/// +/// - `BestBlockRootsIterator` uses best-effort slot. When `start_slot` is greater than the latest available block root +/// on `beacon_state`, returns `Some(root, slot)` where `slot` is the latest available block +/// root. +/// - `BlockRootsIterator` is strict about `start_slot`. When `start_slot` is greater than the latest available block root +/// on `beacon_state`, returns `None`. +/// +/// This is distinct from `BestBlockRootsIterator`. +/// +/// Uses the `latest_block_roots` field of `BeaconState` to as the source of block roots and will +/// perform a lookup on the `Store` for a prior `BeaconState` if `latest_block_roots` has been +/// exhausted. +/// +/// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. +#[derive(Clone)] +pub struct BestBlockRootsIterator<'a, T: EthSpec, U> { + store: Arc, + beacon_state: Cow<'a, BeaconState>, + slot: Slot, +} + +impl<'a, T: EthSpec, U: Store> BestBlockRootsIterator<'a, T, U> { + /// Create a new iterator over all block roots in the given `beacon_state` and prior states. + pub fn new(store: Arc, beacon_state: &'a BeaconState, start_slot: Slot) -> Self { + let mut slot = start_slot; + if slot >= beacon_state.slot { + // Slot may be too high. + slot = beacon_state.slot; + if beacon_state.get_block_root(slot).is_err() { + slot -= 1; + } + } + + Self { + store, + beacon_state: Cow::Borrowed(beacon_state), + slot: slot + 1, + } + } + + /// Create a new iterator over all block roots in the given `beacon_state` and prior states. + pub fn owned(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { + let mut slot = start_slot; + if slot >= beacon_state.slot { + // Slot may be too high. + slot = beacon_state.slot; + // TODO: Use a function other than `get_block_root` as this will always return `Err()` + // for slot = state.slot. + if beacon_state.get_block_root(slot).is_err() { + slot -= 1; + } + } + + Self { + store, + beacon_state: Cow::Owned(beacon_state), + slot: slot + 1, + } + } +} + +impl<'a, T: EthSpec, U: Store> Iterator for BestBlockRootsIterator<'a, T, U> { + type Item = (Hash256, Slot); + + fn next(&mut self) -> Option { + if self.slot == 0 { + // End of Iterator + return None; + } + + self.slot -= 1; + + match self.beacon_state.get_block_root(self.slot) { + Ok(root) => Some((*root, self.slot)), + Err(BeaconStateError::SlotOutOfBounds) => { + // Read a `BeaconState` from the store that has access to prior historical root. + let beacon_state: BeaconState = { + // Load the earliest state from disk. + let new_state_root = self.beacon_state.get_oldest_state_root().ok()?; + + self.store.get(&new_state_root).ok()? + }?; + + self.beacon_state = Cow::Owned(beacon_state); + + let root = self.beacon_state.get_block_root(self.slot).ok()?; + + Some((*root, self.slot)) + } + _ => None, + } + } +} + #[cfg(test)] mod test { use super::*; @@ -206,7 +310,50 @@ mod test { let mut collected: Vec<(Hash256, Slot)> = iter.collect(); collected.reverse(); - let expected_len = 2 * MainnetEthSpec::slots_per_historical_root() - 1; + let expected_len = 2 * MainnetEthSpec::slots_per_historical_root(); + + assert_eq!(collected.len(), expected_len); + + for i in 0..expected_len { + assert_eq!(collected[i].0, Hash256::from(i as u64)); + } + } + + #[test] + fn best_block_root_iter() { + let store = Arc::new(MemoryStore::open()); + let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root(); + + let mut state_a: BeaconState = get_state(); + let mut state_b: BeaconState = get_state(); + + state_a.slot = Slot::from(slots_per_historical_root); + state_b.slot = Slot::from(slots_per_historical_root * 2); + + let mut hashes = (0..).into_iter().map(|i| Hash256::from(i)); + + for root in &mut state_a.latest_block_roots[..] { + *root = hashes.next().unwrap() + } + for root in &mut state_b.latest_block_roots[..] { + *root = hashes.next().unwrap() + } + + let state_a_root = hashes.next().unwrap(); + state_b.latest_state_roots[0] = state_a_root; + store.put(&state_a_root, &state_a).unwrap(); + + let iter = BestBlockRootsIterator::new(store.clone(), &state_b, state_b.slot); + + assert!( + iter.clone().find(|(_root, slot)| *slot == 0).is_some(), + "iter should contain zero slot" + ); + + let mut collected: Vec<(Hash256, Slot)> = iter.collect(); + collected.reverse(); + + let expected_len = 2 * MainnetEthSpec::slots_per_historical_root(); assert_eq!(collected.len(), expected_len); @@ -255,7 +402,7 @@ mod test { let mut collected: Vec<(Hash256, Slot)> = iter.collect(); collected.reverse(); - let expected_len = MainnetEthSpec::slots_per_historical_root() * 2 - 1; + let expected_len = MainnetEthSpec::slots_per_historical_root() * 2; assert_eq!(collected.len(), expected_len, "collection length incorrect"); diff --git a/eth2/lmd_ghost/src/reduced_tree.rs b/eth2/lmd_ghost/src/reduced_tree.rs index 49e900076..dace2bda6 100644 --- a/eth2/lmd_ghost/src/reduced_tree.rs +++ b/eth2/lmd_ghost/src/reduced_tree.rs @@ -8,7 +8,7 @@ use parking_lot::RwLock; use std::collections::HashMap; use std::marker::PhantomData; use std::sync::Arc; -use store::{iter::BlockRootsIterator, Error as StoreError, Store}; +use store::{iter::BestBlockRootsIterator, Error as StoreError, Store}; use types::{BeaconBlock, BeaconState, EthSpec, Hash256, Slot}; type Result = std::result::Result; @@ -530,14 +530,14 @@ where Ok(a_root) } - fn iter_ancestors(&self, child: Hash256) -> Result> { + fn iter_ancestors(&self, child: Hash256) -> Result> { let block = self.get_block(child)?; let state = self.get_state(block.state_root)?; - Ok(BlockRootsIterator::owned( + Ok(BestBlockRootsIterator::owned( self.store.clone(), state, - block.slot, + block.slot - 1, )) } From bbde06eb9f2b7a5ae00a3c6f41295bb8f1fcd083 Mon Sep 17 00:00:00 2001 From: Luke Schoen Date: Tue, 16 Jul 2019 09:40:53 +0200 Subject: [PATCH 72/72] docs: Fix readme typos (#439) --- validator_client/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/README.md b/validator_client/README.md index e6a01007a..0f30ded73 100644 --- a/validator_client/README.md +++ b/validator_client/README.md @@ -12,7 +12,7 @@ The VC is responsible for the following tasks: duties require. - Completing all the fields on a new block (e.g., RANDAO reveal, signature) and publishing the block to a BN. -- Prompting the BN to produce a new shard atteststation as per a validators +- Prompting the BN to produce a new shard attestation as per a validators duties. - Ensuring that no slashable messages are signed by a validator private key. - Keeping track of the system clock and how it relates to slots/epochs. @@ -40,7 +40,7 @@ index. The outcome of a successful poll is a `EpochDuties` struct: ```rust EpochDuties { validator_index: u64, - block_prodcution_slot: u64, + block_production_slot: u64, } ```