From 9aedb12bfc092be95880cdbe4500a1c7768ff2d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 2 Oct 2019 11:04:56 +1000 Subject: [PATCH] Refactor docs into mdbook (#547) * Refactor documentation, ef_tests makefile * Add makefile to root * Tidy readme * Fix readme badges * Tidy logs * Add terminalize gif * Update readme image * Update readme image * Tidy logs * Update readme image * Update readme * Update readme * Fix book link * Update makefiles * Update book * Fix link in book * Add readme for book * Remove old docs, move api spec YAML * Fix eth2/ dir readme * Add readme for lcli * Add about this book section * Minor formatting improvements * Address mehdi's comments --- Makefile | 47 +++-- README.md | 172 +++++-------------- beacon_node/beacon_chain/src/beacon_chain.rs | 12 +- beacon_node/client/src/lib.rs | 11 +- {docs => beacon_node/rest_api}/api_spec.yaml | 0 beacon_node/rpc/src/attestation.rs | 14 +- beacon_node/rpc/src/beacon_block.rs | 10 +- book/README.md | 17 ++ book/book.toml | 2 +- book/src/SUMMARY.md | 17 +- book/src/api.md | 13 ++ book/src/ci.md | 33 ++++ book/src/cli.md | 62 +++++++ book/src/contributing.md | 118 +++++++++++++ book/src/http.md | 85 +++++++++ book/src/interop-cheat-sheet.md | 149 ---------------- book/src/interop-cli.md | 29 ---- book/src/interop-environment.md | 30 ---- book/src/interop-scenarios.md | 101 ----------- book/src/interop.md | 11 -- book/src/intro.md | 37 ++-- book/src/setup.md | 88 ++-------- book/src/simple-testnet.md | 34 ++-- book/src/testnets.md | 162 ++++++++++++++++- book/src/websockets.md | 2 +- docs/README.md | 69 -------- docs/config_examples/beacon-node.toml | 90 ---------- docs/documentation.md | 14 -- docs/env.md | 53 ------ docs/interop.md | 109 ------------ docs/serenity.md | 161 ----------------- eth2/README.md | 36 +--- lcli/README.md | 11 ++ tests/ef_tests/Makefile | 18 ++ tests/ef_tests/README.md | 30 ++++ validator_client/src/block_producer/mod.rs | 6 +- 36 files changed, 701 insertions(+), 1152 deletions(-) rename {docs => beacon_node/rest_api}/api_spec.yaml (100%) create mode 100644 book/README.md create mode 100644 book/src/api.md create mode 100644 book/src/ci.md create mode 100644 book/src/cli.md create mode 100644 book/src/contributing.md create mode 100644 book/src/http.md delete mode 100644 book/src/interop-cheat-sheet.md delete mode 100644 book/src/interop-cli.md delete mode 100644 book/src/interop-environment.md delete mode 100644 book/src/interop-scenarios.md delete mode 100644 book/src/interop.md delete mode 100644 docs/README.md delete mode 100644 docs/config_examples/beacon-node.toml delete mode 100644 docs/documentation.md delete mode 100644 docs/env.md delete mode 100644 docs/interop.md delete mode 100644 docs/serenity.md create mode 100644 lcli/README.md create mode 100644 tests/ef_tests/Makefile create mode 100644 tests/ef_tests/README.md diff --git a/Makefile b/Makefile index d5517ed23..c08d04a9b 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,30 @@ -TESTS_TAG := v0.8.3 -TESTS = general minimal mainnet +.PHONY: tests -TESTS_BASE_DIR := ./tests/ef_tests -REPO_NAME := eth2.0-spec-tests -OUTPUT_DIR := $(TESTS_BASE_DIR)/$(REPO_NAME) - -BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(SPEC_VERSION) +EF_TESTS = "tests/ef_tests" +# Builds the entire workspace in release (optimized). +# +# Binaries will most likely be found in `./target/release` release: - cargo build --all --release + cargo build --release --all -clean_ef_tests: - rm -r $(OUTPUT_DIR) +# Runs the full workspace tests, without downloading any additional test +# vectors. +test: + cargo test --all --all-features --release -ef_tests: download_tests extract_tests - mkdir $(OUTPUT_DIR) - for test in $(TESTS); do \ - tar -C $(OUTPUT_DIR) -xvf $(TESTS_BASE_DIR)/$$test.tar ;\ - rm $(TESTS_BASE_DIR)/$$test.tar ;\ - done +# Runs the entire test suite, downloading test vectors if required. +test-full: make-ef-tests test -extract_tests: - for test in $(TESTS); do \ - gzip -df $(TESTS_BASE_DIR)/$$test.tar.gz ;\ - done +# Runs the makefile in the `ef_tests` repo. +# +# May download and extract an archive of test vectors from the ethereum +# repositories. At the time of writing, this was several hundred MB of +# downloads which extracts into several GB of test vectors. +make-ef-tests: + make -C $(EF_TESTS) -download_tests: - for test in $(TESTS); do \ - wget -P $(TESTS_BASE_DIR) $(BASE_URL)/$$test.tar.gz; \ - done +# Performs a `cargo` clean and cleans the `ef_tests` directory. +clean: + cargo clean + make -C $(EF_TESTS) clean diff --git a/README.md b/README.md index 238d965a2..05a481b4d 100644 --- a/README.md +++ b/README.md @@ -2,172 +2,77 @@ An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime. -[![Build Status]][Build Link] [![Doc Status]][Doc Link] [![Chat Badge]][Chat Link] +[![Build Status]][Build Link] [![Book Status]][Book Link] [![RustDoc Status]][RustDoc Link] [![Chat Badge]][Chat Link] [![Swagger Badge]][Swagger Link] [Build Status]: https://gitlab.sigmaprime.io/sigp/lighthouse/badges/master/build.svg [Build Link]: https://gitlab.sigmaprime.io/sigp/lighthouse/pipelines [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da [Chat Link]: https://discord.gg/cyAszAh -[Doc Status]: https://img.shields.io/badge/docs-master-blue.svg -[Doc Link]: http://lighthouse-docs.sigmaprime.io/ +[Book Status]:https://img.shields.io/badge/user--docs-master-informational +[Book Link]: http://lighthouse-book.sigmaprime.io/ +[RustDoc Status]:https://img.shields.io/badge/code--docs-master-orange +[RustDoc Link]: http://lighthouse-docs.sigmaprime.io/ +[Swagger Badge]: https://img.shields.io/badge/Open%20API-0.2.0-success +[Swagger Link]: https://app.swaggerhub.com/apis-docs/spble/lighthouse_rest_api/0.2.0 + +![terminalize](https://i.postimg.cc/Y0BQ0z3R/terminalize.gif) ## Overview Lighthouse is: - Fully open-source, licensed under Apache 2.0. -- Security-focused, fuzzing has begun and security reviews are planned +- Security-focused. Fuzzing has begun and security reviews are planned for late-2019. - Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and excellent performance (comparable to C++). - Funded by various organisations, including Sigma Prime, the Ethereum Foundation, ConsenSys and private individuals. -- Actively working to promote an inter-operable, multi-client Ethereum 2.0. +- Actively involved in the specification and security analysis of the emerging + Ethereum 2.0 specification. +Like all Ethereum 2.0 clients, Lighthouse is a work-in-progress. ## Development Status -Lighthouse, like all Ethereum 2.0 clients, is a work-in-progress. Instructions -are provided for running the client, however these instructions are designed -for developers and researchers working on the project. We do not (yet) provide -user-facing functionality. - Current development overview: -- Specification `v0.8.1` implemented, optimized and passing test vectors. +- Specification `v0.8.3` implemented, optimized and passing test vectors. - Rust-native libp2p with Gossipsub and Discv5. +- RESTful JSON API via HTTP server. +- Events via WebSocket. - Metrics via Prometheus. -- Basic gRPC API, soon to be replaced with RESTful HTTP/JSON. ### Roadmap -- **Early-September 2019**: `lighthouse-0.0.1` release: A stable testnet for - developers with a useful HTTP API. -- **September 2019**: Inter-operability with other Ethereum 2.0 clients. -- **October 2019**: Public, multi-client testnet with user-facing functionality. -- **January 2020**: Production Beacon Chain testnet. +- ~~**April 2019**: Inital single-client testnets.~~ +- ~~**September 2019**: Inter-operability with other Ethereum 2.0 clients.~~ +- **Early-October 2019**: `lighthouse-0.0.1` release: All major phase 0 + features implemented. +- **Q4 2019**: Public, multi-client testnet with user-facing functionality. +- **Q4 2019**: Third-party security review. +- **Q1 2020**: Production Beacon Chain testnet (tentative). -## Usage -Lighthouse consists of multiple binaries: +## Documentation -- [`beacon_node/`](beacon_node/): produces and verifies blocks from the P2P - connected validators and the P2P network. Provides an API for external services to - interact with Ethereum 2.0. -- [`validator_client/`](validator_client/): connects to a `beacon_node` and - performs the role of a proof-of-stake validator. -- [`account_manager/`](account_manager/): a stand-alone component providing key - management and creation for validators. +The [Lighthouse Book](http://lighthouse-book.sigmaprime.io/) contains information +for testnet users and developers. -### Simple Local Testnet - -**Note: these instructions are intended for developers and researchers. We do -not yet support end-users.** - -In this example we use the `account_manager` to create some keys, launch two -`beacon_node` instances and connect a `validator_client` to one. The two -`beacon_nodes` should stay in sync and build a Beacon Chain. - -First, clone this repository, [setup a development -environment](docs/env.md) and navigate to the root directory of this repository. - -Then, run `$ cargo build --all --release` and navigate to the `target/release` -directory and follow the steps: - -#### 1. Generate Validator Keys - -Generate 16 validator keys and store them in `~/.lighthouse-validator`: - -``` -$ ./account_manager -d ~/.lighthouse-validator generate_deterministic -i 0 -n 16 -``` - -_Note: these keys are for development only. The secret keys are -deterministically generated from low integers. Assume they are public -knowledge._ - -#### 2. Start a Beacon Node - -This node will act as the boot node and provide an API for the -`validator_client`. - -``` -$ ./beacon_node --recent-genesis --rpc -``` - -_Note: `--recent-genesis` defines the genesis time as either the start of the -current hour, or half-way through the current hour (whichever is most recent). -This makes it very easy to create a testnet, but does not allow nodes to -connect if they were started in separate 30-minute windows._ - -#### 3. Start Another Beacon Node - -In another terminal window, start another boot that will connect to the -running node. - -The running node will display it's ENR as a base64 string. This ENR, by default, has a target address of `127.0.0.1` meaning that any new node will connect to this node via `127.0.0.1`. If a boot node should be connected to on a different address, it should be run with the `--discovery-address` CLI flag to specify how other nodes may connect to it. -``` -$ ./beacon_node -r --boot-nodes --listen-address 127.0.0.1 --port 9001 --datadir /tmp/.lighthouse -``` -Here is the ENR string displayed in the terminal from the first node. The ENR can also be obtained from it's default directory `.lighthouse/network/enr.dat`. - -The `--datadir` flag tells this Beacon Node to store it's files in a different -directory. If you're on a system that doesn't have a `/tmp` dir (e.g., Mac, -Windows), substitute this with any directory that has write access. - -Note that all future created nodes can use the same boot-node ENR. Once connected to the boot node, all nodes should discover and connect with each other. -#### 4. Start a Validator Client - -In a third terminal window, start a validator client: - -``` -$ ./validator_client -``` - -You should be able to observe the validator signing blocks, the boot node -processing these blocks and publishing them to the other node. If you have -issues, try restarting the beacon nodes to ensure they have the same genesis -time. Alternatively, raise an issue and include your terminal output. - -## Further Reading - -- [About Lighthouse](docs/lighthouse.md): Goals, Ideology and Ethos surrounding -this implementation. -- [What is Ethereum Serenity](docs/serenity.md): an introduction to Ethereum Serenity. -- [Lighthouse Technical Documentation](http://lighthouse-docs.sigmaprime.io/): The Rust generated documentation, updated regularly. +Code documentation is generated via `cargo doc` and hosted at +[lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/). If you'd like some background on Sigma Prime, please see the [Lighthouse Update -\#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the -[company website](https://sigmaprime.io). - -## Directory Structure - -- [`beacon_node/`](beacon_node/): the "Beacon Node" binary and crates exclusively - associated with it. -- [`docs/`](docs/): documentation related to the repository. This includes contributor - guides, etc. (It does not include code documentation, which can be produced with `cargo doc`). -- [`eth2/`](eth2/): Crates containing common logic across the Lighthouse project. For - example: Ethereum 2.0 types ([`BeaconBlock`](eth2/types/src/beacon_block.rs), [`BeaconState`](eth2/types/src/beacon_state.rs), etc) and - SimpleSerialize (SSZ). -- [`protos/`](protos/): protobuf/gRPC definitions that are common across the Lighthouse project. -- [`validator_client/`](validator_client/): the "Validator Client" binary and crates exclusively - associated with it. -- [`tests/`](tests/): code specific to testing, most notably contains the - Ethereum Foundation test vectors. +\#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or +[sigmaprime.io](https://sigmaprime.io). ## Contributing -**Lighthouse welcomes contributors.** +Lighthouse welcomes contributors. -If you are looking to contribute, please head to our -[onboarding documentation](https://github.com/sigp/lighthouse/blob/master/docs/onboarding.md). - -If you'd like to contribute, try having a look through the [open -issues](https://github.com/sigp/lighthouse/issues) (tip: look for the [good -first -issue](https://github.com/sigp/lighthouse/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) -tag) and ping us on the [gitter](https://gitter.im/sigp/lighthouse) channel. We need -your support! +If you are looking to contribute, please head to the +[Contributing](http://lighthouse-book.sigmaprime.io/contributing.html) section +of the Lighthouse book. ## Contact @@ -175,8 +80,13 @@ The best place for discussion is the [Lighthouse Discord server](https://discord.gg/cyAszAh). Alternatively, you may use the [sigp/lighthouse gitter](https://gitter.im/sigp/lighthouse). +Encrypt sensitive messages using our [PGP +key](https://keybase.io/sigp/pgp_keys.asc?fingerprint=dcf37e025d6c9d42ea795b119e7c6cf9988604be). + ## Donations -If you support the cause, we accept donations to help fund development: +Lighthouse is an open-source project and a public good. Funding public goods is +hard and we're grateful for the donations we receive from the community via: -`0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b` (donation.sigmaprime.eth) +- [Gitcoin Grants](https://gitcoin.co/grants/25/lighthouse-ethereum-20-client). +- Ethereum address: `0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b` (donation.sigmaprime.eth). diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a0fe86474..df0a3f80d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -10,7 +10,7 @@ use lmd_ghost::LmdGhost; use operation_pool::DepositInsertStatus; use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::RwLock; -use slog::{error, info, trace, warn, Logger}; +use slog::{debug, error, info, trace, warn, Logger}; use slot_clock::SlotClock; use ssz::Encode; use state_processing::per_block_processing::{ @@ -646,8 +646,8 @@ impl BeaconChain { trace!( self.log, "Beacon attestation imported"; - "shard" => attestation.data.crosslink.shard, "target_epoch" => attestation.data.target.epoch, + "shard" => attestation.data.crosslink.shard, ); let _ = self .event_handler @@ -1397,12 +1397,16 @@ impl BeaconChain { info!( self.log, "New head beacon block"; + "root" => format!("{}", beacon_block_root), + "slot" => new_slot, + ); + debug!( + self.log, + "Head beacon block"; "justified_root" => format!("{}", beacon_state.current_justified_checkpoint.root), "justified_epoch" => beacon_state.current_justified_checkpoint.epoch, "finalized_root" => format!("{}", beacon_state.finalized_checkpoint.root), "finalized_epoch" => beacon_state.finalized_checkpoint.epoch, - "root" => format!("{}", beacon_block_root), - "slot" => new_slot, ); }; diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index a92a60e2b..7b75a37ad 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -13,7 +13,7 @@ use exit_future::Signal; use futures::{future::Future, Stream}; use network::Service as NetworkService; use rest_api::NetworkInfo; -use slog::{crit, error, info, o}; +use slog::{crit, debug, error, info, o}; use slot_clock::SlotClock; use std::marker::PhantomData; use std::sync::Arc; @@ -309,11 +309,16 @@ fn log_new_slot(chain: &Arc>, log: &slog::Lo info!( log, "Slot start"; + "best_slot" => best_slot, + "slot" => current_slot, + ); + debug!( + log, + "Slot info"; "skip_slots" => current_slot.saturating_sub(best_slot), "best_block_root" => format!("{}", latest_block_root), - "best_block_slot" => best_slot, "slot" => current_slot, - ) + ); } else { error!( log, diff --git a/docs/api_spec.yaml b/beacon_node/rest_api/api_spec.yaml similarity index 100% rename from docs/api_spec.yaml rename to beacon_node/rest_api/api_spec.yaml diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index aea7119aa..7c737cd11 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -103,8 +103,9 @@ impl AttestationService for AttestationServiceInstance { // Attestation was successfully processed. info!( self.log, - "PublishAttestation"; - "type" => "valid_attestation", + "Valid attestation from RPC"; + "target_epoch" => attestation.data.target.epoch, + "shard" => attestation.data.crosslink.shard, ); // valid attestation, propagate to the network @@ -123,8 +124,7 @@ impl AttestationService for AttestationServiceInstance { .unwrap_or_else(|e| { error!( self.log, - "PublishAttestation"; - "type" => "failed to publish attestation to gossipsub", + "Failed to gossip attestation"; "error" => format!("{:?}", e) ); }); @@ -135,8 +135,7 @@ impl AttestationService for AttestationServiceInstance { // Attestation was invalid warn!( self.log, - "PublishAttestation"; - "type" => "invalid_attestation", + "Invalid attestation from RPC"; "error" => format!("{:?}", e), ); resp.set_success(false); @@ -146,8 +145,7 @@ impl AttestationService for AttestationServiceInstance { // Some other error warn!( self.log, - "PublishAttestation"; - "type" => "beacon_chain_error", + "Failed to process attestation from RPC"; "error" => format!("{:?}", e), ); resp.set_success(false); diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index e9e7490ca..ab7c0aef5 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -105,8 +105,8 @@ impl BeaconBlockService for BeaconBlockServiceInstance { info!( self.log, "Valid block from RPC"; - "block_slot" => block.slot, - "block_root" => format!("{}", block_root), + "root" => format!("{}", block_root), + "slot" => block.slot, ); // create the network topic to send on @@ -126,8 +126,7 @@ impl BeaconBlockService for BeaconBlockServiceInstance { .unwrap_or_else(|e| { error!( self.log, - "PublishBeaconBlock"; - "type" => "failed to publish to gossipsub", + "Failed to gossip beacon block"; "error" => format!("{:?}", e) ); }); @@ -151,8 +150,7 @@ impl BeaconBlockService for BeaconBlockServiceInstance { // Some failure during processing. error!( self.log, - "PublishBeaconBlock"; - "type" => "failed_to_process", + "Failed to process beacon block"; "error" => format!("{:?}", e) ); diff --git a/book/README.md b/book/README.md new file mode 100644 index 000000000..3a5396283 --- /dev/null +++ b/book/README.md @@ -0,0 +1,17 @@ +# Lighthouse Book + +Contains an [mdBook](https://github.com/rust-lang-nursery/mdBook) that serves +as the primary source of Lighthouse user documentation. + +The book is hosted at [lighthouse-book.sigmaprime.io](http://lighthouse-book.sigmaprime.io). + +## Usage + +The [mdBook docs](https://github.com/rust-lang-nursery/mdBook#usage) are the +best source of information for building the book. + +### Example + +1. Install mdBook: `$ cargo install mdbook` +1. Build the book, open it in a browser and build after file changes: `$ mdbook + watch --open` diff --git a/book/book.toml b/book/book.toml index 829c7b99c..f48235fc9 100644 --- a/book/book.toml +++ b/book/book.toml @@ -3,4 +3,4 @@ authors = ["Paul Hauner"] language = "en" multilingual = false src = "src" -title = "Lighthouse" +title = "Lighthouse Book" diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 01613f9fd..9f49ea95e 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -1,11 +1,12 @@ # Summary * [Introduction](./intro.md) -* [Development Environment](./setup.md) -* [Websocket Interface](./websockets.md) -* [Simple Local Testnet](./simple-testnet.md) -* [Interop](./interop.md) - * [Environment](./interop-environment.md) - * [CLI Overview](./interop-cli.md) - * [Scenarios](./interop-scenarios.md) - * [Cheat-sheet](./interop-cheat-sheet.md) +* [CLI](./cli.md) + * [Testnets](./testnets.md) + * [Simple Local Testnet](./simple-testnet.md) +* [API](./api.md) + * [HTTP (RESTful JSON)](./http.md) + * [WebSocket](./websockets.md) +* [Contributing](./contributing.md) + * [Development Environment](./setup.md) + * [CI & Testing](./ci.md) diff --git a/book/src/api.md b/book/src/api.md new file mode 100644 index 000000000..0fa6c3001 --- /dev/null +++ b/book/src/api.md @@ -0,0 +1,13 @@ +# APIs + +The Lighthouse `beacon_node` provides two APIs for local consumption: + +- A [RESTful JSON HTTP API](http.html) which provides beacon chain, node and network + information. +- A read-only [WebSocket API](websockets.html) providing beacon chain events, as they occur. + + +## Security + +These endpoints are not designed to be exposed to the public Internet or +untrusted users. They may pose a considerable DoS attack vector when used improperly. diff --git a/book/src/ci.md b/book/src/ci.md new file mode 100644 index 000000000..e89864276 --- /dev/null +++ b/book/src/ci.md @@ -0,0 +1,33 @@ +# Contiguous Integration (CI) and Testing + +Lighthouse uses a self-hosted Gitlab CI server to run tests and deploy docs. + +For security reasons, **CI will only be run automatically for Lighthouse +maintainers.** Contributors without maintainer privileges will need to have CI +triggered for them prior to a PR being merged. + +You can see the full set of tests we run in the +[gitlab-ci.yml](https://github.com/sigp/lighthouse/blob/master/.gitlab-ci.yml) +file. The following two commands should complete successfully before CI can +pass: + +```bash +$ cargo test --all --all-features +$ cargo fmt --all --check +``` + +_Note: Travis CI is also used, however it does not run the full test suite._ + +### Ethereum 2.0 Spec Tests + +The +[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/) +repository contains a large set of tests that verify Lighthouse behaviour +against the Ethereum Foundation specifications. + +These tests are quite large (100's of MB), so we don't download them by +default. Developers should ensure they have downloaded these tests using the +`Makefile` in +[tests/ef_tests](https://github.com/sigp/lighthouse/tree/master/tests/ef_tests). + +**Failures in these tests should prevent CI from passing.** diff --git a/book/src/cli.md b/book/src/cli.md new file mode 100644 index 000000000..47f85b9cb --- /dev/null +++ b/book/src/cli.md @@ -0,0 +1,62 @@ +# Command-Line Interface (CLI) + +Lighthouse a collection of CLI applications. The two primary binaries are: + +- `beacon_node`: the largest and most fundamental component which connects to + the p2p network, processes messages and tracks the head of the beacon + chain. +- `validator_client`: a lightweight but important component which loads a validators private + key and signs messages using a `beacon_node` as a source-of-truth. + +There are also some ancillary binaries: + +- `account_manager`: generates cryptographic keys. +- `lcli`: a general-purpose utility for troubleshooting Lighthouse state + transitions (developer tool). + +## Installation + +Presently, we recommend building Lighthouse using the `$ cargo build --release +--all` command and executing binaries from the +`/target/release` directory. + +## Documentation + +Each binary supports the `--help` flag, this is the best source of +documentation. + + +```bash +$ ./beacon_node --help +``` + +```bash +$ ./validator_client --help +``` + +```bash +$ ./account_manager --help +``` + +```bash +$ ./lcli --help +``` + +## Beacon Node + +The `beacon_node` CLI has two primary tasks: + +- **Resuming** an existing database with `$ ./beacon_node`. +- **Creating** a new testnet database using `$ ./beacon_node testnet`. + +## Creating a new database + +Use the `$./beacon_node testnet` command (see [testnets](./testnets.md) for more +information). + +## Resuming from an existing database + +Once a database has been created, it can be resumed by running `$ ./beacon_node`. + +Presently, this command will fail if no existing database is found. You must +use the `$ ./beacon_node testnet` command to create a new database. diff --git a/book/src/contributing.md b/book/src/contributing.md new file mode 100644 index 000000000..cb29a0189 --- /dev/null +++ b/book/src/contributing.md @@ -0,0 +1,118 @@ +# Contributing to Lighthouse + +[![Chat Badge]][Chat Link] + +[Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da +[Chat Link]: https://discord.gg/cyAszAh + + +Lighthouse welcomes contributions. If you are interested in contributing to the +Ethereum ecosystem, and you want to learn Rust, Lighthouse is a great project +to work on. + +To start contributing, + +1. Setup a [development environment](./setup.md). +2. Browse through the [open issues](https://github.com/sigp/lighthouse/issues) + (tip: look for the [good first + issue](https://github.com/sigp/lighthouse/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + tag). +3. Comment on an issue before starting work. +4. Share your work via a pull-request. + +If you have questions, please reach out via +[Discord](https://discord.gg/cyAszAh). + +## Ethereum 2.0 + +Lighthouse is an implementation of the Ethereum 2.0 specification, as defined +in the [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs) +repository. + +We recommend reading Danny Ryan's (incomplete) [Phase 0 for +Humans](https://notes.ethereum.org/@djrtwo/Bkn3zpwxB?type=view) before diving +into the canonical spec. + +## Rust + +Lighthouse adheres to Rust code conventions as outlined in the [**Rust +Styleguide**](https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md). + +Please use [clippy](https://github.com/rust-lang/rust-clippy) and +[rustfmt](https://github.com/rust-lang/rustfmt) to detect common mistakes and +inconsistent code formatting: + +```bash +$ cargo clippy --all +$ cargo fmt --all --check +``` + +### Panics + +Generally, **panics should be avoided at all costs**. Lighthouse operates in an +adversarial environment (the Internet) and it's a severe vulnerability if +people on the Internet can cause Lighthouse to crash via a panic. + +Always prefer returning a `Result` or `Option` over causing a panic. For +example, prefer `array.get(1)?` over `array[1]`. + +If you know there won't be a panic but can't express that to the compiler, +use `.expect("Helpful message")` instead of `.unwrap()`. Always provide +detailed reasoning in a nearby comment when making assumptions about panics. + +### TODOs + +All `TODO` statements should be accompanied by a GitHub issue. + +```rust +pub fn my_function(&mut self, _something &[u8]) -> Result { + // TODO: something_here + // https://github.com/sigp/lighthouse/issues/XX +} +``` + +### Comments + +**General Comments** + +* Prefer line (``//``) comments to block comments (``/* ... */``) +* Comments can appear on the line prior to the item or after a trailing space. +```rust +// Comment for this struct +struct Lighthouse {} +fn make_blockchain() {} // A comment on the same line after a space +``` + +**Doc Comments** + +* The ``///`` is used to generate comments for Docs. +* The comments should come before attributes. + +```rust +/// Stores the core configuration for this Lighthouse instance. +/// This struct is general, other components may implement more +/// specialized config structs. +#[derive(Clone)] +pub struct LighthouseConfig { + pub data_dir: PathBuf, + pub p2p_listen_port: u16, +} +``` + +### Rust Resources + +Rust is an extremely powerful, low-level programming language that provides +freedom and performance to create powerful projects. The [Rust +Book](https://doc.rust-lang.org/stable/book/) provides insight into the Rust +language and some of the coding style to follow (As well as acting as a great +introduction and tutorial for the language). + +Rust has a steep learning curve, but there are many resources to help. We +suggest: + +* [Rust Book](https://doc.rust-lang.org/stable/book/) +* [Rust by example](https://doc.rust-lang.org/stable/rust-by-example/) +* [Learning Rust With Entirely Too Many Linked Lists](http://cglab.ca/~abeinges/blah/too-many-lists/book/) +* [Rustlings](https://github.com/rustlings/rustlings) +* [Rust Exercism](https://exercism.io/tracks/rust) +* [Learn X in Y minutes - Rust](https://learnxinyminutes.com/docs/rust/) diff --git a/book/src/http.md b/book/src/http.md new file mode 100644 index 000000000..918f349b0 --- /dev/null +++ b/book/src/http.md @@ -0,0 +1,85 @@ +# HTTP API + +[![Swagger Badge]][Swagger Link] + +[Swagger Badge]: https://img.shields.io/badge/Open%20API-0.2.0-success +[Swagger Link]: https://app.swaggerhub.com/apis-docs/spble/lighthouse_rest_api/0.2.0 + + +**The Lighthouse API is documented in Open API format and is available at +[SwaggerHub: Lighthouse REST +API](https://app.swaggerhub.com/apis-docs/spble/lighthouse_rest_api/0.2.0).** + +By default, a Lighthouse `beacon_node` exposes a HTTP server on `localhost:5052`. + +The following CLI flags control the HTTP server: + +- `--no-api`: disable the HTTP server. +- `--api-port`: specify the listen port of the server. +- `--api-address`: specify the listen address of the server. + +## Examples + +In addition to the complete Open API docs (see above), some examples are +provided below. + +_Examples assume there is a Lighthouse node exposing a HTTP API on +`localhost:5052`. Responses are JSON._ + +### Get the node's beacon chain head + +```bash +$ curl localhost:5052/beacon/head + +{"slot":0,"block_root":"0x827bf71805540aa13f6d8c7d18b41b287b2094a4d7a28cbb8deb061dbf5df4f5","state_root":"0x90a78d73294bc9c7519a64e1912161be0e823eb472012ff54204e15a4d717fa5"}% +``` + +### Get the node's finalized checkpoint + +```bash +$ curl localhost:5052/beacon/latest_finalized_checkpoint + +{"epoch":0,"root":"0x0000000000000000000000000000000000000000000000000000000000000000"}% +``` + +### Get the node's ENR + +```bash +$ curl localhost:5052/network/enr + +"-IW4QFyf1VlY5pZs0xZuvKMRZ9_cdl9WMCDAAJXZiZiuGcfRYoU40VPrYDLQj5prneJIz3zcbTjHp9BbThc-yiymJO8HgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjg0-DsTkQynhJCRnLLttBK1RS78lmUkLa-wgzAi-Ob5"% +``` + +### Get a list of connected peer ids + +```bash +$ curl localhost:5052/network/peers + +["QmeMFRTWfo3KbVG7dEBXGhyRMa29yfmnJBXW84rKuGEhuL"]% +``` + +### Get the node's peer id + +```bash +$ curl localhost:5052/network/peer_id + +"QmRD1qs2AqNNRdBcGHUGpUGkpih5cmdL32mhh22Sy79xsJ"% +``` + +### Get the list of listening libp2p addresses + +Lists all the libp2p multiaddrs that the node is listening on. + +```bash +$ curl localhost:5052/network/listen_addresses + +["/ip4/127.0.0.1/tcp/9000","/ip4/192.168.1.121/tcp/9000","/ip4/172.17.0.1/tcp/9000","/ip4/172.42.0.1/tcp/9000","/ip6/::1/tcp/9000","/ip6/fdd3:c293:1bc::203/tcp/9000","/ip6/fdd3:c293:1bc:0:9aa9:b2ea:c610:44db/tcp/9000"]% +``` + +### Pretty-print the genesis state and state root + +Returns the genesis state and state root in your terminal, in YAML. + +```bash +$ curl --header "Content-Type: application/yaml" "localhost:5052/beacon/state?slot=0" +``` diff --git a/book/src/interop-cheat-sheet.md b/book/src/interop-cheat-sheet.md deleted file mode 100644 index 7fea539ea..000000000 --- a/book/src/interop-cheat-sheet.md +++ /dev/null @@ -1,149 +0,0 @@ -# Interop Cheat-sheet - -This document contains a list of tips and tricks that may be useful during -interop testing. - -- When starting a beacon node: - - [Specify a boot node by multiaddr](#boot-node-multiaddr) - - [Specify a boot node by ENR](#boot-node-enr) - - [Avoid port clashes when starting multiple nodes](#port-bump) - - [Specify a custom slot time](#slot-time) -- Using the beacon node HTTP API: - - [Pretty-print the genesis state and state root](#http-state) - - [Curl a node's ENR](#http-enr) - - [Curl a node's connected peers](#http-peer-ids) - - [Curl a node's local peer id](#http-peer-id) - - [Curl a node's listening multiaddrs](#http-listen-addresses) - - [Curl a node's beacon chain head](#http-head) - - [Curl a node's finalized checkpoint](#http-finalized) - -## Category: CLI - -The `--help` command provides detail on the CLI interface. Here are some -interop-specific CLI commands. - - -### Specify a boot node by multiaddr - -You can specify a static list of multiaddrs when booting Lighthouse using -the `--libp2p-addresses` command. - -#### Example: - -``` -$ ./beacon_node --libp2p-addresses /ip4/192.168.0.1/tcp/9000 -``` - - -### Specify a boot node by ENR - -You can specify a static list of Discv5 addresses when booting Lighthouse using -the `--boot-nodes` command. - -#### Example: - -``` -$ ./beacon_node --boot-nodes -IW4QB2Hi8TPuEzQ41Cdf1r2AUU1FFVFDBJdJyOkWk2qXpZfFZQy2YnJIyoT_5fnbtrXUouoskmydZl4pIg90clIkYUDgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjg0-DsTkQynhJCRnLLttBK1RS78lmUkLa-wgzAi-Ob5 -``` - - -### Avoid port clashes when starting nodes - -Starting a second Lighthouse node on the same machine will fail due to TCP/UDP -port collisions. Use the `-b` (`--port-bump`) flag to increase all listening -ports by some `n`. - -#### Example: - -Increase all ports by `10` (using multiples of `10` is recommended). - -``` -$ ./beacon_node -b 10 -``` - - -### Start a testnet with a custom slot time - -Lighthouse can run at quite low slot times when there are few validators (e.g., -`500 ms` slot times should be fine for 8 validators). - -#### Example - -The `-t` (`--slot-time`) flag specifies the milliseconds per slot. - -``` -$ ./beacon_node testnet -t 500 recent 8 -``` - -> Note: `bootstrap` loads the slot time via HTTP and therefore conflicts with -> this flag. - -## Category: HTTP API - -Examples assume there is a Lighthouse node exposing a HTTP API on -`localhost:5052`. Responses are JSON. - - -### Pretty-print the genesis state and state root - -Returns the genesis state and state root in your terminal, in YAML. - -``` -$ curl --header "Content-Type: application/yaml" "localhost:5052/beacon/state?slot=0" -``` - - -### Get the node's ENR - -``` -$ curl localhost:5052/network/enr - -"-IW4QFyf1VlY5pZs0xZuvKMRZ9_cdl9WMCDAAJXZiZiuGcfRYoU40VPrYDLQj5prneJIz3zcbTjHp9BbThc-yiymJO8HgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjg0-DsTkQynhJCRnLLttBK1RS78lmUkLa-wgzAi-Ob5"% -``` - - -### Get a list of connected peer ids - -``` -$ curl localhost:5052/network/peers - -["QmeMFRTWfo3KbVG7dEBXGhyRMa29yfmnJBXW84rKuGEhuL"]% -``` - - -### Get the node's peer id - -``` -curl localhost:5052/network/peer_id - -"QmRD1qs2AqNNRdBcGHUGpUGkpih5cmdL32mhh22Sy79xsJ"% -``` - - -### Get the list of listening libp2p addresses - -Lists all the libp2p multiaddrs that the node is listening on. - -``` -curl localhost:5052/network/listen_addresses - -["/ip4/127.0.0.1/tcp/9000","/ip4/192.168.1.121/tcp/9000","/ip4/172.17.0.1/tcp/9000","/ip4/172.42.0.1/tcp/9000","/ip6/::1/tcp/9000","/ip6/fdd3:c293:1bc::203/tcp/9000","/ip6/fdd3:c293:1bc:0:9aa9:b2ea:c610:44db/tcp/9000"]% -``` - - -### Get the node's beacon chain head - -``` -curl localhost:5052/beacon/head - -{"slot":0,"block_root":"0x827bf71805540aa13f6d8c7d18b41b287b2094a4d7a28cbb8deb061dbf5df4f5","state_root":"0x90a78d73294bc9c7519a64e1912161be0e823eb472012ff54204e15a4d717fa5"}% -``` - - -### Get the node's finalized checkpoint - -``` -curl localhost:5052/beacon/latest_finalized_checkpoint - -{"epoch":0,"root":"0x0000000000000000000000000000000000000000000000000000000000000000"}% -``` diff --git a/book/src/interop-cli.md b/book/src/interop-cli.md deleted file mode 100644 index 3dad845f3..000000000 --- a/book/src/interop-cli.md +++ /dev/null @@ -1,29 +0,0 @@ -# Interop CLI Overview - -The Lighthouse CLI has two primary tasks: - -- **Resuming** an existing database with `$ ./beacon_node`. -- **Creating** a new testnet database using `$ ./beacon_node testnet`. - -_See [Scenarios](./interop-scenarios.md) for methods we've anticipated will be -used interop._ - -## Creating a new database - -There are several methods for creating a new beacon node database: - -- `quick`: using the `(validator_client, genesis_time)` tuple. -- `recent`: as above but `genesis_time` is set to the start of some recent time - window. -- `file`: loads the genesis file from disk in one of multiple formats. -- `bootstrap`: a Lighthouse-specific method where we connect to a running node - and download it's specification and genesis state via the HTTP API. - -See `$ ./beacon_node testnet --help` for more detail. - -## Resuming from an existing database - -Once a database has been created, it can be resumed by running `$ ./beacon_node`. - -Presently, this command will fail if no existing database is found. You must -use the `$ ./beacon_node testnet` command to create a new database. diff --git a/book/src/interop-environment.md b/book/src/interop-environment.md deleted file mode 100644 index 6d3568e29..000000000 --- a/book/src/interop-environment.md +++ /dev/null @@ -1,30 +0,0 @@ -# Interop Environment - -All that is required for inter-op is a built and tested [development -environment](./setup.md). - -## Repositories - -You will only require the [sigp/lighthouse](http://github.com/sigp/lighthouse) -library. - -To allow for faster build/test iterations we will use the -[`interop`](https://github.com/sigp/lighthouse/tree/interop) branch of -[sigp/lighthouse](https://github.com/sigp/lighthouse/tree/interop) for -September 2019 interop. **Please use ensure you `git checkout interop` after -cloning the repo.** - -## File System - -When lighthouse boots, it will create the following -directories: - -- `~/.lighthouse`: database and configuration for the beacon node. -- `~/.lighthouse-validator`: database and configuration for the validator - client. - -After building the binaries with `cargo build --release --all`, there will be a -`target/release` directory in the root of the Lighthouse repository. This is -where the `beacon_node` and `validator_client` binaries are located. - -You do not need to create any of these directories manually. diff --git a/book/src/interop-scenarios.md b/book/src/interop-scenarios.md deleted file mode 100644 index 5e44d822a..000000000 --- a/book/src/interop-scenarios.md +++ /dev/null @@ -1,101 +0,0 @@ -# Interop Scenarios - -Here we demonstrate some expected interop scenarios. - -All scenarios assume a working [development environment](./setup.md) and -commands are based in the `target/release` directory (this is the build dir for -`cargo`). - -Additional functions can be found in the [interop -cheat-sheet](./interop-cheat-sheet.md). - -### Table of contents - -- [Starting from a`validator_count, genesis_time` tuple](#quick-start) -- [Starting a node from a genesis state file](#state-file) -- [Starting a validator client](#val-client) -- [Exporting a genesis state file](#export) from a running Lighthouse - node - - - -### Start beacon node given a validator count and genesis_time - - -To start a brand-new beacon node (with no history) use: - -``` -$ ./beacon_node testnet -f quick 8 -``` - -Where `GENESIS_TIME` is in [unix time](https://duckduckgo.com/?q=unix+time&t=ffab&ia=answer). - -> Notes: -> -> - This method conforms the ["Quick-start -genesis"](https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#quick-start-genesis) -method in the `ethereum/eth2.0-pm` repository. -> - The `-f` flag ignores any existing database or configuration, backing them -> up before re-initializing. -> - `8` is the validator count and `1567222226` is the genesis time. -> - See `$ ./beacon_node testnet quick --help` for more configuration options. - - -### Start Beacon Node given a genesis state file - -A genesis state can be read from file using the `testnet file` subcommand. -There are three supported formats: - -- `ssz` (default) -- `json` -- `yaml` - -Start a new node using `/tmp/genesis.ssz` as the genesis state: - -``` -$ ./beacon_node testnet --spec minimal -f file ssz /tmp/genesis.ssz -``` - -> Notes: -> -> - The `-f` flag ignores any existing database or configuration, backing them -> up before re-initializing. -> - See `$ ./beacon_node testnet file --help` for more configuration options. -> - The `--spec` flag is required to allow SSZ parsing of fixed-length lists. - - -### Start an auto-configured validator client - -To start a brand-new validator client (with no history) use: - -``` -$ ./validator_client testnet -b insecure 0 8 -``` - -> Notes: -> -> - The `-b` flag means the validator client will "bootstrap" specs and config -> from the beacon node. -> - The `insecure` command dictates that the [interop keypairs](https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#pubkeyprivkey-generation) -> will be used. -> - The `0 8` indicates that this validator client should manage 8 validators, -> starting at validator 0 (the first deposited validator). -> - The validator client will try to connect to the beacon node at `localhost`. -> See `--help` to configure that address and other features. -> - The validator client will operate very unsafely in `testnet` mode, happily -> swapping between chains and creating double-votes. - - -### Exporting a genesis file - -Genesis states can downloaded from a running Lighthouse node via the HTTP API. Three content-types are supported: - -- `application/json` -- `application/yaml` -- `application/ssz` - -Using `curl`, a genesis state can be downloaded to `/tmp/genesis.ssz`: - -``` -$ curl --header "Content-Type: application/ssz" "localhost:5052/beacon/state/genesis" -o /tmp/genesis.ssz -``` diff --git a/book/src/interop.md b/book/src/interop.md deleted file mode 100644 index cb119d59d..000000000 --- a/book/src/interop.md +++ /dev/null @@ -1,11 +0,0 @@ -# Lighthouse Interop Guide - -This guide is intended for other Ethereum 2.0 client developers performing -inter-operability testing with Lighthouse. - -## Chapters - -- Read about the required [development environment](./interop-environment.md). -- Get an [overview](./interop-cli.md) of the Lighthouse CLI. -- See how we expect to handle some [interop scenarios](./interop-scenarios.md). -- See the [interop cheat-sheet](./interop-cheat-sheet.md) for useful CLI tips. diff --git a/book/src/intro.md b/book/src/intro.md index ccf867a54..2e06256bc 100644 --- a/book/src/intro.md +++ b/book/src/intro.md @@ -1,9 +1,9 @@ -# Lighthouse Documentation +# Lighthouse Book -[![Build Status]][Build Link] [![Doc Status]][Doc Link] [![Chat Badge]][Chat Link] +_Documentation for Lighthouse users and developers._ + +[![Doc Status]][Doc Link] [![Chat Badge]][Chat Link] -[Build Status]: https://gitlab.sigmaprime.io/sigp/lighthouse/badges/master/build.svg -[Build Link]: https://gitlab.sigmaprime.io/sigp/lighthouse/pipelines [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da [Chat Link]: https://discord.gg/cyAszAh [Doc Status]:https://img.shields.io/badge/rust--docs-master-orange @@ -12,16 +12,27 @@ Lighthouse is an **Ethereum 2.0 client** that connects to other Ethereum 2.0 clients to form a resilient and decentralized proof-of-stake blockchain. -It is written in Rust, maintained by Sigma Prime and funded by the Ethereum -Foundation, Consensys and other individuals and organisations. +We implement the specification as defined in the +[ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs) repository. -## Developer Resources +## Topics -Documentation is presently targeted at **researchers and developers**. It -assumes significant prior knowledge of Ethereum 2.0. - -Topics: +You may read this book from start to finish, or jump to some of these topics: - Get started with [development environment setup](./setup.md). -- See the [interop docs](./interop.md). -- [Run a simple testnet](./simple-testnet.md) in Only Three CLI Commands™. +- Utilize the whole stack by starting a [simple local testnet](./simple-testnet.md). +- Query the [RESTful HTTP API](./http.md) using `curl`. +- Listen to events with the [JSON WebSocket API](./websockets.md). +- View the [Rust code docs](http://lighthouse-docs.sigmaprime.io/). + + +Prospective contributors can read the [Contributing](./contributing.md) section +to understand how we develop and test Lighthouse. + +## About this Book + +This book is open source, contribute at +[github.com/sigp/lighthouse/book](https://github.com/sigp/lighthouse/tree/master/book). + +The Lighthouse CI/CD system maintains a hosted version of the `master` branch +at [lighthouse-book.sigmaprime.io](http://lighthouse-book.sigmaprime.io). diff --git a/book/src/setup.md b/book/src/setup.md index 532de3fc0..22671477c 100644 --- a/book/src/setup.md +++ b/book/src/setup.md @@ -1,81 +1,29 @@ # Development Environment Setup -Follow this guide to get a Lighthouse development environment up-and-running. +## Linux, MacOS & Windows -See the [Quick instructions](#quick-instructions) for a summary or the -[Detailed instructions](#detailed-instructions) for clarification. - -## Quick instructions - -1. Install Rust + Cargo with [rustup](https://rustup.rs/). +1. Install Rust and Cargo with [rustup](https://rustup.rs/). + - Use the `stable` toolchain (it's the default). 1. Install build dependencies using your package manager. - - `$ clang protobuf libssl-dev cmake` -1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse). -1. In the root of the repo, run the tests with `cargo test --all --release`. -1. Then, build the binaries with `cargo build --all --release`. -1. Lighthouse is now fully built and tested. + - `clang`, `protobuf`, `libssl-dev`, `cmake` +1. Clone the [github.com/sigp/lighthouse](https://github.com/sigp/lighthouse) + repository. +1. Run `$ make` to build Lighthouse. +1. Run `$ make test` to run the test suite + - If you experience any failures, please reach out on + [discord](https://discord.gg/cyAszAh). + - Developers use `$ make test-full` to ensure you have the full set of + test vectors. -_Note: first-time compilation may take several minutes._ +> - The `beacon_node`, `validator_client` and other binaries are created in +> `target/release` directory. +> - First-time compilation may take several minutes. -## Detailed instructions +### Windows -A fully-featured development environment can be achieved with the following -steps: - - 1. Install [rustup](https://rustup.rs/). - 1. Use the command `rustup show` to get information about the Rust - installation. You should see that the active tool-chain is the stable - version. - - Updates can be performed using` rustup update`, Lighthouse generally - requires a recent version of Rust. - 1. Install build dependencies (Arch packages are listed here, your - distribution will likely be similar): - - `clang`: required by RocksDB. - - `protobuf`: required for protobuf serialization (gRPC) - - `libssl-dev`: also gRPC - - `cmake`: required for building protobuf - 1. Clone the repository with submodules: `git clone - https://github.com/sigp/lighthouse`. - 1. Change directory to the root of the repository. - 1. Run the test suite with `cargo test --all --release`. The build and test - process can take several minutes. If you experience any failures on - `master`, please raise an - [issue](https://github.com/sigp/lighthouse/issues). - -### Notes: - -Lighthouse targets Rust `stable` but generally runs on `nightly` too. - -#### Note for Windows users: - -Perl may also be required to build lighthouse. You can install [Strawberry -Perl](http://strawberryperl.com/), or alternatively use a choco install command -`choco install strawberryperl`. +Perl may also be required to build Lighthouse. You can install [Strawberry +Perl](http://strawberryperl.com/), or alternatively if you're using the [Chocolatey](https://chocolatey.org/) package manager for Windows, use the following choco install command: `choco install strawberryperl`. Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues compiling in Windows. You can specify a known working version by editing version in `protos/Cargo.toml` section to `protoc-grpcio = "<=0.3.0"`. - -## eth2.0-spec-tests - -The -[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/) -repository contains a large set of tests that verify Lighthouse behaviour -against the Ethereum Foundation specifications. - -The `tests/ef_tests` crate runs these tests and it has some interesting -behaviours: - -- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests - indicate a `pass` when they did not actually run. -- If that directory _is_ present, the tests are executed faithfully, failing if - a discrepancy is found. - -The `tests/ef_tests/eth2.0-spec-tests` directory is not present by default. To -obtain it, use the Makefile in the root of the repository: - -``` -make ef_tests -``` - -_Note: this will download 100+ MB of test files from the [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/)._ diff --git a/book/src/simple-testnet.md b/book/src/simple-testnet.md index b6fa19d6f..b3b24c1c5 100644 --- a/book/src/simple-testnet.md +++ b/book/src/simple-testnet.md @@ -1,32 +1,24 @@ # Simple Local Testnet -You can setup a local, two-node testnet in **Only Three CLI Commands™**. - -Follow the [Quick instructions](#tldr) version if you're confident, or see -[Detailed instructions](#detail) for more. - - -## Quick instructions - -Setup a development environment, build the project and navigate to the -`target/release` directory. +With a functional [development environment](./setup.md), starting a local multi-node +testnet is easy: 1. Start the first node: `$ ./beacon_node testnet -f recent 8` 1. Start a validator client: `$ ./validator_client testnet -b insecure 0 8` -1. Start another node `$ ./beacon_node -b 10 testnet -f bootstrap http://localhost:5052` +1. Start more nodes with `$ ./beacon_node -b 10 testnet -f bootstrap + http://localhost:5052` + - Increment the `-b` value by `10` for each additional node. -_Repeat #3 to add more nodes._ - -## Detailed instructions +## Detailed Instructions First, setup a Lighthouse development environment and navigate to the `target/release` directory (this is where the binaries are located). -## Starting the Beacon Node +## Starting a beacon node Start a new node (creating a fresh database and configuration in `~/.lighthouse`), using: -``` +```bash $ ./beacon_node testnet -f recent 8 ``` @@ -38,11 +30,11 @@ $ ./beacon_node testnet -f recent 8 > - See `$ ./beacon_node testnet recent --help` for more configuration options, > including `minimal`/`mainnet` specification. -## Starting the Validator Client +## Starting a validator client In a new terminal window, start the validator client with: -``` +```bash $ ./validator_client testnet -b insecure 0 8 ``` @@ -57,15 +49,15 @@ $ ./validator_client testnet -b insecure 0 8 > - The validator client will try to connect to the beacon node at `localhost`. > See `--help` to configure that address and other features. -## Adding another Beacon Node +## Adding another beacon node You may connect another (non-validating) node to your local network using the lighthouse `bootstrap` command. -In a new terminal terminal, run: +In a new terminal window, run: -``` +```bash $ ./beacon_node -b 10 testnet -r bootstrap ``` diff --git a/book/src/testnets.md b/book/src/testnets.md index 180673fb3..211d235c1 100644 --- a/book/src/testnets.md +++ b/book/src/testnets.md @@ -1,10 +1,158 @@ # Testnets -Lighthouse does not offer a public testnet _yet_. In the meantime, it's easy to -start a local testnet: +The Lighthouse CLI has a `testnet` sub-command to allow creating or connecting +to Eth2 beacon chain testnets. -- [Run a simple testnet](testnets.html) in Only Three CLI Commands™. -- Developers of other Eth2 clients should see the [interop guide](interop.html). -- The [sigp/lighthouse-docker](https://github.com/sigp/lighthouse-docker) repo - contains a `docker-compose` setup that runs a multi-node network with - built-in metrics and monitoring dashboards, all from your local machine. +For detailed documentation, use the `--help` flag on the CLI: + +```bash +$ ./beacon_node testnet --help +``` + +```bash +$ ./validator_client testnet --help +``` + +## Examples + +All examples assume a working [development environment](./setup.md) and +commands are based in the `target/release` directory (this is the build dir for +`cargo`). + +### Start a beacon node given a validator count and genesis_time + + +To start a brand-new beacon node (with no history) use: + +```bash +$ ./beacon_node testnet -f quick 8 +``` + +Where `GENESIS_TIME` is in [unix time](https://duckduckgo.com/?q=unix+time&t=ffab&ia=answer). + +> Notes: +> +> - This method conforms the ["Quick-start +genesis"](https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#quick-start-genesis) +method in the `ethereum/eth2.0-pm` repository. +> - The `-f` flag ignores any existing database or configuration, backing them +> up before re-initializing. +> - `8` is the validator count and `1567222226` is the genesis time. +> - See `$ ./beacon_node testnet quick --help` for more configuration options. + +### Start a beacon node given a genesis state file + +A genesis state can be read from file using the `testnet file` subcommand. +There are three supported formats: + +- `ssz` (default) +- `json` +- `yaml` + +Start a new node using `/tmp/genesis.ssz` as the genesis state: + +```bash +$ ./beacon_node testnet --spec minimal -f file ssz /tmp/genesis.ssz +``` + +> Notes: +> +> - The `-f` flag ignores any existing database or configuration, backing them +> up before re-initializing. +> - See `$ ./beacon_node testnet file --help` for more configuration options. +> - The `--spec` flag is required to allow SSZ parsing of fixed-length lists. +> Here the `minimal` eth2 specification is chosen, allowing for lower +> validator counts. See +> [eth2.0-specs/configs](https://github.com/ethereum/eth2.0-specs/tree/dev/configs) +> for more info. + +### Start an auto-configured validator client + +To start a brand-new validator client (with no history) use: + +```bash +$ ./validator_client testnet -b insecure 0 8 +``` + +> Notes: +> +> - The `-b` flag means the validator client will "bootstrap" specs and config +> from the beacon node. +> - The `insecure` command dictates that the [interop keypairs](https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#pubkeyprivkey-generation) +> will be used. +> - The `0 8` indicates that this validator client should manage 8 validators, +> starting at validator 0 (the first deposited validator). +> - The validator client will try to connect to the beacon node at `localhost`. +> See `--help` to configure that address and other features. +> - The validator client will operate very unsafely in `testnet` mode, happily +> swapping between chains and creating double-votes. + +### Exporting a genesis file + +Genesis states can downloaded from a running Lighthouse node via the HTTP API. Three content-types are supported: + +- `application/json` +- `application/yaml` +- `application/ssz` + +Using `curl`, a genesis state can be downloaded to `/tmp/genesis.ssz`: + +```bash +$ curl --header "Content-Type: application/ssz" "localhost:5052/beacon/state/genesis" -o /tmp/genesis.ssz +``` + +## Advanced + +Below are some CLI commands useful when working with testnets. + +### Specify a boot node by multiaddr + +You can specify a static list of multiaddrs when booting Lighthouse using +the `--libp2p-addresses` command. + +#### Example: + +```bash +$ ./beacon_node --libp2p-addresses /ip4/192.168.0.1/tcp/9000 +``` + +### Specify a boot node by ENR (Ethereum Name Record) + +You can specify a static list of Discv5 addresses when booting Lighthouse using +the `--boot-nodes` command. + +#### Example: + +```bash +$ ./beacon_node --boot-nodes -IW4QB2Hi8TPuEzQ41Cdf1r2AUU1FFVFDBJdJyOkWk2qXpZfFZQy2YnJIyoT_5fnbtrXUouoskmydZl4pIg90clIkYUDgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjg0-DsTkQynhJCRnLLttBK1RS78lmUkLa-wgzAi-Ob5 +``` + +### Avoid port clashes when starting nodes + +Starting a second Lighthouse node on the same machine will fail due to TCP/UDP +port collisions. Use the `-b` (`--port-bump`) flag to increase all listening +ports by some `n`. + +#### Example: + +Increase all ports by `10` (using multiples of `10` is recommended). + +```bash +$ ./beacon_node -b 10 +``` + +### Start a testnet with a custom slot time + +Lighthouse can run at quite low slot times when there are few validators (e.g., +`500 ms` slot times should be fine for 8 validators). + +#### Example: + +The `-t` (`--slot-time`) flag specifies the milliseconds per slot. + +```bash +$ ./beacon_node testnet -t 500 recent 8 +``` + +> Note: `bootstrap` loads the slot time via HTTP and therefore conflicts with +> this flag. diff --git a/book/src/websockets.md b/book/src/websockets.md index 2b91bd88f..f1eb27475 100644 --- a/book/src/websockets.md +++ b/book/src/websockets.md @@ -1,4 +1,4 @@ -# Websocket Interface +# Websocket API By default, a Lighthouse `beacon_node` exposes a websocket server on `localhost:5053`. diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 3b69e9dd7..000000000 --- a/docs/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Lighthouse Documentation - -_Lighthouse is a work-in-progress. Instructions are provided for running the -client, however these instructions are designed for developers and researchers -working on the project. We do not (yet) provide user-facing functionality._ - -## Introduction - -- [Overview of Ethereum 2.0](serenity.md) -- [Development Environment Setup](env.md) - -For client implementers looking to inter-op, see the [Inter-Op -Docs](interop.md). - -## Command-line Interface - -With the [development environment](env.md) configured, run `cargo build --all ---release` (this can take several minutes on the first build). Then, -navigate to the `target/release/` directory and read the CLI documentation -using: - -``` -$ ./beacon_node -h -``` - -The main [`README.md`](../README.md#simple-local-testnet) provides instructions -for running a small, local testnet. - -## REST API - -The beacon node provides a RESTful HTTP API which serves information about the -Beacon Chain, the P2P network and more. - -This API is documented in the [`rest_oapi.yaml`](rest_oapi.yaml) Swagger YAML -file. There's an interactive version hosted on -[SwaggerHub](https://app.swaggerhub.com/apis/spble/lighthouse_rest_api/0.1.0). - -The implementation of the Swagger API in Lighthouse is incomplete, we do not -(yet) guarantee that all routes are implemented. - -## Configuration Files - -Lighthouse uses [TOML](https://github.com/toml-lang/toml) files for -configuration. The following binaries use the following config files (they are -generated from defaults if they don't already exist): - -- [Beacon Node](/beacon_node) - - [`~/.lighthouse/beacon_node.toml`](#beacon-nodetoml): the primary - configuration file for a beacon node. - - `~/.lighthouse/eth2-spec.toml`: defines chain-specific "constants" that - define an Ethereum 2.0 network. -- [Validator Client](/validator_client) - - `~/.lighthouse/validator_client.toml`: the primary configuration file for - a validator client. - - `~/.lighthouse/eth2-spec.toml`: defines chain-specific "constants" that - define an Ethereum 2.0 network. - -_Note: default directories are shown, CLI flags can be used to override these -defaults._ - -#### `beacon-node.toml` - -A TOML configuration file that defines the behaviour of the beacon node -runtime. - -- Located in the `datadir` (default `~/.lighthouse`) as `beacon-node.toml`. -- Created from defaults if not present. - -See the [example](config_examples/beacon-node.toml) for more information. diff --git a/docs/config_examples/beacon-node.toml b/docs/config_examples/beacon-node.toml deleted file mode 100644 index f0863934e..000000000 --- a/docs/config_examples/beacon-node.toml +++ /dev/null @@ -1,90 +0,0 @@ -# -# Beacon Node TOML configuration file. -# -# Defines the runtime configuration of a Lighthouse Beacon Node. -# - -# The directory where beacon-node specific files will be placed. Includes the -# database and configuration files. -data_dir = ".lighthouse" -# The type of database used. Can be either: -# -# - "disk": LevelDB (almost always desired). -# - "memory": an in-memory hashmap (only used for testing). -db_type = "disk" -# The name of the LevelDB database directory, if any. -db_name = "chain_db" -# If specified, all logs will be written to this file. -log_file = "" -# Defines the Ethereum 2.0 specification set to be used: -# -# - "mainnet": parameters expected to be used for Eth2 mainnet. -# - "minimal": smaller, more efficient parameters used for testing. -spec_constants = "minimal" - -# -# The "genesis_state" object defines how the genesis state should be created. -# - -# The "RecentGenesis" type assumes that genesis started at the beginning of the -# most-recent 30 minute window (e.g., 08:00, 08:30, 09:00, ...). -[genesis_state] -type = "RecentGenesis" -validator_count = 16 - -# "Generated" is the same as "RecentGenesis", however allows for manual -# specification of the genesis_time. -# -# [genesis_state] -# type = "Generated" -# validator_count = 16 -# genesis_time = 1564620118 - -# "Yaml" loads a full genesis state from YAML file. -# -# [genesis_state] -# type = "Yaml" -# file = "~/genesis_state.yaml" - -# -# P2P networking configuration. -# -[network] -# The directory for storing p2p network related files. E.g., p2p keys, peer -# lists, etc. -network_dir = "/home/paul/.lighthouse/network" -# The address that libp2p should use for incoming connections. -listen_address = "127.0.0.1" -# The port that libp2p should use for incoming connections. -libp2p_port = 9000 -# The address that should listen for UDP peer-discovery. -discovery_address = "127.0.0.1" -# The port that should listen for UDP peer-discovery. -discovery_port = 9000 -# Maximum number of libp2p peers. -max_peers = 10 -# Boot nodes for initial peer discovery. -boot_nodes = [] -# The client version, may be customized. -client_version = "Lighthouse/v0.1.0-unstable/x86_64-linux" -# A list of libp2p topics. Purpose unknown. -topics = [] - -# -# gRPC configuration. To be removed. -# -[rpc] -enabled = false -listen_address = "127.0.0.1" -port = 5051 - -# -# RESTful HTTP API server configuration. -# -[rest_api] -# Set to `true` to enable the gRPC server. -enabled = true -# The listen port for the HTTP server. -listen_address = "127.0.0.1" -# The listen port for the HTTP server. -port = 5052 diff --git a/docs/documentation.md b/docs/documentation.md deleted file mode 100644 index 360055887..000000000 --- a/docs/documentation.md +++ /dev/null @@ -1,14 +0,0 @@ -# Lighthouse Technical Documentation - -The technical documentation, as generated by Rust, is available at [lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/). - -This documentation is generated from Lighthouse and updated regularly. - - -### How to update: - -- `cargo doc`: builds the docs inside the `target/doc/` directory. -- `aws s3 sync target/doc/ s3://lighthouse-docs.sigmaprime.io/`: Uploads all of the docs, as generated with `cargo doc`, to the S3 bucket. - -**Note**: You will need appropriate credentials to make the upload. - diff --git a/docs/env.md b/docs/env.md deleted file mode 100644 index d73c68862..000000000 --- a/docs/env.md +++ /dev/null @@ -1,53 +0,0 @@ -# Development Environment Setup - -_This document describes how to setup a development environment. It is intended -for software developers and researchers who wish to contribute to development._ - -Lighthouse is a Rust project and [`cargo`](https://doc.rust-lang.org/cargo/) is -used extensively. As such, you'll need to install Rust in order to build the -project. Generally, Rust is installed using the -[rustup](https://www.rust-lang.org/tools/install) tool-chain manager. - -## Steps - -A fully-featured development environment can be achieved with the following -steps: - - 1. Install [rustup](https://rustup.rs/). - 1. Use the command `rustup show` to get information about the Rust - installation. You should see that the active tool-chain is the stable - version. - - Updates can be performed using` rustup update`, Lighthouse generally - requires a recent version of Rust. - 1. Install build dependencies (Arch packages are listed here, your - distribution will likely be similar): - - `clang`: required by RocksDB. - - `protobuf`: required for protobuf serialization (gRPC) - - `libssl-dev`: also gRPC - - `cmake`: required for building protobuf - - `git-lfs`: The Git extension for [Large File - Support](https://git-lfs.github.com/) (required for Ethereum Foundation - test vectors). - 1. Clone the repository with submodules: `git clone --recursive - https://github.com/sigp/lighthouse`. If you're already cloned the repo, - ensure testing submodules are present: `$ git submodule init; git - submodule update` - 1. Change directory to the root of the repository. - 1. Run the test suite with `cargo test --all --release`. The build and test - process can take several minutes. If you experience any failures on - `master`, please raise an - [issue](https://github.com/sigp/lighthouse/issues). - -## Notes: - -Lighthouse targets Rust `stable` but generally runs on `nightly` too. - -### Note for Windows users: - -Perl may also be required to build lighthouse. You can install [Strawberry -Perl](http://strawberryperl.com/), or alternatively use a choco install command -`choco install strawberryperl`. - -Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues -compiling in Windows. You can specify a known working version by editing -version in `protos/Cargo.toml` section to `protoc-grpcio = "<=0.3.0"`. diff --git a/docs/interop.md b/docs/interop.md deleted file mode 100644 index 5cd3f472a..000000000 --- a/docs/interop.md +++ /dev/null @@ -1,109 +0,0 @@ -# Lighthouse Inter-Op Docs - -_These documents are intended for a highly technical audience, specifically -Ethereum 2.0 implementers._ - -This document provides details on how to use Lighthouse for inter-op testing. - -## Steps - -_Note: binaries are compiled into the `target/release` directory of the -repository. In this example, we run binaries assuming the user is in this -directory. E.g., running the beacon node binary can be achieved with -`$ ./target/release/beacon_node`. Those familiar with `cargo` may use the -equivalent (and more-convenient) `cargo run --release --` commands._ - -1. Setup a Lighthouse [development environment](env.md). -1. Build all the binaries using `cargo build --all --release` -1. Create default configuration files by running `$ ./beacon_node` and pressing - Ctrl+C after the node has started. -1. Follow the steps in [Genesis](#genesis) to configure the genesis state. -1. Follow the steps in [Networking](#networking) to launch a node with - appropriate networking parameters. - -## Genesis - -Lighthouse supports the following methods for generating a genesis state: - -- [`Yaml`](#yaml): loads the genesis state from some YAML file (recommended - method). -- [`Generated`](#generated): generates a state given a `(validator_count, - genesis_time)` - tuple. _Note: this method is not yet fully specified and the state - generated is almost certainly not identical to other implementations._ -- [`RecentGenesis`](#recentgenesis): identical to `Generated`, however the - `genesis_time` is set - to the previous 30-minute window. For example, if a state is generated at - `0845`, the genesis time will be `0830`. - -You may configure a `beacon_node` to use one of these methods using the -[`beacon_node.toml`](README.md#beacon-nodetoml). There is a [documented -example](config_examples/) configuration file which includes an example for -each of these methods (see the `genesis_state` object). - -### Yaml - -This method involves loading a `BeaconState` from a YAML file. We provide -instructions for generating that YAML file and starting from it. If starting -from a pre-existing YAML file, simply skip the generation steps. - -#### Generating a YAML file - -The [cli_util](/tests/cli_util) generate YAML genesis state files. You can run -`$ ./cli_util genesis_yaml -h` to see documentation. We provide an example to -generate a YAML file with the following properties: - -- 10 initial validators, each with [deterministic - keypairs](https://github.com/ethereum/eth2.0-pm/issues/60#issuecomment-512157915). -- The genesis file is stored in `~/.lighthouse/`, the default data directory - (an absolute path must be supplied). -- Genesis time is set to the time when the command is run (it can be customized - with the `-g` flag). - -``` -$ ./cli_util genesis_yaml -n 10 -f /home/user/.lighthouse/genesis_state.yaml -``` - -#### Configuring the Beacon Node - -Modify the [`beacon-node.toml`](README.md#beacon-nodetoml) file to have the -following `genesiss_state` object (choosing the `file`): - -``` -[genesis_state] -type = "Yaml" -file = "/home/user/.lighthouse/genesis_state.yaml" -``` - -### Generated - -Modify the [`beacon-node.toml`](README.md#beacon-nodetoml) file to have the -following `genesis_state` object (choosing the `validator_count` and -`genesis_time`): - -``` -[genesis_state] -type = "Generated" -validator_count = 16 -genesis_time = 1564620118 -``` - -### RecentGenesis - -Modify the [`beacon-node.toml`](README.md#beacon-nodetoml) file to have the -following `genesis_state` object (choosing the `validator_count`): - -``` -[genesis_state] -type = "RecentGenesis" -validator_count = 16 -``` - -## Networking - -_TODO: provide details on config required to connect to some IP address._ - -## References - -The BLS key generation method used should be identical to [this -implementation](https://github.com/ethereum/eth2.0-pm/issues/60#issuecomment-512157915). diff --git a/docs/serenity.md b/docs/serenity.md deleted file mode 100644 index 9a190449f..000000000 --- a/docs/serenity.md +++ /dev/null @@ -1,161 +0,0 @@ -# Ethereum Serenity - -This document aims at providing a high level understanding of Ethereum and the -Serenity phase of the Ethereum roadmap. - -## The Blockchain - -A blockchain can be seen as a decentralized, distributed ledger. The ledger of -transactions is replicated onto all nodes in the network. When a transaction -occurs, it is first propagated to the nodes. Once the nodes receive the -transaction, and verifies the correctness, the nodes attempt to batch the -transactions into a block and append the block to the ledger. Once the ledger -has been successfully appended onto, they propagate the block to the network. -If accepted, this block now becomes the latest block in the chain. If two people -propose a block at the same time, the one canonical blockchain forks. At this -point it must be resolved, and each system has it's own way of resolving these -forks. - -![Blockchain](http://yuml.me/b0d6b30a.jpg) -
Figure 1. Example blockchain with a resolved fork.
- -
- -The idea of the blockchain was first proposed in the seminal [Bitcoin -whitepaper](https://bitcoin.org/bitcoin.pdf) by Satoshi Nakamoto. Since then, a -vast number of updates and blockchains have taken shape providing different -functionality or properties to the original blockchain. - -## What is Ethereum? - -Ethereum is an open blockchain protocol, allowing for the building and use of -decentralized applications that run on blockchain technology. Ethereum was one -of the initial platforms providing turing-complete code to be run on the -blockchain, allowing for conditional payments to occur through the use of this -code. Since then, Ethereum has advanced to allow for a number of Decentralized -Applications (DApps) to be developed and run completely with the blockchain as -the backbone. - -General Ethereum Introduction: - -* [What is Ethereum](http://ethdocs.org/en/latest/introduction/what-is-ethereum.html) -* [Ethereum Introduction](https://github.com/ethereum/wiki/wiki/Ethereum-introduction) - - -### Proof-of-Work and the current state of Ethereum. - -Currently, Ethereum is based on the Proof-of-Work model, a Sybil resilient -mechanism to allow nodes to propose blocks to the network. Although it provides -properties that allow the blockchain to operate in an open, public -(permissionless) network, it faces it's challenges and as a result impacts -the operation of the blockchain. - -The main goals to advance Ethereum is to (1) increase the scalability and -overall transaction processing power of the Ethereum world computer and (2) -find a suitable replacement for Proof-of-Work that still provides the necessary -properties that we need. - -* [Proof-of-Work in Cryptocurrencies: an accessible introduction](https://blog.sigmaprime.io/what-is-proof-of-work.html) - -## Serenity - -Ethereum Serenity refers to a new blockchain system currently under development -by the Ethereum Foundation and the Ethereum community. - -As part of the original Ethereum roadmap -[\[1\]](https://blog.ethereum.org/2015/03/03/ethereum-launch-process/) -[\[2\]](http://ethdocs.org/en/latest/introduction/the-homestead-release.html), -the Proof-of-Stake integration falls under **Release Step 4: *Serenity***. With -this, a number of changes are to be made to the current Ethereum protocol to -incorporate some of the new Proof-of-Stake mechanisms as well as improve on -some of the hindrances faced by the current Proof-of-Work chain. - -To now advance the current Ethereum, the decision is made to move to a sharded -Beacon chain structure where multiple shard-chains will be operating and -interacting with a central beacon chain.The Serenity blockchain consists of -1,025 proof-of-stake blockchains. This includes the "beacon chain" and 1,024 -"shard chains". - -Ethereum Serenity is also known as "Ethereum 2.0" and "Shasper". We prefer -Serenity as it more accurately reflects the established Ethereum roadmap (plus -we think it's a nice name). - -(Be mindful, the specifications change occasionally, so check these to keep up -to date) - -* Current Specifications: - * [Danny Ryan's "State of the Spec"](https://notes.ethereum.org/s/BJEZWNoyE) (A nice summary of the current specifications) - * [Ethereum Serenity - Phase 0: Beacon Chain Spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md) - * [Ethereum Serenity - Phase 1: Sharded Data Chains](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/1_shard-data-chains.md) - * [Beacon Chain - Vitalik Buterin and Justin Drake explain](https://www.youtube.com/watch?v=GAywmwGToUI) -* Understanding Sharding: - * [Prysmatic Labs: Sharding Explained](https://medium.com/prysmatic-labs/how-to-scale-ethereum-sharding-explained-ba2e283b7fce) -* Other relevant resources - * [Proof of Stake - Casper FFG](https://www.youtube.com/watch?v=uQ3IqLDf-oo) - * [Justin Drake VDF Devcon4 Talk](https://www.youtube.com/watch?v=zqL_cMlPjOI) - - -### Beacon Chain - -The concept of a beacon chain differs from existing blockchains, such as -Bitcoin and Ethereum, in that it doesn't process transactions per se. Instead, -it maintains a set of bonded (staked) validators and coordinates these to -provide services to a static set of *sub-blockchains* (i.e. shards). Each of -these shard blockchains processes normal transactions (e.g. "Transfer 5 ETH -from A to B") in parallel whilst deferring consensus mechanisms to the beacon -chain. - -Major services provided by the beacon chain to its shards include the following: - -- A source of entropy, likely using a [RANDAO + VDF - scheme](https://ethresear.ch/t/minimal-vdf-randomness-beacon/3566). -- Validator management, including: - - Inducting and ejecting validators. - - Assigning randomly-shuffled subsets of validators to particular shards. - - Penalizing and rewarding validators. -- Proof-of-stake consensus for shard chain blocks. - -### Shard Chains - -Shards are analogous to CPU cores - they're a resource where transactions can -execute in series (one-after-another). Presently, Ethereum is single-core and -can only _fully_ process one transaction at a time. Sharding allows processing -of multiple transactions simultaneously, greatly increasing the per-second -transaction capacity of Ethereum. - -Each shard uses a proof-of-stake consensus mechanism and shares its validators -(stakers) with other shards. The beacon chain rotates validators -pseudo-randomly between different shards. Shards will likely be the basis of -layer-2 transaction processing schemes, however, that is not in scope of this -discussion. - -### The Proof-of-Work Chain - -The present-Ethereum proof-of-work (PoW) chain will host a smart contract that -enables accounts to deposit 32 ETH, a BLS public key, and some [other -parameters](https://github.com/ethereum/eth2.0-specs/blob/master/specs/casper_sharding_v2.1.md#pow-chain-changes), -allowing them to become beacon chain validators. Each beacon chain will -reference a PoW block hash allowing PoW clients to use the beacon chain as a -source of [Casper FFG finality](https://arxiv.org/abs/1710.09437), if desired. - -It is a requirement that ETH can move freely between shard chains, as well as between -Serenity and present-Ethereum blockchains. The exact mechanics of these transfers remain -an active topic of research and their details are yet to be confirmed. - -## Serenity Progress - -Ethereum Serenity is not fully specified and a working implementation does not -yet exist. Some teams have demos available which indicate progress, but do not -constitute a complete product. We look forward to providing user functionality -once we are ready to provide a minimum-viable user experience. - -The work-in-progress specifications live in the -[ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs) repository. -There is active discussion about the specification in the -[ethereum/sharding](https://gitter.im/ethereum/sharding) gitter channel. A -proof-of-concept implementation in Python is available at -[ethereum/beacon_chain](https://github.com/ethereum/beacon_chain). - -Presently, the specification focuses almost exclusively on the beacon chain, -as it is the focus of current development efforts. Progress on shard chain -specification will soon follow. diff --git a/eth2/README.md b/eth2/README.md index 5f1264372..1ce1c919d 100644 --- a/eth2/README.md +++ b/eth2/README.md @@ -1,35 +1,3 @@ -# Ethereum 2.0 Common Crates +# eth2 -Rust crates containing logic common across the Lighthouse project. - -## Per-Crate Summary - -- [`attester/`](attester/): Core logic for attesting to beacon and shard blocks. -- [`block_proposer/`](block_proposer/): Core logic for proposing beacon blocks. -- [`fork_choice/`](fork_choice/): A collection of fork-choice algorithms for - the Beacon Chain. -- [`state_processing/`](state_processing/): Provides per-slot, per-block, and - per-epoch state processing. -- [`types/`](types/): Defines base Ethereum 2.0 types (e.g., `BeaconBlock`, - `BeaconState`, etc). -- [`utils/`](utils/): - - [`bls`](utils/bls/): A wrapper for an external BLS encryption library. - - [`fisher-yates-shuffle`](utils/fisher-yates-shuffle/): shuffles a list - pseudo-randomly. - - [`hashing`](utils/hashing/): A wrapper for external hashing libraries. - - [`honey-badger-split`](utils/honey-badger-split/): Splits a list in `n` - parts without giving AF about the length of the list, `n`, or anything - else. - - [`int-to-bytes`](utils/int-to-bytes/): Simple library which converts ints - into byte-strings of various lengths. - - [`slot_clock`](utils/slot_clock/): translates the system time into Beacon - Chain "slots". (Also provides another slot clock that's useful during - testing.) - - [`ssz`](utils/ssz/): an implementation of the SimpleSerialize - serialization/deserialization protocol used by Eth 2.0. - - [`ssz_derive`](utils/ssz_derive/): provides procedural macros for - deriving SSZ `Encode`, `Decode`, and `TreeHash` methods. - - [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling - method which is slow, but allows for a subset of indices to be shuffled. - - [`test_random_derive`](utils/test_random_derive/): provides procedural - macros for deriving the `TestRandom` trait defined in `types`. +Common crates containing eth2-specific logic. diff --git a/lcli/README.md b/lcli/README.md new file mode 100644 index 000000000..50ed6a3aa --- /dev/null +++ b/lcli/README.md @@ -0,0 +1,11 @@ +# lcli + +A command-line debugging tool, inspired by [zcli](https://github.com/protolambda/zcli). + +Allows for replaying state transitions from SSZ files to assist in fault-finding. + +## Usage + +``` +$ cargo run --release -- --help +``` diff --git a/tests/ef_tests/Makefile b/tests/ef_tests/Makefile new file mode 100644 index 000000000..ff7e9e815 --- /dev/null +++ b/tests/ef_tests/Makefile @@ -0,0 +1,18 @@ +TESTS_TAG := v0.8.3 +TESTS = general minimal mainnet + +REPO_NAME := eth2.0-spec-tests +OUTPUT_DIR := ./$(REPO_NAME) + +BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG) + +eth2.0-spec-tests: + mkdir $(OUTPUT_DIR) + for test in $(TESTS); do \ + wget $(BASE_URL)/$$test.tar.gz; \ + tar -xzvf $$test.tar.gz -C $(OUTPUT_DIR);\ + rm $$test.tar.gz;\ + done + +clean: + rm -r $(OUTPUT_DIR) diff --git a/tests/ef_tests/README.md b/tests/ef_tests/README.md new file mode 100644 index 000000000..944891d75 --- /dev/null +++ b/tests/ef_tests/README.md @@ -0,0 +1,30 @@ +# Ethereum 2.0 Specification Tests + +This crate parses and executes the test vectors at [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests). + +Functionality is achieved only via the `$ cargo test` command. + +## Tests + +Because the test vectors are very large, we do not download the +tests vectors or require that the tests pass by default. Specifically; + +- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests + indicate a `pass` when they did not actually run. +- If that directory _is_ present, the tests are executed faithfully, failing if + a discrepancy is found. + +## Downloading Test Vectors + +The `eth2.0-spec-tests` directory is not present by default. To +obtain it, use the Makefile: + +``` +$ make +``` + +_Note: this may download hundreds of MB of compressed archives from the +[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/), +which may expand into several GB of files._ + +Remove the tests to save space or update to a new version with `$ make clean`. diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index b23032e50..0d8b39991 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -6,7 +6,6 @@ pub use self::beacon_node_block::{BeaconNodeError, PublishOutcome}; pub use self::grpc::BeaconBlockGrpcClient; use crate::signer::Signer; use core::marker::PhantomData; -use serde_json; use slog::{error, info, trace, warn}; use std::sync::Arc; use tree_hash::{SignedRoot, TreeHash}; @@ -99,10 +98,7 @@ impl<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> BlockProducer<'a, B, S, E> { warn!(self.log, "Signing rejected"; "message" => format!("{:?}", message)); return Ok(ValidatorEvent::SignerRejection(self.slot)); } - Some(signature) => { - info!(self.log, "Signed tree_hash_root for randao_reveal"; "message" => format!("{:?}", message), "signature" => serde_json::to_string(&signature).expect("We should always be able to serialize a signature as JSON.")); - signature - } + Some(signature) => signature, }; if let Some(block) = self