2021-09-25 07:53:58 +00:00
|
|
|
use beacon_chain::test_utils::RelativeSyncCommittee;
|
2020-09-29 03:46:54 +00:00
|
|
|
use beacon_chain::{
|
2020-10-19 05:58:39 +00:00
|
|
|
test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
|
2023-08-21 05:02:32 +00:00
|
|
|
BeaconChain, ChainConfig, StateSkipConfig, WhenSlotSkipped,
|
2020-09-29 03:46:54 +00:00
|
|
|
};
|
|
|
|
|
use environment::null_logger;
|
2021-10-28 01:18:04 +00:00
|
|
|
use eth2::{
|
|
|
|
|
mixin::{RequestAccept, ResponseForkName, ResponseOptional},
|
|
|
|
|
reqwest::RequestBuilder,
|
2023-03-29 02:56:37 +00:00
|
|
|
types::{BlockId as CoreBlockId, ForkChoiceNode, StateId as CoreStateId, *},
|
2023-08-28 00:55:31 +00:00
|
|
|
BeaconNodeHttpClient, Error, StatusCode, Timeouts,
|
2021-10-28 01:18:04 +00:00
|
|
|
};
|
2023-02-02 01:37:46 +00:00
|
|
|
use execution_layer::test_utils::{
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
MockBuilder, Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_BUILDER_THRESHOLD_WEI,
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
|
2023-02-02 01:37:46 +00:00
|
|
|
};
|
2020-12-04 00:18:58 +00:00
|
|
|
use futures::stream::{Stream, StreamExt};
|
2021-02-10 23:29:49 +00:00
|
|
|
use futures::FutureExt;
|
2023-04-03 05:35:11 +00:00
|
|
|
use http_api::{
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
test_utils::{create_api_server, ApiServer},
|
2023-04-03 05:35:11 +00:00
|
|
|
BlockId, StateId,
|
|
|
|
|
};
|
Rename eth2_libp2p to lighthouse_network (#2702)
## Description
The `eth2_libp2p` crate was originally named and designed to incorporate a simple libp2p integration into lighthouse. Since its origins the crates purpose has expanded dramatically. It now houses a lot more sophistication that is specific to lighthouse and no longer just a libp2p integration.
As of this writing it currently houses the following high-level lighthouse-specific logic:
- Lighthouse's implementation of the eth2 RPC protocol and specific encodings/decodings
- Integration and handling of ENRs with respect to libp2p and eth2
- Lighthouse's discovery logic, its integration with discv5 and logic about searching and handling peers.
- Lighthouse's peer manager - This is a large module handling various aspects of Lighthouse's network, such as peer scoring, handling pings and metadata, connection maintenance and recording, etc.
- Lighthouse's peer database - This is a collection of information stored for each individual peer which is specific to lighthouse. We store connection state, sync state, last seen ips and scores etc. The data stored for each peer is designed for various elements of the lighthouse code base such as syncing and the http api.
- Gossipsub scoring - This stores a collection of gossipsub 1.1 scoring mechanisms that are continuously analyssed and updated based on the ethereum 2 networks and how Lighthouse performs on these networks.
- Lighthouse specific types for managing gossipsub topics, sync status and ENR fields
- Lighthouse's network HTTP API metrics - A collection of metrics for lighthouse network monitoring
- Lighthouse's custom configuration of all networking protocols, RPC, gossipsub, discovery, identify and libp2p.
Therefore it makes sense to rename the crate to be more akin to its current purposes, simply that it manages the majority of Lighthouse's network stack. This PR renames this crate to `lighthouse_network`
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2021-10-19 00:30:39 +00:00
|
|
|
use lighthouse_network::{Enr, EnrExt, PeerId};
|
2022-08-30 05:47:31 +00:00
|
|
|
use network::NetworkReceivers;
|
2022-07-25 08:23:00 +00:00
|
|
|
use proto_array::ExecutionStatus;
|
2021-05-04 01:59:51 +00:00
|
|
|
use sensitive_url::SensitiveUrl;
|
2021-03-29 23:42:35 +00:00
|
|
|
use slot_clock::SlotClock;
|
2023-02-10 19:30:14 +00:00
|
|
|
use state_processing::per_block_processing::get_expected_withdrawals;
|
2020-09-29 03:46:54 +00:00
|
|
|
use state_processing::per_slot_processing;
|
2023-08-24 05:54:36 +00:00
|
|
|
use state_processing::state_advance::partial_state_advance;
|
2020-09-29 03:46:54 +00:00
|
|
|
use std::convert::TryInto;
|
|
|
|
|
use std::sync::Arc;
|
2020-12-04 00:18:58 +00:00
|
|
|
use tokio::time::Duration;
|
2020-09-29 03:46:54 +00:00
|
|
|
use tree_hash::TreeHash;
|
2022-06-30 00:49:21 +00:00
|
|
|
use types::application_domain::ApplicationDomain;
|
2020-09-29 03:46:54 +00:00
|
|
|
use types::{
|
2022-07-25 08:23:00 +00:00
|
|
|
AggregateSignature, BitList, Domain, EthSpec, ExecutionBlockHash, Hash256, Keypair,
|
|
|
|
|
MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, Slot,
|
2020-09-29 03:46:54 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
type E = MainnetEthSpec;
|
|
|
|
|
|
2021-07-12 01:47:48 +00:00
|
|
|
const SECONDS_PER_SLOT: u64 = 12;
|
2020-09-29 03:46:54 +00:00
|
|
|
const SLOTS_PER_EPOCH: u64 = 32;
|
|
|
|
|
const VALIDATOR_COUNT: usize = SLOTS_PER_EPOCH as usize;
|
2020-12-04 00:18:58 +00:00
|
|
|
const CHAIN_LENGTH: u64 = SLOTS_PER_EPOCH * 5 - 1; // Make `next_block` an epoch transition
|
2020-09-29 03:46:54 +00:00
|
|
|
const JUSTIFIED_EPOCH: u64 = 4;
|
|
|
|
|
const FINALIZED_EPOCH: u64 = 3;
|
2020-11-09 04:01:03 +00:00
|
|
|
const EXTERNAL_ADDR: &str = "/ip4/0.0.0.0/tcp/9000";
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
/// Skipping the slots around the epoch boundary allows us to check that we're obtaining states
|
|
|
|
|
/// from skipped slots for the finalized and justified checkpoints (instead of the state from the
|
|
|
|
|
/// block that those roots point to).
|
|
|
|
|
const SKIPPED_SLOTS: &[u64] = &[
|
|
|
|
|
JUSTIFIED_EPOCH * SLOTS_PER_EPOCH - 1,
|
|
|
|
|
JUSTIFIED_EPOCH * SLOTS_PER_EPOCH,
|
|
|
|
|
FINALIZED_EPOCH * SLOTS_PER_EPOCH - 1,
|
|
|
|
|
FINALIZED_EPOCH * SLOTS_PER_EPOCH,
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
struct ApiTester {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
harness: Arc<BeaconChainHarness<EphemeralHarnessType<E>>>,
|
2020-10-19 05:58:39 +00:00
|
|
|
chain: Arc<BeaconChain<EphemeralHarnessType<E>>>,
|
2020-09-29 03:46:54 +00:00
|
|
|
client: BeaconNodeHttpClient,
|
2023-03-15 20:34:00 +00:00
|
|
|
next_block: SignedBlockContents<E>,
|
|
|
|
|
reorg_block: SignedBlockContents<E>,
|
2020-09-29 03:46:54 +00:00
|
|
|
attestations: Vec<Attestation<E>>,
|
2021-09-25 07:53:58 +00:00
|
|
|
contribution_and_proofs: Vec<SignedContributionAndProof<E>>,
|
2020-09-29 03:46:54 +00:00
|
|
|
attester_slashing: AttesterSlashing<E>,
|
|
|
|
|
proposer_slashing: ProposerSlashing,
|
|
|
|
|
voluntary_exit: SignedVoluntaryExit,
|
2022-08-30 05:47:31 +00:00
|
|
|
network_rx: NetworkReceivers<E>,
|
2020-10-22 02:59:42 +00:00
|
|
|
local_enr: Enr,
|
|
|
|
|
external_peer_id: PeerId,
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
mock_builder: Option<Arc<MockBuilder<E>>>,
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
struct ApiTesterConfig {
|
|
|
|
|
spec: ChainSpec,
|
2023-08-21 05:02:32 +00:00
|
|
|
retain_historic_states: bool,
|
2023-09-26 02:39:58 +00:00
|
|
|
builder_threshold: Option<u128>,
|
2023-02-02 01:37:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Default for ApiTesterConfig {
|
|
|
|
|
fn default() -> Self {
|
|
|
|
|
let mut spec = E::default_spec();
|
|
|
|
|
spec.shard_committee_period = 2;
|
|
|
|
|
Self {
|
|
|
|
|
spec,
|
2023-08-21 05:02:32 +00:00
|
|
|
retain_historic_states: false,
|
2023-09-26 02:39:58 +00:00
|
|
|
builder_threshold: None,
|
2023-02-02 01:37:46 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-21 05:02:32 +00:00
|
|
|
impl ApiTesterConfig {
|
|
|
|
|
fn retain_historic_states(mut self) -> Self {
|
|
|
|
|
self.retain_historic_states = true;
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
impl ApiTester {
|
2021-10-11 02:45:06 +00:00
|
|
|
pub async fn new() -> Self {
|
2021-08-06 00:47:31 +00:00
|
|
|
// This allows for testing voluntary exits without building out a massive chain.
|
2023-02-02 01:37:46 +00:00
|
|
|
Self::new_from_config(ApiTesterConfig::default()).await
|
2021-09-25 07:53:58 +00:00
|
|
|
}
|
2021-08-06 00:47:31 +00:00
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
pub async fn new_with_hard_forks(altair: bool, bellatrix: bool) -> Self {
|
2023-02-02 01:37:46 +00:00
|
|
|
let mut config = ApiTesterConfig::default();
|
2022-07-25 08:23:00 +00:00
|
|
|
// Set whether the chain has undergone each hard fork.
|
|
|
|
|
if altair {
|
2023-02-02 01:37:46 +00:00
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
2022-07-25 08:23:00 +00:00
|
|
|
}
|
|
|
|
|
if bellatrix {
|
2023-02-02 01:37:46 +00:00
|
|
|
config.spec.bellatrix_fork_epoch = Some(Epoch::new(0));
|
2022-07-25 08:23:00 +00:00
|
|
|
}
|
2023-02-02 01:37:46 +00:00
|
|
|
Self::new_from_config(config).await
|
2022-07-25 08:23:00 +00:00
|
|
|
}
|
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
pub async fn new_from_config(config: ApiTesterConfig) -> Self {
|
|
|
|
|
let spec = config.spec;
|
2022-07-30 00:22:37 +00:00
|
|
|
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
let mut harness = BeaconChainHarness::builder(MainnetEthSpec)
|
|
|
|
|
.spec(spec.clone())
|
|
|
|
|
.chain_config(ChainConfig {
|
|
|
|
|
reconstruct_historic_states: config.retain_historic_states,
|
|
|
|
|
..ChainConfig::default()
|
|
|
|
|
})
|
|
|
|
|
.logger(logging::test_logger())
|
|
|
|
|
.deterministic_keypairs(VALIDATOR_COUNT)
|
|
|
|
|
.fresh_ephemeral_store()
|
2023-09-26 02:39:58 +00:00
|
|
|
.mock_execution_layer_with_config(config.builder_threshold)
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
.build();
|
|
|
|
|
|
|
|
|
|
harness
|
|
|
|
|
.mock_execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.server
|
|
|
|
|
.execution_block_generator()
|
|
|
|
|
.move_to_terminal_block()
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
harness.advance_slot();
|
|
|
|
|
|
|
|
|
|
for _ in 0..CHAIN_LENGTH {
|
|
|
|
|
let slot = harness.chain.slot().unwrap().as_u64();
|
|
|
|
|
|
|
|
|
|
if !SKIPPED_SLOTS.contains(&slot) {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
harness
|
|
|
|
|
.extend_chain(
|
|
|
|
|
1,
|
|
|
|
|
BlockStrategy::OnCanonicalHead,
|
|
|
|
|
AttestationStrategy::AllValidators,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
harness.advance_slot();
|
|
|
|
|
}
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let head = harness.chain.head_snapshot();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
harness.chain.slot().unwrap(),
|
|
|
|
|
head.beacon_block.slot() + 1,
|
|
|
|
|
"precondition: current slot is one after head"
|
|
|
|
|
);
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let (next_block, _next_state) = harness
|
|
|
|
|
.make_block(head.beacon_state.clone(), harness.chain.slot().unwrap())
|
|
|
|
|
.await;
|
2023-03-15 20:34:00 +00:00
|
|
|
let next_block = SignedBlockContents::from(next_block);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-06-17 02:10:46 +00:00
|
|
|
// `make_block` adds random graffiti, so this will produce an alternate block
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let (reorg_block, _reorg_state) = harness
|
Add broadcast validation routes to Beacon Node HTTP API (#4316)
## Issue Addressed
- #4293
- #4264
## Proposed Changes
*Changes largely follow those suggested in the main issue*.
- Add new routes to HTTP API
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Add new routes to `BeaconNodeHttpClient`
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Define new Eth2 common types
- `BroadcastValidation`, enum representing the level of validation to apply to blocks prior to broadcast
- `BroadcastValidationQuery`, the corresponding HTTP query string type for the above type
- ~~Define `_checked` variants of both `publish_block` and `publish_blinded_block` that enforce a validation level at a type level~~
- Add interactive tests to the `bn_http_api_tests` test target covering each validation level (to their own test module, `broadcast_validation_tests`)
- `beacon/blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- Only consensus pass (i.e., equivocates) (200)
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- `beacon/blinded_blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- ~~Only consensus pass (i.e., equivocates) (200)~~
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- Add a new trait, `IntoGossipVerifiedBlock`, which allows type-level guarantees to be made as to gossip validity
- Modify the structure of the `ObservedBlockProducers` cache from a `(slot, validator_index)` mapping to a `((slot, validator_index), block_root)` mapping
- Modify `ObservedBlockProducers::proposer_has_been_observed` to return a `SeenBlock` rather than a boolean on success
- Punish gossip peer (low) for submitting equivocating blocks
- Rename `BlockError::SlashablePublish` to `BlockError::SlashableProposal`
## Additional Info
This PR contains changes that directly modify how blocks are verified within the client. For more context, consult [comments in-thread](https://github.com/sigp/lighthouse/pull/4316#discussion_r1234724202).
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
2023-06-29 12:02:38 +00:00
|
|
|
.make_block(head.beacon_state.clone(), harness.chain.slot().unwrap() + 1)
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.await;
|
2023-03-15 20:34:00 +00:00
|
|
|
let reorg_block = SignedBlockContents::from(reorg_block);
|
2021-06-17 02:10:46 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let head_state_root = head.beacon_state_root();
|
2020-09-29 03:46:54 +00:00
|
|
|
let attestations = harness
|
|
|
|
|
.get_unaggregated_attestations(
|
|
|
|
|
&AttestationStrategy::AllValidators,
|
|
|
|
|
&head.beacon_state,
|
2021-03-17 05:09:57 +00:00
|
|
|
head_state_root,
|
2020-09-29 03:46:54 +00:00
|
|
|
head.beacon_block_root,
|
|
|
|
|
harness.chain.slot().unwrap(),
|
|
|
|
|
)
|
|
|
|
|
.into_iter()
|
2022-05-16 01:59:47 +00:00
|
|
|
.flat_map(|vec| vec.into_iter().map(|(attestation, _subnet_id)| attestation))
|
2020-09-29 03:46:54 +00:00
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
assert!(
|
|
|
|
|
!attestations.is_empty(),
|
|
|
|
|
"precondition: attestations for testing"
|
|
|
|
|
);
|
|
|
|
|
|
2021-09-25 07:53:58 +00:00
|
|
|
let current_epoch = harness
|
|
|
|
|
.chain
|
|
|
|
|
.slot()
|
|
|
|
|
.expect("should get current slot")
|
|
|
|
|
.epoch(E::slots_per_epoch());
|
|
|
|
|
let is_altair = spec
|
|
|
|
|
.altair_fork_epoch
|
|
|
|
|
.map(|epoch| epoch <= current_epoch)
|
|
|
|
|
.unwrap_or(false);
|
|
|
|
|
let contribution_and_proofs = if is_altair {
|
|
|
|
|
harness
|
|
|
|
|
.make_sync_contributions(
|
|
|
|
|
&head.beacon_state,
|
|
|
|
|
head_state_root,
|
|
|
|
|
harness.chain.slot().unwrap(),
|
|
|
|
|
RelativeSyncCommittee::Current,
|
|
|
|
|
)
|
|
|
|
|
.into_iter()
|
|
|
|
|
.filter_map(|(_, contribution)| contribution)
|
|
|
|
|
.collect::<Vec<_>>()
|
|
|
|
|
} else {
|
|
|
|
|
vec![]
|
|
|
|
|
};
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
let attester_slashing = harness.make_attester_slashing(vec![0, 1]);
|
|
|
|
|
let proposer_slashing = harness.make_proposer_slashing(2);
|
|
|
|
|
let voluntary_exit = harness.make_voluntary_exit(3, harness.chain.epoch().unwrap());
|
|
|
|
|
|
2021-08-06 00:47:31 +00:00
|
|
|
let chain = harness.chain.clone();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
chain
|
|
|
|
|
.canonical_head
|
|
|
|
|
.cached_head()
|
|
|
|
|
.finalized_checkpoint()
|
|
|
|
|
.epoch,
|
2020-12-04 00:18:58 +00:00
|
|
|
2,
|
2020-09-29 03:46:54 +00:00
|
|
|
"precondition: finality"
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
chain
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.canonical_head
|
|
|
|
|
.cached_head()
|
|
|
|
|
.justified_checkpoint()
|
2020-09-29 03:46:54 +00:00
|
|
|
.epoch,
|
2020-12-04 00:18:58 +00:00
|
|
|
3,
|
2020-09-29 03:46:54 +00:00
|
|
|
"precondition: justification"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let log = null_logger().unwrap();
|
|
|
|
|
|
2021-08-06 00:47:31 +00:00
|
|
|
let ApiServer {
|
|
|
|
|
server,
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
listening_socket,
|
2021-08-06 00:47:31 +00:00
|
|
|
network_rx,
|
|
|
|
|
local_enr,
|
|
|
|
|
external_peer_id,
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
} = create_api_server(chain.clone(), &harness.runtime, log).await;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-05-16 08:35:59 +00:00
|
|
|
harness.runtime.task_executor.spawn(server, "api_server");
|
2020-09-29 03:46:54 +00:00
|
|
|
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
// Late-initalize the mock builder now that the mock execution node and beacon API ports
|
|
|
|
|
// have been allocated.
|
|
|
|
|
let beacon_api_port = listening_socket.port();
|
|
|
|
|
let beacon_url =
|
|
|
|
|
SensitiveUrl::parse(format!("http://127.0.0.1:{beacon_api_port}").as_str()).unwrap();
|
|
|
|
|
let mock_builder_server = harness.set_mock_builder(beacon_url.clone());
|
2020-09-29 03:46:54 +00:00
|
|
|
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
// Start the mock builder service prior to building the chain out.
|
2022-07-30 00:22:37 +00:00
|
|
|
harness.runtime.task_executor.spawn(
|
2023-10-03 17:59:28 +00:00
|
|
|
async move { mock_builder_server.await },
|
2022-07-30 00:22:37 +00:00
|
|
|
"mock_builder_server",
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let mock_builder = harness.mock_builder.clone();
|
|
|
|
|
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
let client = BeaconNodeHttpClient::new(
|
|
|
|
|
beacon_url,
|
|
|
|
|
Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)),
|
|
|
|
|
);
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
Self {
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
harness: Arc::new(harness),
|
2020-09-29 03:46:54 +00:00
|
|
|
chain,
|
|
|
|
|
client,
|
|
|
|
|
next_block,
|
2021-06-17 02:10:46 +00:00
|
|
|
reorg_block,
|
2020-09-29 03:46:54 +00:00
|
|
|
attestations,
|
2021-09-25 07:53:58 +00:00
|
|
|
contribution_and_proofs,
|
2020-09-29 03:46:54 +00:00
|
|
|
attester_slashing,
|
|
|
|
|
proposer_slashing,
|
|
|
|
|
voluntary_exit,
|
|
|
|
|
network_rx,
|
2021-08-06 00:47:31 +00:00
|
|
|
local_enr,
|
|
|
|
|
external_peer_id,
|
2022-07-30 00:22:37 +00:00
|
|
|
mock_builder,
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-11 02:45:06 +00:00
|
|
|
pub async fn new_from_genesis() -> Self {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let harness = Arc::new(
|
|
|
|
|
BeaconChainHarness::builder(MainnetEthSpec)
|
|
|
|
|
.default_spec()
|
|
|
|
|
.deterministic_keypairs(VALIDATOR_COUNT)
|
|
|
|
|
.fresh_ephemeral_store()
|
|
|
|
|
.build(),
|
|
|
|
|
);
|
2020-12-04 00:18:58 +00:00
|
|
|
|
|
|
|
|
harness.advance_slot();
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let head = harness.chain.head_snapshot();
|
2020-12-04 00:18:58 +00:00
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let (next_block, _next_state) = harness
|
|
|
|
|
.make_block(head.beacon_state.clone(), harness.chain.slot().unwrap())
|
|
|
|
|
.await;
|
2023-03-15 20:34:00 +00:00
|
|
|
let next_block = SignedBlockContents::from(next_block);
|
2020-12-04 00:18:58 +00:00
|
|
|
|
2021-06-17 02:10:46 +00:00
|
|
|
// `make_block` adds random graffiti, so this will produce an alternate block
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let (reorg_block, _reorg_state) = harness
|
|
|
|
|
.make_block(head.beacon_state.clone(), harness.chain.slot().unwrap())
|
|
|
|
|
.await;
|
2023-03-15 20:34:00 +00:00
|
|
|
let reorg_block = SignedBlockContents::from(reorg_block);
|
2021-06-17 02:10:46 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let head_state_root = head.beacon_state_root();
|
2020-12-04 00:18:58 +00:00
|
|
|
let attestations = harness
|
|
|
|
|
.get_unaggregated_attestations(
|
|
|
|
|
&AttestationStrategy::AllValidators,
|
|
|
|
|
&head.beacon_state,
|
2021-03-17 05:09:57 +00:00
|
|
|
head_state_root,
|
2020-12-04 00:18:58 +00:00
|
|
|
head.beacon_block_root,
|
|
|
|
|
harness.chain.slot().unwrap(),
|
|
|
|
|
)
|
|
|
|
|
.into_iter()
|
2022-05-16 01:59:47 +00:00
|
|
|
.flat_map(|vec| vec.into_iter().map(|(attestation, _subnet_id)| attestation))
|
2020-12-04 00:18:58 +00:00
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
let attester_slashing = harness.make_attester_slashing(vec![0, 1]);
|
|
|
|
|
let proposer_slashing = harness.make_proposer_slashing(2);
|
|
|
|
|
let voluntary_exit = harness.make_voluntary_exit(3, harness.chain.epoch().unwrap());
|
|
|
|
|
|
2021-08-06 00:47:31 +00:00
|
|
|
let chain = harness.chain.clone();
|
2020-12-04 00:18:58 +00:00
|
|
|
|
|
|
|
|
let log = null_logger().unwrap();
|
|
|
|
|
|
2021-08-06 00:47:31 +00:00
|
|
|
let ApiServer {
|
|
|
|
|
server,
|
|
|
|
|
listening_socket,
|
|
|
|
|
network_rx,
|
|
|
|
|
local_enr,
|
|
|
|
|
external_peer_id,
|
Use `BeaconProcessor` for API requests (#4462)
## Issue Addressed
NA
## Proposed Changes
Rather than spawning new tasks on the tokio executor to process each HTTP API request, send the tasks to the `BeaconProcessor`. This achieves:
1. Places a bound on how many concurrent requests are being served (i.e., how many we are actually trying to compute at one time).
1. Places a bound on how many requests can be awaiting a response at one time (i.e., starts dropping requests when we have too many queued).
1. Allows the BN prioritise HTTP requests with respect to messages coming from the P2P network (i.e., proiritise importing gossip blocks rather than serving API requests).
Presently there are two levels of priorities:
- `Priority::P0`
- The beacon processor will prioritise these above everything other than importing new blocks.
- Roughly all validator-sensitive endpoints.
- `Priority::P1`
- The beacon processor will prioritise practically all other P2P messages over these, except for historical backfill things.
- Everything that's not `Priority::P0`
The `--http-enable-beacon-processor false` flag can be supplied to revert back to the old behaviour of spawning new `tokio` tasks for each request:
```
--http-enable-beacon-processor <BOOLEAN>
The beacon processor is a scheduler which provides quality-of-service and DoS protection. When set to
"true", HTTP API requests will queued and scheduled alongside other tasks. When set to "false", HTTP API
responses will be executed immediately. [default: true]
```
## New CLI Flags
I added some other new CLI flags:
```
--beacon-processor-aggregate-batch-size <INTEGER>
Specifies the number of gossip aggregate attestations in a signature verification batch. Higher values may
reduce CPU usage in a healthy network while lower values may increase CPU usage in an unhealthy or hostile
network. [default: 64]
--beacon-processor-attestation-batch-size <INTEGER>
Specifies the number of gossip attestations in a signature verification batch. Higher values may reduce CPU
usage in a healthy network whilst lower values may increase CPU usage in an unhealthy or hostile network.
[default: 64]
--beacon-processor-max-workers <INTEGER>
Specifies the maximum concurrent tasks for the task scheduler. Increasing this value may increase resource
consumption. Reducing the value may result in decreased resource usage and diminished performance. The
default value is the number of logical CPU cores on the host.
--beacon-processor-reprocess-queue-len <INTEGER>
Specifies the length of the queue for messages requiring delayed processing. Higher values may prevent
messages from being dropped while lower values may help protect the node from becoming overwhelmed.
[default: 12288]
```
I needed to add the max-workers flag since the "simulator" flavor tests started failing with HTTP timeouts on the test assertions. I believe they were failing because the Github runners only have 2 cores and there just weren't enough workers available to process our requests in time. I added the other flags since they seem fun to fiddle with.
## Additional Info
I bumped the timeouts on the "simulator" flavor test from 4s to 8s. The prioritisation of consensus messages seems to be causing slower responses, I guess this is what we signed up for 🤷
The `validator/register` validator has some special handling because the relays have a bad habit of timing out on these calls. It seems like a waste of a `BeaconProcessor` worker to just wait for the builder API HTTP response, so we spawn a new `tokio` task to wait for a builder response.
I've added an optimisation for the `GET beacon/states/{state_id}/validators/{validator_id}` endpoint in [efbabe3](https://github.com/sigp/lighthouse/pull/4462/commits/efbabe32521ed6eb3564764da4e507d26a1c4bd0). That's the endpoint the VC uses to resolve pubkeys to validator indices, and it's the endpoint that was causing us grief. Perhaps I should move that into a new PR, not sure.
2023-08-08 23:30:15 +00:00
|
|
|
} = create_api_server(chain.clone(), &harness.runtime, log).await;
|
2020-12-04 00:18:58 +00:00
|
|
|
|
2022-05-16 08:35:59 +00:00
|
|
|
harness.runtime.task_executor.spawn(server, "api_server");
|
2020-12-04 00:18:58 +00:00
|
|
|
|
|
|
|
|
let client = BeaconNodeHttpClient::new(
|
2021-05-04 01:59:51 +00:00
|
|
|
SensitiveUrl::parse(&format!(
|
2020-12-04 00:18:58 +00:00
|
|
|
"http://{}:{}",
|
|
|
|
|
listening_socket.ip(),
|
|
|
|
|
listening_socket.port()
|
|
|
|
|
))
|
|
|
|
|
.unwrap(),
|
2021-07-12 01:47:48 +00:00
|
|
|
Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)),
|
2020-12-04 00:18:58 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
|
|
Self {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
harness,
|
2020-12-04 00:18:58 +00:00
|
|
|
chain,
|
|
|
|
|
client,
|
|
|
|
|
next_block,
|
2021-06-17 02:10:46 +00:00
|
|
|
reorg_block,
|
2020-12-04 00:18:58 +00:00
|
|
|
attestations,
|
2021-09-25 07:53:58 +00:00
|
|
|
contribution_and_proofs: vec![],
|
2020-12-04 00:18:58 +00:00
|
|
|
attester_slashing,
|
|
|
|
|
proposer_slashing,
|
|
|
|
|
voluntary_exit,
|
|
|
|
|
network_rx,
|
2021-08-06 00:47:31 +00:00
|
|
|
local_enr,
|
|
|
|
|
external_peer_id,
|
2022-07-30 00:22:37 +00:00
|
|
|
mock_builder: None,
|
2020-12-04 00:18:58 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
fn validator_keypairs(&self) -> &[Keypair] {
|
|
|
|
|
&self.harness.validator_keypairs
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
pub async fn new_mev_tester() -> Self {
|
2022-09-05 04:50:49 +00:00
|
|
|
let tester = Self::new_with_hard_forks(true, true)
|
2022-07-30 00:22:37 +00:00
|
|
|
.await
|
|
|
|
|
.test_post_validator_register_validator()
|
2022-09-05 04:50:49 +00:00
|
|
|
.await;
|
|
|
|
|
// Make sure bids always meet the minimum threshold.
|
|
|
|
|
tester
|
|
|
|
|
.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_BUILDER_THRESHOLD_WEI,
|
|
|
|
|
)));
|
|
|
|
|
tester
|
2022-07-30 00:22:37 +00:00
|
|
|
}
|
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
pub async fn new_mev_tester_no_builder_threshold() -> Self {
|
|
|
|
|
let mut config = ApiTesterConfig {
|
2023-09-26 02:39:58 +00:00
|
|
|
builder_threshold: Some(0),
|
2023-08-21 05:02:32 +00:00
|
|
|
retain_historic_states: false,
|
2023-02-02 01:37:46 +00:00
|
|
|
spec: E::default_spec(),
|
|
|
|
|
};
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.bellatrix_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
let tester = Self::new_from_config(config)
|
|
|
|
|
.await
|
|
|
|
|
.test_post_validator_register_validator()
|
|
|
|
|
.await;
|
|
|
|
|
tester
|
|
|
|
|
.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_BUILDER_PAYLOAD_VALUE_WEI,
|
|
|
|
|
)));
|
|
|
|
|
tester
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
fn skip_slots(self, count: u64) -> Self {
|
|
|
|
|
for _ in 0..count {
|
|
|
|
|
self.chain
|
|
|
|
|
.slot_clock
|
|
|
|
|
.set_slot(self.chain.slot().unwrap().as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn interesting_state_ids(&self) -> Vec<StateId> {
|
|
|
|
|
let mut ids = vec![
|
2022-07-25 08:23:00 +00:00
|
|
|
StateId(CoreStateId::Head),
|
|
|
|
|
StateId(CoreStateId::Genesis),
|
|
|
|
|
StateId(CoreStateId::Finalized),
|
|
|
|
|
StateId(CoreStateId::Justified),
|
|
|
|
|
StateId(CoreStateId::Slot(Slot::new(0))),
|
|
|
|
|
StateId(CoreStateId::Slot(Slot::new(32))),
|
|
|
|
|
StateId(CoreStateId::Slot(Slot::from(SKIPPED_SLOTS[0]))),
|
|
|
|
|
StateId(CoreStateId::Slot(Slot::from(SKIPPED_SLOTS[1]))),
|
|
|
|
|
StateId(CoreStateId::Slot(Slot::from(SKIPPED_SLOTS[2]))),
|
|
|
|
|
StateId(CoreStateId::Slot(Slot::from(SKIPPED_SLOTS[3]))),
|
|
|
|
|
StateId(CoreStateId::Root(Hash256::zero())),
|
2020-09-29 03:46:54 +00:00
|
|
|
];
|
2022-07-25 08:23:00 +00:00
|
|
|
ids.push(StateId(CoreStateId::Root(
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
self.chain.canonical_head.cached_head().head_state_root(),
|
2022-07-25 08:23:00 +00:00
|
|
|
)));
|
2020-09-29 03:46:54 +00:00
|
|
|
ids
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn interesting_block_ids(&self) -> Vec<BlockId> {
|
|
|
|
|
let mut ids = vec![
|
2022-07-25 08:23:00 +00:00
|
|
|
BlockId(CoreBlockId::Head),
|
|
|
|
|
BlockId(CoreBlockId::Genesis),
|
|
|
|
|
BlockId(CoreBlockId::Finalized),
|
|
|
|
|
BlockId(CoreBlockId::Justified),
|
|
|
|
|
BlockId(CoreBlockId::Slot(Slot::new(0))),
|
|
|
|
|
BlockId(CoreBlockId::Slot(Slot::new(32))),
|
|
|
|
|
BlockId(CoreBlockId::Slot(Slot::from(SKIPPED_SLOTS[0]))),
|
|
|
|
|
BlockId(CoreBlockId::Slot(Slot::from(SKIPPED_SLOTS[1]))),
|
|
|
|
|
BlockId(CoreBlockId::Slot(Slot::from(SKIPPED_SLOTS[2]))),
|
|
|
|
|
BlockId(CoreBlockId::Slot(Slot::from(SKIPPED_SLOTS[3]))),
|
|
|
|
|
BlockId(CoreBlockId::Root(Hash256::zero())),
|
2020-09-29 03:46:54 +00:00
|
|
|
];
|
2022-07-25 08:23:00 +00:00
|
|
|
ids.push(BlockId(CoreBlockId::Root(
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
self.chain.canonical_head.cached_head().head_block_root(),
|
2022-07-25 08:23:00 +00:00
|
|
|
)));
|
2020-09-29 03:46:54 +00:00
|
|
|
ids
|
|
|
|
|
}
|
|
|
|
|
pub async fn test_beacon_genesis(self) -> Self {
|
|
|
|
|
let result = self.client.get_beacon_genesis().await.unwrap().data;
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let state = &self.chain.head_snapshot().beacon_state;
|
2020-09-29 03:46:54 +00:00
|
|
|
let expected = GenesisData {
|
2021-07-09 06:15:32 +00:00
|
|
|
genesis_time: state.genesis_time(),
|
|
|
|
|
genesis_validators_root: state.genesis_validators_root(),
|
2020-09-29 03:46:54 +00:00
|
|
|
genesis_fork_version: self.chain.spec.genesis_fork_version,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-03-30 06:08:37 +00:00
|
|
|
// finalization tests
|
|
|
|
|
pub async fn test_beacon_states_root_finalized(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let state_root = state_id.root(&self.chain);
|
|
|
|
|
let state = state_id.state(&self.chain);
|
|
|
|
|
|
|
|
|
|
// if .root or .state fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_state
|
|
|
|
|
// occurs after the state_root and state calls, and that the state_root and state calls
|
|
|
|
|
// were correct.
|
|
|
|
|
if state_root.is_err() || state.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the state is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_root(state_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (state_root, _, _) = state_root.unwrap();
|
|
|
|
|
let (state, _, _) = state.unwrap();
|
|
|
|
|
let state_slot = state.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_state(&state_root, state_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_states_fork_finalized(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let state_root = state_id.root(&self.chain);
|
|
|
|
|
let state = state_id.state(&self.chain);
|
|
|
|
|
|
|
|
|
|
// if .root or .state fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_state
|
|
|
|
|
// occurs after the state_root and state calls, and that the state_root and state calls
|
|
|
|
|
// were correct.
|
|
|
|
|
if state_root.is_err() || state.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the state is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_fork(state_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (state_root, _, _) = state_root.unwrap();
|
|
|
|
|
let (state, _, _) = state.unwrap();
|
|
|
|
|
let state_slot = state.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_state(&state_root, state_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_states_finality_checkpoints_finalized(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let state_root = state_id.root(&self.chain);
|
|
|
|
|
let state = state_id.state(&self.chain);
|
|
|
|
|
|
|
|
|
|
// if .root or .state fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_state
|
|
|
|
|
// occurs after the state_root and state calls, and that the state_root and state calls
|
|
|
|
|
// were correct.
|
|
|
|
|
if state_root.is_err() || state.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the state is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_finality_checkpoints(state_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (state_root, _, _) = state_root.unwrap();
|
|
|
|
|
let (state, _, _) = state.unwrap();
|
|
|
|
|
let state_slot = state.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_state(&state_root, state_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_headers_block_id_finalized(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let block_root = block_id.root(&self.chain);
|
|
|
|
|
let block = block_id.full_block(&self.chain).await;
|
|
|
|
|
|
|
|
|
|
// if .root or .state fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_state
|
|
|
|
|
// occurs after the state_root and state calls, and that the state_root and state calls
|
|
|
|
|
// were correct.
|
|
|
|
|
if block_root.is_err() || block.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the block is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_headers_block_id(block_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (block_root, _, _) = block_root.unwrap();
|
|
|
|
|
let (block, _, _) = block.unwrap();
|
|
|
|
|
let block_slot = block.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_block(&block_root, block_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", block_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_blocks_finalized<T: EthSpec>(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let block_root = block_id.root(&self.chain);
|
|
|
|
|
let block = block_id.full_block(&self.chain).await;
|
|
|
|
|
|
|
|
|
|
// if .root or .full_block fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_block
|
|
|
|
|
// occurs after those calls, and that they were correct.
|
|
|
|
|
if block_root.is_err() || block.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the block is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_blocks::<MainnetEthSpec>(block_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (block_root, _, _) = block_root.unwrap();
|
|
|
|
|
let (block, _, _) = block.unwrap();
|
|
|
|
|
let block_slot = block.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_block(&block_root, block_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", block_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_blinded_blocks_finalized<T: EthSpec>(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let block_root = block_id.root(&self.chain);
|
|
|
|
|
let block = block_id.full_block(&self.chain).await;
|
|
|
|
|
|
|
|
|
|
// if .root or .full_block fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_block
|
|
|
|
|
// occurs after those calls, and that they were correct.
|
|
|
|
|
if block_root.is_err() || block.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the block is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_blinded_blocks::<MainnetEthSpec>(block_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (block_root, _, _) = block_root.unwrap();
|
|
|
|
|
let (block, _, _) = block.unwrap();
|
|
|
|
|
let block_slot = block.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_block(&block_root, block_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", block_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_debug_beacon_states_finalized(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let state_root = state_id.root(&self.chain);
|
|
|
|
|
let state = state_id.state(&self.chain);
|
|
|
|
|
|
|
|
|
|
// if .root or .state fail, skip the test. those would be errors outside the scope
|
|
|
|
|
// of this test, here we're testing the finalized field assuming the call to .is_finalized_state
|
|
|
|
|
// occurs after the state_root and state calls, and that the state_root and state calls
|
|
|
|
|
// were correct.
|
|
|
|
|
if state_root.is_err() || state.is_err() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// now that we know the state is valid, we can unwrap() everything we need
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_debug_beacon_states::<MainnetEthSpec>(state_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.finalized
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (state_root, _, _) = state_root.unwrap();
|
|
|
|
|
let (state, _, _) = state.unwrap();
|
|
|
|
|
let state_slot = state.slot();
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.is_finalized_state(&state_root, state_slot)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_beacon_states_root(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_states_root(state_id.0)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data.root);
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let expected = state_id
|
|
|
|
|
.root(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(root, _execution_optimistic, _finalized)| root);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_states_fork(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_states_fork(state_id.0)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let expected = state_id.fork(&self.chain).ok();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_states_finality_checkpoints(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_states_finality_checkpoints(state_id.0)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
2023-03-30 06:08:37 +00:00
|
|
|
let expected = state_id.state(&self.chain).ok().map(
|
|
|
|
|
|(state, _execution_optimistic, _finalized)| FinalityCheckpointsData {
|
|
|
|
|
previous_justified: state.previous_justified_checkpoint(),
|
|
|
|
|
current_justified: state.current_justified_checkpoint(),
|
|
|
|
|
finalized: state.finalized_checkpoint(),
|
|
|
|
|
},
|
|
|
|
|
);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-09 23:13:56 +00:00
|
|
|
pub async fn test_beacon_states_validator_balances(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
for validator_indices in self.interesting_validator_indices() {
|
2022-07-25 08:23:00 +00:00
|
|
|
let state_opt = state_id.state(&self.chain).ok();
|
2020-11-09 23:13:56 +00:00
|
|
|
let validators: Vec<Validator> = match state_opt.as_ref() {
|
2023-03-30 06:08:37 +00:00
|
|
|
Some((state, _execution_optimistic, _finalized)) => {
|
|
|
|
|
state.validators().clone().into()
|
|
|
|
|
}
|
2020-11-09 23:13:56 +00:00
|
|
|
None => vec![],
|
|
|
|
|
};
|
|
|
|
|
let validator_index_ids = validator_indices
|
|
|
|
|
.iter()
|
|
|
|
|
.cloned()
|
|
|
|
|
.map(|i| ValidatorId::Index(i))
|
|
|
|
|
.collect::<Vec<ValidatorId>>();
|
|
|
|
|
let validator_pubkey_ids = validator_indices
|
|
|
|
|
.iter()
|
|
|
|
|
.cloned()
|
|
|
|
|
.map(|i| {
|
|
|
|
|
ValidatorId::PublicKey(
|
|
|
|
|
validators
|
|
|
|
|
.get(i as usize)
|
|
|
|
|
.map_or(PublicKeyBytes::empty(), |val| val.pubkey.clone()),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
.collect::<Vec<ValidatorId>>();
|
|
|
|
|
|
|
|
|
|
let result_index_ids = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_validator_balances(
|
2022-07-25 08:23:00 +00:00
|
|
|
state_id.0,
|
2020-11-09 23:13:56 +00:00
|
|
|
Some(validator_index_ids.as_slice()),
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
let result_pubkey_ids = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_validator_balances(
|
2022-07-25 08:23:00 +00:00
|
|
|
state_id.0,
|
2020-11-09 23:13:56 +00:00
|
|
|
Some(validator_pubkey_ids.as_slice()),
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
2023-03-30 06:08:37 +00:00
|
|
|
let expected = state_opt.map(|(state, _execution_optimistic, _finalized)| {
|
2020-11-09 23:13:56 +00:00
|
|
|
let mut validators = Vec::with_capacity(validator_indices.len());
|
|
|
|
|
|
|
|
|
|
for i in validator_indices {
|
2021-07-09 06:15:32 +00:00
|
|
|
if i < state.balances().len() as u64 {
|
2020-11-09 23:13:56 +00:00
|
|
|
validators.push(ValidatorBalanceData {
|
|
|
|
|
index: i as u64,
|
2021-07-09 06:15:32 +00:00
|
|
|
balance: state.balances()[i as usize],
|
2020-11-09 23:13:56 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
validators
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
assert_eq!(result_index_ids, expected, "{:?}", state_id);
|
|
|
|
|
assert_eq!(result_pubkey_ids, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_beacon_states_validators(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
2020-10-29 05:13:04 +00:00
|
|
|
for statuses in self.interesting_validator_statuses() {
|
|
|
|
|
for validator_indices in self.interesting_validator_indices() {
|
2022-07-25 08:23:00 +00:00
|
|
|
let state_opt = state_id
|
|
|
|
|
.state(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(state, _execution_optimistic, _finalized)| state);
|
2020-10-29 05:13:04 +00:00
|
|
|
let validators: Vec<Validator> = match state_opt.as_ref() {
|
2021-07-09 06:15:32 +00:00
|
|
|
Some(state) => state.validators().clone().into(),
|
2020-10-29 05:13:04 +00:00
|
|
|
None => vec![],
|
|
|
|
|
};
|
|
|
|
|
let validator_index_ids = validator_indices
|
|
|
|
|
.iter()
|
|
|
|
|
.cloned()
|
|
|
|
|
.map(|i| ValidatorId::Index(i))
|
|
|
|
|
.collect::<Vec<ValidatorId>>();
|
|
|
|
|
let validator_pubkey_ids = validator_indices
|
|
|
|
|
.iter()
|
|
|
|
|
.cloned()
|
|
|
|
|
.map(|i| {
|
|
|
|
|
ValidatorId::PublicKey(
|
|
|
|
|
validators
|
|
|
|
|
.get(i as usize)
|
|
|
|
|
.map_or(PublicKeyBytes::empty(), |val| val.pubkey.clone()),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
.collect::<Vec<ValidatorId>>();
|
|
|
|
|
|
|
|
|
|
let result_index_ids = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_validators(
|
2022-07-25 08:23:00 +00:00
|
|
|
state_id.0,
|
2020-10-29 05:13:04 +00:00
|
|
|
Some(validator_index_ids.as_slice()),
|
|
|
|
|
None,
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-10-29 05:13:04 +00:00
|
|
|
let result_pubkey_ids = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_validators(
|
2022-07-25 08:23:00 +00:00
|
|
|
state_id.0,
|
2020-10-29 05:13:04 +00:00
|
|
|
Some(validator_pubkey_ids.as_slice()),
|
|
|
|
|
None,
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
|
|
|
|
let expected = state_opt.map(|state| {
|
|
|
|
|
let epoch = state.current_epoch();
|
|
|
|
|
let far_future_epoch = self.chain.spec.far_future_epoch;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-10-29 05:13:04 +00:00
|
|
|
let mut validators = Vec::with_capacity(validator_indices.len());
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-10-29 05:13:04 +00:00
|
|
|
for i in validator_indices {
|
2021-07-09 06:15:32 +00:00
|
|
|
if i >= state.validators().len() as u64 {
|
2020-10-29 05:13:04 +00:00
|
|
|
continue;
|
|
|
|
|
}
|
2021-07-09 06:15:32 +00:00
|
|
|
let validator = state.validators()[i as usize].clone();
|
2020-10-29 05:13:04 +00:00
|
|
|
let status = ValidatorStatus::from_validator(
|
2021-02-24 04:15:13 +00:00
|
|
|
&validator,
|
2020-10-29 05:13:04 +00:00
|
|
|
epoch,
|
|
|
|
|
far_future_epoch,
|
|
|
|
|
);
|
2021-02-24 04:15:13 +00:00
|
|
|
if statuses.contains(&status)
|
|
|
|
|
|| statuses.is_empty()
|
|
|
|
|
|| statuses.contains(&status.superstatus())
|
|
|
|
|
{
|
2020-10-29 05:13:04 +00:00
|
|
|
validators.push(ValidatorData {
|
|
|
|
|
index: i as u64,
|
2021-07-09 06:15:32 +00:00
|
|
|
balance: state.balances()[i as usize],
|
2020-10-29 05:13:04 +00:00
|
|
|
status,
|
|
|
|
|
validator,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
validators
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
assert_eq!(result_index_ids, expected, "{:?}", state_id);
|
|
|
|
|
assert_eq!(result_pubkey_ids, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_states_validator_id(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
2022-07-25 08:23:00 +00:00
|
|
|
let state_opt = state_id
|
|
|
|
|
.state(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(state, _execution_optimistic, _finalized)| state);
|
2020-09-29 03:46:54 +00:00
|
|
|
let validators = match state_opt.as_ref() {
|
2021-07-09 06:15:32 +00:00
|
|
|
Some(state) => state.validators().clone().into(),
|
2020-09-29 03:46:54 +00:00
|
|
|
None => vec![],
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
for (i, validator) in validators.into_iter().enumerate() {
|
|
|
|
|
let validator_ids = &[
|
|
|
|
|
ValidatorId::PublicKey(validator.pubkey.clone()),
|
|
|
|
|
ValidatorId::Index(i as u64),
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
for validator_id in validator_ids {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_states_validator_id(state_id.0, validator_id)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
|
|
|
|
if result.is_none() && state_opt.is_none() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let state = state_opt.as_ref().expect("result should be none");
|
|
|
|
|
|
|
|
|
|
let expected = {
|
|
|
|
|
let epoch = state.current_epoch();
|
|
|
|
|
let far_future_epoch = self.chain.spec.far_future_epoch;
|
|
|
|
|
|
|
|
|
|
ValidatorData {
|
|
|
|
|
index: i as u64,
|
2021-07-09 06:15:32 +00:00
|
|
|
balance: state.balances()[i],
|
2020-09-29 03:46:54 +00:00
|
|
|
status: ValidatorStatus::from_validator(
|
2021-02-24 04:15:13 +00:00
|
|
|
&validator,
|
2020-09-29 03:46:54 +00:00
|
|
|
epoch,
|
|
|
|
|
far_future_epoch,
|
|
|
|
|
),
|
|
|
|
|
validator: validator.clone(),
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, Some(expected), "{:?}, {:?}", state_id, validator_id);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_states_committees(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
2022-07-25 08:23:00 +00:00
|
|
|
let mut state_opt = state_id
|
|
|
|
|
.state(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(state, _execution_optimistic, _finalized)| state);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-11-18 23:31:39 +00:00
|
|
|
let epoch_opt = state_opt.as_ref().map(|state| state.current_epoch());
|
2020-09-29 03:46:54 +00:00
|
|
|
let results = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_states_committees(state_id.0, None, None, epoch_opt)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
|
|
|
|
if results.is_none() && state_opt.is_none() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let state = state_opt.as_mut().expect("result should be none");
|
2020-11-18 23:31:39 +00:00
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
state.build_all_committee_caches(&self.chain.spec).unwrap();
|
|
|
|
|
let committees = state
|
2020-11-18 23:31:39 +00:00
|
|
|
.get_beacon_committees_at_epoch(RelativeEpoch::Current)
|
2020-09-29 03:46:54 +00:00
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
for (i, result) in results.unwrap().into_iter().enumerate() {
|
|
|
|
|
let expected = &committees[i];
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.index, expected.index, "{}", state_id);
|
|
|
|
|
assert_eq!(result.slot, expected.slot, "{}", state_id);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
result
|
|
|
|
|
.validators
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(|i| i as usize)
|
|
|
|
|
.collect::<Vec<_>>(),
|
|
|
|
|
expected.committee.to_vec(),
|
|
|
|
|
"{}",
|
|
|
|
|
state_id
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-11-11 00:38:27 +00:00
|
|
|
pub async fn test_beacon_states_randao(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let mut state_opt = state_id
|
|
|
|
|
.state(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(state, _execution_optimistic, _finalized)| state);
|
2022-11-11 00:38:27 +00:00
|
|
|
|
|
|
|
|
let epoch_opt = state_opt.as_ref().map(|state| state.current_epoch());
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_states_randao(state_id.0, epoch_opt)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
|
|
|
|
if result.is_none() && state_opt.is_none() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let state = state_opt.as_mut().expect("result should be none");
|
|
|
|
|
let randao_mix = state
|
|
|
|
|
.get_randao_mix(state.slot().epoch(E::slots_per_epoch()))
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.unwrap().randao, *randao_mix);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_beacon_headers_all_slots(self) -> Self {
|
|
|
|
|
for slot in 0..CHAIN_LENGTH {
|
|
|
|
|
let slot = Slot::from(slot);
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_headers(Some(slot), None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
Use the forwards iterator more often (#2376)
## Issue Addressed
NA
## Primary Change
When investigating memory usage, I noticed that retrieving a block from an early slot (e.g., slot 900) would cause a sharp increase in the memory footprint (from 400mb to 800mb+) which seemed to be ever-lasting.
After some investigation, I found that the reverse iteration from the head back to that slot was the likely culprit. To counter this, I've switched the `BeaconChain::block_root_at_slot` to use the forwards iterator, instead of the reverse one.
I also noticed that the networking stack is using `BeaconChain::root_at_slot` to check if a peer is relevant (`check_peer_relevance`). Perhaps the steep, seemingly-random-but-consistent increases in memory usage are caused by the use of this function.
Using the forwards iterator with the HTTP API alleviated the sharp increases in memory usage. It also made the response much faster (before it felt like to took 1-2s, now it feels instant).
## Additional Changes
In the process I also noticed that we have two functions for getting block roots:
- `BeaconChain::block_root_at_slot`: returns `None` for a skip slot.
- `BeaconChain::root_at_slot`: returns the previous root for a skip slot.
I unified these two functions into `block_root_at_slot` and added the `WhenSlotSkipped` enum. Now, the caller must be explicit about the skip-slot behaviour when requesting a root.
Additionally, I replaced `vec![]` with `Vec::with_capacity` in `store::chunked_vector::range_query`. I stumbled across this whilst debugging and made this modification to see what effect it would have (not much). It seems like a decent change to keep around, but I'm not concerned either way.
Also, `BeaconChain::get_ancestor_block_root` is unused, so I got rid of it :wastebasket:.
## Additional Info
I haven't also done the same for state roots here. Whilst it's possible and a good idea, it's more work since the fwds iterators are presently block-roots-specific.
Whilst there's a few places a reverse iteration of state roots could be triggered (e.g., attestation production, HTTP API), they're no where near as common as the `check_peer_relevance` call. As such, I think we should get this PR merged first, then come back for the state root iters. I made an issue here https://github.com/sigp/lighthouse/issues/2377.
2021-05-31 04:18:20 +00:00
|
|
|
let root = self
|
|
|
|
|
.chain
|
|
|
|
|
.block_root_at_slot(slot, WhenSlotSkipped::None)
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
if root.is_none() && result.is_none() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let root = root.unwrap();
|
Use the forwards iterator more often (#2376)
## Issue Addressed
NA
## Primary Change
When investigating memory usage, I noticed that retrieving a block from an early slot (e.g., slot 900) would cause a sharp increase in the memory footprint (from 400mb to 800mb+) which seemed to be ever-lasting.
After some investigation, I found that the reverse iteration from the head back to that slot was the likely culprit. To counter this, I've switched the `BeaconChain::block_root_at_slot` to use the forwards iterator, instead of the reverse one.
I also noticed that the networking stack is using `BeaconChain::root_at_slot` to check if a peer is relevant (`check_peer_relevance`). Perhaps the steep, seemingly-random-but-consistent increases in memory usage are caused by the use of this function.
Using the forwards iterator with the HTTP API alleviated the sharp increases in memory usage. It also made the response much faster (before it felt like to took 1-2s, now it feels instant).
## Additional Changes
In the process I also noticed that we have two functions for getting block roots:
- `BeaconChain::block_root_at_slot`: returns `None` for a skip slot.
- `BeaconChain::root_at_slot`: returns the previous root for a skip slot.
I unified these two functions into `block_root_at_slot` and added the `WhenSlotSkipped` enum. Now, the caller must be explicit about the skip-slot behaviour when requesting a root.
Additionally, I replaced `vec![]` with `Vec::with_capacity` in `store::chunked_vector::range_query`. I stumbled across this whilst debugging and made this modification to see what effect it would have (not much). It seems like a decent change to keep around, but I'm not concerned either way.
Also, `BeaconChain::get_ancestor_block_root` is unused, so I got rid of it :wastebasket:.
## Additional Info
I haven't also done the same for state roots here. Whilst it's possible and a good idea, it's more work since the fwds iterators are presently block-roots-specific.
Whilst there's a few places a reverse iteration of state roots could be triggered (e.g., attestation production, HTTP API), they're no where near as common as the `check_peer_relevance` call. As such, I think we should get this PR merged first, then come back for the state root iters. I made an issue here https://github.com/sigp/lighthouse/issues/2377.
2021-05-31 04:18:20 +00:00
|
|
|
let block = self
|
|
|
|
|
.chain
|
|
|
|
|
.block_at_slot(slot, WhenSlotSkipped::Prev)
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
let header = BlockHeaderData {
|
|
|
|
|
root,
|
|
|
|
|
canonical: true,
|
|
|
|
|
header: BlockHeaderAndSignature {
|
2021-07-09 06:15:32 +00:00
|
|
|
message: block.message().block_header(),
|
|
|
|
|
signature: block.signature().clone().into(),
|
2020-09-29 03:46:54 +00:00
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
let expected = vec![header];
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.unwrap(), expected, "slot {:?}", slot);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_headers_all_parents(self) -> Self {
|
|
|
|
|
let mut roots = self
|
|
|
|
|
.chain
|
2021-07-06 02:38:53 +00:00
|
|
|
.forwards_iter_block_roots(Slot::new(0))
|
2020-09-29 03:46:54 +00:00
|
|
|
.unwrap()
|
|
|
|
|
.map(Result::unwrap)
|
|
|
|
|
.map(|(root, _slot)| root)
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
// The iterator natively returns duplicate roots for skipped slots.
|
|
|
|
|
roots.dedup();
|
|
|
|
|
|
|
|
|
|
for i in 1..roots.len() {
|
|
|
|
|
let parent_root = roots[i - 1];
|
|
|
|
|
let child_root = roots[i];
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_headers(None, Some(parent_root))
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.len(), 1, "i {}", i);
|
|
|
|
|
assert_eq!(result[0].root, child_root, "i {}", i);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_headers_block_id(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_headers_block_id(block_id.0)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let block_root_opt = block_id
|
|
|
|
|
.root(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(root, _execution_optimistic, _finalized)| root);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
if let CoreBlockId::Slot(slot) = block_id.0 {
|
2021-04-25 03:59:59 +00:00
|
|
|
if block_root_opt.is_none() {
|
|
|
|
|
assert!(SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
} else {
|
|
|
|
|
assert!(!SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let block_opt = block_id
|
|
|
|
|
.full_block(&self.chain)
|
|
|
|
|
.await
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(block, _execution_optimistic, _finalized)| block);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
if block_opt.is_none() && result.is_none() {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let result = result.unwrap();
|
|
|
|
|
let block = block_opt.unwrap();
|
|
|
|
|
let block_root = block_root_opt.unwrap();
|
|
|
|
|
let canonical = self
|
|
|
|
|
.chain
|
Use the forwards iterator more often (#2376)
## Issue Addressed
NA
## Primary Change
When investigating memory usage, I noticed that retrieving a block from an early slot (e.g., slot 900) would cause a sharp increase in the memory footprint (from 400mb to 800mb+) which seemed to be ever-lasting.
After some investigation, I found that the reverse iteration from the head back to that slot was the likely culprit. To counter this, I've switched the `BeaconChain::block_root_at_slot` to use the forwards iterator, instead of the reverse one.
I also noticed that the networking stack is using `BeaconChain::root_at_slot` to check if a peer is relevant (`check_peer_relevance`). Perhaps the steep, seemingly-random-but-consistent increases in memory usage are caused by the use of this function.
Using the forwards iterator with the HTTP API alleviated the sharp increases in memory usage. It also made the response much faster (before it felt like to took 1-2s, now it feels instant).
## Additional Changes
In the process I also noticed that we have two functions for getting block roots:
- `BeaconChain::block_root_at_slot`: returns `None` for a skip slot.
- `BeaconChain::root_at_slot`: returns the previous root for a skip slot.
I unified these two functions into `block_root_at_slot` and added the `WhenSlotSkipped` enum. Now, the caller must be explicit about the skip-slot behaviour when requesting a root.
Additionally, I replaced `vec![]` with `Vec::with_capacity` in `store::chunked_vector::range_query`. I stumbled across this whilst debugging and made this modification to see what effect it would have (not much). It seems like a decent change to keep around, but I'm not concerned either way.
Also, `BeaconChain::get_ancestor_block_root` is unused, so I got rid of it :wastebasket:.
## Additional Info
I haven't also done the same for state roots here. Whilst it's possible and a good idea, it's more work since the fwds iterators are presently block-roots-specific.
Whilst there's a few places a reverse iteration of state roots could be triggered (e.g., attestation production, HTTP API), they're no where near as common as the `check_peer_relevance` call. As such, I think we should get this PR merged first, then come back for the state root iters. I made an issue here https://github.com/sigp/lighthouse/issues/2377.
2021-05-31 04:18:20 +00:00
|
|
|
.block_root_at_slot(block.slot(), WhenSlotSkipped::None)
|
2020-09-29 03:46:54 +00:00
|
|
|
.unwrap()
|
|
|
|
|
.map_or(false, |canonical| block_root == canonical);
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.canonical, canonical, "{:?}", block_id);
|
|
|
|
|
assert_eq!(result.root, block_root, "{:?}", block_id);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
result.header.message,
|
2021-07-09 06:15:32 +00:00
|
|
|
block.message().block_header(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"{:?}",
|
|
|
|
|
block_id
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
result.header.signature,
|
2021-07-09 06:15:32 +00:00
|
|
|
block.signature().clone().into(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"{:?}",
|
|
|
|
|
block_id
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_beacon_blocks_root(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_blocks_root(block_id.0)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data.root);
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let expected = block_id
|
|
|
|
|
.root(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(root, _execution_optimistic, _finalized)| root);
|
2022-07-25 08:23:00 +00:00
|
|
|
if let CoreBlockId::Slot(slot) = block_id.0 {
|
2021-04-25 03:59:59 +00:00
|
|
|
if expected.is_none() {
|
|
|
|
|
assert!(SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
} else {
|
|
|
|
|
assert!(!SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
assert_eq!(result, expected, "{:?}", block_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_blocks_valid(mut self) -> Self {
|
2023-03-15 20:34:00 +00:00
|
|
|
let next_block = self.next_block.clone();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2023-03-15 20:34:00 +00:00
|
|
|
self.client.post_beacon_blocks(&next_block).await.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"valid blocks should be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-31 23:51:37 +00:00
|
|
|
pub async fn test_post_beacon_blocks_ssz_valid(mut self) -> Self {
|
|
|
|
|
let next_block = &self.next_block;
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_blocks_ssz(next_block)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert!(
|
|
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
|
|
|
|
"valid blocks should be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_post_beacon_blocks_invalid(mut self) -> Self {
|
Add broadcast validation routes to Beacon Node HTTP API (#4316)
## Issue Addressed
- #4293
- #4264
## Proposed Changes
*Changes largely follow those suggested in the main issue*.
- Add new routes to HTTP API
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Add new routes to `BeaconNodeHttpClient`
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Define new Eth2 common types
- `BroadcastValidation`, enum representing the level of validation to apply to blocks prior to broadcast
- `BroadcastValidationQuery`, the corresponding HTTP query string type for the above type
- ~~Define `_checked` variants of both `publish_block` and `publish_blinded_block` that enforce a validation level at a type level~~
- Add interactive tests to the `bn_http_api_tests` test target covering each validation level (to their own test module, `broadcast_validation_tests`)
- `beacon/blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- Only consensus pass (i.e., equivocates) (200)
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- `beacon/blinded_blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- ~~Only consensus pass (i.e., equivocates) (200)~~
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- Add a new trait, `IntoGossipVerifiedBlock`, which allows type-level guarantees to be made as to gossip validity
- Modify the structure of the `ObservedBlockProducers` cache from a `(slot, validator_index)` mapping to a `((slot, validator_index), block_root)` mapping
- Modify `ObservedBlockProducers::proposer_has_been_observed` to return a `SeenBlock` rather than a boolean on success
- Punish gossip peer (low) for submitting equivocating blocks
- Rename `BlockError::SlashablePublish` to `BlockError::SlashableProposal`
## Additional Info
This PR contains changes that directly modify how blocks are verified within the client. For more context, consult [comments in-thread](https://github.com/sigp/lighthouse/pull/4316#discussion_r1234724202).
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
2023-06-29 12:02:38 +00:00
|
|
|
let block = self
|
|
|
|
|
.harness
|
|
|
|
|
.make_block_with_modifier(
|
|
|
|
|
self.harness.get_current_state(),
|
|
|
|
|
self.harness.get_current_slot(),
|
|
|
|
|
|b| {
|
|
|
|
|
*b.state_root_mut() = Hash256::zero();
|
|
|
|
|
},
|
|
|
|
|
)
|
2023-03-15 20:34:00 +00:00
|
|
|
.await
|
Add broadcast validation routes to Beacon Node HTTP API (#4316)
## Issue Addressed
- #4293
- #4264
## Proposed Changes
*Changes largely follow those suggested in the main issue*.
- Add new routes to HTTP API
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Add new routes to `BeaconNodeHttpClient`
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Define new Eth2 common types
- `BroadcastValidation`, enum representing the level of validation to apply to blocks prior to broadcast
- `BroadcastValidationQuery`, the corresponding HTTP query string type for the above type
- ~~Define `_checked` variants of both `publish_block` and `publish_blinded_block` that enforce a validation level at a type level~~
- Add interactive tests to the `bn_http_api_tests` test target covering each validation level (to their own test module, `broadcast_validation_tests`)
- `beacon/blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- Only consensus pass (i.e., equivocates) (200)
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- `beacon/blinded_blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- ~~Only consensus pass (i.e., equivocates) (200)~~
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- Add a new trait, `IntoGossipVerifiedBlock`, which allows type-level guarantees to be made as to gossip validity
- Modify the structure of the `ObservedBlockProducers` cache from a `(slot, validator_index)` mapping to a `((slot, validator_index), block_root)` mapping
- Modify `ObservedBlockProducers::proposer_has_been_observed` to return a `SeenBlock` rather than a boolean on success
- Punish gossip peer (low) for submitting equivocating blocks
- Rename `BlockError::SlashablePublish` to `BlockError::SlashableProposal`
## Additional Info
This PR contains changes that directly modify how blocks are verified within the client. For more context, consult [comments in-thread](https://github.com/sigp/lighthouse/pull/4316#discussion_r1234724202).
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
2023-06-29 12:02:38 +00:00
|
|
|
.0;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2023-07-07 14:17:04 +00:00
|
|
|
assert!(self
|
|
|
|
|
.client
|
|
|
|
|
.post_beacon_blocks(&SignedBlockContents::from(block))
|
|
|
|
|
.await
|
|
|
|
|
.is_err());
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
Add broadcast validation routes to Beacon Node HTTP API (#4316)
## Issue Addressed
- #4293
- #4264
## Proposed Changes
*Changes largely follow those suggested in the main issue*.
- Add new routes to HTTP API
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Add new routes to `BeaconNodeHttpClient`
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Define new Eth2 common types
- `BroadcastValidation`, enum representing the level of validation to apply to blocks prior to broadcast
- `BroadcastValidationQuery`, the corresponding HTTP query string type for the above type
- ~~Define `_checked` variants of both `publish_block` and `publish_blinded_block` that enforce a validation level at a type level~~
- Add interactive tests to the `bn_http_api_tests` test target covering each validation level (to their own test module, `broadcast_validation_tests`)
- `beacon/blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- Only consensus pass (i.e., equivocates) (200)
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- `beacon/blinded_blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- ~~Only consensus pass (i.e., equivocates) (200)~~
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- Add a new trait, `IntoGossipVerifiedBlock`, which allows type-level guarantees to be made as to gossip validity
- Modify the structure of the `ObservedBlockProducers` cache from a `(slot, validator_index)` mapping to a `((slot, validator_index), block_root)` mapping
- Modify `ObservedBlockProducers::proposer_has_been_observed` to return a `SeenBlock` rather than a boolean on success
- Punish gossip peer (low) for submitting equivocating blocks
- Rename `BlockError::SlashablePublish` to `BlockError::SlashableProposal`
## Additional Info
This PR contains changes that directly modify how blocks are verified within the client. For more context, consult [comments in-thread](https://github.com/sigp/lighthouse/pull/4316#discussion_r1234724202).
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
2023-06-29 12:02:38 +00:00
|
|
|
"gossip valid blocks should be sent to network"
|
2020-09-29 03:46:54 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-31 23:51:37 +00:00
|
|
|
pub async fn test_post_beacon_blocks_ssz_invalid(mut self) -> Self {
|
|
|
|
|
let block = self
|
|
|
|
|
.harness
|
|
|
|
|
.make_block_with_modifier(
|
|
|
|
|
self.harness.get_current_state(),
|
|
|
|
|
self.harness.get_current_slot(),
|
|
|
|
|
|b| {
|
|
|
|
|
*b.state_root_mut() = Hash256::zero();
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.0;
|
|
|
|
|
|
2023-08-08 07:33:25 +00:00
|
|
|
assert!(self
|
|
|
|
|
.client
|
|
|
|
|
.post_beacon_blocks_ssz(&SignedBlockContents::from(block))
|
|
|
|
|
.await
|
|
|
|
|
.is_err());
|
2023-07-31 23:51:37 +00:00
|
|
|
|
|
|
|
|
assert!(
|
|
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
|
|
|
|
"gossip valid blocks should be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-28 00:55:31 +00:00
|
|
|
pub async fn test_post_beacon_blocks_duplicate(self) -> Self {
|
2023-09-11 01:59:13 +00:00
|
|
|
let block_contents = self
|
2023-08-28 00:55:31 +00:00
|
|
|
.harness
|
|
|
|
|
.make_block(
|
|
|
|
|
self.harness.get_current_state(),
|
|
|
|
|
self.harness.get_current_slot(),
|
|
|
|
|
)
|
|
|
|
|
.await
|
2023-09-11 01:59:13 +00:00
|
|
|
.0
|
|
|
|
|
.into();
|
2023-08-28 00:55:31 +00:00
|
|
|
|
2023-09-11 01:59:13 +00:00
|
|
|
assert!(self
|
|
|
|
|
.client
|
|
|
|
|
.post_beacon_blocks(&block_contents)
|
|
|
|
|
.await
|
|
|
|
|
.is_ok());
|
2023-08-28 00:55:31 +00:00
|
|
|
|
2023-09-11 01:59:13 +00:00
|
|
|
let blinded_block_contents = block_contents.clone_as_blinded();
|
2023-08-28 00:55:31 +00:00
|
|
|
|
|
|
|
|
// Test all the POST methods in sequence, they should all behave the same.
|
|
|
|
|
let responses = vec![
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blocks(&block_contents)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_blocks_v2(&block_contents, None)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blocks_ssz(&block_contents)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blocks_v2_ssz(&block_contents, None)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blinded_blocks(&blinded_block_contents)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blinded_blocks_v2(&blinded_block_contents, None)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blinded_blocks_ssz(&blinded_block_contents)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
self.client
|
2023-09-11 01:59:13 +00:00
|
|
|
.post_beacon_blinded_blocks_v2_ssz(&blinded_block_contents, None)
|
2023-08-28 00:55:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err(),
|
|
|
|
|
];
|
|
|
|
|
for (i, response) in responses.into_iter().enumerate() {
|
|
|
|
|
assert_eq!(
|
|
|
|
|
response.status().unwrap(),
|
|
|
|
|
StatusCode::ACCEPTED,
|
|
|
|
|
"response {i}"
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_beacon_blocks(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
2022-07-25 08:23:00 +00:00
|
|
|
let expected = block_id
|
|
|
|
|
.full_block(&self.chain)
|
|
|
|
|
.await
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(block, _execution_optimistic, _finalized)| block);
|
2021-02-24 04:15:14 +00:00
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
if let CoreBlockId::Slot(slot) = block_id.0 {
|
2021-04-25 03:59:59 +00:00
|
|
|
if expected.is_none() {
|
|
|
|
|
assert!(SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
} else {
|
|
|
|
|
assert!(!SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-28 01:18:04 +00:00
|
|
|
// Check the JSON endpoint.
|
2022-07-25 08:23:00 +00:00
|
|
|
let json_result = self.client.get_beacon_blocks(block_id.0).await.unwrap();
|
2021-08-06 00:47:31 +00:00
|
|
|
|
|
|
|
|
if let (Some(json), Some(expected)) = (&json_result, &expected) {
|
2022-07-25 08:23:00 +00:00
|
|
|
assert_eq!(&json.data, expected.as_ref(), "{:?}", block_id);
|
2021-08-06 00:47:31 +00:00
|
|
|
assert_eq!(
|
|
|
|
|
json.version,
|
|
|
|
|
Some(expected.fork_name(&self.chain.spec).unwrap())
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert_eq!(json_result, None);
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-10-28 01:18:04 +00:00
|
|
|
// Check the SSZ endpoint.
|
2021-07-09 06:15:32 +00:00
|
|
|
let ssz_result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_blocks_ssz(block_id.0, &self.chain.spec)
|
2021-07-09 06:15:32 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
2022-07-25 08:23:00 +00:00
|
|
|
assert_eq!(
|
|
|
|
|
ssz_result.as_ref(),
|
|
|
|
|
expected.as_ref().map(|b| b.as_ref()),
|
|
|
|
|
"{:?}",
|
|
|
|
|
block_id
|
|
|
|
|
);
|
2021-08-06 00:47:31 +00:00
|
|
|
|
|
|
|
|
// Check that the legacy v1 API still works but doesn't return a version field.
|
2022-07-25 08:23:00 +00:00
|
|
|
let v1_result = self.client.get_beacon_blocks_v1(block_id.0).await.unwrap();
|
2021-08-06 00:47:31 +00:00
|
|
|
if let (Some(v1_result), Some(expected)) = (&v1_result, &expected) {
|
|
|
|
|
assert_eq!(v1_result.version, None);
|
2022-07-25 08:23:00 +00:00
|
|
|
assert_eq!(&v1_result.data, expected.as_ref());
|
2021-08-06 00:47:31 +00:00
|
|
|
} else {
|
|
|
|
|
assert_eq!(v1_result, None);
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
2021-10-28 01:18:04 +00:00
|
|
|
|
|
|
|
|
// Check that version headers are provided.
|
2022-07-25 08:23:00 +00:00
|
|
|
let url = self.client.get_beacon_blocks_path(block_id.0).unwrap();
|
2021-10-28 01:18:04 +00:00
|
|
|
|
|
|
|
|
let builders: Vec<fn(RequestBuilder) -> RequestBuilder> = vec![
|
|
|
|
|
|b| b,
|
|
|
|
|
|b| b.accept(Accept::Ssz),
|
|
|
|
|
|b| b.accept(Accept::Json),
|
|
|
|
|
|b| b.accept(Accept::Any),
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
for req_builder in builders {
|
|
|
|
|
let raw_res = self
|
|
|
|
|
.client
|
|
|
|
|
.get_response(url.clone(), req_builder)
|
|
|
|
|
.await
|
|
|
|
|
.optional()
|
|
|
|
|
.unwrap();
|
|
|
|
|
if let (Some(raw_res), Some(expected)) = (&raw_res, &expected) {
|
|
|
|
|
assert_eq!(
|
|
|
|
|
raw_res.fork_name_from_header().unwrap(),
|
|
|
|
|
Some(expected.fork_name(&self.chain.spec).unwrap())
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert!(raw_res.is_none());
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-11-11 00:38:27 +00:00
|
|
|
pub async fn test_beacon_blinded_blocks(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let expected = block_id
|
|
|
|
|
.blinded_block(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(block, _execution_optimistic, _finalized)| block);
|
2022-11-11 00:38:27 +00:00
|
|
|
|
|
|
|
|
if let CoreBlockId::Slot(slot) = block_id.0 {
|
|
|
|
|
if expected.is_none() {
|
|
|
|
|
assert!(SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
} else {
|
|
|
|
|
assert!(!SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Check the JSON endpoint.
|
|
|
|
|
let json_result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_blinded_blocks(block_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
if let (Some(json), Some(expected)) = (&json_result, &expected) {
|
|
|
|
|
assert_eq!(&json.data, expected, "{:?}", block_id);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
json.version,
|
|
|
|
|
Some(expected.fork_name(&self.chain.spec).unwrap())
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert_eq!(json_result, None);
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Check the SSZ endpoint.
|
|
|
|
|
let ssz_result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_blinded_blocks_ssz(block_id.0, &self.chain.spec)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
assert_eq!(ssz_result.as_ref(), expected.as_ref(), "{:?}", block_id);
|
|
|
|
|
|
|
|
|
|
// Check that version headers are provided.
|
|
|
|
|
let url = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_blinded_blocks_path(block_id.0)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let builders: Vec<fn(RequestBuilder) -> RequestBuilder> = vec![
|
|
|
|
|
|b| b,
|
|
|
|
|
|b| b.accept(Accept::Ssz),
|
|
|
|
|
|b| b.accept(Accept::Json),
|
|
|
|
|
|b| b.accept(Accept::Any),
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
for req_builder in builders {
|
|
|
|
|
let raw_res = self
|
|
|
|
|
.client
|
|
|
|
|
.get_response(url.clone(), req_builder)
|
|
|
|
|
.await
|
|
|
|
|
.optional()
|
|
|
|
|
.unwrap();
|
|
|
|
|
if let (Some(raw_res), Some(expected)) = (&raw_res, &expected) {
|
|
|
|
|
assert_eq!(
|
|
|
|
|
raw_res.fork_name_from_header().unwrap(),
|
|
|
|
|
Some(expected.fork_name(&self.chain.spec).unwrap())
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert!(raw_res.is_none());
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_beacon_blocks_attestations(self) -> Self {
|
|
|
|
|
for block_id in self.interesting_block_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_beacon_blocks_attestations(block_id.0)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.map(|res| res.data);
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let expected = block_id.full_block(&self.chain).await.ok().map(
|
2023-03-30 06:08:37 +00:00
|
|
|
|(block, _execution_optimistic, _finalized)| {
|
2022-07-25 08:23:00 +00:00
|
|
|
block.message().body().attestations().clone().into()
|
|
|
|
|
},
|
|
|
|
|
);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
if let CoreBlockId::Slot(slot) = block_id.0 {
|
2021-04-25 03:59:59 +00:00
|
|
|
if expected.is_none() {
|
|
|
|
|
assert!(SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
} else {
|
|
|
|
|
assert!(!SKIPPED_SLOTS.contains(&slot.as_u64()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
assert_eq!(result, expected, "{:?}", block_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_attestations_valid(mut self) -> Self {
|
2020-11-18 23:31:39 +00:00
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attestations(self.attestations.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-11-18 23:31:39 +00:00
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
2020-11-18 23:31:39 +00:00
|
|
|
"valid attestation should be sent to network"
|
|
|
|
|
);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_attestations_invalid(mut self) -> Self {
|
2020-11-18 23:31:39 +00:00
|
|
|
let mut attestations = Vec::new();
|
2020-09-29 03:46:54 +00:00
|
|
|
for attestation in &self.attestations {
|
2020-11-18 23:31:39 +00:00
|
|
|
let mut invalid_attestation = attestation.clone();
|
|
|
|
|
invalid_attestation.data.slot += 1;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-11-18 23:31:39 +00:00
|
|
|
// add both to ensure we only fail on invalid attestations
|
|
|
|
|
attestations.push(attestation.clone());
|
|
|
|
|
attestations.push(invalid_attestation);
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2020-11-18 23:31:39 +00:00
|
|
|
let err = self
|
|
|
|
|
.client
|
|
|
|
|
.post_beacon_pool_attestations(attestations.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
|
|
|
|
match err {
|
|
|
|
|
Error::ServerIndexedMessage(IndexedErrorMessage {
|
|
|
|
|
code,
|
|
|
|
|
message: _,
|
|
|
|
|
failures,
|
|
|
|
|
}) => {
|
|
|
|
|
assert_eq!(code, 400);
|
|
|
|
|
assert_eq!(failures.len(), self.attestations.len());
|
|
|
|
|
}
|
|
|
|
|
_ => panic!("query did not fail correctly"),
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-18 23:31:39 +00:00
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
2020-11-18 23:31:39 +00:00
|
|
|
"if some attestations are valid, we should send them to the network"
|
|
|
|
|
);
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_beacon_pool_attestations(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2020-11-18 23:31:39 +00:00
|
|
|
.get_beacon_pool_attestations(None, None)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let mut expected = self.chain.op_pool.get_all_attestations();
|
|
|
|
|
expected.extend(self.chain.naive_aggregation_pool.read().iter().cloned());
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_attester_slashings_valid(mut self) -> Self {
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attester_slashings(&self.attester_slashing)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"valid attester slashing should be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_attester_slashings_invalid(mut self) -> Self {
|
|
|
|
|
let mut slashing = self.attester_slashing.clone();
|
|
|
|
|
slashing.attestation_1.data.slot += 1;
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attester_slashings(&slashing)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().now_or_never().is_none(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"invalid attester slashing should not be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_beacon_pool_attester_slashings(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_pool_attester_slashings()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = self.chain.op_pool.get_all_attester_slashings();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_proposer_slashings_valid(mut self) -> Self {
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_proposer_slashings(&self.proposer_slashing)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"valid proposer slashing should be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_proposer_slashings_invalid(mut self) -> Self {
|
|
|
|
|
let mut slashing = self.proposer_slashing.clone();
|
|
|
|
|
slashing.signed_header_1.message.slot += 1;
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_proposer_slashings(&slashing)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().now_or_never().is_none(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"invalid proposer slashing should not be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_beacon_pool_proposer_slashings(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_pool_proposer_slashings()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = self.chain.op_pool.get_all_proposer_slashings();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_voluntary_exits_valid(mut self) -> Self {
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_voluntary_exits(&self.voluntary_exit)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().await.is_some(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"valid exit should be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_beacon_pool_voluntary_exits_invalid(mut self) -> Self {
|
|
|
|
|
let mut exit = self.voluntary_exit.clone();
|
|
|
|
|
exit.message.epoch += 1;
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_voluntary_exits(&exit)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
|
|
|
|
assert!(
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx.network_recv.recv().now_or_never().is_none(),
|
2020-09-29 03:46:54 +00:00
|
|
|
"invalid exit should not be sent to network"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_beacon_pool_voluntary_exits(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_pool_voluntary_exits()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = self.chain.op_pool.get_all_voluntary_exits();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_config_fork_schedule(self) -> Self {
|
|
|
|
|
let result = self.client.get_config_fork_schedule().await.unwrap().data;
|
|
|
|
|
|
2021-08-24 01:36:27 +00:00
|
|
|
let expected: Vec<Fork> = ForkName::list_all()
|
|
|
|
|
.into_iter()
|
|
|
|
|
.filter_map(|fork| self.chain.spec.fork_for_name(fork))
|
|
|
|
|
.collect();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_config_spec(self) -> Self {
|
2022-08-10 07:52:59 +00:00
|
|
|
let result = self
|
|
|
|
|
.client
|
2023-09-09 06:09:50 +00:00
|
|
|
.get_config_spec::<ConfigAndPresetDeneb>()
|
2022-08-10 07:52:59 +00:00
|
|
|
.await
|
2023-09-09 06:09:50 +00:00
|
|
|
.map(|res| ConfigAndPreset::Deneb(res.data))
|
2022-08-10 07:52:59 +00:00
|
|
|
.unwrap();
|
|
|
|
|
let expected = ConfigAndPreset::from_chain_spec::<E>(&self.chain.spec, None);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_config_deposit_contract(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_config_deposit_contract()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = DepositContractData {
|
|
|
|
|
address: self.chain.spec.deposit_contract_address,
|
2021-10-01 06:32:38 +00:00
|
|
|
chain_id: self.chain.spec.deposit_chain_id,
|
2020-09-29 03:46:54 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_node_version(self) -> Self {
|
|
|
|
|
let result = self.client.get_node_version().await.unwrap().data;
|
|
|
|
|
|
|
|
|
|
let expected = VersionData {
|
|
|
|
|
version: lighthouse_version::version_with_platform(),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_node_syncing(self) -> Self {
|
|
|
|
|
let result = self.client.get_node_syncing().await.unwrap().data;
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let head_slot = self.chain.canonical_head.cached_head().head_slot();
|
2020-09-29 03:46:54 +00:00
|
|
|
let sync_distance = self.chain.slot().unwrap() - head_slot;
|
|
|
|
|
|
|
|
|
|
let expected = SyncingData {
|
|
|
|
|
is_syncing: false,
|
2022-08-02 23:20:51 +00:00
|
|
|
is_optimistic: Some(false),
|
Implement `el_offline` and use it in the VC (#4295)
## Issue Addressed
Closes https://github.com/sigp/lighthouse/issues/4291, part of #3613.
## Proposed Changes
- Implement the `el_offline` field on `/eth/v1/node/syncing`. We set `el_offline=true` if:
- The EL's internal status is `Offline` or `AuthFailed`, _or_
- The most recent call to `newPayload` resulted in an error (more on this in a moment).
- Use the `el_offline` field in the VC to mark nodes with offline ELs as _unsynced_. These nodes will still be used, but only after synced nodes.
- Overhaul the usage of `RequireSynced` so that `::No` is used almost everywhere. The `--allow-unsynced` flag was broken and had the opposite effect to intended, so it has been deprecated.
- Add tests for the EL being offline on the upcheck call, and being offline due to the newPayload check.
## Why track `newPayload` errors?
Tracking the EL's online/offline status is too coarse-grained to be useful in practice, because:
- If the EL is timing out to some calls, it's unlikely to timeout on the `upcheck` call, which is _just_ `eth_syncing`. Every failed call is followed by an upcheck [here](https://github.com/sigp/lighthouse/blob/693886b94176faa4cb450f024696cb69cda2fe58/beacon_node/execution_layer/src/engines.rs#L372-L380), which would have the effect of masking the failure and keeping the status _online_.
- The `newPayload` call is the most likely to time out. It's the call in which ELs tend to do most of their work (often 1-2 seconds), with `forkchoiceUpdated` usually returning much faster (<50ms).
- If `newPayload` is failing consistently (e.g. timing out) then this is a good indication that either the node's EL is in trouble, or the network as a whole is. In the first case validator clients _should_ prefer other BNs if they have one available. In the second case, all of their BNs will likely report `el_offline` and they'll just have to proceed with trying to use them.
## Additional Changes
- Add utility method `ForkName::latest` which is quite convenient for test writing, but probably other things too.
- Delete some stale comments from when we used to support multiple execution nodes.
2023-05-17 05:51:56 +00:00
|
|
|
// these tests run without the Bellatrix fork enabled
|
|
|
|
|
el_offline: Some(true),
|
2020-09-29 03:46:54 +00:00
|
|
|
head_slot,
|
|
|
|
|
sync_distance,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-22 02:59:42 +00:00
|
|
|
pub async fn test_get_node_identity(self) -> Self {
|
|
|
|
|
let result = self.client.get_node_identity().await.unwrap().data;
|
|
|
|
|
|
|
|
|
|
let expected = IdentityData {
|
|
|
|
|
peer_id: self.local_enr.peer_id().to_string(),
|
|
|
|
|
enr: self.local_enr.clone(),
|
|
|
|
|
p2p_addresses: self.local_enr.multiaddr_p2p_tcp(),
|
|
|
|
|
discovery_addresses: self.local_enr.multiaddr_p2p_udp(),
|
|
|
|
|
metadata: eth2::types::MetaData {
|
|
|
|
|
seq_number: 0,
|
|
|
|
|
attnets: "0x0000000000000000".to_string(),
|
2021-08-04 01:44:57 +00:00
|
|
|
syncnets: "0x00".to_string(),
|
2020-10-22 02:59:42 +00:00
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_node_health(self) -> Self {
|
2023-06-30 01:13:04 +00:00
|
|
|
let status = self.client.get_node_health().await;
|
|
|
|
|
match status {
|
|
|
|
|
Ok(_) => {
|
|
|
|
|
panic!("should return 503 error status code");
|
|
|
|
|
}
|
|
|
|
|
Err(e) => {
|
|
|
|
|
assert_eq!(e.status().unwrap(), 503);
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-10-22 02:59:42 +00:00
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_node_peers_by_id(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_node_peers_by_id(self.external_peer_id.clone())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = PeerData {
|
|
|
|
|
peer_id: self.external_peer_id.to_string(),
|
|
|
|
|
enr: None,
|
2020-11-13 02:02:41 +00:00
|
|
|
last_seen_p2p_address: EXTERNAL_ADDR.to_string(),
|
2020-10-22 02:59:42 +00:00
|
|
|
state: PeerState::Connected,
|
|
|
|
|
direction: PeerDirection::Inbound,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_node_peers(self) -> Self {
|
2020-11-13 02:02:41 +00:00
|
|
|
let peer_states: Vec<Option<&[PeerState]>> = vec![
|
|
|
|
|
Some(&[PeerState::Connected]),
|
|
|
|
|
Some(&[PeerState::Connecting]),
|
|
|
|
|
Some(&[PeerState::Disconnected]),
|
|
|
|
|
Some(&[PeerState::Disconnecting]),
|
|
|
|
|
None,
|
|
|
|
|
Some(&[PeerState::Connected, PeerState::Connecting]),
|
|
|
|
|
];
|
|
|
|
|
let peer_dirs: Vec<Option<&[PeerDirection]>> = vec![
|
|
|
|
|
Some(&[PeerDirection::Outbound]),
|
|
|
|
|
Some(&[PeerDirection::Inbound]),
|
|
|
|
|
Some(&[PeerDirection::Inbound, PeerDirection::Outbound]),
|
|
|
|
|
None,
|
|
|
|
|
];
|
2020-10-22 02:59:42 +00:00
|
|
|
|
2020-11-13 02:02:41 +00:00
|
|
|
for states in peer_states {
|
|
|
|
|
for dirs in peer_dirs.clone() {
|
|
|
|
|
let result = self.client.get_node_peers(states, dirs).await.unwrap();
|
|
|
|
|
let expected_peer = PeerData {
|
|
|
|
|
peer_id: self.external_peer_id.to_string(),
|
|
|
|
|
enr: None,
|
|
|
|
|
last_seen_p2p_address: EXTERNAL_ADDR.to_string(),
|
|
|
|
|
state: PeerState::Connected,
|
|
|
|
|
direction: PeerDirection::Inbound,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let state_match =
|
|
|
|
|
states.map_or(true, |states| states.contains(&PeerState::Connected));
|
|
|
|
|
let dir_match = dirs.map_or(true, |dirs| dirs.contains(&PeerDirection::Inbound));
|
2020-10-22 02:59:42 +00:00
|
|
|
|
2020-11-13 02:02:41 +00:00
|
|
|
let mut expected_peers = Vec::new();
|
|
|
|
|
if state_match && dir_match {
|
|
|
|
|
expected_peers.push(expected_peer);
|
|
|
|
|
}
|
2020-10-22 02:59:42 +00:00
|
|
|
|
2020-11-13 02:02:41 +00:00
|
|
|
assert_eq!(
|
|
|
|
|
result,
|
|
|
|
|
PeersData {
|
|
|
|
|
meta: PeersMetaData {
|
|
|
|
|
count: expected_peers.len() as u64
|
|
|
|
|
},
|
|
|
|
|
data: expected_peers,
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_node_peer_count(self) -> Self {
|
|
|
|
|
let result = self.client.get_node_peer_count().await.unwrap().data;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
result,
|
|
|
|
|
PeerCount {
|
|
|
|
|
connected: 1,
|
|
|
|
|
connecting: 0,
|
|
|
|
|
disconnected: 0,
|
|
|
|
|
disconnecting: 0,
|
|
|
|
|
}
|
|
|
|
|
);
|
2020-10-22 02:59:42 +00:00
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_get_debug_beacon_states(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
2022-07-25 08:23:00 +00:00
|
|
|
let result_json = self
|
|
|
|
|
.client
|
|
|
|
|
.get_debug_beacon_states(state_id.0)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
2021-08-06 00:47:31 +00:00
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let mut expected = state_id
|
|
|
|
|
.state(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(state, _execution_optimistic, _finalized)| state);
|
2021-08-06 00:47:31 +00:00
|
|
|
expected.as_mut().map(|state| state.drop_all_caches());
|
|
|
|
|
|
|
|
|
|
if let (Some(json), Some(expected)) = (&result_json, &expected) {
|
|
|
|
|
assert_eq!(json.data, *expected, "{:?}", state_id);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
json.version,
|
|
|
|
|
Some(expected.fork_name(&self.chain.spec).unwrap())
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert_eq!(result_json, None);
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Check SSZ API.
|
2021-01-06 03:01:46 +00:00
|
|
|
let result_ssz = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_debug_beacon_states_ssz(state_id.0, &self.chain.spec)
|
2021-01-06 03:01:46 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
2021-08-06 00:47:31 +00:00
|
|
|
assert_eq!(result_ssz, expected, "{:?}", state_id);
|
|
|
|
|
|
2021-10-28 01:18:04 +00:00
|
|
|
// Check that version headers are provided.
|
2022-07-25 08:23:00 +00:00
|
|
|
let url = self
|
|
|
|
|
.client
|
|
|
|
|
.get_debug_beacon_states_path(state_id.0)
|
|
|
|
|
.unwrap();
|
2021-10-28 01:18:04 +00:00
|
|
|
|
|
|
|
|
let builders: Vec<fn(RequestBuilder) -> RequestBuilder> =
|
|
|
|
|
vec![|b| b, |b| b.accept(Accept::Ssz)];
|
|
|
|
|
|
|
|
|
|
for req_builder in builders {
|
|
|
|
|
let raw_res = self
|
|
|
|
|
.client
|
|
|
|
|
.get_response(url.clone(), req_builder)
|
|
|
|
|
.await
|
|
|
|
|
.optional()
|
|
|
|
|
.unwrap();
|
|
|
|
|
if let (Some(raw_res), Some(expected)) = (&raw_res, &expected) {
|
|
|
|
|
assert_eq!(
|
|
|
|
|
raw_res.fork_name_from_header().unwrap(),
|
|
|
|
|
Some(expected.fork_name(&self.chain.spec).unwrap())
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert!(raw_res.is_none());
|
|
|
|
|
assert_eq!(expected, None);
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_debug_beacon_heads(self) -> Self {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_debug_beacon_heads()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(|head| (head.root, head.slot))
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
let expected = self.chain.heads();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-03-29 02:56:37 +00:00
|
|
|
pub async fn test_get_debug_fork_choice(self) -> Self {
|
|
|
|
|
let result = self.client.get_debug_fork_choice().await.unwrap();
|
|
|
|
|
|
|
|
|
|
let beacon_fork_choice = self.chain.canonical_head.fork_choice_read_lock();
|
|
|
|
|
|
|
|
|
|
let expected_proto_array = beacon_fork_choice.proto_array().core_proto_array();
|
|
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
result.justified_checkpoint,
|
|
|
|
|
expected_proto_array.justified_checkpoint
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
result.finalized_checkpoint,
|
|
|
|
|
expected_proto_array.finalized_checkpoint
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let expected_fork_choice_nodes: Vec<ForkChoiceNode> = expected_proto_array
|
|
|
|
|
.nodes
|
|
|
|
|
.iter()
|
|
|
|
|
.map(|node| {
|
|
|
|
|
let execution_status = if node.execution_status.is_execution_enabled() {
|
|
|
|
|
Some(node.execution_status.to_string())
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
};
|
|
|
|
|
ForkChoiceNode {
|
|
|
|
|
slot: node.slot,
|
|
|
|
|
block_root: node.root,
|
|
|
|
|
parent_root: node
|
|
|
|
|
.parent
|
|
|
|
|
.and_then(|index| expected_proto_array.nodes.get(index))
|
|
|
|
|
.map(|parent| parent.root),
|
2023-05-15 02:10:42 +00:00
|
|
|
justified_epoch: node.justified_checkpoint.epoch,
|
|
|
|
|
finalized_epoch: node.finalized_checkpoint.epoch,
|
2023-03-29 02:56:37 +00:00
|
|
|
weight: node.weight,
|
|
|
|
|
validity: execution_status,
|
|
|
|
|
execution_block_hash: node
|
|
|
|
|
.execution_status
|
|
|
|
|
.block_hash()
|
|
|
|
|
.map(|block_hash| block_hash.into_root()),
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.fork_choice_nodes, expected_fork_choice_nodes);
|
|
|
|
|
|
|
|
|
|
// need to drop beacon_fork_choice here, else borrow checker will complain
|
|
|
|
|
// that self cannot be moved out since beacon_fork_choice borrowed self.chain
|
|
|
|
|
// and might still live after self is moved out
|
|
|
|
|
drop(beacon_fork_choice);
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
fn validator_count(&self) -> usize {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
self.chain.head_snapshot().beacon_state.validators().len()
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn interesting_validator_indices(&self) -> Vec<Vec<u64>> {
|
|
|
|
|
let validator_count = self.validator_count() as u64;
|
|
|
|
|
|
|
|
|
|
let mut interesting = vec![
|
|
|
|
|
vec![],
|
|
|
|
|
vec![0],
|
|
|
|
|
vec![0, 1],
|
|
|
|
|
vec![0, 1, 3],
|
|
|
|
|
vec![validator_count],
|
|
|
|
|
vec![validator_count, 1],
|
|
|
|
|
vec![validator_count, 1, 3],
|
|
|
|
|
vec![u64::max_value()],
|
|
|
|
|
vec![u64::max_value(), 1],
|
|
|
|
|
vec![u64::max_value(), 1, 3],
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
interesting.push((0..validator_count).collect());
|
|
|
|
|
|
|
|
|
|
interesting
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-29 05:13:04 +00:00
|
|
|
fn interesting_validator_statuses(&self) -> Vec<Vec<ValidatorStatus>> {
|
|
|
|
|
let interesting = vec![
|
|
|
|
|
vec![],
|
|
|
|
|
vec![ValidatorStatus::Active],
|
|
|
|
|
vec![
|
2021-02-24 04:15:13 +00:00
|
|
|
ValidatorStatus::PendingInitialized,
|
|
|
|
|
ValidatorStatus::PendingQueued,
|
|
|
|
|
ValidatorStatus::ActiveOngoing,
|
|
|
|
|
ValidatorStatus::ActiveExiting,
|
|
|
|
|
ValidatorStatus::ActiveSlashed,
|
|
|
|
|
ValidatorStatus::ExitedUnslashed,
|
2020-10-29 05:13:04 +00:00
|
|
|
ValidatorStatus::ExitedSlashed,
|
2021-02-24 04:15:13 +00:00
|
|
|
ValidatorStatus::WithdrawalPossible,
|
|
|
|
|
ValidatorStatus::WithdrawalDone,
|
|
|
|
|
ValidatorStatus::Active,
|
|
|
|
|
ValidatorStatus::Pending,
|
|
|
|
|
ValidatorStatus::Exited,
|
|
|
|
|
ValidatorStatus::Withdrawal,
|
2020-10-29 05:13:04 +00:00
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
interesting
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_get_validator_duties_attester(self) -> Self {
|
|
|
|
|
let current_epoch = self.chain.epoch().unwrap().as_u64();
|
|
|
|
|
|
|
|
|
|
let half = current_epoch / 2;
|
|
|
|
|
let first = current_epoch - half;
|
|
|
|
|
let last = current_epoch + half;
|
|
|
|
|
|
|
|
|
|
for epoch in first..=last {
|
|
|
|
|
for indices in self.interesting_validator_indices() {
|
|
|
|
|
let epoch = Epoch::from(epoch);
|
|
|
|
|
|
|
|
|
|
// The endpoint does not allow getting duties past the next epoch.
|
|
|
|
|
if epoch > current_epoch + 1 {
|
|
|
|
|
assert_eq!(
|
|
|
|
|
self.client
|
2020-11-09 23:13:56 +00:00
|
|
|
.post_validator_duties_attester(epoch, indices.as_slice())
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err()
|
|
|
|
|
.status()
|
|
|
|
|
.map(Into::into),
|
|
|
|
|
Some(400)
|
|
|
|
|
);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let results = self
|
|
|
|
|
.client
|
2020-11-09 23:13:56 +00:00
|
|
|
.post_validator_duties_attester(epoch, indices.as_slice())
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
2020-12-04 00:18:58 +00:00
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let dependent_root = self
|
|
|
|
|
.chain
|
Use the forwards iterator more often (#2376)
## Issue Addressed
NA
## Primary Change
When investigating memory usage, I noticed that retrieving a block from an early slot (e.g., slot 900) would cause a sharp increase in the memory footprint (from 400mb to 800mb+) which seemed to be ever-lasting.
After some investigation, I found that the reverse iteration from the head back to that slot was the likely culprit. To counter this, I've switched the `BeaconChain::block_root_at_slot` to use the forwards iterator, instead of the reverse one.
I also noticed that the networking stack is using `BeaconChain::root_at_slot` to check if a peer is relevant (`check_peer_relevance`). Perhaps the steep, seemingly-random-but-consistent increases in memory usage are caused by the use of this function.
Using the forwards iterator with the HTTP API alleviated the sharp increases in memory usage. It also made the response much faster (before it felt like to took 1-2s, now it feels instant).
## Additional Changes
In the process I also noticed that we have two functions for getting block roots:
- `BeaconChain::block_root_at_slot`: returns `None` for a skip slot.
- `BeaconChain::root_at_slot`: returns the previous root for a skip slot.
I unified these two functions into `block_root_at_slot` and added the `WhenSlotSkipped` enum. Now, the caller must be explicit about the skip-slot behaviour when requesting a root.
Additionally, I replaced `vec![]` with `Vec::with_capacity` in `store::chunked_vector::range_query`. I stumbled across this whilst debugging and made this modification to see what effect it would have (not much). It seems like a decent change to keep around, but I'm not concerned either way.
Also, `BeaconChain::get_ancestor_block_root` is unused, so I got rid of it :wastebasket:.
## Additional Info
I haven't also done the same for state roots here. Whilst it's possible and a good idea, it's more work since the fwds iterators are presently block-roots-specific.
Whilst there's a few places a reverse iteration of state roots could be triggered (e.g., attestation production, HTTP API), they're no where near as common as the `check_peer_relevance` call. As such, I think we should get this PR merged first, then come back for the state root iters. I made an issue here https://github.com/sigp/lighthouse/issues/2377.
2021-05-31 04:18:20 +00:00
|
|
|
.block_root_at_slot(
|
|
|
|
|
(epoch - 1).start_slot(E::slots_per_epoch()) - 1,
|
|
|
|
|
WhenSlotSkipped::Prev,
|
|
|
|
|
)
|
2020-09-29 03:46:54 +00:00
|
|
|
.unwrap()
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.unwrap_or(self.chain.head_beacon_block_root());
|
2020-12-04 00:18:58 +00:00
|
|
|
|
|
|
|
|
assert_eq!(results.dependent_root, dependent_root);
|
|
|
|
|
|
|
|
|
|
let result_duties = results.data;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
let mut state = self
|
|
|
|
|
.chain
|
|
|
|
|
.state_at_slot(
|
|
|
|
|
epoch.start_slot(E::slots_per_epoch()),
|
|
|
|
|
StateSkipConfig::WithStateRoots,
|
|
|
|
|
)
|
|
|
|
|
.unwrap();
|
|
|
|
|
state
|
|
|
|
|
.build_committee_cache(RelativeEpoch::Current, &self.chain.spec)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let expected_len = indices
|
|
|
|
|
.iter()
|
2021-07-09 06:15:32 +00:00
|
|
|
.filter(|i| **i < state.validators().len() as u64)
|
2020-09-29 03:46:54 +00:00
|
|
|
.count();
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
assert_eq!(result_duties.len(), expected_len);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
for (indices_set, &i) in indices.iter().enumerate() {
|
|
|
|
|
if let Some(duty) = state
|
|
|
|
|
.get_attestation_duties(i as usize, RelativeEpoch::Current)
|
|
|
|
|
.unwrap()
|
|
|
|
|
{
|
|
|
|
|
let expected = AttesterData {
|
2021-07-09 06:15:32 +00:00
|
|
|
pubkey: state.validators()[i as usize].pubkey.clone().into(),
|
2020-09-29 03:46:54 +00:00
|
|
|
validator_index: i,
|
|
|
|
|
committees_at_slot: duty.committees_at_slot,
|
|
|
|
|
committee_index: duty.index,
|
|
|
|
|
committee_length: duty.committee_len as u64,
|
|
|
|
|
validator_committee_index: duty.committee_position as u64,
|
|
|
|
|
slot: duty.slot,
|
|
|
|
|
};
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
let result = result_duties
|
2020-09-29 03:46:54 +00:00
|
|
|
.iter()
|
|
|
|
|
.find(|duty| duty.validator_index == i)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
*result, expected,
|
|
|
|
|
"epoch: {}, indices_set: {}",
|
|
|
|
|
epoch, indices_set
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert!(
|
2020-12-04 00:18:58 +00:00
|
|
|
!result_duties.iter().any(|duty| duty.validator_index == i),
|
2020-09-29 03:46:54 +00:00
|
|
|
"validator index should not exist in response"
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_validator_duties_proposer(self) -> Self {
|
|
|
|
|
let current_epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
2022-02-18 05:32:00 +00:00
|
|
|
for epoch in 0..=self.chain.epoch().unwrap().as_u64() + 1 {
|
2021-03-17 05:09:57 +00:00
|
|
|
let epoch = Epoch::from(epoch);
|
2020-12-04 00:18:58 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let dependent_root = self
|
|
|
|
|
.chain
|
Use the forwards iterator more often (#2376)
## Issue Addressed
NA
## Primary Change
When investigating memory usage, I noticed that retrieving a block from an early slot (e.g., slot 900) would cause a sharp increase in the memory footprint (from 400mb to 800mb+) which seemed to be ever-lasting.
After some investigation, I found that the reverse iteration from the head back to that slot was the likely culprit. To counter this, I've switched the `BeaconChain::block_root_at_slot` to use the forwards iterator, instead of the reverse one.
I also noticed that the networking stack is using `BeaconChain::root_at_slot` to check if a peer is relevant (`check_peer_relevance`). Perhaps the steep, seemingly-random-but-consistent increases in memory usage are caused by the use of this function.
Using the forwards iterator with the HTTP API alleviated the sharp increases in memory usage. It also made the response much faster (before it felt like to took 1-2s, now it feels instant).
## Additional Changes
In the process I also noticed that we have two functions for getting block roots:
- `BeaconChain::block_root_at_slot`: returns `None` for a skip slot.
- `BeaconChain::root_at_slot`: returns the previous root for a skip slot.
I unified these two functions into `block_root_at_slot` and added the `WhenSlotSkipped` enum. Now, the caller must be explicit about the skip-slot behaviour when requesting a root.
Additionally, I replaced `vec![]` with `Vec::with_capacity` in `store::chunked_vector::range_query`. I stumbled across this whilst debugging and made this modification to see what effect it would have (not much). It seems like a decent change to keep around, but I'm not concerned either way.
Also, `BeaconChain::get_ancestor_block_root` is unused, so I got rid of it :wastebasket:.
## Additional Info
I haven't also done the same for state roots here. Whilst it's possible and a good idea, it's more work since the fwds iterators are presently block-roots-specific.
Whilst there's a few places a reverse iteration of state roots could be triggered (e.g., attestation production, HTTP API), they're no where near as common as the `check_peer_relevance` call. As such, I think we should get this PR merged first, then come back for the state root iters. I made an issue here https://github.com/sigp/lighthouse/issues/2377.
2021-05-31 04:18:20 +00:00
|
|
|
.block_root_at_slot(
|
|
|
|
|
epoch.start_slot(E::slots_per_epoch()) - 1,
|
|
|
|
|
WhenSlotSkipped::Prev,
|
|
|
|
|
)
|
2021-03-17 05:09:57 +00:00
|
|
|
.unwrap()
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.unwrap_or(self.chain.head_beacon_block_root());
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
// Presently, the beacon chain harness never runs the code that primes the proposer
|
|
|
|
|
// cache. If this changes in the future then we'll need some smarter logic here, but
|
|
|
|
|
// this is succinct and effective for the time being.
|
|
|
|
|
assert!(
|
|
|
|
|
self.chain
|
|
|
|
|
.beacon_proposer_cache
|
|
|
|
|
.lock()
|
|
|
|
|
.get_epoch::<E>(dependent_root, epoch)
|
|
|
|
|
.is_none(),
|
|
|
|
|
"the proposer cache should miss initially"
|
|
|
|
|
);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
// Check that current-epoch requests prime the proposer cache, whilst non-current
|
|
|
|
|
// requests don't.
|
|
|
|
|
if epoch == current_epoch {
|
|
|
|
|
assert!(
|
|
|
|
|
self.chain
|
|
|
|
|
.beacon_proposer_cache
|
|
|
|
|
.lock()
|
|
|
|
|
.get_epoch::<E>(dependent_root, epoch)
|
|
|
|
|
.is_some(),
|
|
|
|
|
"a current-epoch request should prime the proposer cache"
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
assert!(
|
|
|
|
|
self.chain
|
|
|
|
|
.beacon_proposer_cache
|
|
|
|
|
.lock()
|
|
|
|
|
.get_epoch::<E>(dependent_root, epoch)
|
|
|
|
|
.is_none(),
|
|
|
|
|
"a non-current-epoch request should not prime the proposer cache"
|
|
|
|
|
);
|
|
|
|
|
}
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let mut state = self
|
|
|
|
|
.chain
|
|
|
|
|
.state_at_slot(
|
|
|
|
|
epoch.start_slot(E::slots_per_epoch()),
|
|
|
|
|
StateSkipConfig::WithStateRoots,
|
|
|
|
|
)
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
state
|
|
|
|
|
.build_committee_cache(RelativeEpoch::Current, &self.chain.spec)
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let expected_duties = epoch
|
|
|
|
|
.slot_iter(E::slots_per_epoch())
|
|
|
|
|
.map(|slot| {
|
|
|
|
|
let index = state
|
|
|
|
|
.get_beacon_proposer_index(slot, &self.chain.spec)
|
|
|
|
|
.unwrap();
|
2021-07-09 06:15:32 +00:00
|
|
|
let pubkey = state.validators()[index].pubkey.clone().into();
|
2021-03-17 05:09:57 +00:00
|
|
|
|
|
|
|
|
ProposerData {
|
|
|
|
|
pubkey,
|
|
|
|
|
validator_index: index as u64,
|
|
|
|
|
slot,
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.collect::<Vec<_>>();
|
2020-12-04 00:18:58 +00:00
|
|
|
|
2021-03-17 05:09:57 +00:00
|
|
|
let expected = DutiesResponse {
|
|
|
|
|
data: expected_duties,
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: Some(false),
|
2021-03-17 05:09:57 +00:00
|
|
|
dependent_root,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
// If it's the current epoch, check the function with a primed proposer cache.
|
|
|
|
|
if epoch == current_epoch {
|
|
|
|
|
// This is technically a double-check, but it's defensive.
|
|
|
|
|
assert!(
|
|
|
|
|
self.chain
|
|
|
|
|
.beacon_proposer_cache
|
|
|
|
|
.lock()
|
|
|
|
|
.get_epoch::<E>(dependent_root, epoch)
|
|
|
|
|
.is_some(),
|
|
|
|
|
"the request should prime the proposer cache"
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-18 05:32:00 +00:00
|
|
|
// Requests to the epochs after the next epoch should fail.
|
2021-03-17 05:09:57 +00:00
|
|
|
self.client
|
2022-02-18 05:32:00 +00:00
|
|
|
.get_validator_duties_proposer(current_epoch + 2)
|
2021-03-17 05:09:57 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-29 23:42:35 +00:00
|
|
|
pub async fn test_get_validator_duties_early(self) -> Self {
|
|
|
|
|
let current_epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let next_epoch = current_epoch + 1;
|
|
|
|
|
let current_epoch_start = self
|
|
|
|
|
.chain
|
|
|
|
|
.slot_clock
|
|
|
|
|
.start_of(current_epoch.start_slot(E::slots_per_epoch()))
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_current_time(
|
2023-08-03 01:51:47 +00:00
|
|
|
current_epoch_start
|
|
|
|
|
- self.chain.spec.maximum_gossip_clock_disparity()
|
|
|
|
|
- Duration::from_millis(1),
|
2021-03-29 23:42:35 +00:00
|
|
|
);
|
|
|
|
|
|
2022-02-18 05:32:00 +00:00
|
|
|
let dependent_root = self
|
|
|
|
|
.chain
|
|
|
|
|
.block_root_at_slot(
|
|
|
|
|
current_epoch.start_slot(E::slots_per_epoch()) - 1,
|
|
|
|
|
WhenSlotSkipped::Prev,
|
|
|
|
|
)
|
|
|
|
|
.unwrap()
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.unwrap_or(self.chain.head_beacon_block_root());
|
2022-02-18 05:32:00 +00:00
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.get_validator_duties_proposer(current_epoch)
|
|
|
|
|
.await
|
|
|
|
|
.expect("should get proposer duties for the next epoch outside of tolerance");
|
|
|
|
|
|
|
|
|
|
assert!(
|
|
|
|
|
self.chain
|
|
|
|
|
.beacon_proposer_cache
|
|
|
|
|
.lock()
|
|
|
|
|
.get_epoch::<E>(dependent_root, current_epoch)
|
|
|
|
|
.is_none(),
|
|
|
|
|
"should not prime the proposer cache outside of tolerance"
|
2021-03-29 23:42:35 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
self.client
|
|
|
|
|
.post_validator_duties_attester(next_epoch, &[0])
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err()
|
|
|
|
|
.status()
|
|
|
|
|
.map(Into::into),
|
|
|
|
|
Some(400),
|
|
|
|
|
"should not get attester duties outside of tolerance"
|
|
|
|
|
);
|
|
|
|
|
|
2023-08-03 01:51:47 +00:00
|
|
|
self.chain.slot_clock.set_current_time(
|
|
|
|
|
current_epoch_start - self.chain.spec.maximum_gossip_clock_disparity(),
|
|
|
|
|
);
|
2021-03-29 23:42:35 +00:00
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.get_validator_duties_proposer(current_epoch)
|
|
|
|
|
.await
|
|
|
|
|
.expect("should get proposer duties within tolerance");
|
2022-02-18 05:32:00 +00:00
|
|
|
|
|
|
|
|
assert!(
|
|
|
|
|
self.chain
|
|
|
|
|
.beacon_proposer_cache
|
|
|
|
|
.lock()
|
|
|
|
|
.get_epoch::<E>(dependent_root, current_epoch)
|
|
|
|
|
.is_some(),
|
|
|
|
|
"should prime the proposer cache inside the tolerance"
|
|
|
|
|
);
|
|
|
|
|
|
2021-03-29 23:42:35 +00:00
|
|
|
self.client
|
|
|
|
|
.post_validator_duties_attester(next_epoch, &[0])
|
|
|
|
|
.await
|
|
|
|
|
.expect("should get attester duties within tolerance");
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_block_production(self) -> Self {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
2020-09-29 03:46:54 +00:00
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
|
|
|
|
for _ in 0..E::slots_per_epoch() * 3 {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let proposer_pubkey_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| duty.pubkey)
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
|
|
|
|
|
|
|
|
|
let sk = self
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.validator_keypairs()
|
2020-09-29 03:46:54 +00:00
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let randao_reveal = {
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
epoch,
|
|
|
|
|
Domain::Randao,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
);
|
|
|
|
|
let message = epoch.signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let block = self
|
|
|
|
|
.client
|
2022-03-31 07:52:23 +00:00
|
|
|
.get_validator_blocks::<E, FullPayload<E>>(slot, &randao_reveal, None)
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
2023-03-15 20:03:36 +00:00
|
|
|
.data
|
|
|
|
|
.deconstruct()
|
|
|
|
|
.0;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec);
|
2023-08-04 00:27:03 +00:00
|
|
|
let signed_block_contents =
|
|
|
|
|
SignedBlockContents::try_from(signed_block.clone()).unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2023-03-15 20:34:00 +00:00
|
|
|
self.client
|
|
|
|
|
.post_beacon_blocks(&signed_block_contents)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
assert_eq!(self.chain.head_beacon_block().as_ref(), &signed_block);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-21 06:38:31 +00:00
|
|
|
pub async fn test_block_production_ssz(self) -> Self {
|
|
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
|
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
|
|
|
|
for _ in 0..E::slots_per_epoch() * 3 {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let proposer_pubkey_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| duty.pubkey)
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
|
|
|
|
|
|
|
|
|
let sk = self
|
|
|
|
|
.validator_keypairs()
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let randao_reveal = {
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
epoch,
|
|
|
|
|
Domain::Randao,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
);
|
|
|
|
|
let message = epoch.signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let block_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blocks_ssz::<E, FullPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.expect("block bytes");
|
|
|
|
|
|
2023-09-26 00:57:21 +00:00
|
|
|
let block_contents =
|
|
|
|
|
BlockContents::<E, FullPayload<E>>::from_ssz_bytes(&block_bytes, &self.chain.spec)
|
|
|
|
|
.expect("block contents bytes can be decoded");
|
2023-09-21 06:38:31 +00:00
|
|
|
|
2023-09-26 00:57:21 +00:00
|
|
|
let signed_block_contents =
|
|
|
|
|
block_contents.sign(&sk, &fork, genesis_validators_root, &self.chain.spec);
|
2023-09-21 06:38:31 +00:00
|
|
|
|
|
|
|
|
self.client
|
2023-09-26 00:57:21 +00:00
|
|
|
.post_beacon_blocks_ssz(&signed_block_contents)
|
2023-09-21 06:38:31 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
2023-09-26 00:57:21 +00:00
|
|
|
assert_eq!(
|
|
|
|
|
self.chain.head_beacon_block().as_ref(),
|
|
|
|
|
signed_block_contents.signed_block()
|
|
|
|
|
);
|
2023-09-21 06:38:31 +00:00
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-28 07:14:13 +00:00
|
|
|
pub async fn test_block_production_no_verify_randao(self) -> Self {
|
|
|
|
|
for _ in 0..E::slots_per_epoch() {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
|
|
|
|
|
let block = self
|
|
|
|
|
.client
|
2022-09-19 07:58:48 +00:00
|
|
|
.get_validator_blocks_modular::<E, FullPayload<E>>(
|
2022-03-31 07:52:23 +00:00
|
|
|
slot,
|
2022-09-19 07:58:48 +00:00
|
|
|
&Signature::infinity().unwrap().into(),
|
2022-03-31 07:52:23 +00:00
|
|
|
None,
|
2022-09-19 07:58:48 +00:00
|
|
|
SkipRandaoVerification::Yes,
|
2022-03-31 07:52:23 +00:00
|
|
|
)
|
2022-03-28 07:14:13 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
2023-03-15 20:34:00 +00:00
|
|
|
.data
|
|
|
|
|
.deconstruct()
|
|
|
|
|
.0;
|
2022-03-28 07:14:13 +00:00
|
|
|
assert_eq!(block.slot(), slot);
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_block_production_verify_randao_invalid(self) -> Self {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
2022-03-28 07:14:13 +00:00
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
|
|
|
|
for _ in 0..E::slots_per_epoch() {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let proposer_pubkey_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| duty.pubkey)
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
|
|
|
|
|
|
|
|
|
let sk = self
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.validator_keypairs()
|
2022-03-28 07:14:13 +00:00
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let bad_randao_reveal = {
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
epoch,
|
|
|
|
|
Domain::Randao,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
);
|
|
|
|
|
let message = (epoch + 1).signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
|
2022-09-19 07:58:48 +00:00
|
|
|
// Check failure with no `skip_randao_verification` passed.
|
2022-03-28 07:14:13 +00:00
|
|
|
self.client
|
2022-03-31 07:52:23 +00:00
|
|
|
.get_validator_blocks::<E, FullPayload<E>>(slot, &bad_randao_reveal, None)
|
2022-03-28 07:14:13 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
2022-09-19 07:58:48 +00:00
|
|
|
// Check failure with `skip_randao_verification` (requires infinity sig).
|
2022-03-28 07:14:13 +00:00
|
|
|
self.client
|
2022-09-19 07:58:48 +00:00
|
|
|
.get_validator_blocks_modular::<E, FullPayload<E>>(
|
2022-03-28 07:14:13 +00:00
|
|
|
slot,
|
2022-09-19 07:58:48 +00:00
|
|
|
&bad_randao_reveal,
|
2022-03-28 07:14:13 +00:00
|
|
|
None,
|
2022-09-19 07:58:48 +00:00
|
|
|
SkipRandaoVerification::Yes,
|
2022-03-31 07:52:23 +00:00
|
|
|
)
|
2022-03-28 07:14:13 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
pub async fn test_blinded_block_production<Payload: AbstractExecPayload<E>>(&self) {
|
2022-07-30 00:22:37 +00:00
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
|
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
|
|
|
|
for _ in 0..E::slots_per_epoch() * 3 {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let proposer_pubkey_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| duty.pubkey)
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
|
|
|
|
|
|
|
|
|
let sk = self
|
|
|
|
|
.validator_keypairs()
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let randao_reveal = {
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
epoch,
|
|
|
|
|
Domain::Randao,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
);
|
|
|
|
|
let message = epoch.signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let block = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, Payload>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
2023-09-11 02:49:31 +00:00
|
|
|
.data;
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2023-08-10 13:32:49 +00:00
|
|
|
let signed_block_contents =
|
2023-09-11 02:49:31 +00:00
|
|
|
block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec);
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
self.client
|
2023-08-10 13:32:49 +00:00
|
|
|
.post_beacon_blinded_blocks(&signed_block_contents)
|
2022-07-30 00:22:37 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
// This converts the generic `Payload` to a concrete type for comparison.
|
2023-09-11 02:49:31 +00:00
|
|
|
let signed_block = signed_block_contents.deconstruct().0;
|
2022-07-30 00:22:37 +00:00
|
|
|
let head_block = SignedBeaconBlock::from(signed_block.clone());
|
|
|
|
|
assert_eq!(head_block, signed_block);
|
|
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-07 22:53:04 +00:00
|
|
|
pub async fn test_blinded_block_production_ssz<Payload: AbstractExecPayload<E>>(&self) {
|
|
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
|
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
|
|
|
|
for _ in 0..E::slots_per_epoch() * 3 {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let proposer_pubkey_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| duty.pubkey)
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
|
|
|
|
|
|
|
|
|
let sk = self
|
|
|
|
|
.validator_keypairs()
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let randao_reveal = {
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
epoch,
|
|
|
|
|
Domain::Randao,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
);
|
|
|
|
|
let message = epoch.signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
|
2023-09-26 00:32:18 +00:00
|
|
|
let block_contents_bytes = self
|
2023-08-07 22:53:04 +00:00
|
|
|
.client
|
2023-09-21 06:38:31 +00:00
|
|
|
.get_validator_blinded_blocks_ssz::<E, Payload>(slot, &randao_reveal, None)
|
2023-08-07 22:53:04 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
2023-09-21 06:38:31 +00:00
|
|
|
.expect("block bytes");
|
|
|
|
|
|
2023-09-26 00:32:18 +00:00
|
|
|
let block_contents = BlockContents::<E, Payload>::from_ssz_bytes(
|
|
|
|
|
&block_contents_bytes,
|
|
|
|
|
&self.chain.spec,
|
|
|
|
|
)
|
2023-09-26 00:57:21 +00:00
|
|
|
.expect("block contents bytes can be decoded");
|
2023-08-07 22:53:04 +00:00
|
|
|
|
2023-09-11 02:49:31 +00:00
|
|
|
let signed_block_contents =
|
|
|
|
|
block_contents.sign(&sk, &fork, genesis_validators_root, &self.chain.spec);
|
2023-08-07 22:53:04 +00:00
|
|
|
|
|
|
|
|
self.client
|
2023-09-11 02:49:31 +00:00
|
|
|
.post_beacon_blinded_blocks_ssz(&signed_block_contents)
|
2023-08-07 22:53:04 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
// This converts the generic `Payload` to a concrete type for comparison.
|
2023-09-11 02:49:31 +00:00
|
|
|
let signed_block = signed_block_contents.deconstruct().0;
|
2023-08-07 22:53:04 +00:00
|
|
|
let head_block = SignedBeaconBlock::from(signed_block.clone());
|
|
|
|
|
assert_eq!(head_block, signed_block);
|
|
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
pub async fn test_blinded_block_production_no_verify_randao<Payload: AbstractExecPayload<E>>(
|
2022-07-30 00:22:37 +00:00
|
|
|
self,
|
|
|
|
|
) -> Self {
|
|
|
|
|
for _ in 0..E::slots_per_epoch() {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
|
2023-08-10 13:32:49 +00:00
|
|
|
let block_contents = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
2022-09-19 07:58:48 +00:00
|
|
|
.get_validator_blinded_blocks_modular::<E, Payload>(
|
2022-07-30 00:22:37 +00:00
|
|
|
slot,
|
2022-09-19 07:58:48 +00:00
|
|
|
&Signature::infinity().unwrap().into(),
|
2022-07-30 00:22:37 +00:00
|
|
|
None,
|
2022-09-19 07:58:48 +00:00
|
|
|
SkipRandaoVerification::Yes,
|
2022-07-30 00:22:37 +00:00
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
2023-08-10 13:32:49 +00:00
|
|
|
assert_eq!(block_contents.block().slot(), slot);
|
2022-07-30 00:22:37 +00:00
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
pub async fn test_blinded_block_production_verify_randao_invalid<
|
|
|
|
|
Payload: AbstractExecPayload<E>,
|
|
|
|
|
>(
|
2022-07-30 00:22:37 +00:00
|
|
|
self,
|
|
|
|
|
) -> Self {
|
|
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
|
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
|
|
|
|
for _ in 0..E::slots_per_epoch() {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let proposer_pubkey_bytes = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| duty.pubkey)
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
|
|
|
|
|
|
|
|
|
let sk = self
|
|
|
|
|
.validator_keypairs()
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let bad_randao_reveal = {
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
epoch,
|
|
|
|
|
Domain::Randao,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
);
|
|
|
|
|
let message = (epoch + 1).signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
|
2022-09-19 07:58:48 +00:00
|
|
|
// Check failure with full randao verification enabled.
|
2022-07-30 00:22:37 +00:00
|
|
|
self.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, Payload>(slot, &bad_randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
2022-09-19 07:58:48 +00:00
|
|
|
// Check failure with `skip_randao_verification` (requires infinity sig).
|
2022-07-30 00:22:37 +00:00
|
|
|
self.client
|
2022-09-19 07:58:48 +00:00
|
|
|
.get_validator_blinded_blocks_modular::<E, Payload>(
|
2022-07-30 00:22:37 +00:00
|
|
|
slot,
|
2022-09-19 07:58:48 +00:00
|
|
|
&bad_randao_reveal,
|
2022-07-30 00:22:37 +00:00
|
|
|
None,
|
2022-09-19 07:58:48 +00:00
|
|
|
SkipRandaoVerification::Yes,
|
2022-07-30 00:22:37 +00:00
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
|
|
|
|
self.chain.slot_clock.set_slot(slot.as_u64() + 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
pub async fn test_get_validator_attestation_data(self) -> Self {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let mut state = self.chain.head_beacon_state_cloned();
|
2021-07-09 06:15:32 +00:00
|
|
|
let slot = state.slot();
|
2020-09-29 03:46:54 +00:00
|
|
|
state
|
|
|
|
|
.build_committee_cache(RelativeEpoch::Current, &self.chain.spec)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
for index in 0..state.get_committee_count_at_slot(slot).unwrap() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_attestation_data(slot, index)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = self
|
|
|
|
|
.chain
|
|
|
|
|
.produce_unaggregated_attestation(slot, index)
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_validator_aggregate_attestation(self) -> Self {
|
|
|
|
|
let attestation = self
|
|
|
|
|
.chain
|
|
|
|
|
.head_beacon_block()
|
2021-07-09 06:15:32 +00:00
|
|
|
.message()
|
|
|
|
|
.body()
|
|
|
|
|
.attestations()[0]
|
2020-09-29 03:46:54 +00:00
|
|
|
.clone();
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_aggregate_attestation(
|
|
|
|
|
attestation.data.slot,
|
|
|
|
|
attestation.data.tree_hash_root(),
|
|
|
|
|
)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let expected = attestation;
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn get_aggregate(&mut self) -> SignedAggregateAndProof<E> {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let mut head = self.chain.head_snapshot().as_ref().clone();
|
2020-09-29 03:46:54 +00:00
|
|
|
while head.beacon_state.current_epoch() < epoch {
|
|
|
|
|
per_slot_processing(&mut head.beacon_state, None, &self.chain.spec).unwrap();
|
|
|
|
|
}
|
|
|
|
|
head.beacon_state
|
|
|
|
|
.build_committee_cache(RelativeEpoch::Current, &self.chain.spec)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let committee_len = head.beacon_state.get_committee_count_at_slot(slot).unwrap();
|
2021-07-09 06:15:32 +00:00
|
|
|
let fork = head.beacon_state.fork();
|
2020-09-29 03:46:54 +00:00
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
|
|
|
|
|
2020-11-09 23:13:56 +00:00
|
|
|
let duties = self
|
|
|
|
|
.client
|
|
|
|
|
.post_validator_duties_attester(
|
|
|
|
|
epoch,
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
(0..self.validator_keypairs().len() as u64)
|
2020-11-09 23:13:56 +00:00
|
|
|
.collect::<Vec<u64>>()
|
|
|
|
|
.as_slice(),
|
2020-09-29 03:46:54 +00:00
|
|
|
)
|
2020-11-09 23:13:56 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
let (i, kp, duty, proof) = self
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.validator_keypairs()
|
2020-09-29 03:46:54 +00:00
|
|
|
.iter()
|
|
|
|
|
.enumerate()
|
|
|
|
|
.find_map(|(i, kp)| {
|
|
|
|
|
let duty = duties[i].clone();
|
|
|
|
|
|
|
|
|
|
let proof = SelectionProof::new::<E>(
|
|
|
|
|
duty.slot,
|
|
|
|
|
&kp.sk,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
&self.chain.spec,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
if proof
|
|
|
|
|
.is_aggregator(committee_len as usize, &self.chain.spec)
|
|
|
|
|
.unwrap()
|
|
|
|
|
{
|
|
|
|
|
Some((i, kp, duty, proof))
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.expect("there is at least one aggregator for this epoch")
|
|
|
|
|
.clone();
|
|
|
|
|
|
|
|
|
|
if duty.slot > slot {
|
|
|
|
|
self.chain.slot_clock.set_slot(duty.slot.into());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let attestation_data = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_attestation_data(duty.slot, duty.committee_index)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let mut attestation = Attestation {
|
|
|
|
|
aggregation_bits: BitList::with_capacity(duty.committee_length as usize).unwrap(),
|
|
|
|
|
data: attestation_data,
|
|
|
|
|
signature: AggregateSignature::infinity(),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
attestation
|
|
|
|
|
.sign(
|
|
|
|
|
&kp.sk,
|
|
|
|
|
duty.validator_committee_index as usize,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
&self.chain.spec,
|
|
|
|
|
)
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
SignedAggregateAndProof::from_aggregate(
|
|
|
|
|
i as u64,
|
|
|
|
|
attestation,
|
|
|
|
|
Some(proof),
|
|
|
|
|
&kp.sk,
|
|
|
|
|
&fork,
|
|
|
|
|
genesis_validators_root,
|
|
|
|
|
&self.chain.spec,
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_validator_aggregate_and_proofs_valid(mut self) -> Self {
|
|
|
|
|
let aggregate = self.get_aggregate().await;
|
|
|
|
|
|
|
|
|
|
self.client
|
2020-11-09 23:13:56 +00:00
|
|
|
.post_validator_aggregate_and_proof::<E>(&[aggregate])
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
2022-08-30 05:47:31 +00:00
|
|
|
assert!(self.network_rx.network_recv.recv().await.is_some());
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_validator_aggregate_and_proofs_invalid(mut self) -> Self {
|
|
|
|
|
let mut aggregate = self.get_aggregate().await;
|
|
|
|
|
|
|
|
|
|
aggregate.message.aggregate.data.slot += 1;
|
|
|
|
|
|
|
|
|
|
self.client
|
2020-11-09 23:13:56 +00:00
|
|
|
.post_validator_aggregate_and_proof::<E>(&[aggregate])
|
2020-09-29 03:46:54 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap_err();
|
|
|
|
|
|
2022-08-30 05:47:31 +00:00
|
|
|
assert!(self.network_rx.network_recv.recv().now_or_never().is_none());
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_validator_beacon_committee_subscriptions(mut self) -> Self {
|
|
|
|
|
let subscription = BeaconCommitteeSubscription {
|
|
|
|
|
validator_index: 0,
|
|
|
|
|
committee_index: 0,
|
|
|
|
|
committees_at_slot: 1,
|
|
|
|
|
slot: Slot::new(1),
|
|
|
|
|
is_aggregator: true,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_validator_beacon_committee_subscriptions(&[subscription])
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
2022-08-30 05:47:31 +00:00
|
|
|
self.network_rx
|
|
|
|
|
.validator_subscription_recv
|
|
|
|
|
.recv()
|
|
|
|
|
.now_or_never()
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-30 00:49:21 +00:00
|
|
|
pub async fn test_post_validator_register_validator(self) -> Self {
|
|
|
|
|
let mut registrations = vec![];
|
|
|
|
|
let mut fee_recipients = vec![];
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let genesis_epoch = self.chain.spec.genesis_slot.epoch(E::slots_per_epoch());
|
|
|
|
|
let fork = Fork {
|
|
|
|
|
current_version: self.chain.spec.genesis_fork_version,
|
|
|
|
|
previous_version: self.chain.spec.genesis_fork_version,
|
|
|
|
|
epoch: genesis_epoch,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let expected_gas_limit = 11_111_111;
|
2022-06-30 00:49:21 +00:00
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
for (val_index, keypair) in self.validator_keypairs().iter().enumerate() {
|
2022-06-30 00:49:21 +00:00
|
|
|
let pubkey = keypair.pk.compress();
|
|
|
|
|
let fee_recipient = Address::from_low_u64_be(val_index as u64);
|
|
|
|
|
|
|
|
|
|
let data = ValidatorRegistrationData {
|
|
|
|
|
fee_recipient,
|
2022-07-30 00:22:37 +00:00
|
|
|
gas_limit: expected_gas_limit,
|
2022-06-30 00:49:21 +00:00
|
|
|
timestamp: 0,
|
|
|
|
|
pubkey,
|
|
|
|
|
};
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2022-06-30 00:49:21 +00:00
|
|
|
let domain = self.chain.spec.get_domain(
|
2022-07-30 00:22:37 +00:00
|
|
|
genesis_epoch,
|
2022-06-30 00:49:21 +00:00
|
|
|
Domain::ApplicationMask(ApplicationDomain::Builder),
|
|
|
|
|
&fork,
|
|
|
|
|
Hash256::zero(),
|
|
|
|
|
);
|
|
|
|
|
let message = data.signing_root(domain);
|
|
|
|
|
let signature = keypair.sk.sign(message);
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let signed = SignedValidatorRegistrationData {
|
2022-06-30 00:49:21 +00:00
|
|
|
message: data,
|
|
|
|
|
signature,
|
2022-07-30 00:22:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
fee_recipients.push(fee_recipient);
|
|
|
|
|
registrations.push(signed);
|
2022-06-30 00:49:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_validator_register_validator(®istrations)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
for (val_index, (_, fee_recipient)) in self
|
|
|
|
|
.chain
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
.head_snapshot()
|
2022-06-30 00:49:21 +00:00
|
|
|
.beacon_state
|
|
|
|
|
.validators()
|
|
|
|
|
.into_iter()
|
|
|
|
|
.zip(fee_recipients.into_iter())
|
|
|
|
|
.enumerate()
|
|
|
|
|
{
|
|
|
|
|
let actual = self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_suggested_fee_recipient(val_index as u64)
|
|
|
|
|
.await;
|
|
|
|
|
assert_eq!(actual, fee_recipient);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-24 23:34:58 +00:00
|
|
|
pub async fn test_post_validator_register_validator_slashed(self) -> Self {
|
|
|
|
|
// slash a validator
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attester_slashings(&self.attester_slashing)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
self.harness
|
|
|
|
|
.extend_chain(
|
|
|
|
|
1,
|
|
|
|
|
BlockStrategy::OnCanonicalHead,
|
|
|
|
|
AttestationStrategy::AllValidators,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
|
|
|
|
|
let mut registrations = vec![];
|
|
|
|
|
let mut fee_recipients = vec![];
|
|
|
|
|
|
|
|
|
|
let genesis_epoch = self.chain.spec.genesis_slot.epoch(E::slots_per_epoch());
|
|
|
|
|
let fork = Fork {
|
|
|
|
|
current_version: self.chain.spec.genesis_fork_version,
|
|
|
|
|
previous_version: self.chain.spec.genesis_fork_version,
|
|
|
|
|
epoch: genesis_epoch,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let expected_gas_limit = 11_111_111;
|
|
|
|
|
|
|
|
|
|
for (val_index, keypair) in self.validator_keypairs().iter().enumerate() {
|
|
|
|
|
let pubkey = keypair.pk.compress();
|
|
|
|
|
let fee_recipient = Address::from_low_u64_be(val_index as u64);
|
|
|
|
|
|
|
|
|
|
let data = ValidatorRegistrationData {
|
|
|
|
|
fee_recipient,
|
|
|
|
|
gas_limit: expected_gas_limit,
|
|
|
|
|
timestamp: 0,
|
|
|
|
|
pubkey,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let domain = self.chain.spec.get_domain(
|
|
|
|
|
genesis_epoch,
|
|
|
|
|
Domain::ApplicationMask(ApplicationDomain::Builder),
|
|
|
|
|
&fork,
|
|
|
|
|
Hash256::zero(),
|
|
|
|
|
);
|
|
|
|
|
let message = data.signing_root(domain);
|
|
|
|
|
let signature = keypair.sk.sign(message);
|
|
|
|
|
|
|
|
|
|
let signed = SignedValidatorRegistrationData {
|
|
|
|
|
message: data,
|
|
|
|
|
signature,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
fee_recipients.push(fee_recipient);
|
|
|
|
|
registrations.push(signed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_validator_register_validator(®istrations)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
for (val_index, (_, fee_recipient)) in self
|
|
|
|
|
.chain
|
|
|
|
|
.head_snapshot()
|
|
|
|
|
.beacon_state
|
|
|
|
|
.validators()
|
|
|
|
|
.into_iter()
|
|
|
|
|
.zip(fee_recipients.into_iter())
|
|
|
|
|
.enumerate()
|
|
|
|
|
{
|
|
|
|
|
let actual = self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_suggested_fee_recipient(val_index as u64)
|
|
|
|
|
.await;
|
|
|
|
|
if val_index == 0 || val_index == 1 {
|
|
|
|
|
assert_eq!(actual, Address::from_low_u64_be(val_index as u64));
|
|
|
|
|
} else {
|
|
|
|
|
assert_eq!(actual, fee_recipient);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-31 01:53:03 +00:00
|
|
|
pub async fn test_post_validator_liveness_epoch(self) -> Self {
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let head_state = self.chain.head_beacon_state_cloned();
|
|
|
|
|
let indices = (0..head_state.validators().len())
|
|
|
|
|
.map(|i| i as u64)
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
// Construct the expected response
|
|
|
|
|
let expected: Vec<StandardLivenessResponseData> = head_state
|
|
|
|
|
.validators()
|
|
|
|
|
.iter()
|
|
|
|
|
.enumerate()
|
|
|
|
|
.map(|(index, _)| StandardLivenessResponseData {
|
|
|
|
|
index: index as u64,
|
|
|
|
|
is_live: false,
|
|
|
|
|
})
|
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.post_validator_liveness_epoch(epoch, indices.clone())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
// Attest to the current slot
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attestations(self.attestations.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.post_validator_liveness_epoch(epoch, indices.clone())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let committees = head_state
|
|
|
|
|
.get_beacon_committees_at_slot(self.chain.slot().unwrap())
|
|
|
|
|
.unwrap();
|
|
|
|
|
let attesting_validators: Vec<usize> = committees
|
|
|
|
|
.into_iter()
|
|
|
|
|
.flat_map(|committee| committee.committee.iter().cloned())
|
|
|
|
|
.collect();
|
|
|
|
|
// All attesters should now be considered live
|
|
|
|
|
let expected = expected
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(|mut a| {
|
|
|
|
|
if attesting_validators.contains(&(a.index as usize)) {
|
|
|
|
|
a.is_live = true;
|
|
|
|
|
}
|
|
|
|
|
a
|
|
|
|
|
})
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
// Helper function for tests that require a valid RANDAO signature.
|
|
|
|
|
async fn get_test_randao(&self, slot: Slot, epoch: Epoch) -> (u64, SignatureBytes) {
|
|
|
|
|
let fork = self.chain.canonical_head.cached_head().head_fork();
|
|
|
|
|
let genesis_validators_root = self.chain.genesis_validators_root;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let (proposer_pubkey_bytes, proposer_index) = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_duties_proposer(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
|
|
|
|
.into_iter()
|
|
|
|
|
.find(|duty| duty.slot == slot)
|
|
|
|
|
.map(|duty| (duty.pubkey, duty.validator_index))
|
|
|
|
|
.unwrap();
|
|
|
|
|
let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let sk = self
|
|
|
|
|
.validator_keypairs()
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|kp| kp.pk == proposer_pubkey)
|
|
|
|
|
.map(|kp| kp.sk.clone())
|
|
|
|
|
.unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let randao_reveal = {
|
|
|
|
|
let domain =
|
|
|
|
|
self.chain
|
|
|
|
|
.spec
|
|
|
|
|
.get_domain(epoch, Domain::Randao, &fork, genesis_validators_root);
|
|
|
|
|
let message = epoch.signing_root(domain);
|
|
|
|
|
sk.sign(message).into()
|
|
|
|
|
};
|
|
|
|
|
(proposer_index, randao_reveal)
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
pub async fn test_payload_respects_registration(self) -> Self {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let (proposer_index, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
let expected_fee_recipient = Address::from_low_u64_be(proposer_index as u64);
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.fee_recipient(), expected_fee_recipient);
|
|
|
|
|
assert_eq!(payload.gas_limit(), 11_111_111);
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
// If this cache is empty, it indicates fallback was not used, so the payload came from the
|
|
|
|
|
// mock builder.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
|
2020-09-29 03:46:54 +00:00
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
pub async fn test_payload_accepts_mutated_gas_limit(self) -> Self {
|
|
|
|
|
// Mutate gas limit.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::GasLimit(30_000_000));
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let (proposer_index, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
2020-09-29 03:46:54 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2020-10-22 06:05:49 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let expected_fee_recipient = Address::from_low_u64_be(proposer_index as u64);
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.fee_recipient(), expected_fee_recipient);
|
|
|
|
|
assert_eq!(payload.gas_limit(), 30_000_000);
|
2020-11-02 00:37:30 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
// This cache should not be populated because fallback should not have been used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
2020-11-02 00:37:30 +00:00
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
pub async fn test_payload_accepts_changed_fee_recipient(self) -> Self {
|
|
|
|
|
let test_fee_recipient = "0x4242424242424242424242424242424242424242"
|
|
|
|
|
.parse::<Address>()
|
|
|
|
|
.unwrap();
|
2020-11-02 00:37:30 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
// Mutate fee recipient.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::FeeRecipient(test_fee_recipient));
|
2020-11-02 00:37:30 +00:00
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.fee_recipient(), test_fee_recipient);
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// This cache should not be populated because fallback should not have been used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_payload_rejects_invalid_parent_hash(self) -> Self {
|
|
|
|
|
let invalid_parent_hash =
|
|
|
|
|
"0x4242424242424242424242424242424242424242424242424242424242424242"
|
|
|
|
|
.parse::<Hash256>()
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
// Mutate parent hash.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::ParentHash(invalid_parent_hash));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let expected_parent_hash = self
|
|
|
|
|
.chain
|
|
|
|
|
.head_snapshot()
|
|
|
|
|
.beacon_state
|
|
|
|
|
.latest_execution_payload_header()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.block_hash();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.parent_hash(), expected_parent_hash);
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_payload_rejects_invalid_prev_randao(self) -> Self {
|
|
|
|
|
let invalid_prev_randao =
|
|
|
|
|
"0x4242424242424242424242424242424242424242424242424242424242424242"
|
|
|
|
|
.parse::<Hash256>()
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
// Mutate prev randao.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::PrevRandao(invalid_prev_randao));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let expected_prev_randao = self
|
|
|
|
|
.chain
|
|
|
|
|
.canonical_head
|
|
|
|
|
.cached_head()
|
|
|
|
|
.head_random()
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.prev_randao(), expected_prev_randao);
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_payload_rejects_invalid_block_number(self) -> Self {
|
|
|
|
|
let invalid_block_number = 2;
|
|
|
|
|
|
|
|
|
|
// Mutate block number.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::BlockNumber(invalid_block_number));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let expected_block_number = self
|
|
|
|
|
.chain
|
|
|
|
|
.head_snapshot()
|
|
|
|
|
.beacon_state
|
|
|
|
|
.latest_execution_payload_header()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.block_number()
|
2022-07-30 00:22:37 +00:00
|
|
|
+ 1;
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.block_number(), expected_block_number);
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_payload_rejects_invalid_timestamp(self) -> Self {
|
|
|
|
|
let invalid_timestamp = 2;
|
|
|
|
|
|
|
|
|
|
// Mutate timestamp.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Timestamp(invalid_timestamp));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let min_expected_timestamp = self
|
|
|
|
|
.chain
|
|
|
|
|
.head_snapshot()
|
|
|
|
|
.beacon_state
|
|
|
|
|
.latest_execution_payload_header()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.timestamp();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
assert!(payload.timestamp() > min_expected_timestamp);
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_payload_rejects_invalid_signature(self) -> Self {
|
Prevent port re-use in HTTP API tests (#4745)
## Issue Addressed
CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.
This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers:
There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.
2023-09-20 01:19:03 +00:00
|
|
|
self.mock_builder.as_ref().unwrap().invalid_signatures();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_builder_chain_health_skips(self) -> Self {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
|
|
|
|
|
// Since we are proposing this slot, start the count from the previous slot.
|
|
|
|
|
let prev_slot = slot - Slot::new(1);
|
|
|
|
|
let head_slot = self.chain.canonical_head.cached_head().head_slot();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
// Inclusive here to make sure we advance one slot past the threshold.
|
|
|
|
|
for _ in (prev_slot - head_slot).as_usize()..=self.chain.config.builder_fallback_skips {
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_builder_chain_health_skips_per_epoch(self) -> Self {
|
|
|
|
|
// Fill an epoch with `builder_fallback_skips_per_epoch` skip slots.
|
|
|
|
|
for i in 0..E::slots_per_epoch() {
|
|
|
|
|
if i == 0 || i as usize > self.chain.config.builder_fallback_skips_per_epoch {
|
|
|
|
|
self.harness
|
|
|
|
|
.extend_chain(
|
|
|
|
|
1,
|
|
|
|
|
BlockStrategy::OnCanonicalHead,
|
|
|
|
|
AttestationStrategy::AllValidators,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let next_slot = self.chain.slot().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self
|
|
|
|
|
.get_test_randao(next_slot, next_slot.epoch(E::slots_per_epoch()))
|
|
|
|
|
.await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(next_slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// This cache should not be populated because fallback should not have been used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
|
|
|
|
|
|
|
|
|
// Without proposing, advance into the next slot, this should make us cross the threshold
|
|
|
|
|
// number of skips, causing us to use the fallback.
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
let next_slot = self.chain.slot().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self
|
|
|
|
|
.get_test_randao(next_slot, next_slot.epoch(E::slots_per_epoch()))
|
|
|
|
|
.await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(next_slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_builder_chain_health_epochs_since_finalization(self) -> Self {
|
|
|
|
|
let skips = E::slots_per_epoch()
|
|
|
|
|
* self.chain.config.builder_fallback_epochs_since_finalization as u64;
|
|
|
|
|
|
|
|
|
|
for _ in 0..skips {
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Fill the next epoch with blocks, should be enough to justify, not finalize.
|
|
|
|
|
for _ in 0..E::slots_per_epoch() {
|
|
|
|
|
self.harness
|
|
|
|
|
.extend_chain(
|
|
|
|
|
1,
|
|
|
|
|
BlockStrategy::OnCanonicalHead,
|
|
|
|
|
AttestationStrategy::AllValidators,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let next_slot = self.chain.slot().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self
|
|
|
|
|
.get_test_randao(next_slot, next_slot.epoch(E::slots_per_epoch()))
|
|
|
|
|
.await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(next_slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
|
|
|
|
|
// Fill another epoch with blocks, should be enough to finalize. (Sneaky plus 1 because this
|
|
|
|
|
// scenario starts at an epoch boundary).
|
|
|
|
|
for _ in 0..E::slots_per_epoch() + 1 {
|
|
|
|
|
self.harness
|
|
|
|
|
.extend_chain(
|
|
|
|
|
1,
|
|
|
|
|
BlockStrategy::OnCanonicalHead,
|
|
|
|
|
AttestationStrategy::AllValidators,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let next_slot = self.chain.slot().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self
|
|
|
|
|
.get_test_randao(next_slot, next_slot.epoch(E::slots_per_epoch()))
|
|
|
|
|
.await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-07-30 00:22:37 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(next_slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-07-30 00:22:37 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-07-30 00:22:37 +00:00
|
|
|
|
|
|
|
|
// This cache should not be populated because fallback should not have been used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-09 06:05:16 +00:00
|
|
|
pub async fn test_builder_chain_health_optimistic_head(self) -> Self {
|
|
|
|
|
// Make sure the next payload verification will return optimistic before advancing the chain.
|
|
|
|
|
self.harness.mock_execution_layer.as_ref().map(|el| {
|
|
|
|
|
el.server.all_payloads_syncing(true);
|
|
|
|
|
el
|
|
|
|
|
});
|
|
|
|
|
self.harness
|
|
|
|
|
.extend_chain(
|
|
|
|
|
1,
|
|
|
|
|
BlockStrategy::OnCanonicalHead,
|
|
|
|
|
AttestationStrategy::AllValidators,
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
self.harness.advance_slot();
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (proposer_index, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-08-09 06:05:16 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-08-09 06:05:16 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-08-09 06:05:16 +00:00
|
|
|
|
|
|
|
|
let expected_fee_recipient = Address::from_low_u64_be(proposer_index as u64);
|
2022-12-04 22:08:55 +00:00
|
|
|
assert_eq!(payload.fee_recipient(), expected_fee_recipient);
|
2022-08-09 06:05:16 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-09-05 04:50:49 +00:00
|
|
|
pub async fn test_payload_rejects_inadequate_builder_threshold(self) -> Self {
|
|
|
|
|
// Mutate value.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_BUILDER_THRESHOLD_WEI - 1,
|
|
|
|
|
)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
2022-12-04 22:08:55 +00:00
|
|
|
let payload: BlindedPayload<E> = self
|
2022-09-05 04:50:49 +00:00
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2022-09-05 04:50:49 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
2022-12-04 22:08:55 +00:00
|
|
|
.into();
|
2022-09-05 04:50:49 +00:00
|
|
|
|
|
|
|
|
// If this cache is populated, it indicates fallback to the local EE was correctly used.
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
pub async fn test_builder_payload_chosen_when_more_profitable(self) -> Self {
|
|
|
|
|
// Mutate value.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
|
|
|
|
|
)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
|
|
|
|
let payload: BlindedPayload<E> = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2023-02-02 01:37:46 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.into();
|
|
|
|
|
|
|
|
|
|
// The builder's payload should've been chosen, so this cache should not be populated
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_local_payload_chosen_when_equally_profitable(self) -> Self {
|
|
|
|
|
// Mutate value.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
|
|
|
|
|
)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
|
|
|
|
let payload: BlindedPayload<E> = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2023-02-02 01:37:46 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.into();
|
|
|
|
|
|
|
|
|
|
// The local payload should've been chosen, so this cache should be populated
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_local_payload_chosen_when_more_profitable(self) -> Self {
|
|
|
|
|
// Mutate value.
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI - 1,
|
|
|
|
|
)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
|
|
|
|
let payload: BlindedPayload<E> = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2023-02-02 01:37:46 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.into();
|
|
|
|
|
|
|
|
|
|
// The local payload should've been chosen, so this cache should be populated
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-10 19:30:14 +00:00
|
|
|
pub async fn test_builder_works_post_capella(self) -> Self {
|
|
|
|
|
// Ensure builder payload is chosen
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
|
|
|
|
|
)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
|
|
|
|
let payload: BlindedPayload<E> = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2023-02-10 19:30:14 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.into();
|
|
|
|
|
|
|
|
|
|
// The builder's payload should've been chosen, so this cache should not be populated
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-19 00:12:09 +00:00
|
|
|
pub async fn test_builder_works_post_deneb(self) -> Self {
|
|
|
|
|
// Ensure builder payload is chosen
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
|
|
|
|
|
)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
|
|
|
|
let block_contents = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
let (block, maybe_sidecars) = block_contents.deconstruct();
|
|
|
|
|
|
|
|
|
|
// Response should contain blob sidecars
|
|
|
|
|
assert!(maybe_sidecars.is_some());
|
|
|
|
|
|
|
|
|
|
// The builder's payload should've been chosen, so this cache should not be populated
|
|
|
|
|
let payload: BlindedPayload<E> = block.body().execution_payload().unwrap().into();
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_none());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-10 19:30:14 +00:00
|
|
|
pub async fn test_lighthouse_rejects_invalid_withdrawals_root(self) -> Self {
|
|
|
|
|
// Ensure builder payload *would be* chosen
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::Value(Uint256::from(
|
|
|
|
|
DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
|
|
|
|
|
)));
|
|
|
|
|
// Set withdrawals root to something invalid
|
|
|
|
|
self.mock_builder
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.add_operation(Operation::WithdrawalsRoot(Hash256::repeat_byte(0x42)));
|
|
|
|
|
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
|
|
|
|
let (_, randao_reveal) = self.get_test_randao(slot, epoch).await;
|
|
|
|
|
|
|
|
|
|
let payload: BlindedPayload<E> = self
|
|
|
|
|
.client
|
|
|
|
|
.get_validator_blinded_blocks::<E, BlindedPayload<E>>(slot, &randao_reveal, None)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data
|
2023-08-10 13:32:49 +00:00
|
|
|
.block()
|
2023-02-10 19:30:14 +00:00
|
|
|
.body()
|
|
|
|
|
.execution_payload()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.into();
|
|
|
|
|
|
|
|
|
|
// The local payload should've been chosen because the builder's was invalid
|
|
|
|
|
assert!(self
|
|
|
|
|
.chain
|
|
|
|
|
.execution_layer
|
|
|
|
|
.as_ref()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.get_payload_by_root(&payload.tree_hash_root())
|
|
|
|
|
.is_some());
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
#[cfg(target_os = "linux")]
|
|
|
|
|
pub async fn test_get_lighthouse_health(self) -> Self {
|
|
|
|
|
self.client.get_lighthouse_health().await.unwrap();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[cfg(not(target_os = "linux"))]
|
|
|
|
|
pub async fn test_get_lighthouse_health(self) -> Self {
|
|
|
|
|
self.client.get_lighthouse_health().await.unwrap_err();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_syncing(self) -> Self {
|
|
|
|
|
self.client.get_lighthouse_syncing().await.unwrap();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_proto_array(self) -> Self {
|
|
|
|
|
self.client.get_lighthouse_proto_array().await.unwrap();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_validator_inclusion_global(self) -> Self {
|
|
|
|
|
let epoch = self.chain.epoch().unwrap() - 1;
|
|
|
|
|
self.client
|
|
|
|
|
.get_lighthouse_validator_inclusion_global(epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_validator_inclusion(self) -> Self {
|
|
|
|
|
let epoch = self.chain.epoch().unwrap() - 1;
|
|
|
|
|
self.client
|
|
|
|
|
.get_lighthouse_validator_inclusion(epoch, ValidatorId::Index(0))
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_eth1_syncing(self) -> Self {
|
|
|
|
|
self.client.get_lighthouse_eth1_syncing().await.unwrap();
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_eth1_block_cache(self) -> Self {
|
|
|
|
|
let blocks = self.client.get_lighthouse_eth1_block_cache().await.unwrap();
|
|
|
|
|
|
|
|
|
|
assert!(blocks.data.is_empty());
|
|
|
|
|
|
|
|
|
|
self
|
2020-11-02 00:37:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_eth1_deposit_cache(self) -> Self {
|
|
|
|
|
let deposits = self
|
|
|
|
|
.client
|
|
|
|
|
.get_lighthouse_eth1_deposit_cache()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert!(deposits.data.is_empty());
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-22 06:05:49 +00:00
|
|
|
pub async fn test_get_lighthouse_beacon_states_ssz(self) -> Self {
|
|
|
|
|
for state_id in self.interesting_state_ids() {
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
2022-07-25 08:23:00 +00:00
|
|
|
.get_lighthouse_beacon_states_ssz(&state_id.0, &self.chain.spec)
|
2020-10-22 06:05:49 +00:00
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
2022-07-25 08:23:00 +00:00
|
|
|
let mut expected = state_id
|
|
|
|
|
.state(&self.chain)
|
|
|
|
|
.ok()
|
2023-03-30 06:08:37 +00:00
|
|
|
.map(|(state, _execution_optimistic, _finalized)| state);
|
2020-10-22 06:05:49 +00:00
|
|
|
expected.as_mut().map(|state| state.drop_all_caches());
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected, "{:?}", state_id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
2020-11-23 01:00:22 +00:00
|
|
|
|
|
|
|
|
pub async fn test_get_lighthouse_staking(self) -> Self {
|
|
|
|
|
let result = self.client.get_lighthouse_staking().await.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, self.chain.eth1_chain.is_some());
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
2020-12-04 00:18:58 +00:00
|
|
|
|
2021-09-22 00:37:28 +00:00
|
|
|
pub async fn test_get_lighthouse_database_info(self) -> Self {
|
|
|
|
|
let info = self.client.get_lighthouse_database_info().await.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(info.anchor, self.chain.store.get_anchor_info());
|
|
|
|
|
assert_eq!(info.split, self.chain.store.get_split_info());
|
|
|
|
|
assert_eq!(
|
|
|
|
|
info.schema_version,
|
|
|
|
|
store::metadata::CURRENT_SCHEMA_VERSION.as_u64()
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_post_lighthouse_database_reconstruct(self) -> Self {
|
|
|
|
|
let response = self
|
|
|
|
|
.client
|
|
|
|
|
.post_lighthouse_database_reconstruct()
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
assert_eq!(response, "success");
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2021-07-31 03:50:52 +00:00
|
|
|
pub async fn test_post_lighthouse_liveness(self) -> Self {
|
|
|
|
|
let epoch = self.chain.epoch().unwrap();
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let head_state = self.chain.head_beacon_state_cloned();
|
2021-07-31 03:50:52 +00:00
|
|
|
let indices = (0..head_state.validators().len())
|
|
|
|
|
.map(|i| i as u64)
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
// Construct the expected response
|
|
|
|
|
let expected: Vec<LivenessResponseData> = head_state
|
|
|
|
|
.validators()
|
|
|
|
|
.iter()
|
|
|
|
|
.enumerate()
|
|
|
|
|
.map(|(index, _)| LivenessResponseData {
|
|
|
|
|
index: index as u64,
|
|
|
|
|
is_live: false,
|
|
|
|
|
epoch,
|
|
|
|
|
})
|
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.post_lighthouse_liveness(indices.as_slice(), epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
// Attest to the current slot
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attestations(self.attestations.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.post_lighthouse_liveness(indices.as_slice(), epoch)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.data;
|
|
|
|
|
|
|
|
|
|
let committees = head_state
|
|
|
|
|
.get_beacon_committees_at_slot(self.chain.slot().unwrap())
|
|
|
|
|
.unwrap();
|
|
|
|
|
let attesting_validators: Vec<usize> = committees
|
|
|
|
|
.into_iter()
|
2022-05-16 01:59:47 +00:00
|
|
|
.flat_map(|committee| committee.committee.iter().cloned())
|
2021-07-31 03:50:52 +00:00
|
|
|
.collect();
|
|
|
|
|
// All attesters should now be considered live
|
|
|
|
|
let expected = expected
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(|mut a| {
|
|
|
|
|
if attesting_validators.contains(&(a.index as usize)) {
|
|
|
|
|
a.is_live = true;
|
|
|
|
|
}
|
|
|
|
|
a
|
|
|
|
|
})
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result, expected);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
pub async fn test_get_events(self) -> Self {
|
|
|
|
|
// Subscribe to all events
|
|
|
|
|
let topics = vec![
|
|
|
|
|
EventTopic::Attestation,
|
|
|
|
|
EventTopic::VoluntaryExit,
|
|
|
|
|
EventTopic::Block,
|
|
|
|
|
EventTopic::Head,
|
|
|
|
|
EventTopic::FinalizedCheckpoint,
|
|
|
|
|
];
|
|
|
|
|
let mut events_future = self
|
|
|
|
|
.client
|
|
|
|
|
.get_events::<E>(topics.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let expected_attestation_len = self.attestations.len();
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_attestations(self.attestations.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let attestation_events = poll_events(
|
|
|
|
|
&mut events_future,
|
|
|
|
|
expected_attestation_len,
|
|
|
|
|
Duration::from_millis(10000),
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
attestation_events.as_slice(),
|
|
|
|
|
self.attestations
|
|
|
|
|
.clone()
|
|
|
|
|
.into_iter()
|
2021-12-03 04:44:30 +00:00
|
|
|
.map(|attestation| EventKind::Attestation(Box::new(attestation)))
|
2020-12-04 00:18:58 +00:00
|
|
|
.collect::<Vec<_>>()
|
|
|
|
|
.as_slice()
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Produce a voluntary exit event
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_pool_voluntary_exits(&self.voluntary_exit)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let exit_events = poll_events(&mut events_future, 1, Duration::from_millis(10000)).await;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
exit_events.as_slice(),
|
|
|
|
|
&[EventKind::VoluntaryExit(self.voluntary_exit.clone())]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Submit the next block, which is on an epoch boundary, so this will produce a finalized
|
|
|
|
|
// checkpoint event, head event, and block event
|
2023-03-15 20:34:00 +00:00
|
|
|
let block_root = self.next_block.signed_block().canonical_root();
|
2020-12-04 00:18:58 +00:00
|
|
|
|
|
|
|
|
// current_duty_dependent_root = block root because this is the first slot of the epoch
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let current_duty_dependent_root = self.chain.head_beacon_block_root();
|
2020-12-04 00:18:58 +00:00
|
|
|
let current_slot = self.chain.slot().unwrap();
|
2023-03-15 20:34:00 +00:00
|
|
|
let next_slot = self.next_block.signed_block().slot();
|
2020-12-04 00:18:58 +00:00
|
|
|
let finalization_distance = E::slots_per_epoch() * 2;
|
|
|
|
|
|
|
|
|
|
let expected_block = EventKind::Block(SseBlock {
|
|
|
|
|
block: block_root,
|
|
|
|
|
slot: next_slot,
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: false,
|
2020-12-04 00:18:58 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
let expected_head = EventKind::Head(SseHead {
|
|
|
|
|
block: block_root,
|
|
|
|
|
slot: next_slot,
|
2023-03-15 20:34:00 +00:00
|
|
|
state: self.next_block.signed_block().state_root(),
|
2020-12-04 00:18:58 +00:00
|
|
|
current_duty_dependent_root,
|
|
|
|
|
previous_duty_dependent_root: self
|
|
|
|
|
.chain
|
Use the forwards iterator more often (#2376)
## Issue Addressed
NA
## Primary Change
When investigating memory usage, I noticed that retrieving a block from an early slot (e.g., slot 900) would cause a sharp increase in the memory footprint (from 400mb to 800mb+) which seemed to be ever-lasting.
After some investigation, I found that the reverse iteration from the head back to that slot was the likely culprit. To counter this, I've switched the `BeaconChain::block_root_at_slot` to use the forwards iterator, instead of the reverse one.
I also noticed that the networking stack is using `BeaconChain::root_at_slot` to check if a peer is relevant (`check_peer_relevance`). Perhaps the steep, seemingly-random-but-consistent increases in memory usage are caused by the use of this function.
Using the forwards iterator with the HTTP API alleviated the sharp increases in memory usage. It also made the response much faster (before it felt like to took 1-2s, now it feels instant).
## Additional Changes
In the process I also noticed that we have two functions for getting block roots:
- `BeaconChain::block_root_at_slot`: returns `None` for a skip slot.
- `BeaconChain::root_at_slot`: returns the previous root for a skip slot.
I unified these two functions into `block_root_at_slot` and added the `WhenSlotSkipped` enum. Now, the caller must be explicit about the skip-slot behaviour when requesting a root.
Additionally, I replaced `vec![]` with `Vec::with_capacity` in `store::chunked_vector::range_query`. I stumbled across this whilst debugging and made this modification to see what effect it would have (not much). It seems like a decent change to keep around, but I'm not concerned either way.
Also, `BeaconChain::get_ancestor_block_root` is unused, so I got rid of it :wastebasket:.
## Additional Info
I haven't also done the same for state roots here. Whilst it's possible and a good idea, it's more work since the fwds iterators are presently block-roots-specific.
Whilst there's a few places a reverse iteration of state roots could be triggered (e.g., attestation production, HTTP API), they're no where near as common as the `check_peer_relevance` call. As such, I think we should get this PR merged first, then come back for the state root iters. I made an issue here https://github.com/sigp/lighthouse/issues/2377.
2021-05-31 04:18:20 +00:00
|
|
|
.block_root_at_slot(current_slot - E::slots_per_epoch(), WhenSlotSkipped::Prev)
|
2020-12-04 00:18:58 +00:00
|
|
|
.unwrap()
|
|
|
|
|
.unwrap(),
|
|
|
|
|
epoch_transition: true,
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: false,
|
2020-12-04 00:18:58 +00:00
|
|
|
});
|
|
|
|
|
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
let finalized_block_root = self
|
|
|
|
|
.chain
|
|
|
|
|
.block_root_at_slot(next_slot - finalization_distance, WhenSlotSkipped::Prev)
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap();
|
|
|
|
|
let finalized_block = self
|
|
|
|
|
.chain
|
|
|
|
|
.get_blinded_block(&finalized_block_root)
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap();
|
|
|
|
|
let finalized_state_root = finalized_block.state_root();
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
let expected_finalized = EventKind::FinalizedCheckpoint(SseFinalizedCheckpoint {
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
block: finalized_block_root,
|
|
|
|
|
state: finalized_state_root,
|
2020-12-04 00:18:58 +00:00
|
|
|
epoch: Epoch::new(3),
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: false,
|
2020-12-04 00:18:58 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_blocks(&self.next_block)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let block_events = poll_events(&mut events_future, 3, Duration::from_millis(10000)).await;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
block_events.as_slice(),
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
&[expected_block, expected_head, expected_finalized]
|
2020-12-04 00:18:58 +00:00
|
|
|
);
|
|
|
|
|
|
2021-06-17 02:10:46 +00:00
|
|
|
// Test a reorg event
|
|
|
|
|
let mut chain_reorg_event_future = self
|
|
|
|
|
.client
|
|
|
|
|
.get_events::<E>(&[EventTopic::ChainReorg])
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let expected_reorg = EventKind::ChainReorg(SseChainReorg {
|
2023-07-11 20:05:05 +00:00
|
|
|
slot: self.reorg_block.signed_block().slot(),
|
2021-06-17 02:10:46 +00:00
|
|
|
depth: 1,
|
2023-03-15 20:34:00 +00:00
|
|
|
old_head_block: self.next_block.signed_block().canonical_root(),
|
|
|
|
|
old_head_state: self.next_block.signed_block().state_root(),
|
|
|
|
|
new_head_block: self.reorg_block.signed_block().canonical_root(),
|
|
|
|
|
new_head_state: self.reorg_block.signed_block().state_root(),
|
|
|
|
|
epoch: self
|
|
|
|
|
.next_block
|
|
|
|
|
.signed_block()
|
|
|
|
|
.slot()
|
|
|
|
|
.epoch(E::slots_per_epoch()),
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: false,
|
2021-06-17 02:10:46 +00:00
|
|
|
});
|
|
|
|
|
|
Add broadcast validation routes to Beacon Node HTTP API (#4316)
## Issue Addressed
- #4293
- #4264
## Proposed Changes
*Changes largely follow those suggested in the main issue*.
- Add new routes to HTTP API
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Add new routes to `BeaconNodeHttpClient`
- `post_beacon_blocks_v2`
- `post_blinded_beacon_blocks_v2`
- Define new Eth2 common types
- `BroadcastValidation`, enum representing the level of validation to apply to blocks prior to broadcast
- `BroadcastValidationQuery`, the corresponding HTTP query string type for the above type
- ~~Define `_checked` variants of both `publish_block` and `publish_blinded_block` that enforce a validation level at a type level~~
- Add interactive tests to the `bn_http_api_tests` test target covering each validation level (to their own test module, `broadcast_validation_tests`)
- `beacon/blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- Only consensus pass (i.e., equivocates) (200)
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- `beacon/blinded_blocks`
- `broadcast_validation=gossip`
- Invalid (400)
- Full Pass (200)
- Partial Pass (202)
- `broadcast_validation=consensus`
- Invalid (400)
- Only gossip (400)
- ~~Only consensus pass (i.e., equivocates) (200)~~
- Full pass (200)
- `broadcast_validation=consensus_and_equivocation`
- Invalid (400)
- Invalid due to early equivocation (400)
- Only gossip (400)
- Only consensus (400)
- Pass (200)
- Add a new trait, `IntoGossipVerifiedBlock`, which allows type-level guarantees to be made as to gossip validity
- Modify the structure of the `ObservedBlockProducers` cache from a `(slot, validator_index)` mapping to a `((slot, validator_index), block_root)` mapping
- Modify `ObservedBlockProducers::proposer_has_been_observed` to return a `SeenBlock` rather than a boolean on success
- Punish gossip peer (low) for submitting equivocating blocks
- Rename `BlockError::SlashablePublish` to `BlockError::SlashableProposal`
## Additional Info
This PR contains changes that directly modify how blocks are verified within the client. For more context, consult [comments in-thread](https://github.com/sigp/lighthouse/pull/4316#discussion_r1234724202).
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
2023-06-29 12:02:38 +00:00
|
|
|
self.harness.advance_slot();
|
|
|
|
|
|
2021-06-17 02:10:46 +00:00
|
|
|
self.client
|
|
|
|
|
.post_beacon_blocks(&self.reorg_block)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let reorg_event = poll_events(
|
|
|
|
|
&mut chain_reorg_event_future,
|
|
|
|
|
1,
|
|
|
|
|
Duration::from_millis(10000),
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
assert_eq!(reorg_event.as_slice(), &[expected_reorg]);
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-24 05:54:36 +00:00
|
|
|
pub async fn test_get_expected_withdrawals_invalid_state(self) -> Self {
|
|
|
|
|
let state_id = CoreStateId::Root(Hash256::zero());
|
|
|
|
|
|
|
|
|
|
let result = self.client.get_expected_withdrawals(&state_id).await;
|
|
|
|
|
|
|
|
|
|
match result {
|
|
|
|
|
Err(e) => {
|
|
|
|
|
assert_eq!(e.status().unwrap(), 404);
|
|
|
|
|
}
|
|
|
|
|
_ => panic!("query did not fail correctly"),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_expected_withdrawals_capella(self) -> Self {
|
|
|
|
|
let slot = self.chain.slot().unwrap();
|
|
|
|
|
let state_id = CoreStateId::Slot(slot);
|
|
|
|
|
|
|
|
|
|
// calculate the expected withdrawals
|
|
|
|
|
let (mut state, _, _) = StateId(state_id).state(&self.chain).unwrap();
|
|
|
|
|
let proposal_slot = state.slot() + 1;
|
|
|
|
|
let proposal_epoch = proposal_slot.epoch(E::slots_per_epoch());
|
|
|
|
|
let (state_root, _, _) = StateId(state_id).root(&self.chain).unwrap();
|
|
|
|
|
if proposal_epoch != state.current_epoch() {
|
|
|
|
|
let _ = partial_state_advance(
|
|
|
|
|
&mut state,
|
|
|
|
|
Some(state_root),
|
|
|
|
|
proposal_slot,
|
|
|
|
|
&self.chain.spec,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
let expected_withdrawals = get_expected_withdrawals(&state, &self.chain.spec).unwrap();
|
|
|
|
|
|
|
|
|
|
// fetch expected withdrawals from the client
|
|
|
|
|
let result = self.client.get_expected_withdrawals(&state_id).await;
|
|
|
|
|
match result {
|
|
|
|
|
Ok(withdrawal_response) => {
|
|
|
|
|
assert_eq!(withdrawal_response.execution_optimistic, Some(false));
|
|
|
|
|
assert_eq!(withdrawal_response.finalized, Some(false));
|
|
|
|
|
assert_eq!(withdrawal_response.data, expected_withdrawals.to_vec());
|
|
|
|
|
}
|
|
|
|
|
Err(e) => {
|
|
|
|
|
println!("{:?}", e);
|
|
|
|
|
panic!("query failed incorrectly");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub async fn test_get_expected_withdrawals_pre_capella(self) -> Self {
|
|
|
|
|
let state_id = CoreStateId::Head;
|
|
|
|
|
|
|
|
|
|
let result = self.client.get_expected_withdrawals(&state_id).await;
|
|
|
|
|
|
|
|
|
|
match result {
|
|
|
|
|
Err(e) => {
|
|
|
|
|
assert_eq!(e.status().unwrap(), 400);
|
|
|
|
|
}
|
|
|
|
|
_ => panic!("query did not fail correctly"),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-25 07:53:58 +00:00
|
|
|
pub async fn test_get_events_altair(self) -> Self {
|
|
|
|
|
let topics = vec![EventTopic::ContributionAndProof];
|
|
|
|
|
let mut events_future = self
|
|
|
|
|
.client
|
|
|
|
|
.get_events::<E>(topics.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let expected_contribution_len = self.contribution_and_proofs.len();
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_validator_contribution_and_proofs(self.contribution_and_proofs.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let contribution_events = poll_events(
|
|
|
|
|
&mut events_future,
|
|
|
|
|
expected_contribution_len,
|
|
|
|
|
Duration::from_millis(10000),
|
|
|
|
|
)
|
|
|
|
|
.await;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
contribution_events.as_slice(),
|
|
|
|
|
self.contribution_and_proofs
|
|
|
|
|
.clone()
|
|
|
|
|
.into_iter()
|
|
|
|
|
.map(|contribution| EventKind::ContributionAndProof(Box::new(contribution)))
|
|
|
|
|
.collect::<Vec<_>>()
|
|
|
|
|
.as_slice()
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
pub async fn test_get_events_from_genesis(self) -> Self {
|
|
|
|
|
let topics = vec![EventTopic::Block, EventTopic::Head];
|
|
|
|
|
let mut events_future = self
|
|
|
|
|
.client
|
|
|
|
|
.get_events::<E>(topics.as_slice())
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
2023-03-15 20:34:00 +00:00
|
|
|
let block_root = self.next_block.signed_block().canonical_root();
|
|
|
|
|
let next_slot = self.next_block.signed_block().slot();
|
2020-12-04 00:18:58 +00:00
|
|
|
|
|
|
|
|
let expected_block = EventKind::Block(SseBlock {
|
|
|
|
|
block: block_root,
|
|
|
|
|
slot: next_slot,
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: false,
|
2020-12-04 00:18:58 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
let expected_head = EventKind::Head(SseHead {
|
|
|
|
|
block: block_root,
|
|
|
|
|
slot: next_slot,
|
2023-03-15 20:34:00 +00:00
|
|
|
state: self.next_block.signed_block().state_root(),
|
2020-12-04 00:18:58 +00:00
|
|
|
current_duty_dependent_root: self.chain.genesis_block_root,
|
|
|
|
|
previous_duty_dependent_root: self.chain.genesis_block_root,
|
|
|
|
|
epoch_transition: false,
|
2022-07-25 08:23:00 +00:00
|
|
|
execution_optimistic: false,
|
2020-12-04 00:18:58 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
self.client
|
|
|
|
|
.post_beacon_blocks(&self.next_block)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
let block_events = poll_events(&mut events_future, 2, Duration::from_millis(10000)).await;
|
|
|
|
|
assert_eq!(block_events.as_slice(), &[expected_block, expected_head]);
|
|
|
|
|
|
|
|
|
|
self
|
|
|
|
|
}
|
2022-07-25 08:23:00 +00:00
|
|
|
|
|
|
|
|
pub async fn test_check_optimistic_responses(&mut self) {
|
|
|
|
|
// Check responses are not optimistic.
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_headers_block_id(CoreBlockId::Head)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.execution_optimistic, Some(false));
|
|
|
|
|
|
|
|
|
|
// Change head to be optimistic.
|
|
|
|
|
self.chain
|
|
|
|
|
.canonical_head
|
|
|
|
|
.fork_choice_write_lock()
|
|
|
|
|
.proto_array_mut()
|
|
|
|
|
.core_proto_array_mut()
|
|
|
|
|
.nodes
|
|
|
|
|
.last_mut()
|
|
|
|
|
.map(|head_node| {
|
|
|
|
|
head_node.execution_status = ExecutionStatus::Optimistic(ExecutionBlockHash::zero())
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Check responses are now optimistic.
|
|
|
|
|
let result = self
|
|
|
|
|
.client
|
|
|
|
|
.get_beacon_headers_block_id(CoreBlockId::Head)
|
|
|
|
|
.await
|
|
|
|
|
.unwrap()
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
assert_eq!(result.execution_optimistic, Some(true));
|
|
|
|
|
}
|
2020-12-04 00:18:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async fn poll_events<S: Stream<Item = Result<EventKind<T>, eth2::Error>> + Unpin, T: EthSpec>(
|
|
|
|
|
stream: &mut S,
|
|
|
|
|
num_events: usize,
|
|
|
|
|
timeout: Duration,
|
|
|
|
|
) -> Vec<EventKind<T>> {
|
|
|
|
|
let mut events = Vec::new();
|
|
|
|
|
|
|
|
|
|
let collect_stream_fut = async {
|
|
|
|
|
loop {
|
|
|
|
|
if let Some(result) = stream.next().await {
|
|
|
|
|
events.push(result.unwrap());
|
|
|
|
|
if events.len() == num_events {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
tokio::select! {
|
2021-06-18 05:58:01 +00:00
|
|
|
_ = collect_stream_fut => {events}
|
2020-12-04 00:18:58 +00:00
|
|
|
_ = tokio::time::sleep(timeout) => { return events; }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_events() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new().await.test_get_events().await;
|
2020-12-04 00:18:58 +00:00
|
|
|
}
|
|
|
|
|
|
2021-09-25 07:53:58 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_events_altair() {
|
2023-02-02 01:37:46 +00:00
|
|
|
let mut config = ApiTesterConfig::default();
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
ApiTester::new_from_config(config)
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2021-09-25 07:53:58 +00:00
|
|
|
.test_get_events_altair()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-04 00:18:58 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_events_from_genesis() {
|
|
|
|
|
ApiTester::new_from_genesis()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-12-04 00:18:58 +00:00
|
|
|
.test_get_events_from_genesis()
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-10-29 22:25:20 +00:00
|
|
|
async fn beacon_get() {
|
2021-02-10 23:29:49 +00:00
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2021-02-10 23:29:49 +00:00
|
|
|
.test_beacon_genesis()
|
|
|
|
|
.await
|
2023-03-30 06:08:37 +00:00
|
|
|
.test_beacon_states_root_finalized()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_fork_finalized()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_finality_checkpoints_finalized()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_headers_block_id_finalized()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_blocks_finalized::<MainnetEthSpec>()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_blinded_blocks_finalized::<MainnetEthSpec>()
|
|
|
|
|
.await
|
|
|
|
|
.test_debug_beacon_states_finalized()
|
|
|
|
|
.await
|
2021-02-10 23:29:49 +00:00
|
|
|
.test_beacon_states_root()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_fork()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_finality_checkpoints()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_validators()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_validator_balances()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_committees()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_states_validator_id()
|
|
|
|
|
.await
|
2022-11-11 00:38:27 +00:00
|
|
|
.test_beacon_states_randao()
|
|
|
|
|
.await
|
2021-02-10 23:29:49 +00:00
|
|
|
.test_beacon_headers_all_slots()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_headers_all_parents()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_headers_block_id()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_blocks()
|
|
|
|
|
.await
|
2022-11-11 00:38:27 +00:00
|
|
|
.test_beacon_blinded_blocks()
|
|
|
|
|
.await
|
2021-02-10 23:29:49 +00:00
|
|
|
.test_beacon_blocks_attestations()
|
|
|
|
|
.await
|
|
|
|
|
.test_beacon_blocks_root()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_beacon_pool_attestations()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_beacon_pool_attester_slashings()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_beacon_pool_proposer_slashings()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_beacon_pool_voluntary_exits()
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn post_beacon_blocks_valid() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new().await.test_post_beacon_blocks_valid().await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2023-07-31 23:51:37 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_beacon_blocks_ssz_valid() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_beacon_blocks_ssz_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn test_post_beacon_blocks_ssz_invalid() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_beacon_blocks_ssz_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn post_beacon_blocks_invalid() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_beacon_blocks_invalid()
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2023-08-28 00:55:31 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_beacon_blocks_duplicate() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_beacon_blocks_duplicate()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_attestations_valid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_attestations_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_attestations_invalid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_attestations_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_attester_slashings_valid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_attester_slashings_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_attester_slashings_invalid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_attester_slashings_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_proposer_slashings_valid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_proposer_slashings_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_proposer_slashings_invalid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_proposer_slashings_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_voluntary_exits_valid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_voluntary_exits_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn beacon_pools_post_voluntary_exits_invalid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_post_beacon_pool_voluntary_exits_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn config_get() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_config_fork_schedule()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_config_spec()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_config_deposit_contract()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn debug_get() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_debug_beacon_states()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_debug_beacon_heads()
|
2023-03-29 02:56:37 +00:00
|
|
|
.await
|
|
|
|
|
.test_get_debug_fork_choice()
|
2020-09-29 03:46:54 +00:00
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn node_get() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_node_version()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_node_syncing()
|
2020-10-22 02:59:42 +00:00
|
|
|
.await
|
|
|
|
|
.test_get_node_identity()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_node_health()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_node_peers_by_id()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_node_peers()
|
2020-11-13 02:02:41 +00:00
|
|
|
.await
|
|
|
|
|
.test_get_node_peer_count()
|
2020-09-29 03:46:54 +00:00
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-29 23:42:35 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_validator_duties_early() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_validator_duties_early()
|
|
|
|
|
.await;
|
2021-03-29 23:42:35 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_duties_attester() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_validator_duties_attester()
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_duties_attester_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_get_validator_duties_attester()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_duties_proposer() {
|
2023-08-21 05:02:32 +00:00
|
|
|
ApiTester::new_from_config(ApiTesterConfig::default().retain_historic_states())
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
|
|
|
|
.test_get_validator_duties_proposer()
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_duties_proposer_with_skip_slots() {
|
2023-08-21 05:02:32 +00:00
|
|
|
ApiTester::new_from_config(ApiTesterConfig::default().retain_historic_states())
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_get_validator_duties_proposer()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn block_production() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new().await.test_block_production().await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn block_production_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_block_production()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-28 07:14:13 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn block_production_no_verify_randao() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_block_production_no_verify_randao()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn block_production_verify_randao_invalid() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_block_production_verify_randao_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-21 06:38:31 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn block_production_ssz_full_payload() {
|
|
|
|
|
ApiTester::new().await.test_block_production_ssz().await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn block_production_ssz_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_block_production_ssz()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_full_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production::<FullPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-07 22:53:04 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_ssz_full_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production_ssz::<FullPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_with_skip_slots_full_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_blinded_block_production::<FullPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-07 22:53:04 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_ssz_with_skip_slots_full_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_blinded_block_production_ssz::<FullPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_no_verify_randao_full_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production_no_verify_randao::<FullPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_verify_randao_invalid_full_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production_verify_randao_invalid::<FullPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_blinded_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production::<BlindedPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_with_skip_slots_blinded_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_blinded_block_production::<BlindedPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_no_verify_randao_blinded_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production_no_verify_randao::<BlindedPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn blinded_block_production_verify_randao_invalid_blinded_payload_premerge() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_blinded_block_production_verify_randao_invalid::<BlindedPayload<_>>()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_attestation_data() {
|
2021-10-11 02:45:06 +00:00
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_validator_attestation_data()
|
|
|
|
|
.await;
|
2020-09-29 03:46:54 +00:00
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_attestation_data_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_get_validator_attestation_data()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_aggregate_attestation() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_validator_aggregate_attestation()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_aggregate_attestation_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_get_validator_aggregate_attestation()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_aggregate_and_proofs_valid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_validator_aggregate_and_proofs_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_aggregate_and_proofs_valid_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_get_validator_aggregate_and_proofs_valid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_aggregate_and_proofs_invalid() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_validator_aggregate_and_proofs_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.skip_slots(E::slots_per_epoch() * 2)
|
|
|
|
|
.test_get_validator_aggregate_and_proofs_invalid()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn get_validator_beacon_committee_subscriptions() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_validator_beacon_committee_subscriptions()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-30 00:49:21 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_validator_register_validator() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_validator_register_validator()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-24 23:34:58 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_validator_register_validator_slashed() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_validator_register_validator_slashed()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-30 00:22:37 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_validator_register_valid() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_respects_registration()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_validator_register_gas_limit_mutation() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_accepts_mutated_gas_limit()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_validator_register_fee_recipient_mutation() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_accepts_changed_fee_recipient()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_blinded_block_invalid_parent_hash() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_rejects_invalid_parent_hash()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_blinded_block_invalid_prev_randao() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_rejects_invalid_prev_randao()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_blinded_block_invalid_block_number() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_rejects_invalid_block_number()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_blinded_block_invalid_timestamp() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_rejects_invalid_timestamp()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn get_blinded_block_invalid_signature() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_rejects_invalid_signature()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_chain_health_skips() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_chain_health_skips()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_chain_health_skips_per_epoch() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_chain_health_skips_per_epoch()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_chain_health_epochs_since_finalization() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_chain_health_epochs_since_finalization()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-09 06:05:16 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_chain_health_optimistic_head() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_chain_health_optimistic_head()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2022-09-05 04:50:49 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_inadequate_builder_threshold() {
|
|
|
|
|
ApiTester::new_mev_tester()
|
|
|
|
|
.await
|
|
|
|
|
.test_payload_rejects_inadequate_builder_threshold()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_payload_chosen_by_profit() {
|
|
|
|
|
ApiTester::new_mev_tester_no_builder_threshold()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_payload_chosen_when_more_profitable()
|
|
|
|
|
.await
|
|
|
|
|
.test_local_payload_chosen_when_equally_profitable()
|
|
|
|
|
.await
|
|
|
|
|
.test_local_payload_chosen_when_more_profitable()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-02-10 19:30:14 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_works_post_capella() {
|
|
|
|
|
let mut config = ApiTesterConfig {
|
2023-09-26 02:39:58 +00:00
|
|
|
builder_threshold: Some(0),
|
2023-08-21 05:02:32 +00:00
|
|
|
retain_historic_states: false,
|
2023-02-10 19:30:14 +00:00
|
|
|
spec: E::default_spec(),
|
|
|
|
|
};
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.bellatrix_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.capella_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
|
|
|
|
|
ApiTester::new_from_config(config)
|
|
|
|
|
.await
|
|
|
|
|
.test_post_validator_register_validator()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_works_post_capella()
|
|
|
|
|
.await
|
|
|
|
|
.test_lighthouse_rejects_invalid_withdrawals_root()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-19 00:12:09 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn builder_works_post_deneb() {
|
|
|
|
|
let mut config = ApiTesterConfig {
|
2023-09-26 02:39:58 +00:00
|
|
|
builder_threshold: Some(0),
|
2023-08-22 13:20:58 +00:00
|
|
|
retain_historic_states: false,
|
2023-08-19 00:12:09 +00:00
|
|
|
spec: E::default_spec(),
|
|
|
|
|
};
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.bellatrix_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.capella_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.deneb_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
|
|
|
|
|
ApiTester::new_from_config(config)
|
|
|
|
|
.await
|
|
|
|
|
.test_post_validator_register_validator()
|
|
|
|
|
.await
|
|
|
|
|
.test_builder_works_post_deneb()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-31 01:53:03 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn post_validator_liveness_epoch() {
|
|
|
|
|
ApiTester::new()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_validator_liveness_epoch()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-28 05:30:57 +00:00
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
2020-09-29 03:46:54 +00:00
|
|
|
async fn lighthouse_endpoints() {
|
|
|
|
|
ApiTester::new()
|
2021-10-11 02:45:06 +00:00
|
|
|
.await
|
2020-09-29 03:46:54 +00:00
|
|
|
.test_get_lighthouse_health()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_syncing()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_proto_array()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_validator_inclusion()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_validator_inclusion_global()
|
2020-10-22 06:05:49 +00:00
|
|
|
.await
|
2020-11-02 00:37:30 +00:00
|
|
|
.test_get_lighthouse_eth1_syncing()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_eth1_block_cache()
|
|
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_eth1_deposit_cache()
|
|
|
|
|
.await
|
2020-10-22 06:05:49 +00:00
|
|
|
.test_get_lighthouse_beacon_states_ssz()
|
2020-11-23 01:00:22 +00:00
|
|
|
.await
|
|
|
|
|
.test_get_lighthouse_staking()
|
2021-07-31 03:50:52 +00:00
|
|
|
.await
|
2021-09-22 00:37:28 +00:00
|
|
|
.test_get_lighthouse_database_info()
|
|
|
|
|
.await
|
|
|
|
|
.test_post_lighthouse_database_reconstruct()
|
|
|
|
|
.await
|
2021-07-31 03:50:52 +00:00
|
|
|
.test_post_lighthouse_liveness()
|
2020-09-29 03:46:54 +00:00
|
|
|
.await;
|
|
|
|
|
}
|
2022-07-25 08:23:00 +00:00
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn optimistic_responses() {
|
|
|
|
|
ApiTester::new_with_hard_forks(true, true)
|
|
|
|
|
.await
|
|
|
|
|
.test_check_optimistic_responses()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
2023-08-24 05:54:36 +00:00
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn expected_withdrawals_invalid_pre_capella() {
|
|
|
|
|
let mut config = ApiTesterConfig::default();
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
ApiTester::new_from_config(config)
|
|
|
|
|
.await
|
|
|
|
|
.test_get_expected_withdrawals_pre_capella()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn expected_withdrawals_invalid_state() {
|
|
|
|
|
let mut config = ApiTesterConfig::default();
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.bellatrix_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.capella_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
ApiTester::new_from_config(config)
|
|
|
|
|
.await
|
|
|
|
|
.test_get_expected_withdrawals_invalid_state()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
|
|
|
async fn expected_withdrawals_valid_capella() {
|
|
|
|
|
let mut config = ApiTesterConfig::default();
|
|
|
|
|
config.spec.altair_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.bellatrix_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
config.spec.capella_fork_epoch = Some(Epoch::new(0));
|
|
|
|
|
ApiTester::new_from_config(config)
|
|
|
|
|
.await
|
|
|
|
|
.test_get_expected_withdrawals_capella()
|
|
|
|
|
.await;
|
|
|
|
|
}
|
