lighthouse/consensus/state_processing/src/per_block_processing/tests.rs

1015 lines
32 KiB
Rust
Raw Normal View History

#![cfg(all(test, not(feature = "fake_crypto")))]
2019-08-30 05:49:06 +00:00
use crate::per_block_processing;
use crate::per_block_processing::errors::{
AttestationInvalid, AttesterSlashingInvalid, BlockOperationError, BlockProcessingError,
DepositInvalid, HeaderInvalid, IndexedAttestationInvalid, IntoWithIndex,
ProposerSlashingInvalid,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
};
use crate::{
per_block_processing::{process_operations, verify_exit::verify_exit},
BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot, VerifySignatures,
};
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use lazy_static::lazy_static;
use ssz_types::Bitfield;
use test_utils::generate_deterministic_keypairs;
use types::*;
2019-04-10 04:42:31 +00:00
pub const MAX_VALIDATOR_COUNT: usize = 97;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
pub const NUM_DEPOSITS: u64 = 1;
pub const VALIDATOR_COUNT: usize = 64;
pub const EPOCH_OFFSET: u64 = 4;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
pub const NUM_ATTESTATIONS: u64 = 1;
2019-04-10 04:42:31 +00:00
lazy_static! {
/// A cached set of keys.
static ref KEYPAIRS: Vec<Keypair> = generate_deterministic_keypairs(MAX_VALIDATOR_COUNT);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
async fn get_harness<E: EthSpec>(
epoch_offset: u64,
num_validators: usize,
) -> BeaconChainHarness<EphemeralHarnessType<E>> {
// Set the state and block to be in the last slot of the `epoch_offset`th epoch.
let last_slot_of_epoch =
(MainnetEthSpec::genesis_epoch() + epoch_offset).end_slot(E::slots_per_epoch());
let harness = BeaconChainHarness::builder(E::default())
.default_spec()
.keypairs(KEYPAIRS[0..num_validators].to_vec())
.fresh_ephemeral_store()
.build();
let state = harness.get_current_state();
if last_slot_of_epoch > Slot::new(0) {
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
harness
.add_attested_blocks_at_slots(
state,
Hash256::zero(),
(1..last_slot_of_epoch.as_u64())
.map(Slot::new)
.collect::<Vec<_>>()
.as_slice(),
(0..num_validators).collect::<Vec<_>>().as_slice(),
)
.await;
}
harness
}
Update to Spec v0.10 (#817) * Start updating types * WIP * Signature hacking * Existing EF tests passing with fake_crypto * Updates * Delete outdated API spec * The refactor continues * It compiles * WIP test fixes * All release tests passing bar genesis state parsing * Update and test YamlConfig * Update to spec v0.10 compatible BLS * Updates to BLS EF tests * Add EF test for AggregateVerify And delete unused hash2curve tests for uncompressed points * Update EF tests to v0.10.1 * Use optional block root correctly in block proc * Use genesis fork in deposit domain. All tests pass * Cargo fmt * Fast aggregate verify test * Update REST API docs * Cargo fmt * Fix unused import * Bump spec tags to v0.10.1 * Add `seconds_per_eth1_block` to chainspec * Update to timestamp based eth1 voting scheme * Return None from `get_votes_to_consider` if block cache is empty * Handle overflows in `is_candidate_block` * Revert to failing tests * Fix eth1 data sets test * Choose default vote according to spec * Fix collect_valid_votes tests * Fix `get_votes_to_consider` to choose all eligible blocks * Uncomment winning_vote tests * Add comments; remove unused code * Reduce seconds_per_eth1_block for simulation * Addressed review comments * Add test for default vote case * Fix logs * Remove unused functions * Meter default eth1 votes * Fix comments * Address review comments; remove unused dependency * Disable/delete two outdated tests * Bump eth1 default vote warn to error * Delete outdated eth1 test Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>
2020-02-10 23:19:36 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn valid_block_ok() {
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let state = harness.get_current_state();
let slot = state.slot();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let (block, mut state) = harness
.make_block_return_pre_state(state, slot + Slot::new(1))
.await;
2019-04-10 11:56:31 +00:00
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&block,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
2019-04-15 00:38:13 +00:00
assert!(result.is_ok());
2019-04-10 11:56:31 +00:00
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_block_header_state_slot() {
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
2019-04-10 11:56:31 +00:00
let state = harness.get_current_state();
let slot = state.slot() + Slot::new(1);
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let (signed_block, mut state) = harness.make_block_return_pre_state(state, slot).await;
let (mut block, signature) = signed_block.deconstruct();
*block.slot_mut() = slot + Slot::new(1);
2019-04-10 11:56:31 +00:00
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&SignedBeaconBlock::from_block(block, signature),
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
2019-04-10 11:56:31 +00:00
2019-04-15 00:38:13 +00:00
assert_eq!(
result,
Err(BlockProcessingError::HeaderInvalid {
reason: HeaderInvalid::StateSlotMismatch
})
2019-04-15 00:38:13 +00:00
);
2019-04-10 11:56:31 +00:00
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_parent_block_root() {
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let state = harness.get_current_state();
let slot = state.slot();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let (signed_block, mut state) = harness
.make_block_return_pre_state(state, slot + Slot::new(1))
.await;
let (mut block, signature) = signed_block.deconstruct();
*block.parent_root_mut() = Hash256::from([0xAA; 32]);
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&SignedBeaconBlock::from_block(block, signature),
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
assert_eq!(
result,
Err(BlockProcessingError::HeaderInvalid {
reason: HeaderInvalid::ParentBlockRootMismatch {
state: state.latest_block_header().canonical_root(),
block: Hash256::from([0xAA; 32])
}
})
);
2019-04-10 11:56:31 +00:00
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_block_signature() {
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let state = harness.get_current_state();
let slot = state.slot();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let (signed_block, mut state) = harness
.make_block_return_pre_state(state, slot + Slot::new(1))
.await;
let (block, _) = signed_block.deconstruct();
2019-04-15 00:38:13 +00:00
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&SignedBeaconBlock::from_block(block, Signature::empty()),
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
2019-04-15 00:38:13 +00:00
// should get a BadSignature error
assert_eq!(
result,
Err(BlockProcessingError::HeaderInvalid {
reason: HeaderInvalid::ProposalSignatureInvalid
})
2019-04-15 00:38:13 +00:00
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_randao_reveal_signature() {
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let state = harness.get_current_state();
let slot = state.slot();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let (signed_block, mut state) = harness
.make_block_with_modifier(state, slot + 1, |block| {
*block.body_mut().randao_reveal_mut() = Signature::empty();
})
.await;
2019-05-20 06:47:44 +00:00
let mut ctxt = ConsensusContext::new(signed_block.slot());
let result = per_block_processing(
&mut state,
&signed_block,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
// should get a BadRandaoSignature error
assert_eq!(result, Err(BlockProcessingError::RandaoSignatureInvalid));
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn valid_4_deposits() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut state = harness.get_current_state();
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let (deposits, state) = harness.make_deposits(&mut state, 4, None, None);
let deposits = VariableList::from(deposits);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
*head_block.to_mut().body_mut().deposits_mut() = deposits;
let result = process_operations::process_deposits(state, head_block.body().deposits(), &spec);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting Ok because these are valid deposits.
assert_eq!(result, Ok(()));
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_deposit_deposit_count_too_big() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut state = harness.get_current_state();
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let (deposits, state) = harness.make_deposits(&mut state, 1, None, None);
let deposits = VariableList::from(deposits);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
*head_block.to_mut().body_mut().deposits_mut() = deposits;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let big_deposit_count = NUM_DEPOSITS + 1;
state.eth1_data_mut().deposit_count = big_deposit_count;
let result = process_operations::process_deposits(state, head_block.body().deposits(), &spec);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting DepositCountInvalid because we incremented the deposit_count
assert_eq!(
result,
Err(BlockProcessingError::DepositCountInvalid {
expected: big_deposit_count as usize,
found: 1
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_deposit_count_too_small() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut state = harness.get_current_state();
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let (deposits, state) = harness.make_deposits(&mut state, 1, None, None);
let deposits = VariableList::from(deposits);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
*head_block.to_mut().body_mut().deposits_mut() = deposits;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let small_deposit_count = NUM_DEPOSITS - 1;
state.eth1_data_mut().deposit_count = small_deposit_count;
let result = process_operations::process_deposits(state, head_block.body().deposits(), &spec);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting DepositCountInvalid because we decremented the deposit_count
assert_eq!(
result,
Err(BlockProcessingError::DepositCountInvalid {
expected: small_deposit_count as usize,
found: 1
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_deposit_bad_merkle_proof() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut state = harness.get_current_state();
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let (deposits, state) = harness.make_deposits(&mut state, 1, None, None);
let deposits = VariableList::from(deposits);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
*head_block.to_mut().body_mut().deposits_mut() = deposits;
let bad_index = state.eth1_deposit_index() as usize;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Manually offsetting deposit count and index to trigger bad merkle proof
state.eth1_data_mut().deposit_count += 1;
*state.eth1_deposit_index_mut() += 1;
let result = process_operations::process_deposits(state, head_block.body().deposits(), &spec);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting BadMerkleProof because the proofs were created with different indices
assert_eq!(
result,
Err(BlockProcessingError::DepositInvalid {
index: bad_index,
reason: DepositInvalid::BadMerkleProof
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_deposit_wrong_sig() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut state = harness.get_current_state();
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let (deposits, state) =
harness.make_deposits(&mut state, 1, None, Some(SignatureBytes::empty()));
let deposits = VariableList::from(deposits);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
*head_block.to_mut().body_mut().deposits_mut() = deposits;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let result = process_operations::process_deposits(state, head_block.body().deposits(), &spec);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting Ok(()) even though the block signature does not correspond to the correct public key
assert_eq!(result, Ok(()));
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_deposit_invalid_pub_key() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut state = harness.get_current_state();
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let (deposits, state) =
harness.make_deposits(&mut state, 1, Some(PublicKeyBytes::empty()), None);
let deposits = VariableList::from(deposits);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
*head_block.to_mut().body_mut().deposits_mut() = deposits;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let result = process_operations::process_deposits(state, head_block.body().deposits(), &spec);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting Ok(()) even though we passed in invalid publickeybytes in the public key field of the deposit data.
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
assert_eq!(result, Ok(()));
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_no_committee_for_index() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
head_block.to_mut().body_mut().attestations_mut()[0]
.data
.index += 1;
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting NoCommittee because we manually set the attestation's index to be invalid
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
assert_eq!(
result,
Err(BlockProcessingError::AttestationInvalid {
index: 0,
reason: AttestationInvalid::BadCommitteeIndex
})
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_wrong_justified_checkpoint() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
let old_justified_checkpoint = head_block.body().attestations()[0].data.source;
let mut new_justified_checkpoint = old_justified_checkpoint;
new_justified_checkpoint.epoch += Epoch::new(1);
head_block.to_mut().body_mut().attestations_mut()[0]
.data
.source = new_justified_checkpoint;
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting WrongJustifiedCheckpoint because we manually set the
// source field of the AttestationData object to be invalid
assert_eq!(
result,
Err(BlockProcessingError::AttestationInvalid {
index: 0,
reason: AttestationInvalid::WrongJustifiedCheckpoint {
state: old_justified_checkpoint,
attestation: new_justified_checkpoint,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
is_current: true,
}
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_bad_aggregation_bitfield_len() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
head_block.to_mut().body_mut().attestations_mut()[0].aggregation_bits =
Bitfield::with_capacity(spec.target_committee_size).unwrap();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting InvalidBitfield because the size of the aggregation_bitfield is bigger than the committee size.
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
assert_eq!(
result,
Err(BlockProcessingError::BeaconStateError(
BeaconStateError::InvalidBitfield
))
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_bad_signature() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, 97).await; // minimal number of required validators for this test
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
head_block.to_mut().body_mut().attestations_mut()[0].signature = AggregateSignature::empty();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting BadSignature because we're signing with invalid secret_keys
assert_eq!(
result,
Err(BlockProcessingError::AttestationInvalid {
index: 0,
reason: AttestationInvalid::BadIndexedAttestation(
IndexedAttestationInvalid::BadSignature
)
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_included_too_early() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
let new_attesation_slot = head_block.body().attestations()[0].data.slot
+ Slot::new(MainnetEthSpec::slots_per_epoch());
head_block.to_mut().body_mut().attestations_mut()[0]
.data
.slot = new_attesation_slot;
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting IncludedTooEarly because the shard included in the crosslink is bigger than expected
assert_eq!(
result,
Err(BlockProcessingError::AttestationInvalid {
index: 0,
reason: AttestationInvalid::IncludedTooEarly {
state: state.slot(),
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
delay: spec.min_attestation_inclusion_delay,
attestation: new_attesation_slot,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
}
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_included_too_late() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
// note to maintainer: might need to increase validator count if we get NoCommittee
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
let new_attesation_slot = head_block.body().attestations()[0].data.slot
- Slot::new(MainnetEthSpec::slots_per_epoch());
head_block.to_mut().body_mut().attestations_mut()[0]
.data
.slot = new_attesation_slot;
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
assert_eq!(
result,
Err(BlockProcessingError::AttestationInvalid {
index: 0,
reason: AttestationInvalid::IncludedTooLate {
state: state.slot(),
attestation: new_attesation_slot,
}
})
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attestation_target_epoch_slot_mismatch() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
// note to maintainer: might need to increase validator count if we get NoCommittee
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut state = harness.get_current_state();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let mut head_block = harness
.chain
.head_beacon_block()
.as_ref()
.clone()
.deconstruct()
.0;
head_block.to_mut().body_mut().attestations_mut()[0]
.data
.target
.epoch += Epoch::new(1);
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attestations(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
head_block.body(),
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
Update to Spec v0.10 (#817) * Start updating types * WIP * Signature hacking * Existing EF tests passing with fake_crypto * Updates * Delete outdated API spec * The refactor continues * It compiles * WIP test fixes * All release tests passing bar genesis state parsing * Update and test YamlConfig * Update to spec v0.10 compatible BLS * Updates to BLS EF tests * Add EF test for AggregateVerify And delete unused hash2curve tests for uncompressed points * Update EF tests to v0.10.1 * Use optional block root correctly in block proc * Use genesis fork in deposit domain. All tests pass * Cargo fmt * Fast aggregate verify test * Update REST API docs * Cargo fmt * Fix unused import * Bump spec tags to v0.10.1 * Add `seconds_per_eth1_block` to chainspec * Update to timestamp based eth1 voting scheme * Return None from `get_votes_to_consider` if block cache is empty * Handle overflows in `is_candidate_block` * Revert to failing tests * Fix eth1 data sets test * Choose default vote according to spec * Fix collect_valid_votes tests * Fix `get_votes_to_consider` to choose all eligible blocks * Uncomment winning_vote tests * Add comments; remove unused code * Reduce seconds_per_eth1_block for simulation * Addressed review comments * Add test for default vote case * Fix logs * Remove unused functions * Meter default eth1 votes * Fix comments * Address review comments; remove unused dependency * Disable/delete two outdated tests * Bump eth1 default vote warn to error * Delete outdated eth1 test Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>
2020-02-10 23:19:36 +00:00
assert_eq!(
result,
Err(BlockProcessingError::AttestationInvalid {
index: 0,
reason: AttestationInvalid::TargetEpochSlotMismatch {
target_epoch: Epoch::new(EPOCH_OFFSET + 1),
slot_epoch: Epoch::new(EPOCH_OFFSET),
Update to Spec v0.10 (#817) * Start updating types * WIP * Signature hacking * Existing EF tests passing with fake_crypto * Updates * Delete outdated API spec * The refactor continues * It compiles * WIP test fixes * All release tests passing bar genesis state parsing * Update and test YamlConfig * Update to spec v0.10 compatible BLS * Updates to BLS EF tests * Add EF test for AggregateVerify And delete unused hash2curve tests for uncompressed points * Update EF tests to v0.10.1 * Use optional block root correctly in block proc * Use genesis fork in deposit domain. All tests pass * Cargo fmt * Fast aggregate verify test * Update REST API docs * Cargo fmt * Fix unused import * Bump spec tags to v0.10.1 * Add `seconds_per_eth1_block` to chainspec * Update to timestamp based eth1 voting scheme * Return None from `get_votes_to_consider` if block cache is empty * Handle overflows in `is_candidate_block` * Revert to failing tests * Fix eth1 data sets test * Choose default vote according to spec * Fix collect_valid_votes tests * Fix `get_votes_to_consider` to choose all eligible blocks * Uncomment winning_vote tests * Add comments; remove unused code * Reduce seconds_per_eth1_block for simulation * Addressed review comments * Add test for default vote case * Fix logs * Remove unused functions * Meter default eth1 votes * Fix comments * Address review comments; remove unused dependency * Disable/delete two outdated tests * Bump eth1 default vote warn to error * Delete outdated eth1 test Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>
2020-02-10 23:19:36 +00:00
}
})
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn valid_insert_attester_slashing() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let attester_slashing = harness.make_attester_slashing(vec![1, 2]);
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attester_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[attester_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting Ok(()) because attester slashing is valid
assert_eq!(result, Ok(()));
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attester_slashing_not_slashable() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]);
attester_slashing.attestation_1 = attester_slashing.attestation_2.clone();
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attester_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[attester_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting NotSlashable because the two attestations are the same
assert_eq!(
result,
Err(BlockProcessingError::AttesterSlashingInvalid {
index: 0,
reason: AttesterSlashingInvalid::NotSlashable
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attester_slashing_1_invalid() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]);
attester_slashing.attestation_1.attesting_indices = VariableList::from(vec![2, 1]);
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attester_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[attester_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
assert_eq!(
result,
Err(
BlockOperationError::Invalid(AttesterSlashingInvalid::IndexedAttestation1Invalid(
BlockOperationError::Invalid(
IndexedAttestationInvalid::BadValidatorIndicesOrdering(0)
)
))
.into_with_index(0)
)
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_attester_slashing_2_invalid() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut attester_slashing = harness.make_attester_slashing(vec![1, 2]);
attester_slashing.attestation_2.attesting_indices = VariableList::from(vec![2, 1]);
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_attester_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[attester_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
assert_eq!(
result,
Err(
BlockOperationError::Invalid(AttesterSlashingInvalid::IndexedAttestation2Invalid(
BlockOperationError::Invalid(
IndexedAttestationInvalid::BadValidatorIndicesOrdering(0)
)
))
.into_with_index(0)
)
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn valid_insert_proposer_slashing() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let proposer_slashing = harness.make_proposer_slashing(1);
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting Ok(_) because we inserted a valid proposer slashing
assert!(result.is_ok());
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_proposer_slashing_proposals_identical() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut proposer_slashing = harness.make_proposer_slashing(1);
proposer_slashing.signed_header_1.message = proposer_slashing.signed_header_2.message.clone();
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
// Expecting ProposalsIdentical because we the two headers are identical
assert_eq!(
result,
Err(BlockProcessingError::ProposerSlashingInvalid {
index: 0,
reason: ProposerSlashingInvalid::ProposalsIdentical
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_proposer_slashing_proposer_unknown() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let mut proposer_slashing = harness.make_proposer_slashing(1);
proposer_slashing.signed_header_1.message.proposer_index = 3_141_592;
proposer_slashing.signed_header_2.message.proposer_index = 3_141_592;
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting ProposerUnknown because validator_index is unknown
assert_eq!(
result,
Err(BlockProcessingError::ProposerSlashingInvalid {
index: 0,
reason: ProposerSlashingInvalid::ProposerUnknown(3_141_592)
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_proposer_slashing_duplicate_slashing() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let proposer_slashing = harness.make_proposer_slashing(1);
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result_1 = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing.clone()],
VerifySignatures::False,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
assert!(result_1.is_ok());
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let result_2 = process_operations::process_proposer_slashings(
&mut state,
&[proposer_slashing],
VerifySignatures::False,
&mut ctxt,
&spec,
);
// Expecting ProposerNotSlashable because we've already slashed the validator
assert_eq!(
result_2,
Err(BlockProcessingError::ProposerSlashingInvalid {
index: 0,
reason: ProposerSlashingInvalid::ProposerNotSlashable(1)
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_bad_proposal_1_signature() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut proposer_slashing = harness.make_proposer_slashing(1);
proposer_slashing.signed_header_1.signature = Signature::empty();
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting BadProposal1Signature because signature of proposal 1 is invalid
assert_eq!(
result,
Err(BlockProcessingError::ProposerSlashingInvalid {
index: 0,
reason: ProposerSlashingInvalid::BadProposal1Signature
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_bad_proposal_2_signature() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut proposer_slashing = harness.make_proposer_slashing(1);
proposer_slashing.signed_header_2.signature = Signature::empty();
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing],
VerifySignatures::True,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting BadProposal2Signature because signature of proposal 2 is invalid
assert_eq!(
result,
Err(BlockProcessingError::ProposerSlashingInvalid {
index: 0,
reason: ProposerSlashingInvalid::BadProposal2Signature
})
);
}
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
#[tokio::test]
async fn invalid_proposer_slashing_proposal_epoch_mismatch() {
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
let spec = MainnetEthSpec::default_spec();
Use async code when interacting with EL (#3244) ## Overview This rather extensive PR achieves two primary goals: 1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state. 2. Refactors fork choice, block production and block processing to `async` functions. Additionally, it achieves: - Concurrent forkchoice updates to the EL and cache pruning after a new head is selected. - Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production. - Concurrent per-block-processing and execution payload verification during block processing. - The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?): - I had to do this to deal with sending blocks into spawned tasks. - Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones. - We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap. - Avoids cloning *all the blocks* in *every chain segment* during sync. - It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:) - The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs. For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273 ## Changes to `canonical_head` and `fork_choice` Previously, the `BeaconChain` had two separate fields: ``` canonical_head: RwLock<Snapshot>, fork_choice: RwLock<BeaconForkChoice> ``` Now, we have grouped these values under a single struct: ``` canonical_head: CanonicalHead { cached_head: RwLock<Arc<Snapshot>>, fork_choice: RwLock<BeaconForkChoice> } ``` Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously. ## Breaking Changes ### The `state` (root) field in the `finalized_checkpoint` SSE event Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event: 1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`. 4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots. Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1). I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku. ## Notes for Reviewers I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct. I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking". I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it. I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around. Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2. You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests: - Changing tests to be `tokio::async` tests. - Adding `.await` to fork choice, block processing and block production functions. - Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`. - Wrapping `SignedBeaconBlock` in an `Arc`. - In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant. I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic. Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
let harness = get_harness::<MainnetEthSpec>(EPOCH_OFFSET, VALIDATOR_COUNT).await;
let mut proposer_slashing = harness.make_proposer_slashing(1);
proposer_slashing.signed_header_1.message.slot = Slot::new(0);
proposer_slashing.signed_header_2.message.slot = Slot::new(128);
let mut state = harness.get_current_state();
let mut ctxt = ConsensusContext::new(state.slot());
let result = process_operations::process_proposer_slashings(
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&mut state,
&[proposer_slashing],
VerifySignatures::False,
&mut ctxt,
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
&spec,
);
// Expecting ProposalEpochMismatch because the two epochs are different
assert_eq!(
result,
Err(BlockProcessingError::ProposerSlashingInvalid {
index: 0,
reason: ProposerSlashingInvalid::ProposalSlotMismatch(
Slot::from(0_u64),
Slot::from(128_u64)
BlockProcessing testing (#559) * Add valid_deposit test and build_with_deposit method * Insert_deposit takes a num_deposit param * Deposit with spec.max_effective_balance * Copy int_to_bytes32 implem from beacon_chain_builder * Add debug information to insert_deposit * Remove length-proof assertion * Insert_deposit displays error * Batch deposits now pass tests * Optimize insert_deposit * Rename insert_deposits and set num_deposits to 2 in valid_deposit test * update test_utils to pass tests * fix typo in test_utils * update cast in test_utils * Add DepositCountInvalid tests * Add tests for bad deposit signature * Add tests and test utils in test_builder * Return error instead of ok on bad signature * Update DepositTestTask enum * Add comment about manually setting deposit_count and deposit_index * add badblsbytes test * add bad_index var for clarity ; remove underflow test * cargo fmt * Add insert 3 exits tests * Add validator_unknwon test * Add far_future_epoch test and already exited * Add MaxVoluntaryExits + 1 test * Add exit_already_initiated test * Add exit_not_active test * Add too_young_to_leave test * Cargo fmt * Confirm already_anitiated test * Fix typo in enum variant * Adjust some tests to return ok(()) and revert changes for early return in per_block_processing.rs * cargo fmt * Adjust AlreadyIniated test to expect Ok(()) and revert changes in per_block_processing.rs * Remove extraneous newline * Add insert_valid_attester_slashing * Initial cargo fmt * Add NotSlashable test * Cargo fmt * Remove AttestationDataIdentical * Make test_task pass through reference ; fix max_attester_slashing_plus_one test * Initial cargo fmt * Add InvalidIndexedAttestation1 and 2 * Add comments * Add ProposalsIdenticalTest * Add ProposalsIdentical test * Cargo fmt * Add ProposerUnknown test * Add ProposalEpochMismatch test * Add BadProposal1Signature and Badproposal2Signature tests * Add ProposerNotSlashable test * Derive PartialEq and use if instead of match * Merge attestation tests * Remove useless AlreadyInitiated variant in beacon_state * Remove MaxOperations plus one tests for each operation * Clean comments * add IncludedTooLate and BadTargetEpoch tests * Update AttestationDataBuilder call in operation_pool testing * Cargo fmt * Remove BadIndex enum variant, unused in the code * Cargo fmt * Cargo fmt updated * simply increment deposit_count instead of hardsetting deposit_index in insert_deposits * Fix bad_merkle_proof when calling insert_deposits
2019-11-12 05:09:33 +00:00
)
})
);
}
#[tokio::test]
async fn fork_spanning_exit() {
let mut spec = MainnetEthSpec::default_spec();
let slots_per_epoch = MainnetEthSpec::slots_per_epoch();
spec.altair_fork_epoch = Some(Epoch::new(2));
spec.bellatrix_fork_epoch = Some(Epoch::new(4));
spec.shard_committee_period = 0;
let harness = BeaconChainHarness::builder(MainnetEthSpec::default())
.spec(spec.clone())
.deterministic_keypairs(VALIDATOR_COUNT)
.mock_execution_layer()
.fresh_ephemeral_store()
.build();
harness.extend_to_slot(slots_per_epoch.into()).await;
/*
* Produce an exit *before* Altair.
*/
let signed_exit = harness.make_voluntary_exit(0, Epoch::new(1));
assert!(signed_exit.message.epoch < spec.altair_fork_epoch.unwrap());
/*
* Ensure the exit verifies before Altair.
*/
let head = harness.chain.canonical_head.cached_head();
let head_state = &head.snapshot.beacon_state;
assert!(head_state.current_epoch() < spec.altair_fork_epoch.unwrap());
verify_exit(head_state, &signed_exit, VerifySignatures::True, &spec)
.expect("phase0 exit verifies against phase0 state");
/*
* Ensure the exit verifies after Altair.
*/
harness
.extend_to_slot(spec.altair_fork_epoch.unwrap().start_slot(slots_per_epoch))
.await;
let head = harness.chain.canonical_head.cached_head();
let head_state = &head.snapshot.beacon_state;
assert!(head_state.current_epoch() >= spec.altair_fork_epoch.unwrap());
assert!(head_state.current_epoch() < spec.bellatrix_fork_epoch.unwrap());
verify_exit(head_state, &signed_exit, VerifySignatures::True, &spec)
.expect("phase0 exit verifies against altair state");
/*
* Ensure the exit no longer verifies after Bellatrix.
*/
harness
.extend_to_slot(
spec.bellatrix_fork_epoch
.unwrap()
.start_slot(slots_per_epoch),
)
.await;
let head = harness.chain.canonical_head.cached_head();
let head_state = &head.snapshot.beacon_state;
assert!(head_state.current_epoch() >= spec.bellatrix_fork_epoch.unwrap());
verify_exit(head_state, &signed_exit, VerifySignatures::True, &spec)
.expect_err("phase0 exit does not verify against bellatrix state");
}