## Issue Addressed Closes #2052 ## Proposed Changes - Refactor the attester/proposer duties endpoints in the BN - Performance improvements - Fixes some potential inconsistencies with the dependent root fields. - Removes `http_api::beacon_proposer_cache` and just uses the one on the `BeaconChain` instead. - Move the code for the proposer/attester duties endpoints into separate files, for readability. - Refactor the `DutiesService` in the VC - Required to reduce the delay on broadcasting new blocks. - Gets rid of the `ValidatorDuty` shim struct that came about when we adopted the standard API. - Separate block/attestation duty tasks so that they don't block each other when one is slow. - In the VC, use `PublicKeyBytes` to represent validators instead of `PublicKey`. `PublicKey` is a legit crypto object whilst `PublicKeyBytes` is just a byte-array, it's much faster to clone/hash `PublicKeyBytes` and this change has had a significant impact on runtimes. - Unfortunately this has created lots of dust changes. - In the BN, store `PublicKeyBytes` in the `beacon_proposer_cache` and allow access to them. The HTTP API always sends `PublicKeyBytes` over the wire and the conversion from `PublicKey` -> `PublickeyBytes` is non-trivial, especially when queries have 100s/1000s of validators (like Pyrmont). - Add the `state_processing::state_advance` mod which dedups a lot of the "apply `n` skip slots to the state" code. - This also fixes a bug with some functions which were failing to include a state root as per [this comment](072695284f/consensus/state_processing/src/state_advance.rs (L69-L74)). I couldn't find any instance of this bug that resulted in anything more severe than keying a shuffling cache by the wrong block root. - Swap the VC block service to use `mpsc` from `tokio` instead of `futures`. This is consistent with the rest of the code base. ~~This PR *reduces* the size of the codebase 🎉~~ It *used* to reduce the size of the code base before I added more comments. ## Observations on Prymont - Proposer duties times down from peaks of 450ms to consistent <1ms. - Current epoch attester duties times down from >1s peaks to a consistent 20-30ms. - Block production down from +600ms to 100-200ms. ## Additional Info - ~~Blocked on #2241~~ - ~~Blocked on #2234~~ ## TODO - [x] ~~Refactor this into some smaller PRs?~~ Leaving this as-is for now. - [x] Address `per_slot_processing` roots. - [x] Investigate slow next epoch times. Not getting added to cache on block processing? - [x] Consider [this](072695284f/beacon_node/store/src/hot_cold_store.rs (L811-L812)) in the scenario of replacing the state roots Co-authored-by: pawan <pawandhananjay@gmail.com> Co-authored-by: Michael Sproul <michael@sigmaprime.io>
219 lines
7.1 KiB
Rust
219 lines
7.1 KiB
Rust
use crate::{
|
|
interchange::Interchange,
|
|
test_utils::{pubkey, DEFAULT_GENESIS_VALIDATORS_ROOT},
|
|
SigningRoot, SlashingDatabase,
|
|
};
|
|
use serde_derive::{Deserialize, Serialize};
|
|
use tempfile::tempdir;
|
|
use types::{Epoch, Hash256, PublicKeyBytes, Slot};
|
|
|
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
|
pub struct MultiTestCase {
|
|
pub name: String,
|
|
pub genesis_validators_root: Hash256,
|
|
pub steps: Vec<TestCase>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
|
pub struct TestCase {
|
|
pub should_succeed: bool,
|
|
pub allow_partial_import: bool,
|
|
pub interchange: Interchange,
|
|
pub blocks: Vec<TestBlock>,
|
|
pub attestations: Vec<TestAttestation>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
|
pub struct TestBlock {
|
|
pub pubkey: PublicKeyBytes,
|
|
pub slot: Slot,
|
|
pub signing_root: Hash256,
|
|
pub should_succeed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
|
pub struct TestAttestation {
|
|
pub pubkey: PublicKeyBytes,
|
|
pub source_epoch: Epoch,
|
|
pub target_epoch: Epoch,
|
|
pub signing_root: Hash256,
|
|
pub should_succeed: bool,
|
|
}
|
|
|
|
impl MultiTestCase {
|
|
pub fn new(name: &str, steps: Vec<TestCase>) -> Self {
|
|
MultiTestCase {
|
|
name: name.into(),
|
|
genesis_validators_root: DEFAULT_GENESIS_VALIDATORS_ROOT,
|
|
steps,
|
|
}
|
|
}
|
|
|
|
pub fn single(name: &str, test_case: TestCase) -> Self {
|
|
Self::new(name, vec![test_case])
|
|
}
|
|
|
|
pub fn gvr(mut self, genesis_validators_root: Hash256) -> Self {
|
|
self.genesis_validators_root = genesis_validators_root;
|
|
self
|
|
}
|
|
|
|
pub fn run(&self) {
|
|
let dir = tempdir().unwrap();
|
|
let slashing_db_file = dir.path().join("slashing_protection.sqlite");
|
|
let slashing_db = SlashingDatabase::create(&slashing_db_file).unwrap();
|
|
|
|
for test_case in &self.steps {
|
|
match slashing_db.import_interchange_info(
|
|
test_case.interchange.clone(),
|
|
self.genesis_validators_root,
|
|
) {
|
|
Ok(import_outcomes) => {
|
|
let failed_records = import_outcomes
|
|
.iter()
|
|
.filter(|o| o.failed())
|
|
.collect::<Vec<_>>();
|
|
if !test_case.should_succeed {
|
|
panic!(
|
|
"test `{}` succeeded on import when it should have failed",
|
|
self.name
|
|
);
|
|
}
|
|
if !failed_records.is_empty() && !test_case.allow_partial_import {
|
|
panic!(
|
|
"test `{}` failed to import some records but should have succeeded: {:#?}",
|
|
self.name, failed_records,
|
|
);
|
|
}
|
|
}
|
|
Err(e) if test_case.should_succeed => {
|
|
panic!(
|
|
"test `{}` failed on import when it should have succeeded, error: {:?}",
|
|
self.name, e
|
|
);
|
|
}
|
|
_ => (),
|
|
}
|
|
|
|
for (i, block) in test_case.blocks.iter().enumerate() {
|
|
match slashing_db.check_and_insert_block_signing_root(
|
|
&block.pubkey,
|
|
block.slot,
|
|
SigningRoot::from(block.signing_root),
|
|
) {
|
|
Ok(safe) if !block.should_succeed => {
|
|
panic!(
|
|
"block {} from `{}` succeeded when it should have failed: {:?}",
|
|
i, self.name, safe
|
|
);
|
|
}
|
|
Err(e) if block.should_succeed => {
|
|
panic!(
|
|
"block {} from `{}` failed when it should have succeeded: {:?}",
|
|
i, self.name, e
|
|
);
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
for (i, att) in test_case.attestations.iter().enumerate() {
|
|
match slashing_db.check_and_insert_attestation_signing_root(
|
|
&att.pubkey,
|
|
att.source_epoch,
|
|
att.target_epoch,
|
|
SigningRoot::from(att.signing_root),
|
|
) {
|
|
Ok(safe) if !att.should_succeed => {
|
|
panic!(
|
|
"attestation {} from `{}` succeeded when it should have failed: {:?}",
|
|
i, self.name, safe
|
|
);
|
|
}
|
|
Err(e) if att.should_succeed => {
|
|
panic!(
|
|
"attestation {} from `{}` failed when it should have succeeded: {:?}",
|
|
i, self.name, e
|
|
);
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl TestCase {
|
|
pub fn new(interchange: Interchange) -> Self {
|
|
TestCase {
|
|
should_succeed: true,
|
|
allow_partial_import: false,
|
|
interchange,
|
|
blocks: vec![],
|
|
attestations: vec![],
|
|
}
|
|
}
|
|
|
|
pub fn should_fail(mut self) -> Self {
|
|
self.should_succeed = false;
|
|
self
|
|
}
|
|
|
|
pub fn allow_partial_import(mut self) -> Self {
|
|
self.allow_partial_import = true;
|
|
self
|
|
}
|
|
|
|
pub fn with_blocks(self, blocks: impl IntoIterator<Item = (usize, u64, bool)>) -> Self {
|
|
self.with_signing_root_blocks(
|
|
blocks
|
|
.into_iter()
|
|
.map(|(index, slot, should_succeed)| (index, slot, 0, should_succeed)),
|
|
)
|
|
}
|
|
|
|
pub fn with_signing_root_blocks(
|
|
mut self,
|
|
blocks: impl IntoIterator<Item = (usize, u64, u64, bool)>,
|
|
) -> Self {
|
|
self.blocks.extend(
|
|
blocks
|
|
.into_iter()
|
|
.map(|(pk, slot, signing_root, should_succeed)| TestBlock {
|
|
pubkey: pubkey(pk),
|
|
slot: Slot::new(slot),
|
|
signing_root: Hash256::from_low_u64_be(signing_root),
|
|
should_succeed,
|
|
}),
|
|
);
|
|
self
|
|
}
|
|
|
|
pub fn with_attestations(
|
|
self,
|
|
attestations: impl IntoIterator<Item = (usize, u64, u64, bool)>,
|
|
) -> Self {
|
|
self.with_signing_root_attestations(
|
|
attestations
|
|
.into_iter()
|
|
.map(|(id, source, target, succeed)| (id, source, target, 0, succeed)),
|
|
)
|
|
}
|
|
|
|
pub fn with_signing_root_attestations(
|
|
mut self,
|
|
attestations: impl IntoIterator<Item = (usize, u64, u64, u64, bool)>,
|
|
) -> Self {
|
|
self.attestations.extend(attestations.into_iter().map(
|
|
|(pk, source, target, signing_root, should_succeed)| TestAttestation {
|
|
pubkey: pubkey(pk),
|
|
source_epoch: Epoch::new(source),
|
|
target_epoch: Epoch::new(target),
|
|
signing_root: Hash256::from_low_u64_be(signing_root),
|
|
should_succeed,
|
|
},
|
|
));
|
|
self
|
|
}
|
|
}
|