2022-11-22 18:27:48 +00:00
|
|
|
use crate::engines::ForkchoiceState;
|
2023-01-31 17:26:23 +00:00
|
|
|
use crate::http::{
|
|
|
|
ENGINE_EXCHANGE_TRANSITION_CONFIGURATION_V1, ENGINE_FORKCHOICE_UPDATED_V1,
|
|
|
|
ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2,
|
|
|
|
ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2,
|
|
|
|
};
|
2023-02-05 22:26:36 +00:00
|
|
|
use crate::BlobTxConversionError;
|
Separate execution payloads in the DB (#3157)
## Proposed Changes
Reduce post-merge disk usage by not storing finalized execution payloads in Lighthouse's database.
:warning: **This is achieved in a backwards-incompatible way for networks that have already merged** :warning:. Kiln users and shadow fork enjoyers will be unable to downgrade after running the code from this PR. The upgrade migration may take several minutes to run, and can't be aborted after it begins.
The main changes are:
- New column in the database called `ExecPayload`, keyed by beacon block root.
- The `BeaconBlock` column now stores blinded blocks only.
- Lots of places that previously used full blocks now use blinded blocks, e.g. analytics APIs, block replay in the DB, etc.
- On finalization:
- `prune_abanonded_forks` deletes non-canonical payloads whilst deleting non-canonical blocks.
- `migrate_db` deletes finalized canonical payloads whilst deleting finalized states.
- Conversions between blinded and full blocks are implemented in a compositional way, duplicating some work from Sean's PR #3134.
- The execution layer has a new `get_payload_by_block_hash` method that reconstructs a payload using the EE's `eth_getBlockByHash` call.
- I've tested manually that it works on Kiln, using Geth and Nethermind.
- This isn't necessarily the most efficient method, and new engine APIs are being discussed to improve this: https://github.com/ethereum/execution-apis/pull/146.
- We're depending on the `ethers` master branch, due to lots of recent changes. We're also using a workaround for https://github.com/gakonst/ethers-rs/issues/1134.
- Payload reconstruction is used in the HTTP API via `BeaconChain::get_block`, which is now `async`. Due to the `async` fn, the `blocking_json` wrapper has been removed.
- Payload reconstruction is used in network RPC to serve blocks-by-{root,range} responses. Here the `async` adjustment is messier, although I think I've managed to come up with a reasonable compromise: the handlers take the `SendOnDrop` by value so that they can drop it on _task completion_ (after the `fn` returns). Still, this is introducing disk reads onto core executor threads, which may have a negative performance impact (thoughts appreciated).
## Additional Info
- [x] For performance it would be great to remove the cloning of full blocks when converting them to blinded blocks to write to disk. I'm going to experiment with a `put_block` API that takes the block by value, breaks it into a blinded block and a payload, stores the blinded block, and then re-assembles the full block for the caller.
- [x] We should measure the latency of blocks-by-root and blocks-by-range responses.
- [x] We should add integration tests that stress the payload reconstruction (basic tests done, issue for more extensive tests: https://github.com/sigp/lighthouse/issues/3159)
- [x] We should (manually) test the schema v9 migration from several prior versions, particularly as blocks have changed on disk and some migrations rely on being able to load blocks.
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2022-05-12 00:42:17 +00:00
|
|
|
pub use ethers_core::types::Transaction;
|
2022-12-14 00:52:46 +00:00
|
|
|
use ethers_core::utils::rlp::{self, Decodable, Rlp};
|
2022-06-29 09:07:09 +00:00
|
|
|
use http::deposit_methods::RpcError;
|
2022-12-14 00:52:46 +00:00
|
|
|
pub use json_structures::{JsonWithdrawal, TransitionConfigurationV1};
|
2022-03-08 06:46:24 +00:00
|
|
|
use reqwest::StatusCode;
|
2021-09-29 22:14:15 +00:00
|
|
|
use serde::{Deserialize, Serialize};
|
2022-12-14 00:52:46 +00:00
|
|
|
use std::convert::TryFrom;
|
2022-08-19 04:27:23 +00:00
|
|
|
use strum::IntoStaticStr;
|
2022-10-26 19:15:26 +00:00
|
|
|
use superstruct::superstruct;
|
2022-03-31 07:52:23 +00:00
|
|
|
pub use types::{
|
2023-02-02 01:37:46 +00:00
|
|
|
Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader,
|
|
|
|
ExecutionPayloadRef, FixedVector, ForkName, Hash256, Uint256, VariableList, Withdrawal,
|
2022-03-31 07:52:23 +00:00
|
|
|
};
|
2023-01-19 14:32:08 +00:00
|
|
|
use types::{ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge};
|
2022-02-20 21:54:35 +00:00
|
|
|
|
2022-04-05 20:55:42 +00:00
|
|
|
pub mod auth;
|
2021-09-29 22:14:15 +00:00
|
|
|
pub mod http;
|
2021-11-15 06:13:38 +00:00
|
|
|
pub mod json_structures;
|
2021-09-29 22:14:15 +00:00
|
|
|
|
2022-03-31 07:52:23 +00:00
|
|
|
pub const LATEST_TAG: &str = "latest";
|
|
|
|
|
2021-11-15 06:13:38 +00:00
|
|
|
pub type PayloadId = [u8; 8];
|
2021-09-29 22:14:15 +00:00
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum Error {
|
|
|
|
Reqwest(reqwest::Error),
|
2022-03-08 06:46:24 +00:00
|
|
|
Auth(auth::Error),
|
2021-09-29 22:14:15 +00:00
|
|
|
BadResponse(String),
|
|
|
|
RequestFailed(String),
|
2022-02-28 22:07:48 +00:00
|
|
|
InvalidExecutePayloadResponse(&'static str),
|
2021-09-29 22:14:15 +00:00
|
|
|
JsonRpc(RpcError),
|
|
|
|
Json(serde_json::Error),
|
2022-07-01 01:15:19 +00:00
|
|
|
ServerMessage { code: i64, message: String },
|
2021-09-29 22:14:15 +00:00
|
|
|
Eip155Failure,
|
|
|
|
IsSyncing,
|
2022-02-28 22:07:48 +00:00
|
|
|
ExecutionBlockNotFound(ExecutionBlockHash),
|
2021-09-29 22:14:15 +00:00
|
|
|
ExecutionHeadBlockNotFound,
|
2022-02-28 22:07:48 +00:00
|
|
|
ParentHashEqualsBlockHash(ExecutionBlockHash),
|
2021-11-15 06:13:38 +00:00
|
|
|
PayloadIdUnavailable,
|
2022-03-08 04:40:42 +00:00
|
|
|
TransitionConfigurationMismatch,
|
2022-03-31 07:52:23 +00:00
|
|
|
PayloadConversionLogicFlaw,
|
2023-02-05 21:55:55 +00:00
|
|
|
SszError(ssz_types::Error),
|
2022-10-26 19:15:26 +00:00
|
|
|
DeserializeWithdrawals(ssz_types::Error),
|
2022-07-01 01:15:19 +00:00
|
|
|
BuilderApi(builder_client::Error),
|
2022-10-26 19:15:26 +00:00
|
|
|
IncorrectStateVariant,
|
2022-11-22 18:27:48 +00:00
|
|
|
RequiredMethodUnsupported(&'static str),
|
|
|
|
UnsupportedForkVariant(String),
|
|
|
|
BadConversion(String),
|
2022-12-14 00:52:46 +00:00
|
|
|
RlpDecoderError(rlp::DecoderError),
|
2023-02-05 22:26:36 +00:00
|
|
|
BlobTxConversionError(BlobTxConversionError),
|
2021-09-29 22:14:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl From<reqwest::Error> for Error {
|
|
|
|
fn from(e: reqwest::Error) -> Self {
|
2022-03-08 06:46:24 +00:00
|
|
|
if matches!(
|
|
|
|
e.status(),
|
|
|
|
Some(StatusCode::UNAUTHORIZED) | Some(StatusCode::FORBIDDEN)
|
|
|
|
) {
|
|
|
|
Error::Auth(auth::Error::InvalidToken)
|
|
|
|
} else {
|
|
|
|
Error::Reqwest(e)
|
|
|
|
}
|
2021-09-29 22:14:15 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<serde_json::Error> for Error {
|
|
|
|
fn from(e: serde_json::Error) -> Self {
|
|
|
|
Error::Json(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-08 06:46:24 +00:00
|
|
|
impl From<auth::Error> for Error {
|
|
|
|
fn from(e: auth::Error) -> Self {
|
|
|
|
Error::Auth(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-01 01:15:19 +00:00
|
|
|
impl From<builder_client::Error> for Error {
|
|
|
|
fn from(e: builder_client::Error) -> Self {
|
|
|
|
Error::BuilderApi(e)
|
|
|
|
}
|
2021-09-29 22:14:15 +00:00
|
|
|
}
|
|
|
|
|
2022-12-14 00:52:46 +00:00
|
|
|
impl From<rlp::DecoderError> for Error {
|
|
|
|
fn from(e: rlp::DecoderError) -> Self {
|
|
|
|
Error::RlpDecoderError(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-05 21:55:55 +00:00
|
|
|
impl From<ssz_types::Error> for Error {
|
|
|
|
fn from(e: ssz_types::Error) -> Self {
|
|
|
|
Error::SszError(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-05 22:26:36 +00:00
|
|
|
impl From<BlobTxConversionError> for Error {
|
|
|
|
fn from(e: BlobTxConversionError) -> Self {
|
|
|
|
Error::BlobTxConversionError(e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-19 04:27:23 +00:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, IntoStaticStr)]
|
|
|
|
#[strum(serialize_all = "snake_case")]
|
2022-02-17 21:47:06 +00:00
|
|
|
pub enum PayloadStatusV1Status {
|
2021-09-29 22:14:15 +00:00
|
|
|
Valid,
|
|
|
|
Invalid,
|
|
|
|
Syncing,
|
2022-02-17 21:47:06 +00:00
|
|
|
Accepted,
|
|
|
|
InvalidBlockHash,
|
2021-09-29 22:14:15 +00:00
|
|
|
}
|
|
|
|
|
2021-11-15 06:13:38 +00:00
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
2022-02-17 21:47:06 +00:00
|
|
|
pub struct PayloadStatusV1 {
|
|
|
|
pub status: PayloadStatusV1Status,
|
2022-02-28 22:07:48 +00:00
|
|
|
pub latest_valid_hash: Option<ExecutionBlockHash>,
|
2021-12-12 09:04:21 +00:00
|
|
|
pub validation_error: Option<String>,
|
2021-09-29 22:14:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
|
|
|
|
#[serde(untagged)]
|
|
|
|
pub enum BlockByNumberQuery<'a> {
|
|
|
|
Tag(&'a str),
|
|
|
|
}
|
|
|
|
|
Separate execution payloads in the DB (#3157)
## Proposed Changes
Reduce post-merge disk usage by not storing finalized execution payloads in Lighthouse's database.
:warning: **This is achieved in a backwards-incompatible way for networks that have already merged** :warning:. Kiln users and shadow fork enjoyers will be unable to downgrade after running the code from this PR. The upgrade migration may take several minutes to run, and can't be aborted after it begins.
The main changes are:
- New column in the database called `ExecPayload`, keyed by beacon block root.
- The `BeaconBlock` column now stores blinded blocks only.
- Lots of places that previously used full blocks now use blinded blocks, e.g. analytics APIs, block replay in the DB, etc.
- On finalization:
- `prune_abanonded_forks` deletes non-canonical payloads whilst deleting non-canonical blocks.
- `migrate_db` deletes finalized canonical payloads whilst deleting finalized states.
- Conversions between blinded and full blocks are implemented in a compositional way, duplicating some work from Sean's PR #3134.
- The execution layer has a new `get_payload_by_block_hash` method that reconstructs a payload using the EE's `eth_getBlockByHash` call.
- I've tested manually that it works on Kiln, using Geth and Nethermind.
- This isn't necessarily the most efficient method, and new engine APIs are being discussed to improve this: https://github.com/ethereum/execution-apis/pull/146.
- We're depending on the `ethers` master branch, due to lots of recent changes. We're also using a workaround for https://github.com/gakonst/ethers-rs/issues/1134.
- Payload reconstruction is used in the HTTP API via `BeaconChain::get_block`, which is now `async`. Due to the `async` fn, the `blocking_json` wrapper has been removed.
- Payload reconstruction is used in network RPC to serve blocks-by-{root,range} responses. Here the `async` adjustment is messier, although I think I've managed to come up with a reasonable compromise: the handlers take the `SendOnDrop` by value so that they can drop it on _task completion_ (after the `fn` returns). Still, this is introducing disk reads onto core executor threads, which may have a negative performance impact (thoughts appreciated).
## Additional Info
- [x] For performance it would be great to remove the cloning of full blocks when converting them to blinded blocks to write to disk. I'm going to experiment with a `put_block` API that takes the block by value, breaks it into a blinded block and a payload, stores the blinded block, and then re-assembles the full block for the caller.
- [x] We should measure the latency of blocks-by-root and blocks-by-range responses.
- [x] We should add integration tests that stress the payload reconstruction (basic tests done, issue for more extensive tests: https://github.com/sigp/lighthouse/issues/3159)
- [x] We should (manually) test the schema v9 migration from several prior versions, particularly as blocks have changed on disk and some migrations rely on being able to load blocks.
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2022-05-12 00:42:17 +00:00
|
|
|
/// Representation of an exection block with enough detail to determine the terminal PoW block.
|
|
|
|
///
|
|
|
|
/// See `get_pow_block_hash_at_total_difficulty`.
|
2021-09-29 22:14:15 +00:00
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
pub struct ExecutionBlock {
|
2021-10-02 01:39:11 +00:00
|
|
|
#[serde(rename = "hash")]
|
2022-02-28 22:07:48 +00:00
|
|
|
pub block_hash: ExecutionBlockHash,
|
2021-10-02 01:39:11 +00:00
|
|
|
#[serde(rename = "number", with = "eth2_serde_utils::u64_hex_be")]
|
2021-09-29 22:14:15 +00:00
|
|
|
pub block_number: u64,
|
2022-02-28 22:07:48 +00:00
|
|
|
pub parent_hash: ExecutionBlockHash,
|
2021-09-29 22:14:15 +00:00
|
|
|
pub total_difficulty: Uint256,
|
2022-07-18 23:15:41 +00:00
|
|
|
#[serde(with = "eth2_serde_utils::u64_hex_be")]
|
|
|
|
pub timestamp: u64,
|
2021-09-29 22:14:15 +00:00
|
|
|
}
|
2021-11-15 06:13:38 +00:00
|
|
|
|
2022-10-26 19:15:26 +00:00
|
|
|
/// Representation of an execution block with enough detail to reconstruct a payload.
|
|
|
|
#[superstruct(
|
|
|
|
variants(Merge, Capella, Eip4844),
|
|
|
|
variant_attributes(
|
|
|
|
derive(Clone, Debug, PartialEq, Serialize, Deserialize,),
|
|
|
|
serde(bound = "T: EthSpec", rename_all = "camelCase"),
|
|
|
|
),
|
|
|
|
cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
|
|
|
|
partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
|
|
|
|
)]
|
Separate execution payloads in the DB (#3157)
## Proposed Changes
Reduce post-merge disk usage by not storing finalized execution payloads in Lighthouse's database.
:warning: **This is achieved in a backwards-incompatible way for networks that have already merged** :warning:. Kiln users and shadow fork enjoyers will be unable to downgrade after running the code from this PR. The upgrade migration may take several minutes to run, and can't be aborted after it begins.
The main changes are:
- New column in the database called `ExecPayload`, keyed by beacon block root.
- The `BeaconBlock` column now stores blinded blocks only.
- Lots of places that previously used full blocks now use blinded blocks, e.g. analytics APIs, block replay in the DB, etc.
- On finalization:
- `prune_abanonded_forks` deletes non-canonical payloads whilst deleting non-canonical blocks.
- `migrate_db` deletes finalized canonical payloads whilst deleting finalized states.
- Conversions between blinded and full blocks are implemented in a compositional way, duplicating some work from Sean's PR #3134.
- The execution layer has a new `get_payload_by_block_hash` method that reconstructs a payload using the EE's `eth_getBlockByHash` call.
- I've tested manually that it works on Kiln, using Geth and Nethermind.
- This isn't necessarily the most efficient method, and new engine APIs are being discussed to improve this: https://github.com/ethereum/execution-apis/pull/146.
- We're depending on the `ethers` master branch, due to lots of recent changes. We're also using a workaround for https://github.com/gakonst/ethers-rs/issues/1134.
- Payload reconstruction is used in the HTTP API via `BeaconChain::get_block`, which is now `async`. Due to the `async` fn, the `blocking_json` wrapper has been removed.
- Payload reconstruction is used in network RPC to serve blocks-by-{root,range} responses. Here the `async` adjustment is messier, although I think I've managed to come up with a reasonable compromise: the handlers take the `SendOnDrop` by value so that they can drop it on _task completion_ (after the `fn` returns). Still, this is introducing disk reads onto core executor threads, which may have a negative performance impact (thoughts appreciated).
## Additional Info
- [x] For performance it would be great to remove the cloning of full blocks when converting them to blinded blocks to write to disk. I'm going to experiment with a `put_block` API that takes the block by value, breaks it into a blinded block and a payload, stores the blinded block, and then re-assembles the full block for the caller.
- [x] We should measure the latency of blocks-by-root and blocks-by-range responses.
- [x] We should add integration tests that stress the payload reconstruction (basic tests done, issue for more extensive tests: https://github.com/sigp/lighthouse/issues/3159)
- [x] We should (manually) test the schema v9 migration from several prior versions, particularly as blocks have changed on disk and some migrations rely on being able to load blocks.
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2022-05-12 00:42:17 +00:00
|
|
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
2022-10-26 19:15:26 +00:00
|
|
|
#[serde(bound = "T: EthSpec", rename_all = "camelCase", untagged)]
|
Separate execution payloads in the DB (#3157)
## Proposed Changes
Reduce post-merge disk usage by not storing finalized execution payloads in Lighthouse's database.
:warning: **This is achieved in a backwards-incompatible way for networks that have already merged** :warning:. Kiln users and shadow fork enjoyers will be unable to downgrade after running the code from this PR. The upgrade migration may take several minutes to run, and can't be aborted after it begins.
The main changes are:
- New column in the database called `ExecPayload`, keyed by beacon block root.
- The `BeaconBlock` column now stores blinded blocks only.
- Lots of places that previously used full blocks now use blinded blocks, e.g. analytics APIs, block replay in the DB, etc.
- On finalization:
- `prune_abanonded_forks` deletes non-canonical payloads whilst deleting non-canonical blocks.
- `migrate_db` deletes finalized canonical payloads whilst deleting finalized states.
- Conversions between blinded and full blocks are implemented in a compositional way, duplicating some work from Sean's PR #3134.
- The execution layer has a new `get_payload_by_block_hash` method that reconstructs a payload using the EE's `eth_getBlockByHash` call.
- I've tested manually that it works on Kiln, using Geth and Nethermind.
- This isn't necessarily the most efficient method, and new engine APIs are being discussed to improve this: https://github.com/ethereum/execution-apis/pull/146.
- We're depending on the `ethers` master branch, due to lots of recent changes. We're also using a workaround for https://github.com/gakonst/ethers-rs/issues/1134.
- Payload reconstruction is used in the HTTP API via `BeaconChain::get_block`, which is now `async`. Due to the `async` fn, the `blocking_json` wrapper has been removed.
- Payload reconstruction is used in network RPC to serve blocks-by-{root,range} responses. Here the `async` adjustment is messier, although I think I've managed to come up with a reasonable compromise: the handlers take the `SendOnDrop` by value so that they can drop it on _task completion_ (after the `fn` returns). Still, this is introducing disk reads onto core executor threads, which may have a negative performance impact (thoughts appreciated).
## Additional Info
- [x] For performance it would be great to remove the cloning of full blocks when converting them to blinded blocks to write to disk. I'm going to experiment with a `put_block` API that takes the block by value, breaks it into a blinded block and a payload, stores the blinded block, and then re-assembles the full block for the caller.
- [x] We should measure the latency of blocks-by-root and blocks-by-range responses.
- [x] We should add integration tests that stress the payload reconstruction (basic tests done, issue for more extensive tests: https://github.com/sigp/lighthouse/issues/3159)
- [x] We should (manually) test the schema v9 migration from several prior versions, particularly as blocks have changed on disk and some migrations rely on being able to load blocks.
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2022-05-12 00:42:17 +00:00
|
|
|
pub struct ExecutionBlockWithTransactions<T: EthSpec> {
|
|
|
|
pub parent_hash: ExecutionBlockHash,
|
|
|
|
#[serde(alias = "miner")]
|
|
|
|
pub fee_recipient: Address,
|
|
|
|
pub state_root: Hash256,
|
|
|
|
pub receipts_root: Hash256,
|
|
|
|
#[serde(with = "ssz_types::serde_utils::hex_fixed_vec")]
|
|
|
|
pub logs_bloom: FixedVector<u8, T::BytesPerLogsBloom>,
|
|
|
|
#[serde(alias = "mixHash")]
|
|
|
|
pub prev_randao: Hash256,
|
|
|
|
#[serde(rename = "number", with = "eth2_serde_utils::u64_hex_be")]
|
|
|
|
pub block_number: u64,
|
|
|
|
#[serde(with = "eth2_serde_utils::u64_hex_be")]
|
|
|
|
pub gas_limit: u64,
|
|
|
|
#[serde(with = "eth2_serde_utils::u64_hex_be")]
|
|
|
|
pub gas_used: u64,
|
|
|
|
#[serde(with = "eth2_serde_utils::u64_hex_be")]
|
|
|
|
pub timestamp: u64,
|
|
|
|
#[serde(with = "ssz_types::serde_utils::hex_var_list")]
|
|
|
|
pub extra_data: VariableList<u8, T::MaxExtraDataBytes>,
|
|
|
|
pub base_fee_per_gas: Uint256,
|
2022-10-26 19:15:26 +00:00
|
|
|
#[superstruct(only(Eip4844))]
|
2022-11-24 05:41:35 +00:00
|
|
|
#[serde(with = "eth2_serde_utils::u256_hex_be")]
|
|
|
|
pub excess_data_gas: Uint256,
|
Separate execution payloads in the DB (#3157)
## Proposed Changes
Reduce post-merge disk usage by not storing finalized execution payloads in Lighthouse's database.
:warning: **This is achieved in a backwards-incompatible way for networks that have already merged** :warning:. Kiln users and shadow fork enjoyers will be unable to downgrade after running the code from this PR. The upgrade migration may take several minutes to run, and can't be aborted after it begins.
The main changes are:
- New column in the database called `ExecPayload`, keyed by beacon block root.
- The `BeaconBlock` column now stores blinded blocks only.
- Lots of places that previously used full blocks now use blinded blocks, e.g. analytics APIs, block replay in the DB, etc.
- On finalization:
- `prune_abanonded_forks` deletes non-canonical payloads whilst deleting non-canonical blocks.
- `migrate_db` deletes finalized canonical payloads whilst deleting finalized states.
- Conversions between blinded and full blocks are implemented in a compositional way, duplicating some work from Sean's PR #3134.
- The execution layer has a new `get_payload_by_block_hash` method that reconstructs a payload using the EE's `eth_getBlockByHash` call.
- I've tested manually that it works on Kiln, using Geth and Nethermind.
- This isn't necessarily the most efficient method, and new engine APIs are being discussed to improve this: https://github.com/ethereum/execution-apis/pull/146.
- We're depending on the `ethers` master branch, due to lots of recent changes. We're also using a workaround for https://github.com/gakonst/ethers-rs/issues/1134.
- Payload reconstruction is used in the HTTP API via `BeaconChain::get_block`, which is now `async`. Due to the `async` fn, the `blocking_json` wrapper has been removed.
- Payload reconstruction is used in network RPC to serve blocks-by-{root,range} responses. Here the `async` adjustment is messier, although I think I've managed to come up with a reasonable compromise: the handlers take the `SendOnDrop` by value so that they can drop it on _task completion_ (after the `fn` returns). Still, this is introducing disk reads onto core executor threads, which may have a negative performance impact (thoughts appreciated).
## Additional Info
- [x] For performance it would be great to remove the cloning of full blocks when converting them to blinded blocks to write to disk. I'm going to experiment with a `put_block` API that takes the block by value, breaks it into a blinded block and a payload, stores the blinded block, and then re-assembles the full block for the caller.
- [x] We should measure the latency of blocks-by-root and blocks-by-range responses.
- [x] We should add integration tests that stress the payload reconstruction (basic tests done, issue for more extensive tests: https://github.com/sigp/lighthouse/issues/3159)
- [x] We should (manually) test the schema v9 migration from several prior versions, particularly as blocks have changed on disk and some migrations rely on being able to load blocks.
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2022-05-12 00:42:17 +00:00
|
|
|
#[serde(rename = "hash")]
|
|
|
|
pub block_hash: ExecutionBlockHash,
|
|
|
|
pub transactions: Vec<Transaction>,
|
2022-10-26 19:15:26 +00:00
|
|
|
#[superstruct(only(Capella, Eip4844))]
|
2022-12-14 00:52:46 +00:00
|
|
|
pub withdrawals: Vec<JsonWithdrawal>,
|
Separate execution payloads in the DB (#3157)
## Proposed Changes
Reduce post-merge disk usage by not storing finalized execution payloads in Lighthouse's database.
:warning: **This is achieved in a backwards-incompatible way for networks that have already merged** :warning:. Kiln users and shadow fork enjoyers will be unable to downgrade after running the code from this PR. The upgrade migration may take several minutes to run, and can't be aborted after it begins.
The main changes are:
- New column in the database called `ExecPayload`, keyed by beacon block root.
- The `BeaconBlock` column now stores blinded blocks only.
- Lots of places that previously used full blocks now use blinded blocks, e.g. analytics APIs, block replay in the DB, etc.
- On finalization:
- `prune_abanonded_forks` deletes non-canonical payloads whilst deleting non-canonical blocks.
- `migrate_db` deletes finalized canonical payloads whilst deleting finalized states.
- Conversions between blinded and full blocks are implemented in a compositional way, duplicating some work from Sean's PR #3134.
- The execution layer has a new `get_payload_by_block_hash` method that reconstructs a payload using the EE's `eth_getBlockByHash` call.
- I've tested manually that it works on Kiln, using Geth and Nethermind.
- This isn't necessarily the most efficient method, and new engine APIs are being discussed to improve this: https://github.com/ethereum/execution-apis/pull/146.
- We're depending on the `ethers` master branch, due to lots of recent changes. We're also using a workaround for https://github.com/gakonst/ethers-rs/issues/1134.
- Payload reconstruction is used in the HTTP API via `BeaconChain::get_block`, which is now `async`. Due to the `async` fn, the `blocking_json` wrapper has been removed.
- Payload reconstruction is used in network RPC to serve blocks-by-{root,range} responses. Here the `async` adjustment is messier, although I think I've managed to come up with a reasonable compromise: the handlers take the `SendOnDrop` by value so that they can drop it on _task completion_ (after the `fn` returns). Still, this is introducing disk reads onto core executor threads, which may have a negative performance impact (thoughts appreciated).
## Additional Info
- [x] For performance it would be great to remove the cloning of full blocks when converting them to blinded blocks to write to disk. I'm going to experiment with a `put_block` API that takes the block by value, breaks it into a blinded block and a payload, stores the blinded block, and then re-assembles the full block for the caller.
- [x] We should measure the latency of blocks-by-root and blocks-by-range responses.
- [x] We should add integration tests that stress the payload reconstruction (basic tests done, issue for more extensive tests: https://github.com/sigp/lighthouse/issues/3159)
- [x] We should (manually) test the schema v9 migration from several prior versions, particularly as blocks have changed on disk and some migrations rely on being able to load blocks.
Co-authored-by: Paul Hauner <paul@paulhauner.com>
2022-05-12 00:42:17 +00:00
|
|
|
}
|
|
|
|
|
2022-12-14 00:52:46 +00:00
|
|
|
impl<T: EthSpec> TryFrom<ExecutionPayload<T>> for ExecutionBlockWithTransactions<T> {
|
|
|
|
type Error = Error;
|
|
|
|
|
|
|
|
fn try_from(payload: ExecutionPayload<T>) -> Result<Self, Error> {
|
|
|
|
let json_payload = match payload {
|
2022-10-26 19:15:26 +00:00
|
|
|
ExecutionPayload::Merge(block) => Self::Merge(ExecutionBlockWithTransactionsMerge {
|
|
|
|
parent_hash: block.parent_hash,
|
|
|
|
fee_recipient: block.fee_recipient,
|
|
|
|
state_root: block.state_root,
|
|
|
|
receipts_root: block.receipts_root,
|
|
|
|
logs_bloom: block.logs_bloom,
|
|
|
|
prev_randao: block.prev_randao,
|
|
|
|
block_number: block.block_number,
|
|
|
|
gas_limit: block.gas_limit,
|
|
|
|
gas_used: block.gas_used,
|
|
|
|
timestamp: block.timestamp,
|
|
|
|
extra_data: block.extra_data,
|
|
|
|
base_fee_per_gas: block.base_fee_per_gas,
|
|
|
|
block_hash: block.block_hash,
|
|
|
|
transactions: block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.map(|tx| Transaction::decode(&Rlp::new(tx)))
|
2022-12-14 00:52:46 +00:00
|
|
|
.collect::<Result<Vec<_>, _>>()?,
|
2022-10-26 19:15:26 +00:00
|
|
|
}),
|
|
|
|
ExecutionPayload::Capella(block) => {
|
|
|
|
Self::Capella(ExecutionBlockWithTransactionsCapella {
|
|
|
|
parent_hash: block.parent_hash,
|
|
|
|
fee_recipient: block.fee_recipient,
|
|
|
|
state_root: block.state_root,
|
|
|
|
receipts_root: block.receipts_root,
|
|
|
|
logs_bloom: block.logs_bloom,
|
|
|
|
prev_randao: block.prev_randao,
|
|
|
|
block_number: block.block_number,
|
|
|
|
gas_limit: block.gas_limit,
|
|
|
|
gas_used: block.gas_used,
|
|
|
|
timestamp: block.timestamp,
|
|
|
|
extra_data: block.extra_data,
|
|
|
|
base_fee_per_gas: block.base_fee_per_gas,
|
|
|
|
block_hash: block.block_hash,
|
|
|
|
transactions: block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.map(|tx| Transaction::decode(&Rlp::new(tx)))
|
2022-12-14 00:52:46 +00:00
|
|
|
.collect::<Result<Vec<_>, _>>()?,
|
|
|
|
withdrawals: Vec::from(block.withdrawals)
|
|
|
|
.into_iter()
|
|
|
|
.map(|withdrawal| withdrawal.into())
|
|
|
|
.collect(),
|
2022-10-26 19:15:26 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
ExecutionPayload::Eip4844(block) => {
|
|
|
|
Self::Eip4844(ExecutionBlockWithTransactionsEip4844 {
|
|
|
|
parent_hash: block.parent_hash,
|
|
|
|
fee_recipient: block.fee_recipient,
|
|
|
|
state_root: block.state_root,
|
|
|
|
receipts_root: block.receipts_root,
|
|
|
|
logs_bloom: block.logs_bloom,
|
|
|
|
prev_randao: block.prev_randao,
|
|
|
|
block_number: block.block_number,
|
|
|
|
gas_limit: block.gas_limit,
|
|
|
|
gas_used: block.gas_used,
|
|
|
|
timestamp: block.timestamp,
|
|
|
|
extra_data: block.extra_data,
|
|
|
|
base_fee_per_gas: block.base_fee_per_gas,
|
2022-11-24 05:41:35 +00:00
|
|
|
excess_data_gas: block.excess_data_gas,
|
2022-10-26 19:15:26 +00:00
|
|
|
block_hash: block.block_hash,
|
|
|
|
transactions: block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.map(|tx| Transaction::decode(&Rlp::new(tx)))
|
2022-12-14 00:52:46 +00:00
|
|
|
.collect::<Result<Vec<_>, _>>()?,
|
|
|
|
withdrawals: Vec::from(block.withdrawals)
|
|
|
|
.into_iter()
|
|
|
|
.map(|withdrawal| withdrawal.into())
|
|
|
|
.collect(),
|
2022-10-26 19:15:26 +00:00
|
|
|
})
|
|
|
|
}
|
2022-12-14 00:52:46 +00:00
|
|
|
};
|
|
|
|
Ok(json_payload)
|
2022-10-26 19:15:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[superstruct(
|
|
|
|
variants(V1, V2),
|
2022-11-30 00:01:47 +00:00
|
|
|
variant_attributes(derive(Clone, Debug, Eq, Hash, PartialEq),),
|
2022-10-26 19:15:26 +00:00
|
|
|
cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
|
|
|
|
partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
|
|
|
|
)]
|
2022-11-30 00:01:47 +00:00
|
|
|
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
2021-11-15 06:13:38 +00:00
|
|
|
pub struct PayloadAttributes {
|
2022-10-26 19:15:26 +00:00
|
|
|
#[superstruct(getter(copy))]
|
2021-11-15 06:13:38 +00:00
|
|
|
pub timestamp: u64,
|
2022-10-26 19:15:26 +00:00
|
|
|
#[superstruct(getter(copy))]
|
2022-03-03 02:10:57 +00:00
|
|
|
pub prev_randao: Hash256,
|
2022-10-26 19:15:26 +00:00
|
|
|
#[superstruct(getter(copy))]
|
2021-12-12 09:04:21 +00:00
|
|
|
pub suggested_fee_recipient: Address,
|
2022-10-26 19:15:26 +00:00
|
|
|
#[superstruct(only(V2))]
|
2023-01-19 14:32:08 +00:00
|
|
|
pub withdrawals: Vec<Withdrawal>,
|
2022-11-22 18:27:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl PayloadAttributes {
|
2022-11-30 00:01:47 +00:00
|
|
|
pub fn new(
|
|
|
|
timestamp: u64,
|
|
|
|
prev_randao: Hash256,
|
|
|
|
suggested_fee_recipient: Address,
|
|
|
|
withdrawals: Option<Vec<Withdrawal>>,
|
|
|
|
) -> Self {
|
2023-01-19 14:32:08 +00:00
|
|
|
match withdrawals {
|
|
|
|
Some(withdrawals) => PayloadAttributes::V2(PayloadAttributesV2 {
|
|
|
|
timestamp,
|
|
|
|
prev_randao,
|
|
|
|
suggested_fee_recipient,
|
|
|
|
withdrawals,
|
|
|
|
}),
|
|
|
|
None => PayloadAttributes::V1(PayloadAttributesV1 {
|
|
|
|
timestamp,
|
|
|
|
prev_randao,
|
|
|
|
suggested_fee_recipient,
|
|
|
|
}),
|
2022-11-22 18:27:48 +00:00
|
|
|
}
|
|
|
|
}
|
2021-11-15 06:13:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
|
|
pub struct ForkchoiceUpdatedResponse {
|
2022-02-17 21:47:06 +00:00
|
|
|
pub payload_status: PayloadStatusV1,
|
2021-11-15 06:13:38 +00:00
|
|
|
pub payload_id: Option<PayloadId>,
|
|
|
|
}
|
2022-03-31 07:52:23 +00:00
|
|
|
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq)]
|
|
|
|
pub enum ProposeBlindedBlockResponseStatus {
|
|
|
|
Valid,
|
|
|
|
Invalid,
|
|
|
|
Syncing,
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
|
|
pub struct ProposeBlindedBlockResponse {
|
|
|
|
pub status: ProposeBlindedBlockResponseStatus,
|
|
|
|
pub latest_valid_hash: Option<Hash256>,
|
|
|
|
pub validation_error: Option<String>,
|
|
|
|
}
|
2022-11-22 18:27:48 +00:00
|
|
|
|
2023-01-19 14:32:08 +00:00
|
|
|
#[superstruct(
|
|
|
|
variants(Merge, Capella, Eip4844),
|
|
|
|
variant_attributes(derive(Clone, Debug, PartialEq),),
|
2023-02-02 01:37:46 +00:00
|
|
|
map_into(ExecutionPayload),
|
|
|
|
map_ref_into(ExecutionPayloadRef),
|
2023-01-19 14:32:08 +00:00
|
|
|
cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
|
|
|
|
partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
|
|
|
|
)]
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
|
|
pub struct GetPayloadResponse<T: EthSpec> {
|
|
|
|
#[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))]
|
|
|
|
pub execution_payload: ExecutionPayloadMerge<T>,
|
|
|
|
#[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))]
|
|
|
|
pub execution_payload: ExecutionPayloadCapella<T>,
|
|
|
|
#[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))]
|
|
|
|
pub execution_payload: ExecutionPayloadEip4844<T>,
|
|
|
|
pub block_value: Uint256,
|
|
|
|
}
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
impl<'a, T: EthSpec> From<GetPayloadResponseRef<'a, T>> for ExecutionPayloadRef<'a, T> {
|
|
|
|
fn from(response: GetPayloadResponseRef<'a, T>) -> Self {
|
|
|
|
map_get_payload_response_ref_into_execution_payload_ref!(&'a _, response, |inner, cons| {
|
|
|
|
cons(&inner.execution_payload)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<T: EthSpec> From<GetPayloadResponse<T>> for ExecutionPayload<T> {
|
|
|
|
fn from(response: GetPayloadResponse<T>) -> Self {
|
|
|
|
map_get_payload_response_into_execution_payload!(response, |inner, cons| {
|
|
|
|
cons(inner.execution_payload)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<T: EthSpec> From<GetPayloadResponse<T>> for (ExecutionPayload<T>, Uint256) {
|
|
|
|
fn from(response: GetPayloadResponse<T>) -> Self {
|
|
|
|
match response {
|
|
|
|
GetPayloadResponse::Merge(inner) => (
|
|
|
|
ExecutionPayload::Merge(inner.execution_payload),
|
|
|
|
inner.block_value,
|
|
|
|
),
|
|
|
|
GetPayloadResponse::Capella(inner) => (
|
|
|
|
ExecutionPayload::Capella(inner.execution_payload),
|
|
|
|
inner.block_value,
|
|
|
|
),
|
|
|
|
GetPayloadResponse::Eip4844(inner) => (
|
|
|
|
ExecutionPayload::Eip4844(inner.execution_payload),
|
|
|
|
inner.block_value,
|
|
|
|
),
|
2023-01-19 14:32:08 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-02 01:37:46 +00:00
|
|
|
impl<T: EthSpec> GetPayloadResponse<T> {
|
|
|
|
pub fn execution_payload_ref(&self) -> ExecutionPayloadRef<T> {
|
|
|
|
self.to_ref().into()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-01-12 04:15:08 +00:00
|
|
|
#[derive(Clone, Copy, Debug)]
|
2023-01-31 17:26:23 +00:00
|
|
|
pub struct EngineCapabilities {
|
2022-11-22 18:27:48 +00:00
|
|
|
pub new_payload_v1: bool,
|
|
|
|
pub new_payload_v2: bool,
|
2023-01-19 14:32:08 +00:00
|
|
|
pub new_payload_v3: bool,
|
2022-11-22 18:27:48 +00:00
|
|
|
pub forkchoice_updated_v1: bool,
|
|
|
|
pub forkchoice_updated_v2: bool,
|
|
|
|
pub get_payload_v1: bool,
|
|
|
|
pub get_payload_v2: bool,
|
2023-01-19 14:32:08 +00:00
|
|
|
pub get_payload_v3: bool,
|
2022-11-22 18:27:48 +00:00
|
|
|
pub exchange_transition_configuration_v1: bool,
|
|
|
|
}
|
2023-01-31 17:26:23 +00:00
|
|
|
|
|
|
|
impl EngineCapabilities {
|
|
|
|
pub fn to_response(&self) -> Vec<&str> {
|
|
|
|
let mut response = Vec::new();
|
|
|
|
if self.new_payload_v1 {
|
|
|
|
response.push(ENGINE_NEW_PAYLOAD_V1);
|
|
|
|
}
|
|
|
|
if self.new_payload_v2 {
|
|
|
|
response.push(ENGINE_NEW_PAYLOAD_V2);
|
|
|
|
}
|
|
|
|
if self.forkchoice_updated_v1 {
|
|
|
|
response.push(ENGINE_FORKCHOICE_UPDATED_V1);
|
|
|
|
}
|
|
|
|
if self.forkchoice_updated_v2 {
|
|
|
|
response.push(ENGINE_FORKCHOICE_UPDATED_V2);
|
|
|
|
}
|
|
|
|
if self.get_payload_v1 {
|
|
|
|
response.push(ENGINE_GET_PAYLOAD_V1);
|
|
|
|
}
|
|
|
|
if self.get_payload_v2 {
|
|
|
|
response.push(ENGINE_GET_PAYLOAD_V2);
|
|
|
|
}
|
|
|
|
if self.exchange_transition_configuration_v1 {
|
|
|
|
response.push(ENGINE_EXCHANGE_TRANSITION_CONFIGURATION_V1);
|
|
|
|
}
|
|
|
|
|
|
|
|
response
|
|
|
|
}
|
|
|
|
}
|