2019-02-14 01:09:18 +00:00
|
|
|
[package]
|
|
|
|
|
name = "types"
|
2022-10-30 04:04:24 +00:00
|
|
|
version = "0.2.1"
|
2019-02-14 05:46:33 +00:00
|
|
|
authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
|
2022-02-25 00:10:17 +00:00
|
|
|
edition = "2021"
|
2019-02-14 01:09:18 +00:00
|
|
|
|
2019-12-06 05:44:03 +00:00
|
|
|
[[bench]]
|
|
|
|
|
name = "benches"
|
|
|
|
|
harness = false
|
|
|
|
|
|
2019-02-14 01:09:18 +00:00
|
|
|
[dependencies]
|
2022-02-19 01:29:05 +00:00
|
|
|
serde-big-array = {version = "0.3.2", features = ["const-generics"]}
|
2020-05-18 11:24:23 +00:00
|
|
|
bls = { path = "../../crypto/bls" }
|
|
|
|
|
compare_fields = { path = "../../common/compare_fields" }
|
|
|
|
|
compare_fields_derive = { path = "../../common/compare_fields_derive" }
|
|
|
|
|
eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" }
|
2021-10-14 02:58:11 +00:00
|
|
|
ethereum-types = "0.12.1"
|
2022-04-04 00:26:16 +00:00
|
|
|
eth2_hashing = "0.3.0"
|
2020-05-17 11:16:48 +00:00
|
|
|
hex = "0.4.2"
|
2020-05-18 11:24:23 +00:00
|
|
|
int_to_bytes = { path = "../int_to_bytes" }
|
2020-10-05 08:22:19 +00:00
|
|
|
log = "0.4.11"
|
|
|
|
|
rayon = "1.4.1"
|
2022-04-04 00:26:16 +00:00
|
|
|
rand = "0.8.5"
|
2020-05-18 11:24:23 +00:00
|
|
|
safe_arith = { path = "../safe_arith" }
|
2021-07-15 00:52:02 +00:00
|
|
|
serde = {version = "1.0.116" , features = ["rc"] }
|
2020-10-05 08:22:19 +00:00
|
|
|
serde_derive = "1.0.116"
|
2019-10-30 01:22:18 +00:00
|
|
|
slog = "2.5.2"
|
2021-11-29 03:57:54 +00:00
|
|
|
eth2_ssz = "0.4.1"
|
Implement SSZ union type (#2579)
## Issue Addressed
NA
## Proposed Changes
Implements the "union" type from the SSZ spec for `ssz`, `ssz_derive`, `tree_hash` and `tree_hash_derive` so it may be derived for `enums`:
https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union
The union type is required for the merge, since the `Transaction` type is defined as a single-variant union `Union[OpaqueTransaction]`.
### Crate Updates
This PR will (hopefully) cause CI to publish new versions for the following crates:
- `eth2_ssz_derive`: `0.2.1` -> `0.3.0`
- `eth2_ssz`: `0.3.0` -> `0.4.0`
- `eth2_ssz_types`: `0.2.0` -> `0.2.1`
- `tree_hash`: `0.3.0` -> `0.4.0`
- `tree_hash_derive`: `0.3.0` -> `0.4.0`
These these crates depend on each other, I've had to add a workspace-level `[patch]` for these crates. A follow-up PR will need to remove this patch, ones the new versions are published.
### Union Behaviors
We already had SSZ `Encode` and `TreeHash` derive for enums, however it just did a "transparent" pass-through of the inner value. Since the "union" decoding from the spec is in conflict with the transparent method, I've required that all `enum` have exactly one of the following enum-level attributes:
#### SSZ
- `#[ssz(enum_behaviour = "union")]`
- matches the spec used for the merge
- `#[ssz(enum_behaviour = "transparent")]`
- maintains existing functionality
- not supported for `Decode` (never was)
#### TreeHash
- `#[tree_hash(enum_behaviour = "union")]`
- matches the spec used for the merge
- `#[tree_hash(enum_behaviour = "transparent")]`
- maintains existing functionality
This means that we can maintain the existing transparent behaviour, but all existing users will get a compile-time error until they explicitly opt-in to being transparent.
### Legacy Option Encoding
Before this PR, we already had a union-esque encoding for `Option<T>`. However, this was with the *old* SSZ spec where the union selector was 4 bytes. During merge specification, the spec was changed to use 1 byte for the selector.
Whilst the 4-byte `Option` encoding was never used in the spec, we used it in our database. Writing a migrate script for all occurrences of `Option` in the database would be painful, especially since it's used in the `CommitteeCache`. To avoid the migrate script, I added a serde-esque `#[ssz(with = "module")]` field-level attribute to `ssz_derive` so that we can opt into the 4-byte encoding on a field-by-field basis.
The `ssz::legacy::four_byte_impl!` macro allows a one-liner to define the module required for the `#[ssz(with = "module")]` for some `Option<T> where T: Encode + Decode`.
Notably, **I have removed `Encode` and `Decode` impls for `Option`**. I've done this to force a break on downstream users. Like I mentioned, `Option` isn't used in the spec so I don't think it'll be *that* annoying. I think it's nicer than quietly having two different union implementations or quietly breaking the existing `Option` impl.
### Crate Publish Ordering
I've modified the order in which CI publishes crates to ensure that we don't publish a crate without ensuring we already published a crate that it depends upon.
## TODO
- [ ] Queue a follow-up `[patch]`-removing PR.
2021-09-25 05:58:36 +00:00
|
|
|
eth2_ssz_derive = "0.3.0"
|
2021-11-29 03:57:54 +00:00
|
|
|
eth2_ssz_types = "0.2.2"
|
2020-05-18 11:24:23 +00:00
|
|
|
swap_or_not_shuffle = { path = "../swap_or_not_shuffle" }
|
|
|
|
|
test_random_derive = { path = "../../common/test_random_derive" }
|
2021-11-29 03:57:54 +00:00
|
|
|
tree_hash = "0.4.1"
|
Implement SSZ union type (#2579)
## Issue Addressed
NA
## Proposed Changes
Implements the "union" type from the SSZ spec for `ssz`, `ssz_derive`, `tree_hash` and `tree_hash_derive` so it may be derived for `enums`:
https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union
The union type is required for the merge, since the `Transaction` type is defined as a single-variant union `Union[OpaqueTransaction]`.
### Crate Updates
This PR will (hopefully) cause CI to publish new versions for the following crates:
- `eth2_ssz_derive`: `0.2.1` -> `0.3.0`
- `eth2_ssz`: `0.3.0` -> `0.4.0`
- `eth2_ssz_types`: `0.2.0` -> `0.2.1`
- `tree_hash`: `0.3.0` -> `0.4.0`
- `tree_hash_derive`: `0.3.0` -> `0.4.0`
These these crates depend on each other, I've had to add a workspace-level `[patch]` for these crates. A follow-up PR will need to remove this patch, ones the new versions are published.
### Union Behaviors
We already had SSZ `Encode` and `TreeHash` derive for enums, however it just did a "transparent" pass-through of the inner value. Since the "union" decoding from the spec is in conflict with the transparent method, I've required that all `enum` have exactly one of the following enum-level attributes:
#### SSZ
- `#[ssz(enum_behaviour = "union")]`
- matches the spec used for the merge
- `#[ssz(enum_behaviour = "transparent")]`
- maintains existing functionality
- not supported for `Decode` (never was)
#### TreeHash
- `#[tree_hash(enum_behaviour = "union")]`
- matches the spec used for the merge
- `#[tree_hash(enum_behaviour = "transparent")]`
- maintains existing functionality
This means that we can maintain the existing transparent behaviour, but all existing users will get a compile-time error until they explicitly opt-in to being transparent.
### Legacy Option Encoding
Before this PR, we already had a union-esque encoding for `Option<T>`. However, this was with the *old* SSZ spec where the union selector was 4 bytes. During merge specification, the spec was changed to use 1 byte for the selector.
Whilst the 4-byte `Option` encoding was never used in the spec, we used it in our database. Writing a migrate script for all occurrences of `Option` in the database would be painful, especially since it's used in the `CommitteeCache`. To avoid the migrate script, I added a serde-esque `#[ssz(with = "module")]` field-level attribute to `ssz_derive` so that we can opt into the 4-byte encoding on a field-by-field basis.
The `ssz::legacy::four_byte_impl!` macro allows a one-liner to define the module required for the `#[ssz(with = "module")]` for some `Option<T> where T: Encode + Decode`.
Notably, **I have removed `Encode` and `Decode` impls for `Option`**. I've done this to force a break on downstream users. Like I mentioned, `Option` isn't used in the spec so I don't think it'll be *that* annoying. I think it's nicer than quietly having two different union implementations or quietly breaking the existing `Option` impl.
### Crate Publish Ordering
I've modified the order in which CI publishes crates to ensure that we don't publish a crate without ensuring we already published a crate that it depends upon.
## TODO
- [ ] Queue a follow-up `[patch]`-removing PR.
2021-09-25 05:58:36 +00:00
|
|
|
tree_hash_derive = "0.4.0"
|
2022-04-04 00:26:16 +00:00
|
|
|
rand_xorshift = "0.3.0"
|
2020-05-18 11:24:23 +00:00
|
|
|
cached_tree_hash = { path = "../cached_tree_hash" }
|
2020-10-05 08:22:19 +00:00
|
|
|
serde_yaml = "0.8.13"
|
2019-11-28 02:20:16 +00:00
|
|
|
tempfile = "3.1.0"
|
2020-06-18 11:06:34 +00:00
|
|
|
derivative = "2.1.1"
|
2021-08-17 01:00:24 +00:00
|
|
|
rusqlite = { version = "0.25.3", features = ["bundled"], optional = true }
|
2021-10-14 02:58:11 +00:00
|
|
|
arbitrary = { version = "1.0", features = ["derive"], optional = true }
|
2021-11-29 22:32:53 +00:00
|
|
|
eth2_serde_utils = "0.1.1"
|
2022-03-08 19:48:12 +00:00
|
|
|
regex = "1.5.5"
|
2020-11-18 23:31:37 +00:00
|
|
|
lazy_static = "1.4.0"
|
2022-04-04 00:26:16 +00:00
|
|
|
parking_lot = "0.12.0"
|
2021-07-09 06:15:32 +00:00
|
|
|
itertools = "0.10.0"
|
2022-10-13 14:37:20 +00:00
|
|
|
superstruct = "0.6.0"
|
2022-01-20 09:14:21 +00:00
|
|
|
serde_json = "1.0.74"
|
2022-02-17 23:55:04 +00:00
|
|
|
smallvec = "1.8.0"
|
2022-07-01 01:15:19 +00:00
|
|
|
serde_with = "1.13.0"
|
2022-08-10 07:52:59 +00:00
|
|
|
maplit = "1.0.2"
|
2019-02-15 05:12:24 +00:00
|
|
|
|
|
|
|
|
[dev-dependencies]
|
2020-10-05 08:22:19 +00:00
|
|
|
criterion = "0.3.3"
|
2021-07-09 06:15:32 +00:00
|
|
|
beacon_chain = { path = "../../beacon_node/beacon_chain" }
|
2021-08-06 00:47:31 +00:00
|
|
|
eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" }
|
2021-09-22 00:37:28 +00:00
|
|
|
state_processing = { path = "../state_processing" }
|
Use async code when interacting with EL (#3244)
## Overview
This rather extensive PR achieves two primary goals:
1. Uses the finalized/justified checkpoints of fork choice (FC), rather than that of the head state.
2. Refactors fork choice, block production and block processing to `async` functions.
Additionally, it achieves:
- Concurrent forkchoice updates to the EL and cache pruning after a new head is selected.
- Concurrent "block packing" (attestations, etc) and execution payload retrieval during block production.
- Concurrent per-block-processing and execution payload verification during block processing.
- The `Arc`-ification of `SignedBeaconBlock` during block processing (it's never mutated, so why not?):
- I had to do this to deal with sending blocks into spawned tasks.
- Previously we were cloning the beacon block at least 2 times during each block processing, these clones are either removed or turned into cheaper `Arc` clones.
- We were also `Box`-ing and un-`Box`-ing beacon blocks as they moved throughout the networking crate. This is not a big deal, but it's nice to avoid shifting things between the stack and heap.
- Avoids cloning *all the blocks* in *every chain segment* during sync.
- It also has the potential to clean up our code where we need to pass an *owned* block around so we can send it back in the case of an error (I didn't do much of this, my PR is already big enough :sweat_smile:)
- The `BeaconChain::HeadSafetyStatus` struct was removed. It was an old relic from prior merge specs.
For motivation for this change, see https://github.com/sigp/lighthouse/pull/3244#issuecomment-1160963273
## Changes to `canonical_head` and `fork_choice`
Previously, the `BeaconChain` had two separate fields:
```
canonical_head: RwLock<Snapshot>,
fork_choice: RwLock<BeaconForkChoice>
```
Now, we have grouped these values under a single struct:
```
canonical_head: CanonicalHead {
cached_head: RwLock<Arc<Snapshot>>,
fork_choice: RwLock<BeaconForkChoice>
}
```
Apart from ergonomics, the only *actual* change here is wrapping the canonical head snapshot in an `Arc`. This means that we no longer need to hold the `cached_head` (`canonical_head`, in old terms) lock when we want to pull some values from it. This was done to avoid deadlock risks by preventing functions from acquiring (and holding) the `cached_head` and `fork_choice` locks simultaneously.
## Breaking Changes
### The `state` (root) field in the `finalized_checkpoint` SSE event
Consider the scenario where epoch `n` is just finalized, but `start_slot(n)` is skipped. There are two state roots we might in the `finalized_checkpoint` SSE event:
1. The state root of the finalized block, which is `get_block(finalized_checkpoint.root).state_root`.
4. The state root at slot of `start_slot(n)`, which would be the state from (1), but "skipped forward" through any skip slots.
Previously, Lighthouse would choose (2). However, we can see that when [Teku generates that event](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java#L171-L182) it uses [`getStateRootFromBlockRoot`](https://github.com/ConsenSys/teku/blob/de2b2801c89ef5abf983d6bf37867c37fc47121f/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java#L336-L341) which uses (1).
I have switched Lighthouse from (2) to (1). I think it's a somewhat arbitrary choice between the two, where (1) is easier to compute and is consistent with Teku.
## Notes for Reviewers
I've renamed `BeaconChain::fork_choice` to `BeaconChain::recompute_head`. Doing this helped ensure I broke all previous uses of fork choice and I also find it more descriptive. It describes an action and can't be confused with trying to get a reference to the `ForkChoice` struct.
I've changed the ordering of SSE events when a block is received. It used to be `[block, finalized, head]` and now it's `[block, head, finalized]`. It was easier this way and I don't think we were making any promises about SSE event ordering so it's not "breaking".
I've made it so fork choice will run when it's first constructed. I did this because I wanted to have a cached version of the last call to `get_head`. Ensuring `get_head` has been run *at least once* means that the cached values doesn't need to wrapped in an `Option`. This was fairly simple, it just involved passing a `slot` to the constructor so it knows *when* it's being run. When loading a fork choice from the store and a slot clock isn't handy I've just used the `slot` that was saved in the `fork_choice_store`. That seems like it would be a faithful representation of the slot when we saved it.
I added the `genesis_time: u64` to the `BeaconChain`. It's small, constant and nice to have around.
Since we're using FC for the fin/just checkpoints, we no longer get the `0x00..00` roots at genesis. You can see I had to remove a work-around in `ef-tests` here: b56be3bc2. I can't find any reason why this would be an issue, if anything I think it'll be better since the genesis-alias has caught us out a few times (0x00..00 isn't actually a real root). Edit: I did find a case where the `network` expected the 0x00..00 alias and patched it here: 3f26ac3e2.
You'll notice a lot of changes in tests. Generally, tests should be functionally equivalent. Here are the things creating the most diff-noise in tests:
- Changing tests to be `tokio::async` tests.
- Adding `.await` to fork choice, block processing and block production functions.
- Refactor of the `canonical_head` "API" provided by the `BeaconChain`. E.g., `chain.canonical_head.cached_head()` instead of `chain.canonical_head.read()`.
- Wrapping `SignedBeaconBlock` in an `Arc`.
- In the `beacon_chain/tests/block_verification`, we can't use the `lazy_static` `CHAIN_SEGMENT` variable anymore since it's generated with an async function. We just generate it in each test, not so efficient but hopefully insignificant.
I had to disable `rayon` concurrent tests in the `fork_choice` tests. This is because the use of `rayon` and `block_on` was causing a panic.
Co-authored-by: Mac L <mjladson@pm.me>
2022-07-03 05:36:50 +00:00
|
|
|
tokio = "1.14.0"
|
2020-05-05 23:12:28 +00:00
|
|
|
|
|
|
|
|
[features]
|
Remove saturating arith from state_processing (#1644)
## Issue Addressed
Resolves #1100
## Proposed Changes
* Implement the `SafeArith` trait for `Slot` and `Epoch`, so that methods like `safe_add` become available.
* Tweak the `SafeArith` trait to allow a different `Rhs` type (analagous to `std::ops::Add`, etc).
* Add a `legacy-arith` feature to `types` and `state_processing` that conditionally enables implementations of
the `std` ops with saturating semantics.
* Check compilation of `types` and `state_processing` _without_ `legacy-arith` on CI,
thus guaranteeing that they only use the `SafeArith` primitives :tada:
## Additional Info
The `legacy-arith` feature gets turned on by all higher-level crates that depend on `state_processing` or `types`, thus allowing the beacon chain, networking, and other components to continue to rely on the availability of ops like `+`, `-`, `*`, etc.
**This is a consensus-breaking change**, but brings us in line with the spec, and our incompatibilities shouldn't have been reachable with any valid configuration of Eth2 parameters.
2020-09-25 05:18:21 +00:00
|
|
|
default = ["sqlite", "legacy-arith"]
|
|
|
|
|
# Allow saturating arithmetic on slots and epochs. Enabled by default, but deprecated.
|
|
|
|
|
legacy-arith = []
|
2020-05-18 06:25:16 +00:00
|
|
|
sqlite = ["rusqlite"]
|
2020-05-05 23:12:28 +00:00
|
|
|
arbitrary-fuzz = [
|
|
|
|
|
"arbitrary",
|
|
|
|
|
"ethereum-types/arbitrary",
|
|
|
|
|
"bls/arbitrary",
|
|
|
|
|
"eth2_ssz/arbitrary",
|
|
|
|
|
"eth2_ssz_types/arbitrary",
|
|
|
|
|
"swap_or_not_shuffle/arbitrary",
|
|
|
|
|
"tree_hash/arbitrary",
|
|
|
|
|
]
|
