lighthouse/consensus/ssz/tests/tests.rs

533 lines
14 KiB
Rust
Raw Normal View History

2019-05-06 01:36:00 +00:00
use ethereum_types::H256;
use ssz::{Decode, DecodeError, Encode};
use ssz_derive::{Decode, Encode};
2019-05-03 01:05:51 +00:00
2019-05-06 01:36:00 +00:00
mod round_trip {
use super::*;
use std::collections::BTreeMap;
use std::iter::FromIterator;
2019-05-06 01:36:00 +00:00
fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(items: Vec<T>) {
2019-05-06 01:36:00 +00:00
for item in items {
let encoded = &item.as_ssz_bytes();
assert_eq!(item.ssz_bytes_len(), encoded.len());
assert_eq!(T::from_ssz_bytes(encoded), Ok(item));
2019-05-06 01:36:00 +00:00
}
2019-05-04 08:06:01 +00:00
}
2019-05-03 01:05:51 +00:00
2019-05-06 01:36:00 +00:00
#[test]
fn bool() {
let items: Vec<bool> = vec![true, false];
2019-05-03 01:05:51 +00:00
2019-05-06 01:36:00 +00:00
round_trip(items);
}
2019-05-04 08:06:01 +00:00
Deposit Cache Finalization & Fast WS Sync (#2915) ## Summary The deposit cache now has the ability to finalize deposits. This will cause it to drop unneeded deposit logs and hashes in the deposit Merkle tree that are no longer required to construct deposit proofs. The cache is finalized whenever the latest finalized checkpoint has a new `Eth1Data` with all deposits imported. This has three benefits: 1. Improves the speed of constructing Merkle proofs for deposits as we can just replay deposits since the last finalized checkpoint instead of all historical deposits when re-constructing the Merkle tree. 2. Significantly faster weak subjectivity sync as the deposit cache can be transferred to the newly syncing node in compressed form. The Merkle tree that stores `N` finalized deposits requires a maximum of `log2(N)` hashes. The newly syncing node then only needs to download deposits since the last finalized checkpoint to have a full tree. 3. Future proofing in preparation for [EIP-4444](https://eips.ethereum.org/EIPS/eip-4444) as execution nodes will no longer be required to store logs permanently so we won't always have all historical logs available to us. ## More Details Image to illustrate how the deposit contract merkle tree evolves and finalizes along with the resulting `DepositTreeSnapshot` ![image](https://user-images.githubusercontent.com/37123614/151465302-5fc56284-8a69-4998-b20e-45db3934ac70.png) ## Other Considerations I've changed the structure of the `SszDepositCache` so once you load & save your database from this version of lighthouse, you will no longer be able to load it from older versions. Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>
2022-10-30 04:04:24 +00:00
#[test]
fn option_u16() {
let items: Vec<Option<u16>> = vec![None, Some(2u16)];
round_trip(items);
}
2019-05-06 01:36:00 +00:00
#[test]
fn u8_array_4() {
let items: Vec<[u8; 4]> = vec![[0, 0, 0, 0], [1, 0, 0, 0], [1, 2, 3, 4], [1, 2, 0, 4]];
2019-05-04 08:06:01 +00:00
2019-05-06 01:36:00 +00:00
round_trip(items);
2019-05-04 08:06:01 +00:00
}
2019-05-06 01:36:00 +00:00
#[test]
fn h256() {
let items: Vec<H256> = vec![H256::zero(), H256::from([1; 32]), H256::random()];
2019-05-06 01:36:00 +00:00
round_trip(items);
}
2019-05-06 01:36:00 +00:00
#[test]
fn vec_of_h256() {
let items: Vec<Vec<H256>> = vec![
vec![],
vec![H256::zero(), H256::from([1; 32]), H256::random()],
];
2019-05-04 08:06:01 +00:00
2019-05-06 01:36:00 +00:00
round_trip(items);
}
Deposit Cache Finalization & Fast WS Sync (#2915) ## Summary The deposit cache now has the ability to finalize deposits. This will cause it to drop unneeded deposit logs and hashes in the deposit Merkle tree that are no longer required to construct deposit proofs. The cache is finalized whenever the latest finalized checkpoint has a new `Eth1Data` with all deposits imported. This has three benefits: 1. Improves the speed of constructing Merkle proofs for deposits as we can just replay deposits since the last finalized checkpoint instead of all historical deposits when re-constructing the Merkle tree. 2. Significantly faster weak subjectivity sync as the deposit cache can be transferred to the newly syncing node in compressed form. The Merkle tree that stores `N` finalized deposits requires a maximum of `log2(N)` hashes. The newly syncing node then only needs to download deposits since the last finalized checkpoint to have a full tree. 3. Future proofing in preparation for [EIP-4444](https://eips.ethereum.org/EIPS/eip-4444) as execution nodes will no longer be required to store logs permanently so we won't always have all historical logs available to us. ## More Details Image to illustrate how the deposit contract merkle tree evolves and finalizes along with the resulting `DepositTreeSnapshot` ![image](https://user-images.githubusercontent.com/37123614/151465302-5fc56284-8a69-4998-b20e-45db3934ac70.png) ## Other Considerations I've changed the structure of the `SszDepositCache` so once you load & save your database from this version of lighthouse, you will no longer be able to load it from older versions. Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>
2022-10-30 04:04:24 +00:00
#[test]
fn option_vec_h256() {
let items: Vec<Option<Vec<H256>>> = vec![
None,
Some(vec![]),
Some(vec![H256::zero(), H256::from([1; 32]), H256::random()]),
];
round_trip(items);
}
2019-05-06 01:36:00 +00:00
#[test]
fn vec_u16() {
let items: Vec<Vec<u16>> = vec![
vec![],
vec![255],
vec![0, 1, 2],
vec![100; 64],
vec![255, 0, 255],
];
round_trip(items);
}
#[test]
fn vec_of_vec_u16() {
let items: Vec<Vec<Vec<u16>>> = vec![
vec![],
vec![vec![]],
vec![vec![1, 2, 3]],
vec![vec![], vec![]],
vec![vec![], vec![1, 2, 3]],
vec![vec![1, 2, 3], vec![1, 2, 3]],
vec![vec![1, 2, 3], vec![], vec![1, 2, 3]],
vec![vec![], vec![], vec![1, 2, 3]],
vec![vec![], vec![1], vec![1, 2, 3]],
vec![vec![], vec![1], vec![1, 2, 3]],
];
round_trip(items);
}
#[derive(Debug, PartialEq, Encode, Decode)]
struct FixedLen {
a: u16,
b: u64,
c: u32,
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
2019-05-06 01:36:00 +00:00
fn fixed_len_struct_encoding() {
let items: Vec<FixedLen> = vec![
FixedLen { a: 0, b: 0, c: 0 },
FixedLen { a: 1, b: 1, c: 1 },
FixedLen { a: 1, b: 0, c: 1 },
];
let expected_encodings = vec![
// | u16--| u64----------------------------| u32----------|
vec![00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00],
vec![01, 00, 01, 00, 00, 00, 00, 00, 00, 00, 01, 00, 00, 00],
vec![01, 00, 00, 00, 00, 00, 00, 00, 00, 00, 01, 00, 00, 00],
];
for i in 0..items.len() {
assert_eq!(
items[i].as_ssz_bytes(),
expected_encodings[i],
"Failed on {}",
i
);
}
}
2019-05-13 03:51:58 +00:00
#[test]
fn fixed_len_excess_bytes() {
let fixed = FixedLen { a: 1, b: 2, c: 3 };
let mut bytes = fixed.as_ssz_bytes();
bytes.append(&mut vec![0]);
assert_eq!(
FixedLen::from_ssz_bytes(&bytes),
Err(DecodeError::InvalidByteLength {
len: 15,
expected: 14,
})
);
}
2019-05-06 01:36:00 +00:00
#[test]
fn vec_of_fixed_len_struct() {
let items: Vec<FixedLen> = vec![
FixedLen { a: 0, b: 0, c: 0 },
FixedLen { a: 1, b: 1, c: 1 },
FixedLen { a: 1, b: 0, c: 1 },
];
round_trip(items);
}
#[derive(Debug, PartialEq, Encode, Decode)]
struct VariableLen {
a: u16,
b: Vec<u16>,
c: u32,
}
2019-05-11 09:09:27 +00:00
#[test]
#[allow(clippy::zero_prefixed_literal)]
2019-05-11 09:09:27 +00:00
fn offset_into_fixed_bytes() {
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// | offset | u32 | variable
01, 00, 09, 00, 00, 00, 01, 00, 00, 00, 00, 00, 01, 00, 02, 00,
];
assert_eq!(
VariableLen::from_ssz_bytes(&bytes),
Err(DecodeError::OffsetIntoFixedPortion(9))
2019-05-11 09:09:27 +00:00
);
}
2019-05-13 03:51:58 +00:00
#[test]
fn variable_len_excess_bytes() {
let variable = VariableLen {
a: 1,
b: vec![2],
c: 3,
};
let mut bytes = variable.as_ssz_bytes();
bytes.append(&mut vec![0]);
// The error message triggered is not so helpful, it's caught by a side-effect. Just
// checking there is _some_ error is fine.
assert!(VariableLen::from_ssz_bytes(&bytes).is_err());
}
2019-05-11 09:09:27 +00:00
#[test]
#[allow(clippy::zero_prefixed_literal)]
2019-05-11 09:09:27 +00:00
fn first_offset_skips_byte() {
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// | offset | u32 | variable
01, 00, 11, 00, 00, 00, 01, 00, 00, 00, 00, 00, 01, 00, 02, 00,
];
assert_eq!(
VariableLen::from_ssz_bytes(&bytes),
Err(DecodeError::OffsetSkipsVariableBytes(11))
2019-05-11 09:09:27 +00:00
);
}
2019-05-06 01:36:00 +00:00
#[test]
#[allow(clippy::zero_prefixed_literal)]
2019-05-06 01:36:00 +00:00
fn variable_len_struct_encoding() {
let items: Vec<VariableLen> = vec![
VariableLen {
a: 0,
b: vec![],
c: 0,
},
VariableLen {
a: 1,
b: vec![0],
c: 1,
},
VariableLen {
a: 1,
b: vec![0, 1, 2],
c: 1,
},
];
let expected_encodings = vec![
// 00..................................09
// | u16--| vec offset-----| u32------------| vec payload --------|
vec![00, 00, 10, 00, 00, 00, 00, 00, 00, 00],
vec![01, 00, 10, 00, 00, 00, 01, 00, 00, 00, 00, 00],
vec![
01, 00, 10, 00, 00, 00, 01, 00, 00, 00, 00, 00, 01, 00, 02, 00,
],
];
for i in 0..items.len() {
assert_eq!(
items[i].as_ssz_bytes(),
expected_encodings[i],
"Failed on {}",
i
);
}
}
#[test]
fn vec_of_variable_len_struct() {
let items: Vec<VariableLen> = vec![
VariableLen {
a: 0,
b: vec![],
c: 0,
},
VariableLen {
a: 255,
b: vec![0, 1, 2, 3],
c: 99,
},
VariableLen {
a: 255,
b: vec![0],
c: 99,
},
VariableLen {
a: 50,
b: vec![0],
c: 0,
},
];
round_trip(items);
2019-05-04 08:06:01 +00:00
}
2019-05-11 09:09:27 +00:00
#[derive(Debug, PartialEq, Encode, Decode)]
struct ThreeVariableLen {
a: u16,
b: Vec<u16>,
c: Vec<u16>,
d: Vec<u16>,
}
2019-05-11 12:25:28 +00:00
#[test]
fn three_variable_len() {
let vec: Vec<ThreeVariableLen> = vec![ThreeVariableLen {
a: 42,
b: vec![0],
c: vec![1],
d: vec![2],
}];
round_trip(vec);
}
2019-05-11 09:09:27 +00:00
#[test]
#[allow(clippy::zero_prefixed_literal)]
2019-05-11 09:09:27 +00:00
fn offsets_decreasing() {
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// | offset | offset | offset | variable
2019-05-11 09:09:27 +00:00
01, 00, 14, 00, 00, 00, 15, 00, 00, 00, 14, 00, 00, 00, 00, 00,
];
assert_eq!(
ThreeVariableLen::from_ssz_bytes(&bytes),
Err(DecodeError::OffsetsAreDecreasing(14))
2019-05-11 09:09:27 +00:00
);
}
#[test]
fn tuple_u8_u16() {
let vec: Vec<(u8, u16)> = vec![
(0, 0),
(0, 1),
(1, 0),
(u8::max_value(), u16::max_value()),
(0, u16::max_value()),
(u8::max_value(), 0),
(42, 12301),
];
round_trip(vec);
}
#[test]
fn tuple_vec_vec() {
let vec: Vec<(u64, Vec<u8>, Vec<Vec<u16>>)> = vec![
(0, vec![], vec![vec![]]),
(99, vec![101], vec![vec![], vec![]]),
(
42,
vec![12, 13, 14],
vec![vec![99, 98, 97, 96], vec![42, 44, 46, 48, 50]],
),
];
round_trip(vec);
}
#[test]
fn btree_map_fixed() {
let data = vec![
BTreeMap::new(),
BTreeMap::from_iter(vec![(0u8, 0u16), (1, 2), (2, 4), (4, 6)]),
];
round_trip(data);
}
#[test]
fn btree_map_variable_value() {
let data = vec![
BTreeMap::new(),
BTreeMap::from_iter(vec![
(
0u64,
ThreeVariableLen {
a: 1,
b: vec![3, 5, 7],
c: vec![],
d: vec![0, 0],
},
),
(
1,
ThreeVariableLen {
a: 99,
b: vec![1],
c: vec![2, 3, 4, 5, 6, 7, 8, 9, 10],
d: vec![4, 5, 6, 7, 8],
},
),
(
2,
ThreeVariableLen {
a: 0,
b: vec![],
c: vec![],
d: vec![],
},
),
]),
];
round_trip(data);
}
2019-05-03 01:05:51 +00:00
}
Implement SSZ union type (#2579) ## Issue Addressed NA ## Proposed Changes Implements the "union" type from the SSZ spec for `ssz`, `ssz_derive`, `tree_hash` and `tree_hash_derive` so it may be derived for `enums`: https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union The union type is required for the merge, since the `Transaction` type is defined as a single-variant union `Union[OpaqueTransaction]`. ### Crate Updates This PR will (hopefully) cause CI to publish new versions for the following crates: - `eth2_ssz_derive`: `0.2.1` -> `0.3.0` - `eth2_ssz`: `0.3.0` -> `0.4.0` - `eth2_ssz_types`: `0.2.0` -> `0.2.1` - `tree_hash`: `0.3.0` -> `0.4.0` - `tree_hash_derive`: `0.3.0` -> `0.4.0` These these crates depend on each other, I've had to add a workspace-level `[patch]` for these crates. A follow-up PR will need to remove this patch, ones the new versions are published. ### Union Behaviors We already had SSZ `Encode` and `TreeHash` derive for enums, however it just did a "transparent" pass-through of the inner value. Since the "union" decoding from the spec is in conflict with the transparent method, I've required that all `enum` have exactly one of the following enum-level attributes: #### SSZ - `#[ssz(enum_behaviour = "union")]` - matches the spec used for the merge - `#[ssz(enum_behaviour = "transparent")]` - maintains existing functionality - not supported for `Decode` (never was) #### TreeHash - `#[tree_hash(enum_behaviour = "union")]` - matches the spec used for the merge - `#[tree_hash(enum_behaviour = "transparent")]` - maintains existing functionality This means that we can maintain the existing transparent behaviour, but all existing users will get a compile-time error until they explicitly opt-in to being transparent. ### Legacy Option Encoding Before this PR, we already had a union-esque encoding for `Option<T>`. However, this was with the *old* SSZ spec where the union selector was 4 bytes. During merge specification, the spec was changed to use 1 byte for the selector. Whilst the 4-byte `Option` encoding was never used in the spec, we used it in our database. Writing a migrate script for all occurrences of `Option` in the database would be painful, especially since it's used in the `CommitteeCache`. To avoid the migrate script, I added a serde-esque `#[ssz(with = "module")]` field-level attribute to `ssz_derive` so that we can opt into the 4-byte encoding on a field-by-field basis. The `ssz::legacy::four_byte_impl!` macro allows a one-liner to define the module required for the `#[ssz(with = "module")]` for some `Option<T> where T: Encode + Decode`. Notably, **I have removed `Encode` and `Decode` impls for `Option`**. I've done this to force a break on downstream users. Like I mentioned, `Option` isn't used in the spec so I don't think it'll be *that* annoying. I think it's nicer than quietly having two different union implementations or quietly breaking the existing `Option` impl. ### Crate Publish Ordering I've modified the order in which CI publishes crates to ensure that we don't publish a crate without ensuring we already published a crate that it depends upon. ## TODO - [ ] Queue a follow-up `[patch]`-removing PR.
2021-09-25 05:58:36 +00:00
mod derive_macro {
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use std::fmt::Debug;
fn assert_encode<T: Encode>(item: &T, bytes: &[u8]) {
assert_eq!(item.as_ssz_bytes(), bytes);
}
fn assert_encode_decode<T: Encode + Decode + PartialEq + Debug>(item: &T, bytes: &[u8]) {
assert_encode(item, bytes);
assert_eq!(T::from_ssz_bytes(bytes).unwrap(), *item);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoFixedUnion {
U8(u8),
U16(u16),
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct TwoFixedUnionStruct {
a: TwoFixedUnion,
}
#[test]
fn two_fixed_union() {
let eight = TwoFixedUnion::U8(1);
let sixteen = TwoFixedUnion::U16(1);
assert_encode_decode(&eight, &[0, 1]);
assert_encode_decode(&sixteen, &[1, 1, 0]);
assert_encode_decode(&TwoFixedUnionStruct { a: eight }, &[4, 0, 0, 0, 0, 1]);
assert_encode_decode(&TwoFixedUnionStruct { a: sixteen }, &[4, 0, 0, 0, 1, 1, 0]);
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct VariableA {
a: u8,
b: Vec<u8>,
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct VariableB {
a: Vec<u8>,
b: u8,
}
#[derive(PartialEq, Debug, Encode)]
#[ssz(enum_behaviour = "transparent")]
enum TwoVariableTrans {
A(VariableA),
B(VariableB),
}
#[derive(PartialEq, Debug, Encode)]
struct TwoVariableTransStruct {
a: TwoVariableTrans,
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoVariableUnion {
A(VariableA),
B(VariableB),
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct TwoVariableUnionStruct {
a: TwoVariableUnion,
}
#[test]
fn two_variable_trans() {
let trans_a = TwoVariableTrans::A(VariableA {
a: 1,
b: vec![2, 3],
});
let trans_b = TwoVariableTrans::B(VariableB {
a: vec![1, 2],
b: 3,
});
assert_encode(&trans_a, &[1, 5, 0, 0, 0, 2, 3]);
assert_encode(&trans_b, &[5, 0, 0, 0, 3, 1, 2]);
assert_encode(
&TwoVariableTransStruct { a: trans_a },
&[4, 0, 0, 0, 1, 5, 0, 0, 0, 2, 3],
);
assert_encode(
&TwoVariableTransStruct { a: trans_b },
&[4, 0, 0, 0, 5, 0, 0, 0, 3, 1, 2],
);
}
#[test]
fn two_variable_union() {
let union_a = TwoVariableUnion::A(VariableA {
a: 1,
b: vec![2, 3],
});
let union_b = TwoVariableUnion::B(VariableB {
a: vec![1, 2],
b: 3,
});
assert_encode_decode(&union_a, &[0, 1, 5, 0, 0, 0, 2, 3]);
assert_encode_decode(&union_b, &[1, 5, 0, 0, 0, 3, 1, 2]);
assert_encode_decode(
&TwoVariableUnionStruct { a: union_a },
&[4, 0, 0, 0, 0, 1, 5, 0, 0, 0, 2, 3],
);
assert_encode_decode(
&TwoVariableUnionStruct { a: union_b },
&[4, 0, 0, 0, 1, 5, 0, 0, 0, 3, 1, 2],
);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoVecUnion {
A(Vec<u8>),
B(Vec<u8>),
}
#[test]
fn two_vec_union() {
assert_encode_decode(&TwoVecUnion::A(vec![]), &[0]);
assert_encode_decode(&TwoVecUnion::B(vec![]), &[1]);
assert_encode_decode(&TwoVecUnion::A(vec![0]), &[0, 0]);
assert_encode_decode(&TwoVecUnion::B(vec![0]), &[1, 0]);
assert_encode_decode(&TwoVecUnion::A(vec![0, 1]), &[0, 0, 1]);
assert_encode_decode(&TwoVecUnion::B(vec![0, 1]), &[1, 0, 1]);
}
}