Update deterministic subnets upgrade to allow prefix computation (#4959)

* add new spec field to spec

* add spec changes

* fix bug and update expected subnets with validation from pyspec

* fix values and make test prettier (?)

* fix style

---------

Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com>

common/eth2_network_config/built_in_network_configs/gnosis/config.yaml

@@ -105,3 +105,4 @@ MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000
 ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3

common/eth2_network_config/built_in_network_configs/holesky/config.yaml

@@ -105,6 +105,7 @@ ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3
 
 # Deneb
 # `2**7` (=128)

common/eth2_network_config/built_in_network_configs/mainnet/config.yaml

@@ -105,3 +105,4 @@ MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000
 ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3

common/eth2_network_config/built_in_network_configs/prater/config.yaml

@@ -115,6 +115,7 @@ ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3
 
 # Deneb
 # `2**7` (=128)

common/eth2_network_config/built_in_network_configs/sepolia/config.yaml

@@ -104,6 +104,7 @@ ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3
 
 # Deneb
 # `2**7` (=128)

consensus/types/src/chain_spec.rs

@@ -189,6 +189,7 @@ pub struct ChainSpec {
     pub attestation_subnet_count: u64,
     pub attestation_subnet_extra_bits: u8,
     pub attestation_subnet_prefix_bits: u8,
+    pub attestation_subnet_shuffling_prefix_bits: u8,
 
     /*
      * Networking Deneb
@@ -701,6 +702,8 @@ impl ChainSpec {
             message_domain_valid_snappy: default_message_domain_valid_snappy(),
             attestation_subnet_extra_bits: default_attestation_subnet_extra_bits(),
             attestation_subnet_prefix_bits: default_attestation_subnet_prefix_bits(),
+            attestation_subnet_shuffling_prefix_bits:
+                default_attestation_subnet_shuffling_prefix_bits(),
             max_request_blocks: default_max_request_blocks(),
 
             /*
@@ -962,6 +965,8 @@ impl ChainSpec {
             message_domain_valid_snappy: default_message_domain_valid_snappy(),
             attestation_subnet_extra_bits: default_attestation_subnet_extra_bits(),
             attestation_subnet_prefix_bits: default_attestation_subnet_prefix_bits(),
+            attestation_subnet_shuffling_prefix_bits:
+                default_attestation_subnet_shuffling_prefix_bits(),
             max_request_blocks: default_max_request_blocks(),
 
             /*
@@ -1139,6 +1144,9 @@ pub struct Config {
     #[serde(default = "default_attestation_subnet_prefix_bits")]
     #[serde(with = "serde_utils::quoted_u8")]
     attestation_subnet_prefix_bits: u8,
+    #[serde(default = "default_attestation_subnet_shuffling_prefix_bits")]
+    #[serde(with = "serde_utils::quoted_u8")]
+    attestation_subnet_shuffling_prefix_bits: u8,
     #[serde(default = "default_max_request_blocks_deneb")]
     #[serde(with = "serde_utils::quoted_u64")]
     max_request_blocks_deneb: u64,
@@ -1236,6 +1244,10 @@ const fn default_attestation_subnet_prefix_bits() -> u8 {
     6
 }
 
+const fn default_attestation_subnet_shuffling_prefix_bits() -> u8 {
+    3
+}
+
 const fn default_max_request_blocks() -> u64 {
     1024
 }
@@ -1414,6 +1426,7 @@ impl Config {
             message_domain_valid_snappy: spec.message_domain_valid_snappy,
             attestation_subnet_extra_bits: spec.attestation_subnet_extra_bits,
             attestation_subnet_prefix_bits: spec.attestation_subnet_prefix_bits,
+            attestation_subnet_shuffling_prefix_bits: spec.attestation_subnet_shuffling_prefix_bits,
             max_request_blocks_deneb: spec.max_request_blocks_deneb,
             max_request_blob_sidecars: spec.max_request_blob_sidecars,
             min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests,
@@ -1474,6 +1487,7 @@ impl Config {
             message_domain_valid_snappy,
             attestation_subnet_extra_bits,
             attestation_subnet_prefix_bits,
+            attestation_subnet_shuffling_prefix_bits,
             max_request_blocks,
             epochs_per_subnet_subscription,
             attestation_propagation_slot_range,
@@ -1531,6 +1545,7 @@ impl Config {
             message_domain_valid_snappy,
             attestation_subnet_extra_bits,
             attestation_subnet_prefix_bits,
+            attestation_subnet_shuffling_prefix_bits,
             max_request_blocks,
             epochs_per_subnet_subscription,
             attestation_propagation_slot_range,
@@ -1817,6 +1832,7 @@ mod yaml_tests {
         check_default!(message_domain_valid_snappy);
         check_default!(attestation_subnet_extra_bits);
         check_default!(attestation_subnet_prefix_bits);
+        check_default!(attestation_subnet_shuffling_prefix_bits);
 
         assert_eq!(chain_spec.bellatrix_fork_epoch, None);
     }

consensus/types/src/subnet_id.rs

@@ -72,36 +72,43 @@ impl SubnetId {
             .into())
     }
 
-    #[allow(clippy::arithmetic_side_effects)]
     /// Computes the set of subnets the node should be subscribed to during the current epoch,
     /// along with the first epoch in which these subscriptions are no longer valid.
+    #[allow(clippy::arithmetic_side_effects)]
     pub fn compute_subnets_for_epoch<T: EthSpec>(
         node_id: ethereum_types::U256,
         epoch: Epoch,
         spec: &ChainSpec,
     ) -> Result<(impl Iterator<Item = SubnetId>, Epoch), &'static str> {
-        // Simplify the variable name
+        // simplify variable naming
         let subscription_duration = spec.epochs_per_subnet_subscription;
+        let prefix_bits = spec.attestation_subnet_prefix_bits as u64;
+        let shuffling_prefix_bits = spec.attestation_subnet_shuffling_prefix_bits as u64;
 
-        let node_id_prefix =
+        // calculate the prefixes used to compute the subnet and shuffling
-            (node_id >> (256 - spec.attestation_subnet_prefix_bits as usize)).as_usize();
+        let node_id_prefix = (node_id >> (256 - prefix_bits)).as_u64();
+        let shuffling_prefix = (node_id >> (256 - (prefix_bits + shuffling_prefix_bits))).as_u64();
 
-        // NOTE: The as_u64() panics if the number is larger than u64::max_value(). This cannot be
+        // number of groups the shuffling creates
-        // true as spec.epochs_per_subnet_subscription is a u64.
+        let shuffling_groups = 1 << shuffling_prefix_bits;
-        let node_offset = (node_id % ethereum_types::U256::from(subscription_duration)).as_u64();
+        // shuffling group for this node
+        let shuffling_bits = shuffling_prefix % shuffling_groups;
+        let epoch_transition = (node_id_prefix
+            + (shuffling_bits * (subscription_duration >> shuffling_prefix_bits)))
+            % subscription_duration;
 
         // Calculate at which epoch this node needs to re-evaluate
         let valid_until_epoch = epoch.as_u64()
             + subscription_duration
-                .saturating_sub((epoch.as_u64() + node_offset) % subscription_duration);
+                .saturating_sub((epoch.as_u64() + epoch_transition) % subscription_duration);
 
-        let subscription_event_idx = (epoch.as_u64() + node_offset) / subscription_duration;
+        let subscription_event_idx = (epoch.as_u64() + epoch_transition) / subscription_duration;
         let permutation_seed =
             ethereum_hashing::hash(&int_to_bytes::int_to_bytes8(subscription_event_idx));
 
         let num_subnets = 1 << spec.attestation_subnet_prefix_bits;
         let permutated_prefix = compute_shuffled_index(
-            node_id_prefix,
+            node_id_prefix as usize,
             num_subnets,
             &permutation_seed,
             spec.shuffle_round_count,
@@ -180,38 +187,33 @@ mod tests {
             "60930578857433095740782970114409273483106482059893286066493409689627770333527",
             "103822458477361691467064888613019442068586830412598673713899771287914656699997",
         ]
-        .into_iter()
+        .map(|v| ethereum_types::U256::from_dec_str(v).unwrap());
-        .map(|v| ethereum_types::U256::from_dec_str(v).unwrap())
-        .collect::<Vec<_>>();
 
         let epochs = [
             54321u64, 1017090249, 1827566880, 846255942, 766597383, 1204990115, 1616209495,
             1774367616, 1484598751, 3525502229,
         ]
-        .into_iter()
+        .map(Epoch::from);
-        .map(Epoch::from)
-        .collect::<Vec<_>>();
 
         // Test mainnet
         let spec = ChainSpec::mainnet();
 
         // Calculated by hand
-        let expected_valid_time: Vec<u64> = [
+        let expected_valid_time = [
-            54528, 1017090371, 1827567108, 846256076, 766597570, 1204990135, 1616209582,
+            54528u64, 1017090255, 1827567030, 846256049, 766597387, 1204990287, 1616209536,
-            1774367723, 1484598953, 3525502371,
+            1774367857, 1484598847, 3525502311,
-        ]
+        ];
-        .into();
 
         // Calculated from pyspec
-        let expected_subnets = vec![
+        let expected_subnets = [
             vec![4u64, 5u64],
-            vec![61, 62],
+            vec![31, 32],
-            vec![23, 24],
+            vec![39, 40],
             vec![38, 39],
             vec![53, 54],
-            vec![39, 40],
+            vec![57, 58],
             vec![48, 49],
-            vec![39, 40],
+            vec![1, 2],
             vec![34, 35],
             vec![37, 38],
         ];
@@ -228,11 +230,11 @@ mod tests {
             >(node_ids[x], epochs[x], &spec)
             .unwrap();
 
-            assert_eq!(Epoch::from(expected_valid_time[x]), valid_time);
             assert_eq!(
                 expected_subnets[x],
                 computed_subnets.map(SubnetId::into).collect::<Vec<u64>>()
             );
+            assert_eq!(Epoch::from(expected_valid_time[x]), valid_time);
         }
     }
 }

lighthouse/environment/tests/testnet_dir/config.yaml

@@ -97,3 +97,4 @@ MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000
 ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3