Prevent port re-use in HTTP API tests (#4745)

## Issue Addressed

CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports.

This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb 🤞

There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work.

Cargo.lock

@@ -601,7 +601,6 @@ dependencies = [
  "tree_hash",
  "tree_hash_derive",
  "types",
- "unused_port",
 ]
 
 [[package]]
@@ -2583,7 +2582,9 @@ dependencies = [
 name = "execution_layer"
 version = "0.1.0"
 dependencies = [
+ "arc-swap",
  "async-trait",
+ "axum",
  "builder_client",
  "bytes",
  "environment",
@@ -2598,6 +2599,7 @@ dependencies = [
  "hash-db",
  "hash256-std-hasher",
  "hex",
+ "hyper",
  "jsonwebtoken",
  "keccak-hash",
  "lazy_static",
@@ -3381,7 +3383,6 @@ dependencies = [
  "tokio-stream",
  "tree_hash",
  "types",
- "unused_port",
  "warp",
  "warp_utils",
 ]

beacon_node/beacon_chain/Cargo.toml

@@ -65,7 +65,6 @@ sensitive_url = { path = "../../common/sensitive_url" }
 superstruct = "0.5.0"
 hex = "0.4.2"
 exit-future = "0.2.0"
-unused_port = {path = "../../common/unused_port"}
 oneshot_broadcast = { path = "../../common/oneshot_broadcast" }
 
 [[test]]

beacon_node/beacon_chain/src/test_utils.rs

@@ -17,8 +17,8 @@ use bls::get_withdrawal_credentials;
 use execution_layer::{
     auth::JwtKey,
     test_utils::{
-        ExecutionBlockGenerator, MockExecutionLayer, TestingBuilder, DEFAULT_JWT_SECRET,
+        ExecutionBlockGenerator, MockBuilder, MockBuilderServer, MockExecutionLayer,
-        DEFAULT_TERMINAL_BLOCK,
+        DEFAULT_JWT_SECRET, DEFAULT_TERMINAL_BLOCK,
     },
     ExecutionLayer,
 };
@@ -167,7 +167,6 @@ pub struct Builder<T: BeaconChainTypes> {
     store_mutator: Option<BoxedMutator<T::EthSpec, T::HotStore, T::ColdStore>>,
     execution_layer: Option<ExecutionLayer<T::EthSpec>>,
     mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
-    mock_builder: Option<TestingBuilder<T::EthSpec>>,
     testing_slot_clock: Option<TestingSlotClock>,
     runtime: TestRuntime,
     log: Logger,
@@ -301,7 +300,6 @@ where
             store_mutator: None,
             execution_layer: None,
             mock_execution_layer: None,
-            mock_builder: None,
             testing_slot_clock: None,
             runtime,
             log,
@@ -433,7 +431,11 @@ where
         self
     }
 
-    pub fn mock_execution_layer(mut self) -> Self {
+    pub fn mock_execution_layer(self) -> Self {
+        self.mock_execution_layer_with_config(None)
+    }
+
+    pub fn mock_execution_layer_with_config(mut self, builder_threshold: Option<u128>) -> Self {
         let spec = self.spec.clone().expect("cannot build without spec");
         let shanghai_time = spec.capella_fork_epoch.map(|epoch| {
             HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
@@ -442,55 +444,15 @@ where
             self.runtime.task_executor.clone(),
             DEFAULT_TERMINAL_BLOCK,
             shanghai_time,
-            None,
+            builder_threshold,
             Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
             spec,
-            None,
         );
         self.execution_layer = Some(mock.el.clone());
         self.mock_execution_layer = Some(mock);
         self
     }
 
-    pub fn mock_execution_layer_with_builder(
-        mut self,
-        beacon_url: SensitiveUrl,
-        builder_threshold: Option<u128>,
-    ) -> Self {
-        // Get a random unused port
-        let port = unused_port::unused_tcp4_port().unwrap();
-        let builder_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap();
-
-        let spec = self.spec.clone().expect("cannot build without spec");
-        let shanghai_time = spec.capella_fork_epoch.map(|epoch| {
-            HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
-        });
-        let mock_el = MockExecutionLayer::new(
-            self.runtime.task_executor.clone(),
-            DEFAULT_TERMINAL_BLOCK,
-            shanghai_time,
-            builder_threshold,
-            Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
-            spec.clone(),
-            Some(builder_url.clone()),
-        )
-        .move_to_terminal_block();
-
-        let mock_el_url = SensitiveUrl::parse(mock_el.server.url().as_str()).unwrap();
-
-        self.mock_builder = Some(TestingBuilder::new(
-            mock_el_url,
-            builder_url,
-            beacon_url,
-            spec,
-            self.runtime.task_executor.clone(),
-        ));
-        self.execution_layer = Some(mock_el.el.clone());
-        self.mock_execution_layer = Some(mock_el);
-
-        self
-    }
-
     /// Instruct the mock execution engine to always return a "valid" response to any payload it is
     /// asked to execute.
     pub fn mock_execution_layer_all_payloads_valid(self) -> Self {
@@ -572,7 +534,7 @@ where
             shutdown_receiver: Arc::new(Mutex::new(shutdown_receiver)),
             runtime: self.runtime,
             mock_execution_layer: self.mock_execution_layer,
-            mock_builder: self.mock_builder.map(Arc::new),
+            mock_builder: None,
             rng: make_rng(),
         }
     }
@@ -597,7 +559,7 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
     pub runtime: TestRuntime,
 
     pub mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
-    pub mock_builder: Option<Arc<TestingBuilder<T::EthSpec>>>,
+    pub mock_builder: Option<Arc<MockBuilder<T::EthSpec>>>,
 
     pub rng: Mutex<StdRng>,
 }
@@ -633,6 +595,49 @@ where
             .execution_block_generator()
     }
 
+    pub fn set_mock_builder(&mut self, beacon_url: SensitiveUrl) -> MockBuilderServer {
+        let mock_el = self
+            .mock_execution_layer
+            .as_ref()
+            .expect("harness was not built with mock execution layer");
+
+        let mock_el_url = SensitiveUrl::parse(mock_el.server.url().as_str()).unwrap();
+
+        // Create the builder, listening on a free port.
+        let (mock_builder, mock_builder_server) = MockBuilder::new_for_testing(
+            mock_el_url,
+            beacon_url,
+            self.spec.clone(),
+            self.runtime.task_executor.clone(),
+        );
+
+        // Set the builder URL in the execution layer now that its port is known.
+        let builder_listen_addr = mock_builder_server.local_addr();
+        let port = builder_listen_addr.port();
+        mock_el
+            .el
+            .set_builder_url(
+                SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap(),
+                None,
+            )
+            .unwrap();
+
+        self.mock_builder = Some(Arc::new(mock_builder));
+
+        // Sanity check.
+        let el_builder = self
+            .chain
+            .execution_layer
+            .as_ref()
+            .unwrap()
+            .builder()
+            .unwrap();
+        let mock_el_builder = mock_el.el.builder().unwrap();
+        assert!(Arc::ptr_eq(&el_builder, &mock_el_builder));
+
+        mock_builder_server
+    }
+
     pub fn get_all_validators(&self) -> Vec<usize> {
         (0..self.validator_keypairs.len()).collect()
     }

beacon_node/execution_layer/Cargo.toml

@@ -42,6 +42,8 @@ ethers-core = "1.0.2"
 builder_client = { path = "../builder_client" }
 fork_choice = { path = "../../consensus/fork_choice" }
 mev-rs = { git = "https://github.com/ralexstokes/mev-rs", rev = "216657016d5c0889b505857c89ae42c7aa2764af" }
+axum = "0.6"
+hyper = "0.14"
 ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus", rev = "e380108" }
 ssz_rs = "0.9.0"
 tokio-stream = { version = "0.1.9", features = [ "sync" ] }
@@ -51,3 +53,4 @@ hash256-std-hasher = "0.15.2"
 triehash = "0.8.4"
 hash-db = "0.15.2"
 pretty_reqwest_error = { path = "../../common/pretty_reqwest_error" }
+arc-swap = "1.6.0"

beacon_node/execution_layer/src/lib.rs

@@ -5,6 +5,7 @@
 //! deposit-contract functionality that the `beacon_node/eth1` crate already provides.
 
 use crate::payload_cache::PayloadCache;
+use arc_swap::ArcSwapOption;
 use auth::{strip_prefix, Auth, JwtKey};
 use builder_client::BuilderHttpClient;
 pub use engine_api::EngineCapabilities;
@@ -209,7 +210,7 @@ pub enum FailedCondition {
 
 struct Inner<E: EthSpec> {
     engine: Arc<Engine>,
-    builder: Option<BuilderHttpClient>,
+    builder: ArcSwapOption<BuilderHttpClient>,
     execution_engine_forkchoice_lock: Mutex<()>,
     suggested_fee_recipient: Option<Address>,
     proposer_preparation_data: Mutex<HashMap<u64, ProposerPreparationDataEntry>>,
@@ -324,25 +325,9 @@ impl<T: EthSpec> ExecutionLayer<T> {
             Engine::new(api, executor.clone(), &log)
         };
 
-        let builder = builder_url
-            .map(|url| {
-                let builder_client = BuilderHttpClient::new(url.clone(), builder_user_agent)
-                    .map_err(Error::Builder)?;
-
-                info!(
-                    log,
-                    "Using external block builder";
-                    "builder_url" => ?url,
-                    "builder_profit_threshold" => builder_profit_threshold,
-                    "local_user_agent" => builder_client.get_user_agent(),
-                );
-                Ok::<_, Error>(builder_client)
-            })
-            .transpose()?;
-
         let inner = Inner {
             engine: Arc::new(engine),
-            builder,
+            builder: ArcSwapOption::empty(),
             execution_engine_forkchoice_lock: <_>::default(),
             suggested_fee_recipient,
             proposer_preparation_data: Mutex::new(HashMap::new()),
@@ -356,19 +341,45 @@ impl<T: EthSpec> ExecutionLayer<T> {
             last_new_payload_errored: RwLock::new(false),
         };
 
-        Ok(Self {
+        let el = Self {
             inner: Arc::new(inner),
-        })
+        };
-    }
+
-}
+        if let Some(builder_url) = builder_url {
+            el.set_builder_url(builder_url, builder_user_agent)?;
+        }
+
+        Ok(el)
+    }
 
-impl<T: EthSpec> ExecutionLayer<T> {
     fn engine(&self) -> &Arc<Engine> {
         &self.inner.engine
     }
 
-    pub fn builder(&self) -> &Option<BuilderHttpClient> {
+    pub fn builder(&self) -> Option<Arc<BuilderHttpClient>> {
-        &self.inner.builder
+        self.inner.builder.load_full()
+    }
+
+    /// Set the builder URL after initialization.
+    ///
+    /// This is useful for breaking circular dependencies between mock ELs and mock builders in
+    /// tests.
+    pub fn set_builder_url(
+        &self,
+        builder_url: SensitiveUrl,
+        builder_user_agent: Option<String>,
+    ) -> Result<(), Error> {
+        let builder_client = BuilderHttpClient::new(builder_url.clone(), builder_user_agent)
+            .map_err(Error::Builder)?;
+        info!(
+            self.log(),
+            "Using external block builder";
+            "builder_url" => ?builder_url,
+            "builder_profit_threshold" => self.inner.builder_profit_threshold.as_u128(),
+            "local_user_agent" => builder_client.get_user_agent(),
+        );
+        self.inner.builder.swap(Some(Arc::new(builder_client)));
+        Ok(())
     }
 
     /// Cache a full payload, keyed on the `tree_hash_root` of the payload

beacon_node/execution_layer/src/test_utils/mock_builder.rs

@@ -40,6 +40,11 @@ use types::{
     Uint256,
 };
 
+pub type MockBuilderServer = axum::Server<
+    hyper::server::conn::AddrIncoming,
+    axum::routing::IntoMakeService<axum::routing::Router>,
+>;
+
 #[derive(Clone)]
 pub enum Operation {
     FeeRecipient(Address),
@@ -170,19 +175,25 @@ impl BidStuff for BuilderBid {
     }
 }
 
-pub struct TestingBuilder<E: EthSpec> {
+#[derive(Clone)]
-    server: BlindedBlockProviderServer<MockBuilder<E>>,
+pub struct MockBuilder<E: EthSpec> {
-    pub builder: MockBuilder<E>,
+    el: ExecutionLayer<E>,
+    beacon_client: BeaconNodeHttpClient,
+    spec: ChainSpec,
+    context: Arc<Context>,
+    val_registration_cache: Arc<RwLock<HashMap<BlsPublicKey, SignedValidatorRegistration>>>,
+    builder_sk: SecretKey,
+    operations: Arc<RwLock<Vec<Operation>>>,
+    invalidate_signatures: Arc<RwLock<bool>>,
 }
 
-impl<E: EthSpec> TestingBuilder<E> {
+impl<E: EthSpec> MockBuilder<E> {
-    pub fn new(
+    pub fn new_for_testing(
         mock_el_url: SensitiveUrl,
-        builder_url: SensitiveUrl,
         beacon_url: SensitiveUrl,
         spec: ChainSpec,
         executor: TaskExecutor,
-    ) -> Self {
+    ) -> (Self, MockBuilderServer) {
         let file = NamedTempFile::new().unwrap();
         let path = file.path().into();
         std::fs::write(&path, hex::encode(DEFAULT_JWT_SECRET)).unwrap();
@@ -211,39 +222,13 @@ impl<E: EthSpec> TestingBuilder<E> {
             spec,
             context,
         );
-        let port = builder_url.full.port().unwrap();
+        let host: Ipv4Addr = Ipv4Addr::LOCALHOST;
-        let host: Ipv4Addr = builder_url
+        let port = 0;
-            .full
+        let provider = BlindedBlockProviderServer::new(host, port, builder.clone());
-            .host_str()
+        let server = provider.serve();
-            .unwrap()
+        (builder, server)
-            .to_string()
-            .parse()
-            .unwrap();
-        let server = BlindedBlockProviderServer::new(host, port, builder.clone());
-        Self { server, builder }
     }
 
-    pub async fn run(&self) {
-        let server = self.server.serve();
-        if let Err(err) = server.await {
-            println!("error while listening for incoming: {err}")
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct MockBuilder<E: EthSpec> {
-    el: ExecutionLayer<E>,
-    beacon_client: BeaconNodeHttpClient,
-    spec: ChainSpec,
-    context: Arc<Context>,
-    val_registration_cache: Arc<RwLock<HashMap<BlsPublicKey, SignedValidatorRegistration>>>,
-    builder_sk: SecretKey,
-    operations: Arc<RwLock<Vec<Operation>>>,
-    invalidate_signatures: Arc<RwLock<bool>>,
-}
-
-impl<E: EthSpec> MockBuilder<E> {
     pub fn new(
         el: ExecutionLayer<E>,
         beacon_client: BeaconNodeHttpClient,

beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs

@@ -31,7 +31,6 @@ impl<T: EthSpec> MockExecutionLayer<T> {
             None,
             Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
             spec,
-            None,
         )
     }
 
@@ -43,7 +42,6 @@ impl<T: EthSpec> MockExecutionLayer<T> {
         builder_threshold: Option<u128>,
         jwt_key: Option<JwtKey>,
         spec: ChainSpec,
-        builder_url: Option<SensitiveUrl>,
     ) -> Self {
         let handle = executor.handle().unwrap();
 
@@ -65,7 +63,6 @@ impl<T: EthSpec> MockExecutionLayer<T> {
 
         let config = Config {
             execution_endpoints: vec![url],
-            builder_url,
             secret_files: vec![path],
             suggested_fee_recipient: Some(Address::repeat_byte(42)),
             builder_profit_threshold: builder_threshold.unwrap_or(DEFAULT_BUILDER_THRESHOLD_WEI),

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -25,7 +25,7 @@ use warp::{http::StatusCode, Filter, Rejection};
 use crate::EngineCapabilities;
 pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator};
 pub use hook::Hook;
-pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder};
+pub use mock_builder::{Context as MockBuilderContext, MockBuilder, MockBuilderServer, Operation};
 pub use mock_execution_layer::MockExecutionLayer;
 
 pub const DEFAULT_TERMINAL_DIFFICULTY: u64 = 6400;

beacon_node/http_api/Cargo.toml

@@ -40,7 +40,6 @@ logging = { path = "../../common/logging" }
 ethereum_serde_utils = "0.5.0"
 operation_pool = { path = "../operation_pool" }
 sensitive_url = { path = "../../common/sensitive_url" }
-unused_port = { path = "../../common/unused_port" }
 store = { path = "../store" }
 bytes = "1.1.0"
 beacon_processor = { path = "../beacon_processor" }

beacon_node/http_api/src/lib.rs

@@ -3635,12 +3635,13 @@ pub fn serve<T: BeaconChainTypes>(
                         // send the response back to our original HTTP request
                         // task via a channel.
                         let builder_future = async move {
-                            let builder = chain
+                            let arc_builder = chain
                                 .execution_layer
                                 .as_ref()
                                 .ok_or(BeaconChainError::ExecutionLayerMissing)
                                 .map_err(warp_utils::reject::beacon_chain_error)?
-                                .builder()
+                                .builder();
+                            let builder = arc_builder
                                 .as_ref()
                                 .ok_or(BeaconChainError::BuilderMissing)
                                 .map_err(warp_utils::reject::beacon_chain_error)?;

beacon_node/http_api/src/test_utils.rs

@@ -129,17 +129,9 @@ pub async fn create_api_server<T: BeaconChainTypes>(
     test_runtime: &TestRuntime,
     log: Logger,
 ) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
-    // Get a random unused port.
+    // Use port 0 to allocate a new unused port.
-    let port = unused_port::unused_tcp4_port().unwrap();
+    let port = 0;
-    create_api_server_on_port(chain, test_runtime, log, port).await
-}
 
-pub async fn create_api_server_on_port<T: BeaconChainTypes>(
-    chain: Arc<BeaconChain<T>>,
-    test_runtime: &TestRuntime,
-    log: Logger,
-    port: u16,
-) -> ApiServer<T::EthSpec, impl Future<Output = ()>> {
     let (network_senders, network_receivers) = NetworkSenders::new();
 
     // Default metadata

beacon_node/http_api/tests/tests.rs

@@ -10,15 +10,14 @@ use eth2::{
     types::{BlockId as CoreBlockId, ForkChoiceNode, StateId as CoreStateId, *},
     BeaconNodeHttpClient, Error, StatusCode, Timeouts,
 };
-use execution_layer::test_utils::TestingBuilder;
-use execution_layer::test_utils::DEFAULT_BUILDER_THRESHOLD_WEI;
 use execution_layer::test_utils::{
-    Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
+    MockBuilder, Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_BUILDER_THRESHOLD_WEI,
+    DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
 };
 use futures::stream::{Stream, StreamExt};
 use futures::FutureExt;
 use http_api::{
-    test_utils::{create_api_server, create_api_server_on_port, ApiServer},
+    test_utils::{create_api_server, ApiServer},
     BlockId, StateId,
 };
 use lighthouse_network::{Enr, EnrExt, PeerId};
@@ -73,7 +72,7 @@ struct ApiTester {
     network_rx: NetworkReceivers<E>,
     local_enr: Enr,
     external_peer_id: PeerId,
-    mock_builder: Option<Arc<TestingBuilder<E>>>,
+    mock_builder: Option<Arc<MockBuilder<E>>>,
 }
 
 struct ApiTesterConfig {
@@ -120,24 +119,28 @@ impl ApiTester {
     }
 
     pub async fn new_from_config(config: ApiTesterConfig) -> Self {
-        // Get a random unused port
         let spec = config.spec;
-        let port = unused_port::unused_tcp4_port().unwrap();
-        let beacon_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap();
 
-        let harness = Arc::new(
+        let mut harness = BeaconChainHarness::builder(MainnetEthSpec)
-            BeaconChainHarness::builder(MainnetEthSpec)
+            .spec(spec.clone())
-                .spec(spec.clone())
+            .chain_config(ChainConfig {
-                .chain_config(ChainConfig {
+                reconstruct_historic_states: config.retain_historic_states,
-                    reconstruct_historic_states: config.retain_historic_states,
+                ..ChainConfig::default()
-                    ..ChainConfig::default()
+            })
-                })
+            .logger(logging::test_logger())
-                .logger(logging::test_logger())
+            .deterministic_keypairs(VALIDATOR_COUNT)
-                .deterministic_keypairs(VALIDATOR_COUNT)
+            .fresh_ephemeral_store()
-                .fresh_ephemeral_store()
+            .mock_execution_layer_with_config(config.builder_threshold)
-                .mock_execution_layer_with_builder(beacon_url.clone(), config.builder_threshold)
+            .build();
-                .build(),
+
-        );
+        harness
+            .mock_execution_layer
+            .as_ref()
+            .unwrap()
+            .server
+            .execution_block_generator()
+            .move_to_terminal_block()
+            .unwrap();
 
         harness.advance_slot();
 
@@ -245,29 +248,40 @@ impl ApiTester {
 
         let ApiServer {
             server,
-            listening_socket: _,
+            listening_socket,
             network_rx,
             local_enr,
             external_peer_id,
-        } = create_api_server_on_port(chain.clone(), &harness.runtime, log, port).await;
+        } = create_api_server(chain.clone(), &harness.runtime, log).await;
 
         harness.runtime.task_executor.spawn(server, "api_server");
 
+        // Late-initalize the mock builder now that the mock execution node and beacon API ports
+        // have been allocated.
+        let beacon_api_port = listening_socket.port();
+        let beacon_url =
+            SensitiveUrl::parse(format!("http://127.0.0.1:{beacon_api_port}").as_str()).unwrap();
+        let mock_builder_server = harness.set_mock_builder(beacon_url.clone());
+
+        // Start the mock builder service prior to building the chain out.
+        harness.runtime.task_executor.spawn(
+            async move {
+                if let Err(e) = mock_builder_server.await {
+                    panic!("error in mock builder server: {e:?}");
+                }
+            },
+            "mock_builder_server",
+        );
+
+        let mock_builder = harness.mock_builder.clone();
+
         let client = BeaconNodeHttpClient::new(
             beacon_url,
             Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)),
         );
 
-        let builder_ref = harness.mock_builder.as_ref().unwrap().clone();
-        harness.runtime.task_executor.spawn(
-            async move { builder_ref.run().await },
-            "mock_builder_server",
-        );
-
-        let mock_builder = harness.mock_builder.clone();
-
         Self {
-            harness,
+            harness: Arc::new(harness),
             chain,
             client,
             next_block,
@@ -379,7 +393,6 @@ impl ApiTester {
             .mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_BUILDER_THRESHOLD_WEI,
             )));
@@ -402,7 +415,6 @@ impl ApiTester {
             .mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_BUILDER_PAYLOAD_VALUE_WEI,
             )));
@@ -3275,6 +3287,7 @@ impl ApiTester {
             .unwrap()
             .get_payload_by_root(&payload.tree_hash_root())
             .is_none());
+
         self
     }
 
@@ -3283,7 +3296,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::GasLimit(30_000_000));
 
         let slot = self.chain.slot().unwrap();
@@ -3326,7 +3338,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::FeeRecipient(test_fee_recipient));
 
         let slot = self.chain.slot().unwrap();
@@ -3368,7 +3379,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::ParentHash(invalid_parent_hash));
 
         let slot = self.chain.slot().unwrap();
@@ -3417,7 +3427,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::PrevRandao(invalid_prev_randao));
 
         let slot = self.chain.slot().unwrap();
@@ -3462,7 +3471,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::BlockNumber(invalid_block_number));
 
         let slot = self.chain.slot().unwrap();
@@ -3509,7 +3517,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Timestamp(invalid_timestamp));
 
         let slot = self.chain.slot().unwrap();
@@ -3549,11 +3556,7 @@ impl ApiTester {
     }
 
     pub async fn test_payload_rejects_invalid_signature(self) -> Self {
-        self.mock_builder
+        self.mock_builder.as_ref().unwrap().invalid_signatures();
-            .as_ref()
-            .unwrap()
-            .builder
-            .invalid_signatures();
 
         let slot = self.chain.slot().unwrap();
         let epoch = self.chain.epoch().unwrap();
@@ -3831,7 +3834,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_BUILDER_THRESHOLD_WEI - 1,
             )));
@@ -3868,7 +3870,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
             )));
@@ -3905,7 +3906,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI,
             )));
@@ -3942,7 +3942,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI - 1,
             )));
@@ -3979,7 +3978,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
             )));
@@ -3996,7 +3994,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::WithdrawalsRoot(withdrawals_root));
 
         let epoch = self.chain.epoch().unwrap();
@@ -4029,7 +4026,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::Value(Uint256::from(
                 DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1,
             )));
@@ -4037,7 +4033,6 @@ impl ApiTester {
         self.mock_builder
             .as_ref()
             .unwrap()
-            .builder
             .add_operation(Operation::WithdrawalsRoot(Hash256::repeat_byte(0x42)));
 
         let slot = self.chain.slot().unwrap();