From b244fd1723d68225de03f2a6d082be48e9d8e51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:17:39 -0300 Subject: [PATCH 01/19] feat(validator): decode a blinded block from produceBlockV3 When the beacon node's builder wins a slot, produceBlockV3 answers with a blinded block. The validator client needs a ProducedBlock variant for it that keeps the full block's root, so the signing path and the proposal guard stay unchanged, and that encodes as a SignedBlindedBeaconBlock for the blinded_blocks endpoint. --- .../src/beacon_node/block_contents.rs | 99 ++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/crates/validator/src/beacon_node/block_contents.rs b/crates/validator/src/beacon_node/block_contents.rs index c4d2806ca..040143337 100644 --- a/crates/validator/src/beacon_node/block_contents.rs +++ b/crates/validator/src/beacon_node/block_contents.rs @@ -42,8 +42,18 @@ //! | Fork | Produced | Published | //! |---|---|---| //! | Deneb, Electra, Fulu | `BlockContents` | `SignedBlockContents` | +//! | Blinded (electra, fulu) | `BlindedBeaconBlock` | `SignedBlindedBeaconBlock` | //! | Gloas and later | `GloasBlockContents` (block and self-built envelope) or a bare block | bare signed block, then the envelope separately | //! +//! # Blinded blocks +//! +//! When the beacon node's builder wins the slot, `produceBlockV3` answers with +//! a [`BlindedBeaconBlock`] (`Eth-Execution-Payload-Blinded: true`): the block +//! carries the payload's header, and the builder reveals the payload once this +//! client has signed. No blobs travel with it, since the builder publishes +//! them. Its root is the full block's, so signing and the proposal guard work +//! as for any other block. Electra and fulu share one blinded container. +//! //! # Gloas //! //! The block no longer carries the payload (EIP-7732): it commits to a bid, and @@ -85,6 +95,7 @@ //! thirty-two blobs would fail to decode, in a client that had been correct for //! months. +use ethlambda_types::beacon::builder::{BlindedBeaconBlock, SignedBlindedBeaconBlock}; use ethlambda_types::beacon::containers::deneb::Blob; use ethlambda_types::beacon::containers::electra::{BeaconBlock, SignedBeaconBlock}; use ethlambda_types::beacon::containers::gloas; @@ -256,6 +267,10 @@ pub enum Contents { block: gloas::BeaconBlock, payload: Option>, }, + /// Electra or fulu, when a builder supplied the payload: the block carries + /// the payload's header, and the builder reveals the payload after this + /// client signs. No blobs travel with it; the builder publishes them. + Blinded { block: BlindedBeaconBlock }, } impl ProducedBlock { @@ -326,6 +341,37 @@ impl ProducedBlock { } } + /// Decode a blinded `produceBlockV3` body (`Eth-Execution-Payload-Blinded: + /// true`). Electra and fulu share one blinded container; every other fork is + /// refused, as [`Self::from_ssz`] refuses them. + pub fn from_blinded_ssz(fork: ForkName, bytes: &[u8]) -> crate::error::Result { + match fork { + ForkName::Electra | ForkName::Fulu => { + let block = BlindedBeaconBlock::from_ssz_bytes(bytes).map_err(|err| { + Error::Decode(format!( + "produced blinded block for fork {} did not decode: {err:?}", + fork.as_str() + )) + })?; + Ok(Self { + fork, + contents: Contents::Blinded { block }, + }) + } + _ => Err(Error::InconsistentResponse(format!( + "beacon node produced a blinded block for fork {}; blinded blocks exist for \ + electra and fulu only", + fork.as_str() + ))), + } + } + + /// Whether a builder supplied the payload, so the block is published to + /// `blinded_blocks` and its fee recipient is the builder's. + pub fn is_blinded(&self) -> bool { + matches!(self.contents, Contents::Blinded { .. }) + } + /// Decode a gloas `produceBlockV4` body. `payload_included` is the node's /// `Eth-Execution-Payload-Included`, which selects between /// [`GloasBlockContents`] and a bare block: SSZ carries no tag, so a wrong @@ -365,6 +411,7 @@ impl ProducedBlock { block.slot } Contents::Gloas { block, .. } => block.slot, + Contents::Blinded { block } => block.slot, } } @@ -375,6 +422,7 @@ impl ProducedBlock { block.proposer_index } Contents::Gloas { block, .. } => block.proposer_index, + Contents::Blinded { block } => block.proposer_index, } } @@ -387,11 +435,13 @@ impl ProducedBlock { block.hash_tree_root() } Contents::Gloas { block, .. } => block.hash_tree_root(), + Contents::Blinded { block } => block.hash_tree_root(), } } /// Where the block pays its execution-layer rewards: the payload's fee - /// recipient before gloas, the bid's from it on. + /// recipient before gloas, the bid's from it on. For a blinded block, the + /// header's, which is the builder's address rather than the validator's. pub fn fee_recipient(&self) -> ethlambda_types::beacon::primitives::ExecutionAddress { match &self.contents { Contents::WithBlobProofs { block, .. } | Contents::WithCellProofs { block, .. } => { @@ -404,6 +454,7 @@ impl ProducedBlock { .message .fee_recipient } + Contents::Blinded { block } => block.body.execution_payload_header.fee_recipient, } } @@ -441,6 +492,7 @@ impl ProducedBlock { Contents::Gloas { payload, .. } => { payload.as_ref().map_or(0, |payload| payload.blobs.len()) } + Contents::Blinded { block } => block.body.blob_kzg_commitments.len(), } } @@ -485,6 +537,11 @@ impl ProducedBlock { signature, } .to_ssz(), + Contents::Blinded { block } => SignedBlindedBeaconBlock { + message: block, + signature, + } + .to_ssz(), } } } @@ -775,6 +832,46 @@ mod tests { } } + #[test] + fn a_blinded_block_decodes_and_keeps_the_full_blocks_root() { + let full = empty_block_for(5, 7); + let blinded = BlindedBeaconBlock::from(full.clone()); + + let produced = + ProducedBlock::from_blinded_ssz(ForkName::Fulu, &blinded.to_ssz()).expect("decodes"); + + assert!(produced.is_blinded()); + assert_eq!(produced.slot(), 5); + assert_eq!(produced.proposer_index(), 7); + assert_eq!(produced.block_root(), full.hash_tree_root()); + assert_eq!(produced.blob_count(), 0); + } + + #[test] + fn a_signed_blinded_block_encodes_as_signed_blinded_beacon_block() { + let blinded = BlindedBeaconBlock::from(empty_block_for(5, 7)); + let produced = + ProducedBlock::from_blinded_ssz(ForkName::Electra, &blinded.to_ssz()).expect("ok"); + + let signed = produced.into_signed_ssz(BlsSignature([6; 96])); + let decoded = SignedBlindedBeaconBlock::from_ssz_bytes(&signed).expect("decodes"); + assert_eq!(decoded.message, blinded); + assert_eq!(decoded.signature, BlsSignature([6; 96])); + } + + #[test] + fn a_blinded_block_for_gloas_or_deneb_is_refused() { + let bytes = BlindedBeaconBlock::from(empty_block_for(5, 7)).to_ssz(); + for fork in [ForkName::Gloas, ForkName::Deneb] { + let err = ProducedBlock::from_blinded_ssz(fork, &bytes).expect_err("must refuse"); + assert!( + matches!(err, Error::InconsistentResponse(_)), + "{} gave {err:?}", + fork.as_str() + ); + } + } + #[test] fn a_truncated_body_is_a_decode_error_not_a_panic() { let bytes = contents_bytes(1, 1); From a95ebe8c697bcfebe23fba80a2f304578af1f3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:20:19 -0300 Subject: [PATCH 02/19] feat(validator): ask for and publish blinded blocks over the Beacon API Builder proposals need the client to tell the node how to weigh a bid, to accept a blinded answer only when it will publish one, and to hand the signed block to the node's builder. BlockRequest carries an optional builder_boost_factor (None keeps today's behaviour: factor 0, blinded refused), and the trait gains register_validators and publish_blinded_block. A blinded publish that fails with a 5xx or no answer is its own error, BlindedUnconfirmed, because the relay may still publish the block, unlike a 4xx which is a definite refusal. Registrations go to every node, since they install state on each node's builder. --- .../rpc/src/beacon/validator_client_tests.rs | 5 + crates/validator/src/beacon_node/fallback.rs | 28 ++ crates/validator/src/beacon_node/http.rs | 304 +++++++++++++++++- crates/validator/src/beacon_node/mock.rs | 78 +++++ crates/validator/src/beacon_node/mod.rs | 54 +++- crates/validator/src/error.rs | 14 + crates/validator/src/proposal.rs | 5 + 7 files changed, 472 insertions(+), 16 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 654fb6d5d..5acd928a7 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -393,6 +393,7 @@ async fn block_production_without_an_execution_client_is_retryable() { proposer_index: 0, randao_reveal: Default::default(), graffiti: Default::default(), + builder_boost_factor: None, }; let err = client.produce_block(&request).await.unwrap_err(); assert!(matches!(err, Error::BeaconNodeSyncing), "got {err:?}"); @@ -610,6 +611,7 @@ async fn produce_with_pool( proposer_index: proposer, randao_reveal, graffiti: ethlambda_types::primitives::H256(graffiti), + builder_boost_factor: None, }; let produced = client.produce_block(&request).await.unwrap(); assert_eq!(produced.slot(), slot); @@ -644,6 +646,8 @@ fn fulu_block( match &produced.contents { Contents::WithCellProofs { block, .. } | Contents::WithBlobProofs { block, .. } => block, Contents::Gloas { .. } => unreachable!("a fulu proposal"), + // These tests ask with no `builder_boost_factor`, so no blinded answer. + Contents::Blinded { .. } => unreachable!("builder proposals are off"), } } @@ -1060,6 +1064,7 @@ async fn the_validator_client_can_serve_on_the_sync_committee() { proposer_index: proposer, randao_reveal, graffiti: Default::default(), + builder_boost_factor: None, }; let produced = client.produce_block(&request).await.unwrap(); let block_domain = get_domain(&advanced, DOMAIN_BEACON_PROPOSER, Some(epoch)); diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 8251f9559..62ba51b31 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -8,6 +8,7 @@ use std::future::Future; use std::pin::Pin; use async_trait::async_trait; +use ethlambda_types::beacon::builder::SignedValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::gloas; @@ -253,6 +254,32 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Placeholder until the producing node is remembered; see the next commit. + async fn publish_blinded_block( + &self, + slot: Slot, + fork: ForkName, + body: &[u8], + ) -> Result { + self.try_each("publish_blinded_block", |node| { + node.publish_blinded_block(slot, fork, body) + }) + .await + } + + /// To every node, like the other calls that install state on a node: a + /// node that never heard the registrations cannot serve a builder bid for + /// these validators after a failover. + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + self.try_all("register_validators", |node| { + node.register_validators(registrations) + }) + .await + } + /// Fails over like any query: the envelope the node cached is only on the /// node that produced the block, and one that did not answers 404. async fn execution_payload_envelope( @@ -601,6 +628,7 @@ mod tests { proposer_index, randao_reveal: BlsSignature([0; 96]), graffiti: Bytes32::default(), + builder_boost_factor: None, } } diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 013964c74..60a01c50f 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -23,6 +23,7 @@ use std::time::Duration; use async_trait::async_trait; +use ethlambda_types::beacon::builder::SignedValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::electra::Attestation; @@ -593,8 +594,9 @@ impl BeaconNodeApi for HttpBeaconNode { Ok(data) } - /// `builder_boost_factor=0` is how this client says it does not do the - /// builder flow. + /// `builder_boost_factor` is how this client says whether it does the + /// builder flow: the request's factor when builder proposals are on, `0` + /// when they are off (`None`). /// /// The parameter is a bid comparison: the node returns the local execution /// payload when its value is at least `factor / 100` of the builder's, so @@ -607,19 +609,21 @@ impl BeaconNodeApi for HttpBeaconNode { /// answer with a blinded block. The one unconditional guarantee is the /// other half: a node with no builder configured MUST return a full block. /// - /// A blinded answer is therefore rejected below rather than assumed away. - /// This client cannot publish what it cannot unblind, so signing one would - /// burn the slot's proposal guard entry for a block that can never be - /// sent. + /// A blinded answer is therefore never assumed away. With builder proposals + /// off (`None`) it is rejected: this client would not publish it through + /// `blinded_blocks`, so signing one would burn the slot's proposal guard + /// entry for a block that can never be sent. With them on it is decoded as + /// a blinded block, which `publish_blinded_block` sends. async fn produce_block(&self, request: &BlockRequest) -> Result { if request.fork >= ForkName::Gloas { return self.produce_gloas_block(request).await; } let path = format!( - "/eth/v3/validator/blocks/{}?randao_reveal={}&graffiti={}&builder_boost_factor=0", + "/eth/v3/validator/blocks/{}?randao_reveal={}&graffiti={}&builder_boost_factor={}", request.slot, encode_hex(&request.randao_reveal.0), encode_hex(&request.graffiti.0), + request.builder_boost_factor.unwrap_or(0), ); let SszResponse { fork, @@ -639,15 +643,19 @@ impl BeaconNodeApi for HttpBeaconNode { .to_string(), ) })?; - if blinded { + if blinded && request.builder_boost_factor.is_none() { return Err(Error::InconsistentResponse(format!( - "node produced a blinded block for slot {}; this client does not implement the \ - builder flow and cannot publish one", + "node produced a blinded block for slot {}; builder proposals are off, so this \ + client cannot publish one", request.slot ))); } - let block = ProducedBlock::from_ssz(fork, &body)?; + let block = if blinded { + ProducedBlock::from_blinded_ssz(fork, &body)? + } else { + ProducedBlock::from_ssz(fork, &body)? + }; // Enforced here rather than at the call site so failover works: see the // contract on `BeaconNodeApi::produce_block`. validate_produced_block(request, &block)?; @@ -687,6 +695,62 @@ impl BeaconNodeApi for HttpBeaconNode { }) } + /// Maps the answer as the trait's contract says: 2xx is handed to the + /// builder (202 is the node's broadcast-without-import, as for a full + /// block), a 4xx is a definite refusal, and a 5xx or a transport failure + /// is [`Error::BlindedUnconfirmed`], since the relay may still publish. + async fn publish_blinded_block( + &self, + _slot: Slot, + fork: ForkName, + body: &[u8], + ) -> Result { + let url = format!("{}/eth/v2/beacon/blinded_blocks", self.base_url); + let response = self + .client + .post(&url) + .header("Eth-Consensus-Version", fork.as_str()) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body.to_vec()) + .send() + .await + .map_err(|err| Error::BlindedUnconfirmed { + status: None, + detail: format!("{url}: {err}"), + })?; + + let status = response.status(); + if status == StatusCode::ACCEPTED { + return Ok(Published::BroadcastNotImported); + } + if status.is_success() { + return Ok(Published::HandedToBuilder); + } + let body = response.text().await.unwrap_or_default(); + if status.is_server_error() { + return Err(Error::BlindedUnconfirmed { + status: Some(status.as_u16()), + detail: format!("{url} answered {status}: {body}"), + }); + } + Err(Error::BeaconNodeStatus { + status: status.as_u16(), + body, + }) + } + + /// JSON, an empty list never reaching the wire. + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + if registrations.is_empty() { + return Ok(()); + } + self.post_no_content("/eth/v1/validator/register_validator", ®istrations, None) + .await + } + async fn execution_payload_envelope( &self, slot: Slot, @@ -1189,6 +1253,224 @@ mod tests { assert_eq!(value["builders"], serde_json::json!([])); } + /// A one-route Beacon API double: whatever the handler sees is pushed to + /// `seen` as (uri, body), and the answer is `status` with `headers` and + /// `reply` as the body. + type Seen = std::sync::Arc)>>>; + type Canned = (Seen, u16, Vec<(&'static str, &'static str)>, Vec); + + struct Served { + node: HttpBeaconNode, + seen: Seen, + } + + async fn serve( + status: u16, + headers: Vec<(&'static str, &'static str)>, + reply: Vec, + ) -> Served { + use axum::body::Bytes; + use axum::extract::State; + use axum::http::Uri; + use axum::response::IntoResponse; + + let seen: Seen = Default::default(); + let state = (seen.clone(), status, headers, reply); + let handler = |State(state): State, uri: Uri, body: Bytes| async move { + let (seen, status, headers, reply) = state; + seen.lock() + .expect("lock") + .push((uri.to_string(), body.to_vec())); + let mut response = ( + axum::http::StatusCode::from_u16(status).expect("a status"), + reply, + ) + .into_response(); + for (name, value) in headers { + response + .headers_mut() + .insert(name, axum::http::HeaderValue::from_static(value)); + } + response + }; + let app = axum::Router::new().fallback(handler).with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("binds"); + let address = listener.local_addr().expect("address"); + tokio::spawn(async move { axum::serve(listener, app).await }); + Served { + node: HttpBeaconNode::new(format!("http://{address}")).expect("builds"), + seen, + } + } + + fn block_request(factor: Option) -> BlockRequest { + BlockRequest { + slot: 5, + fork: ForkName::Fulu, + proposer_index: 7, + randao_reveal: Default::default(), + graffiti: Default::default(), + builder_boost_factor: factor, + } + } + + const SSZ_HEADERS: [(&str, &str); 3] = [ + ("content-type", "application/octet-stream"), + ("eth-consensus-version", "fulu"), + ("eth-execution-payload-blinded", "true"), + ]; + + fn blinded_bytes() -> Vec { + use libssz::SszEncode as _; + ethlambda_types::beacon::builder::BlindedBeaconBlock::from( + crate::beacon_node::block_contents::empty_block_for(5, 7), + ) + .to_ssz() + } + + #[tokio::test] + async fn the_boost_factor_is_zero_without_builder_proposals_and_the_factor_with() { + for (factor, expected) in [ + (None, "builder_boost_factor=0"), + (Some(250), "builder_boost_factor=250"), + ] { + let served = serve(500, vec![], vec![]).await; + let _ = served.node.produce_block(&block_request(factor)).await; + let seen = served.seen.lock().expect("lock"); + assert_eq!(seen.len(), 1); + assert!( + seen[0].0.contains(expected), + "{} lacks {expected}", + seen[0].0 + ); + } + } + + #[tokio::test] + async fn a_blinded_answer_is_refused_without_builder_proposals() { + let served = serve(200, SSZ_HEADERS.to_vec(), blinded_bytes()).await; + let err = served + .node + .produce_block(&block_request(None)) + .await + .expect_err("a blinded block cannot be published here"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + } + + #[tokio::test] + async fn a_blinded_answer_decodes_as_blinded_with_builder_proposals() { + let served = serve(200, SSZ_HEADERS.to_vec(), blinded_bytes()).await; + let produced = served + .node + .produce_block(&block_request(Some(100))) + .await + .expect("decodes"); + assert!(produced.is_blinded()); + assert_eq!(produced.slot(), 5); + assert_eq!(produced.proposer_index(), 7); + } + + #[tokio::test] + async fn registrations_are_posted_as_json_that_round_trips() { + use ethlambda_types::beacon::builder::{ + SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature, H160}; + + let registrations = vec![SignedValidatorRegistrationV1 { + message: ValidatorRegistrationV1 { + fee_recipient: H160([0xab; 20]), + gas_limit: 36_000_000, + timestamp: 1_700_000_000, + pubkey: BlsPubkey([3; 48]), + }, + signature: BlsSignature([4; 96]), + }]; + let served = serve(200, vec![], vec![]).await; + served + .node + .register_validators(®istrations) + .await + .expect("accepted"); + + let seen = served.seen.lock().expect("lock"); + assert_eq!(seen.len(), 1); + assert!(seen[0].0.ends_with("/eth/v1/validator/register_validator")); + let decoded: Vec = + serde_json::from_slice(&seen[0].1).expect("the body is the JSON a relay takes"); + assert_eq!(decoded, registrations); + } + + #[tokio::test] + async fn an_empty_registration_list_never_reaches_the_wire() { + let served = serve(200, vec![], vec![]).await; + served.node.register_validators(&[]).await.expect("a no-op"); + assert!(served.seen.lock().expect("lock").is_empty()); + } + + #[tokio::test] + async fn a_blinded_publish_maps_each_status() { + for (status, expected) in [ + (200, Published::HandedToBuilder), + (202, Published::BroadcastNotImported), + ] { + let served = serve(status, vec![], vec![]).await; + let published = served + .node + .publish_blinded_block(5, ForkName::Fulu, &[1, 2, 3]) + .await + .expect("accepted"); + assert_eq!(published, expected, "status {status}"); + let seen = served.seen.lock().expect("lock"); + assert!(seen[0].0.ends_with("/eth/v2/beacon/blinded_blocks")); + assert_eq!(seen[0].1, vec![1, 2, 3]); + } + + let served = serve(400, vec![], b"bad".to_vec()).await; + let err = served + .node + .publish_blinded_block(5, ForkName::Fulu, &[1]) + .await + .expect_err("a refusal"); + assert!( + matches!(err, Error::BeaconNodeStatus { status: 400, .. }), + "got {err:?}" + ); + + let served = serve(500, vec![], b"boom".to_vec()).await; + let err = served + .node + .publish_blinded_block(5, ForkName::Fulu, &[1]) + .await + .expect_err("unconfirmed"); + assert!( + matches!( + err, + Error::BlindedUnconfirmed { + status: Some(500), + .. + } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn a_blinded_publish_that_cannot_connect_is_unconfirmed() { + // Nothing listens on port 1. + let node = HttpBeaconNode::new("http://127.0.0.1:1").expect("builds"); + let err = node + .publish_blinded_block(5, ForkName::Fulu, &[1]) + .await + .expect_err("no answer"); + assert!( + matches!(err, Error::BlindedUnconfirmed { status: None, .. }), + "got {err:?}" + ); + } + #[test] fn a_boolean_header_is_read_strictly() { assert_eq!( diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index fe4a952b9..bad9ab471 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::sync::Mutex; use async_trait::async_trait; +use ethlambda_types::beacon::builder::{BlindedBeaconBlock, SignedValidatorRegistrationV1}; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::electra::{Attestation, SignedAggregateAndProof}; @@ -79,6 +80,22 @@ pub struct MockBeaconNode { /// What `publish_block` reports. Defaults to `Imported`; a test that cares /// about the 202 path sets it. pub publish_outcome: Option, + /// When set, `produce_block` answers a pre-gloas request that carries a + /// `builder_boost_factor` with a blinded block (for the slot and proposer + /// `produces_block` names), as a node whose builder won the slot does. A + /// request with no factor still gets a full block, as the contract on + /// `BeaconNodeApi::produce_block` requires. + pub produces_blinded: bool, + /// Every registration `register_validators` received, in call order. + pub registrations: Mutex>, + /// The size of each `register_validators` call, in call order, so a test + /// can see how a send was batched. + pub registration_batches: Mutex>, + /// Every blinded block published, as (slot, fork, body). + pub published_blinded: Mutex)>>, + /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; + /// `Err(detail)` is a `BlindedUnconfirmed` carrying `detail`. + pub blinded_publish_outcome: Option>, /// What the test asserts against. pub submitted: Mutex>, @@ -258,6 +275,20 @@ impl MockBeaconNode { self } + /// Answer a `produce_block` that carries a `builder_boost_factor` with a + /// blinded block, for the slot and proposer `with_block` names. + pub fn with_blinded_block(mut self) -> Self { + self.produces_blinded = true; + self + } + + /// Make `publish_blinded_block` fail as a node that did not confirm the + /// block, carrying `detail`. + pub fn with_unconfirmed_blinded_publish(mut self, detail: &str) -> Self { + self.blinded_publish_outcome = Some(Err(detail.to_string())); + self + } + /// Answer `produce_block` at a gloas slot with a bare gloas block whose /// bid names `builder_index`, with its self-built envelope attached when /// `include_payload` is set. @@ -640,6 +671,16 @@ impl BeaconNodeApi for MockBeaconNode { failure: BeaconNodeFailure::Request, detail: "no block set".into(), })?; + if self.produces_blinded && request.builder_boost_factor.is_some() { + let block = ProducedBlock { + fork: ForkName::Electra, + contents: Contents::Blinded { + block: BlindedBeaconBlock::from(empty_block_for(slot, proposer_index)), + }, + }; + crate::beacon_node::validate_produced_block(request, &block)?; + return Ok(block); + } // Electra, matching the block shape `empty_block_for` builds. A test // that needs a fork mismatch constructs one itself rather than getting // it by accident here. @@ -664,6 +705,43 @@ impl BeaconNodeApi for MockBeaconNode { Ok(self.publish_outcome.unwrap_or(Published::Imported)) } + async fn publish_blinded_block( + &self, + slot: Slot, + fork: ForkName, + body: &[u8], + ) -> Result { + self.guard("publish_blinded_block")?; + self.published_blinded + .lock() + .expect("lock") + .push((slot, fork, body.to_vec())); + match &self.blinded_publish_outcome { + None => Ok(Published::HandedToBuilder), + Some(Ok(published)) => Ok(*published), + Some(Err(detail)) => Err(Error::BlindedUnconfirmed { + status: None, + detail: detail.clone(), + }), + } + } + + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + self.guard("register_validators")?; + self.registration_batches + .lock() + .expect("lock") + .push(registrations.len()); + self.registrations + .lock() + .expect("lock") + .extend(registrations.iter().cloned()); + Ok(()) + } + async fn execution_payload_envelope( &self, slot: Slot, diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index c689d2a1d..ed292c5f4 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -6,6 +6,7 @@ //! (a reorg invalidating duties, a node failing mid-slot) untestable. use async_trait::async_trait; +use ethlambda_types::beacon::builder::SignedValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::fork::ForkName; @@ -130,6 +131,11 @@ pub struct BlockRequest { pub randao_reveal: BlsSignature, /// Thirty-two bytes consensus never reads. pub graffiti: Bytes32, + /// The `builder_boost_factor` to send, or `None` when builder proposals are + /// off. `None` sends `0` and refuses a blinded answer; `Some` sends the + /// factor and accepts one. Ignored from gloas, where `produceBlockV4` takes + /// the builder configuration in its body. + pub builder_boost_factor: Option, } /// What became of a published block. @@ -145,6 +151,14 @@ pub enum Published { Imported, /// 202: broadcast, but the node could not import it. BroadcastNotImported, + /// 200 from `blinded_blocks`: the node handed the signed blinded block to + /// its builder, which publishes the block and reveals the payload. Nothing + /// says the node imported it. + HandedToBuilder, + /// The node failed to confirm the blinded block (5xx or no answer). The + /// relay may still have published it; see `Error::BlindedUnconfirmed`. + /// Produced only by `ProposalService`, never by an implementation. + BuilderUnconfirmed, } /// Check that `block` is the block that was asked for. @@ -351,12 +365,13 @@ pub trait BeaconNodeApi: Send + Sync { /// returning, so that a node answering about the wrong slot or naming the /// wrong proposer is failed over from rather than signed for. /// - /// # Contract: never a blinded block + /// # Contract: a blinded block only when builder proposals are on /// - /// This client does not implement the builder flow, so it cannot publish a - /// blinded block and must not sign one. An implementation must ask for an - /// unblinded block and reject a blinded answer rather than returning - /// something the caller will sign and then be unable to send. + /// A blinded block is returned only when `request.builder_boost_factor` is + /// `Some`, since only then does the caller publish one through + /// [`Self::publish_blinded_block`]. With `None` an implementation must ask + /// for an unblinded block and reject a blinded answer rather than + /// returning something the caller will sign and then be unable to send. async fn produce_block(&self, request: &BlockRequest) -> Result; /// Publish an already-signed block, given the SSZ body and the fork it was @@ -372,6 +387,35 @@ pub trait BeaconNodeApi: Send + Sync { /// proposal, whether or not the first one succeeded. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result; + /// Publish a signed blinded block, produced at `slot` under `fork`, to + /// `POST /eth/v2/beacon/blinded_blocks`. The node hands it to its builder. + /// + /// # Contract: the node that produced it, and no other + /// + /// Only the producing node's builder holds the payload, and only that node + /// claimed the slot. Failover must not offer it elsewhere. + /// + /// # Contract: which error means what + /// + /// A definite refusal (4xx) is an `Err` other than + /// `Error::BlindedUnconfirmed`: the block was not published. A 5xx or a + /// transport failure is `Error::BlindedUnconfirmed`, because the relay may + /// still publish the block. + async fn publish_blinded_block( + &self, + slot: Slot, + fork: ForkName, + body: &[u8], + ) -> Result; + + /// Send signed builder registrations to the node, which forwards them to its + /// builder. JSON, since a node may forward the body unchanged to MEV-Boost. + /// State installed on the node's builder, so failover sends it to every node. + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()>; + /// The envelope the node cached for the self-built payload of the block /// whose root is `block_root`, proposed at `slot`. /// diff --git a/crates/validator/src/error.rs b/crates/validator/src/error.rs index 55b82a0ff..2c0334f87 100644 --- a/crates/validator/src/error.rs +++ b/crates/validator/src/error.rs @@ -62,6 +62,19 @@ pub enum Error { BeaconNodeStatus { status: u16, body: String }, #[error("beacon node is syncing")] BeaconNodeSyncing, + /// A signed blinded block was sent and the node's answer does not say + /// whether it went out: a 5xx, or no answer at all. + /// + /// Distinct from [`Self::BeaconNodeStatus`], which for this call means a + /// definite refusal (a 4xx: the block was not published). Here the node + /// may have handed the block to its builder before failing, and the relay + /// publishes on receipt, so the block may well be on the network. Nothing + /// can safely be retried: the slot is signed, and a second block for it + /// would be an equivocation. `status` is `None` for a transport failure. + #[error( + "the beacon node did not confirm the blinded block (it may still be published): {detail}" + )] + BlindedUnconfirmed { status: Option, detail: String }, /// The slot's attestation work ran past the end of the slot and was /// abandoned. /// @@ -125,6 +138,7 @@ impl Error { | Self::AllBeaconNodesFailed(_) | Self::BeaconNodeStatus { .. } | Self::BeaconNodeSyncing + | Self::BlindedUnconfirmed { .. } | Self::InconsistentResponse(_) | Self::AttestationDeadline { .. } | Self::Decode(_) => true, diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 1c76e4b37..6f493ea47 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -215,6 +215,7 @@ impl ProposalService { proposer_index: duty.validator_index, randao_reveal, graffiti, + builder_boost_factor: None, }; let mut produced = self.beacon_node.produce_block(&request).await?; @@ -431,6 +432,10 @@ impl ProposalService { ); crate::metrics::inc_blocks_broadcast_not_imported(); } + // `publish_block` never answers with these: a blinded block goes + // through `publish_blinded_block`, and `BuilderUnconfirmed` is + // built by this service, not by an implementation. + Published::HandedToBuilder | Published::BuilderUnconfirmed => {} } Ok(published) } From a413a2d988ff0e8d5d21fffaac8c6fbcb8aa85dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:21:37 -0300 Subject: [PATCH 03/19] feat(validator): publish a blinded block only on the node that produced it Only the producing node's builder holds the payload a blinded header commits to, and only that node claimed the slot. Failing over to another node would hand a signed block to a builder that never saw the bid, so the fallback remembers which node produced the latest blinded block and publishes it there alone, returning that node's answer unchanged so the caller can tell a refusal from an unconfirmed send. --- crates/validator/src/beacon_node/fallback.rs | 218 ++++++++++++++++++- 1 file changed, 209 insertions(+), 9 deletions(-) diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 62ba51b31..f4e81cb8f 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -40,12 +40,19 @@ type BoxFuture<'a, T> = Pin + Send + 'a>>; /// first one that succeeds. pub struct FallbackBeaconNode { nodes: Vec, + /// The node that produced the latest blinded block, by slot. A blinded + /// block can only be published where it was produced: that node's builder + /// holds the payload, and that node claimed the slot. + blinded_producer: std::sync::Mutex>, } impl FallbackBeaconNode { pub fn new(nodes: Vec) -> Self { assert!(!nodes.is_empty(), "at least one beacon node is required"); - Self { nodes } + Self { + nodes, + blinded_producer: std::sync::Mutex::new(None), + } } /// Try each node in order, returning the first success. @@ -69,13 +76,23 @@ impl FallbackBeaconNode { /// both `&self` and the future to one named `'p` lets the compiler unify /// it with whatever the call site's shortest borrow actually is. async fn try_each<'p, T, F>(&'p self, what: &'static str, call: F) -> Result + where + F: Fn(&'p B) -> BoxFuture<'p, Result>, + { + self.try_each_at(what, call).await.map(|(_, value)| value) + } + + /// [`Self::try_each`], also returning the position of the node that + /// answered, for the calls whose follow-up must go to the same node. The + /// logging and the final `warn!` live here so the two cannot drift. + async fn try_each_at<'p, T, F>(&'p self, what: &'static str, call: F) -> Result<(usize, T)> where F: Fn(&'p B) -> BoxFuture<'p, Result>, { let mut last = None; for (position, node) in self.nodes.iter().enumerate() { match call(node).await { - Ok(value) => return Ok(value), + Ok(value) => return Ok((position, value)), Err(err) => { // Debug, not warn: with more than one node configured, an // outage on one of them fires this every slot for as long @@ -231,9 +248,19 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Remembers which node answered when the block is blinded, and forgets + /// it when it is not: the record is for the latest block only, so a full + /// block never leaves a stale claim that a blinded publish could match. async fn produce_block(&self, request: &BlockRequest) -> Result { - self.try_each("produce_block", |node| node.produce_block(request)) - .await + let (position, block) = self + .try_each_at("produce_block", |node| node.produce_block(request)) + .await?; + let record = block.is_blinded().then(|| (block.slot(), position)); + *self + .blinded_producer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = record; + Ok(block) } /// First node that accepts it wins, rather than a broadcast to every node. @@ -254,17 +281,36 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } - /// Placeholder until the producing node is remembered; see the next commit. + /// Only on the node that produced the block, with no failover. + /// + /// Unlike [`Self::publish_block`], offering the block to another node + /// would not reach the network: only the producing node's builder holds + /// the payload the header commits to, and only that node claimed the slot + /// with it. A second node would at best refuse it and at worst hand a + /// signed block to a builder that never saw the bid. So if the producing + /// node fails, the publish fails, and the answer is returned unchanged so + /// the caller can tell a refusal from an unconfirmed send. The proposal + /// guard has already recorded the slot, so nothing retries it. async fn publish_blinded_block( &self, slot: Slot, fork: ForkName, body: &[u8], ) -> Result { - self.try_each("publish_blinded_block", |node| { - node.publish_blinded_block(slot, fork, body) - }) - .await + let record = *self + .blinded_producer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match record { + Some((produced_slot, position)) if produced_slot == slot => { + self.nodes[position] + .publish_blinded_block(slot, fork, body) + .await + } + _ => Err(Error::InconsistentResponse(format!( + "no beacon node produced a blinded block for slot {slot}" + ))), + } } /// To every node, like the other calls that install state on a node: a @@ -878,4 +924,158 @@ mod tests { "the healthy second node must be found" ); } + + fn blinded_request(slot: Slot, proposer_index: ValidatorIndex) -> BlockRequest { + BlockRequest { + builder_boost_factor: Some(100), + ..request(slot, proposer_index) + } + } + + fn blinded_node() -> MockBeaconNode { + MockBeaconNode::new().with_block(5, 7).with_blinded_block() + } + + /// Node 0 cannot produce, node 1 can and answers blinded: the block must go + /// back to node 1, and node 0 must never see it. + #[tokio::test] + async fn a_blinded_block_is_published_where_it_was_produced() { + let first = MockBeaconNode::new().failing_call("produce_block", "down"); + let fallback = FallbackBeaconNode::new(vec![first, blinded_node()]); + + let produced = fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("node 1 produces"); + assert!(produced.is_blinded()); + + let published = fallback + .publish_blinded_block(5, ForkName::Electra, &[1, 2, 3]) + .await + .expect("published"); + assert_eq!(published, Published::HandedToBuilder); + + let first_published = fallback.nodes[0].published_blinded.lock().expect("lock"); + let second_published = fallback.nodes[1].published_blinded.lock().expect("lock"); + assert!(first_published.is_empty(), "node 0 never produced it"); + assert_eq!( + *second_published, + vec![(5, ForkName::Electra, vec![1, 2, 3])] + ); + } + + /// The producing node failing the publish is the end of it: another node's + /// builder never saw the bid, so it must not be offered the block. + #[tokio::test] + async fn a_blinded_publish_does_not_fail_over() { + let second = blinded_node().with_unconfirmed_blinded_publish("relay timed out"); + let first = MockBeaconNode::new().failing_call("produce_block", "down"); + let fallback = FallbackBeaconNode::new(vec![first, second]); + + fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("node 1 produces"); + let err = fallback + .publish_blinded_block(5, ForkName::Electra, &[9]) + .await + .expect_err("node 1 failed to confirm"); + assert!( + matches!(err, Error::BlindedUnconfirmed { .. }), + "got {err:?}" + ); + assert!( + fallback.nodes[0] + .published_blinded + .lock() + .expect("lock") + .is_empty(), + "no failover to node 0" + ); + } + + #[tokio::test] + async fn a_blinded_publish_for_another_slot_is_refused() { + let fallback = FallbackBeaconNode::new(vec![blinded_node()]); + fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("produces"); + + let err = fallback + .publish_blinded_block(6, ForkName::Electra, &[1]) + .await + .expect_err("nothing was produced for slot 6"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + assert!( + fallback.nodes[0] + .published_blinded + .lock() + .expect("lock") + .is_empty() + ); + } + + /// A full block produced afterwards clears the claim, so an old blinded + /// block's slot cannot be published against a node that no longer holds it. + #[tokio::test] + async fn a_full_block_clears_the_blinded_record() { + let fallback = FallbackBeaconNode::new(vec![blinded_node()]); + fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("blinded"); + fallback + .produce_block(&request(5, 7)) + .await + .expect("full, since no factor was sent"); + + fallback + .publish_blinded_block(5, ForkName::Electra, &[1]) + .await + .expect_err("the record is gone"); + } + + #[tokio::test] + async fn registrations_reach_every_node() { + use ethlambda_types::beacon::builder::{ + SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature, H160}; + + let registration = SignedValidatorRegistrationV1 { + message: ValidatorRegistrationV1 { + fee_recipient: H160([1; 20]), + gas_limit: 30_000_000, + timestamp: 5, + pubkey: BlsPubkey([2; 48]), + }, + signature: BlsSignature([3; 96]), + }; + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::new(), + MockBeaconNode::failing("middle down"), + MockBeaconNode::new(), + ]); + + fallback + .register_validators(std::slice::from_ref(®istration)) + .await + .expect("two of three accepted"); + + for position in [0, 2] { + assert_eq!( + *fallback.nodes[position].registrations.lock().expect("lock"), + vec![registration.clone()], + "node {position}" + ); + } + assert!( + fallback.nodes[1] + .registrations + .lock() + .expect("lock") + .is_empty() + ); + } } From 7ad2c9e460dd50cd7b13311a1908e81bfdb30257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:23:18 -0300 Subject: [PATCH 04/19] feat(validator): sign and send builder registrations A builder only builds for a validator that registered a fee recipient and gas limit, signed under the builder domain (genesis fork version, zero validators root) so it verifies on any fork. RegistrationService caches each signature until the settings change, since builders keep the newest timestamp and a fresh signature per epoch would cost a BLS signature per validator for nothing, re-sends the full set every epoch because nodes forget registrations, and stops at gloas where proposer preferences replace them. Failures are logged and counted, never propagated: a missed registration costs at most a builder block, not a duty. --- crates/validator/src/lib.rs | 1 + crates/validator/src/metrics.rs | 60 ++++ crates/validator/src/registration.rs | 417 +++++++++++++++++++++++++++ crates/validator/src/signing.rs | 67 ++++- 4 files changed, 544 insertions(+), 1 deletion(-) create mode 100644 crates/validator/src/registration.rs diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index ebec73b1f..fc0cbdad5 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -43,6 +43,7 @@ pub mod payload_attestation; pub mod proposal; pub mod proposal_guard; pub mod proposer_settings; +pub mod registration; pub(crate) mod secure_fs; pub mod signing; pub mod slot_clock; diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index 2da746bb3..8f7a4f254 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -113,6 +113,46 @@ static BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL: LazyLock = LazyLock::new .unwrap() }); +/// Builder registrations a beacon node accepted, counted per registration so a +/// batch of 500 reads as 500. +static BUILDER_REGISTRATIONS_SENT_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_builder_registrations_sent_total", + "Builder registrations a beacon node accepted" + ) + .unwrap() +}); + +/// Registration batches no beacon node accepted. A missed registration costs +/// at most a builder block, never a duty, so this is a warning sign rather +/// than a failure. +static BUILDER_REGISTRATION_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_builder_registration_failures_total", + "Registration batches no beacon node accepted" + ) + .unwrap() +}); + +/// Blinded blocks a beacon node handed to its builder. +static BLINDED_BLOCKS_PROPOSED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_blinded_blocks_proposed_total", + "Blinded blocks a beacon node handed to its builder" + ) + .unwrap() +}); + +/// Blinded blocks the beacon node did not confirm. The relay may still have +/// published them, so this is not a count of missed proposals. +static BLINDED_BLOCKS_UNCONFIRMED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_blinded_blocks_unconfirmed_total", + "Blinded blocks the beacon node did not confirm, which the relay may still have published" + ) + .unwrap() +}); + /// Blocks this process declined to sign because it had already proposed that /// slot for that validator in this run. /// @@ -316,6 +356,10 @@ pub fn init() { LazyLock::force(&ATTESTATION_DEADLINE_MISSED_TOTAL); LazyLock::force(&BLOCKS_PROPOSED_TOTAL); LazyLock::force(&BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL); + LazyLock::force(&BUILDER_REGISTRATIONS_SENT_TOTAL); + LazyLock::force(&BUILDER_REGISTRATION_FAILURES_TOTAL); + LazyLock::force(&BLINDED_BLOCKS_PROPOSED_TOTAL); + LazyLock::force(&BLINDED_BLOCKS_UNCONFIRMED_TOTAL); LazyLock::force(&BLOCKS_REFUSED_TOTAL); LazyLock::force(&BLOCK_PROPOSAL_FAILURES_TOTAL); LazyLock::force(&ENVELOPES_PUBLISHED_TOTAL); @@ -376,6 +420,22 @@ pub fn inc_blocks_broadcast_not_imported() { BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL.inc(); } +pub fn inc_builder_registrations_sent(count: usize) { + BUILDER_REGISTRATIONS_SENT_TOTAL.inc_by(count as u64); +} + +pub fn inc_builder_registration_failures() { + BUILDER_REGISTRATION_FAILURES_TOTAL.inc(); +} + +pub fn inc_blinded_blocks_proposed() { + BLINDED_BLOCKS_PROPOSED_TOTAL.inc(); +} + +pub fn inc_blinded_blocks_unconfirmed() { + BLINDED_BLOCKS_UNCONFIRMED_TOTAL.inc(); +} + pub fn inc_blocks_refused() { BLOCKS_REFUSED_TOTAL.inc(); } diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs new file mode 100644 index 000000000..b2b534e45 --- /dev/null +++ b/crates/validator/src/registration.rs @@ -0,0 +1,417 @@ +//! Builder registrations: what tells a beacon node's builder where to pay each +//! validator and which gas limit to target. +//! +//! A registration is a `ValidatorRegistrationV1` the validator signs, so a +//! relay can trust that the fee recipient and gas limit it builds for were +//! chosen by the key that will sign the block. The beacon node forwards them to +//! its builder, which keeps the newest per validator. +//! +//! # Why a signature is reused until the settings change +//! +//! Builders order a validator's registrations by `timestamp` and keep the +//! latest, so re-sending an unchanged registration with the same timestamp is +//! harmless, and re-signing it with a fresh one every epoch would cost a BLS +//! signature per validator for a message that says nothing new. The signed +//! registration is therefore cached per validator and re-signed, with a new +//! timestamp, only when the fee recipient or gas limit changes. The full set is +//! still re-sent every epoch: a beacon node forgets its registrations when it +//! restarts, and a relay may expire them, so repeating them is what keeps the +//! validators registered without any state beyond this cache. +//! +//! # Why it stops at gloas +//! +//! From gloas the builder market works through proposer preferences, which the +//! beacon node serves itself, so there is nothing for this client to register. +//! Both entry points return at the gloas fork epoch. +//! +//! # Why every node gets them +//! +//! A registration installs state on a node's builder, so the fallback client +//! sends to every node rather than the first that answers: a node that never +//! heard them cannot serve a bid for these validators after a failover. +//! +//! # Failures are never propagated +//! +//! A missed registration costs at most a builder block, never a duty, so a +//! failed send is logged and counted and the caller moves on. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use ethlambda_types::beacon::builder::{SignedValidatorRegistrationV1, ValidatorRegistrationV1}; +use ethlambda_types::beacon::fork::ForkName; +use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch}; +use tokio::sync::RwLock; +use tracing::{debug, warn}; + +use crate::beacon_node::BeaconNodeApi; +use crate::keys::ValidatorStore; +use crate::proposer_settings::ProposerSettings; +use crate::signing::SigningContext; + +/// Registrations per request. Lighthouse's batch size: large enough that a few +/// thousand validators take a handful of requests, small enough that one +/// request stays well under a node's body limit. +pub const REGISTRATION_BATCH: usize = 500; + +/// What [`RegistrationService::current`] found for a set of validators. +struct Current { + /// Every validator's registration, cached or newly signed. + all: Vec, + /// The subset signed in this call, because it had no registration yet or + /// its fee recipient or gas limit changed. + changed: Vec, + /// Validators skipped because no fee recipient is configured for them. + no_fee_recipient: usize, +} + +/// Signs and sends builder registrations. +pub struct RegistrationService { + beacon_node: Arc, + context: Arc, + settings: Arc, + /// The validators the beacon node resolved at the last epoch refresh. + known: std::sync::Mutex>, + /// The latest signed registration per validator. + signed: std::sync::Mutex>, +} + +impl RegistrationService { + pub fn new( + beacon_node: Arc, + context: Arc, + settings: Arc, + ) -> Self { + Self { + beacon_node, + context, + settings, + known: std::sync::Mutex::new(Vec::new()), + signed: std::sync::Mutex::new(HashMap::new()), + } + } + + /// The registration for each of `pubkeys` as of `now_secs`, signing only + /// the ones whose `(fee_recipient, gas_limit)` differs from the cached one. + /// + /// `now_secs` is a parameter so a test can fix the timestamp. + fn current(&self, store: &ValidatorStore, pubkeys: &[BlsPubkey], now_secs: u64) -> Current { + // A poisoned lock is recovered: every write is one map operation, so + // the data behind it is never half-changed. + let mut signed = self + .signed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut current = Current { + all: Vec::with_capacity(pubkeys.len()), + changed: Vec::new(), + no_fee_recipient: 0, + }; + for pubkey in pubkeys { + let Some(fee_recipient) = self.settings.fee_recipient(pubkey) else { + current.no_fee_recipient += 1; + continue; + }; + let gas_limit = self.settings.gas_limit(pubkey); + + if let Some(held) = signed.get(pubkey) + && held.message.fee_recipient == fee_recipient + && held.message.gas_limit == gas_limit + { + current.all.push(held.clone()); + continue; + } + + let message = ValidatorRegistrationV1 { + fee_recipient, + gas_limit, + timestamp: now_secs, + pubkey: *pubkey, + }; + // A key the keymanager removed since the epoch refresh is the one + // way this fails; the next refresh no longer lists it. + let signature = match self.context.sign_validator_registration(store, &message) { + Ok(signature) => signature, + Err(err) => { + warn!(validator = ?pubkey, %err, "Could not sign a builder registration; skipping it"); + continue; + } + }; + let registration = SignedValidatorRegistrationV1 { message, signature }; + signed.insert(*pubkey, registration.clone()); + current.all.push(registration.clone()); + current.changed.push(registration); + } + current + } + + /// Whether `epoch` is at or after gloas, where there is nothing to register. + fn is_gloas(&self, epoch: Epoch) -> bool { + self.context.config.fork_at_epoch(epoch) >= ForkName::Gloas + } + + fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) + } + + /// Register every validator in `pubkeys`, the ones the beacon node resolved + /// at this epoch's refresh. Called once per epoch. + pub async fn register_all( + &self, + epoch: Epoch, + pubkeys: Vec, + store: &RwLock, + ) { + if self.is_gloas(epoch) { + return; + } + *self + .known + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = pubkeys.clone(); + + // Scoped away from the sends below, as `proposal.rs` does: a held read + // guard across a network call lets a queued keymanager writer block + // every reader behind it. + let current = { + let store = store.read().await; + self.current(&store, &pubkeys, Self::now_secs()) + }; + if current.no_fee_recipient > 0 { + warn!( + skipped = current.no_fee_recipient, + "Validators without a fee recipient are not registered with the builder" + ); + } + self.send(current.all, REGISTRATION_BATCH).await; + } + + /// Send only the registrations whose fee recipient or gas limit changed + /// since they were last signed, over the validators the last + /// [`Self::register_all`] was given. Called every slot, so a keymanager + /// change reaches the builder within a slot rather than an epoch. + /// + /// Silent about validators with no fee recipient: that is reported once + /// per epoch by [`Self::register_all`], not repeated every slot. + pub async fn register_changed(&self, epoch: Epoch, store: &RwLock) { + if self.is_gloas(epoch) { + return; + } + let known = self + .known + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if known.is_empty() { + return; + } + let current = { + let store = store.read().await; + self.current(&store, &known, Self::now_secs()) + }; + if current.changed.is_empty() { + return; + } + self.send(current.changed, REGISTRATION_BATCH).await; + } + + /// Send `registrations` to the beacon node in chunks of `batch_size`. + /// + /// The size is a parameter so a test can see the chunking without signing + /// hundreds of keys. + async fn send(&self, registrations: Vec, batch_size: usize) { + for chunk in registrations.chunks(batch_size) { + match self.beacon_node.register_validators(chunk).await { + Ok(()) => { + debug!(count = chunk.len(), "Builder registrations sent"); + crate::metrics::inc_builder_registrations_sent(chunk.len()); + } + Err(err) => { + warn!(count = chunk.len(), %err, "No beacon node accepted the builder registrations"); + crate::metrics::inc_builder_registration_failures(); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon_node::mock::MockBeaconNode; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::primitives::{Bytes32, H160, Root}; + + fn context(config: Config) -> Arc { + Arc::new(SigningContext { + config, + genesis_validators_root: Root::ZERO, + }) + } + + /// `count` distinct keys. The secrets are small integers, which are valid + /// BLS scalars, so no keystore is needed. + fn store_with_keys(count: usize) -> (ValidatorStore, Vec) { + let mut store = ValidatorStore::new(); + let pubkeys = (0..count) + .map(|index| { + let mut secret = [0u8; 32]; + secret[24..].copy_from_slice(&(index as u64 + 1).to_be_bytes()); + store.insert_secret("test", &secret).expect("inserts") + }) + .collect(); + (store, pubkeys) + } + + fn settings(fee_recipient: Option) -> Arc { + Arc::new(ProposerSettings::new(Bytes32::ZERO, fee_recipient)) + } + + fn service( + config: Config, + settings: Arc, + ) -> (RegistrationService, Arc) { + let node = Arc::new(MockBeaconNode::new()); + ( + RegistrationService::new(node.clone(), context(config), settings), + node, + ) + } + + #[test] + fn registrations_are_signed_once_and_reused() { + let (store, pubkeys) = store_with_keys(2); + let (service, _) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + + let first = service.current(&store, &pubkeys, 1_000); + assert_eq!(first.all.len(), 2); + assert_eq!(first.changed.len(), 2); + + let second = service.current(&store, &pubkeys, 2_000); + assert_eq!(second.all, first.all, "same timestamp and signature"); + assert!(second.changed.is_empty()); + assert_eq!(second.all[0].message.timestamp, 1_000); + } + + #[test] + fn a_new_fee_recipient_or_gas_limit_re_signs() { + let (store, pubkeys) = store_with_keys(3); + let settings = settings(Some(H160([0xab; 20]))); + let (service, _) = service(Config::mainnet(), settings.clone()); + let first = service.current(&store, &pubkeys, 1_000); + + settings.set_fee_recipient(&pubkeys[0], H160([0xcd; 20])); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + let second = service.current(&store, &pubkeys, 2_000); + + let changed: Vec<_> = second.changed.iter().map(|r| r.message.pubkey).collect(); + assert_eq!(changed, vec![pubkeys[0], pubkeys[1]]); + assert_eq!(second.all[0].message.fee_recipient, H160([0xcd; 20])); + assert_eq!(second.all[0].message.timestamp, 2_000); + assert_eq!(second.all[1].message.gas_limit, 30_000_000); + assert_eq!(second.all[1].message.timestamp, 2_000); + assert_ne!(second.all[0].signature, first.all[0].signature); + assert_eq!(second.all[2], first.all[2], "the third is untouched"); + } + + #[test] + fn a_validator_without_a_fee_recipient_is_not_registered() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(None); + settings.set_fee_recipient(&pubkeys[1], H160([0xcd; 20])); + let (service, _) = service(Config::mainnet(), settings); + + let current = service.current(&store, &pubkeys, 1_000); + + assert_eq!(current.no_fee_recipient, 1); + assert_eq!(current.all.len(), 1); + assert_eq!(current.all[0].message.pubkey, pubkeys[1]); + } + + #[test] + fn a_removed_key_is_skipped_rather_than_failing_the_rest() { + let (mut store, pubkeys) = store_with_keys(2); + let (service, _) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + store.remove(&pubkeys[0]); + + let current = service.current(&store, &pubkeys, 1_000); + + assert_eq!(current.all.len(), 1); + assert_eq!(current.all[0].message.pubkey, pubkeys[1]); + } + + /// 501 keys is one more than a batch; signing them is a millisecond each. + #[tokio::test] + async fn register_all_sends_in_batches_of_500() { + let (store, pubkeys) = store_with_keys(REGISTRATION_BATCH + 1); + let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let store = RwLock::new(store); + + service.register_all(0, pubkeys, &store).await; + + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![REGISTRATION_BATCH, 1] + ); + } + + #[tokio::test] + async fn send_chunks_by_the_size_it_is_given() { + let (store, pubkeys) = store_with_keys(5); + let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let current = service.current(&store, &pubkeys, 1_000); + + service.send(current.all, 2).await; + + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2, 2, 1] + ); + } + + #[tokio::test] + async fn nothing_is_sent_at_or_after_gloas() { + let (store, pubkeys) = store_with_keys(2); + let config = Config { + gloas_fork_epoch: 4, + ..Config::mainnet() + }; + let (service, node) = service(config, settings(Some(H160([0xab; 20])))); + let store = RwLock::new(store); + + service.register_all(4, pubkeys.clone(), &store).await; + service.register_changed(5, &store).await; + assert!(node.registration_batches.lock().expect("lock").is_empty()); + + // The epoch before the fork still registers. + service.register_all(3, pubkeys, &store).await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2]); + } + + #[tokio::test] + async fn register_changed_sends_only_the_changed_keys() { + let (store, pubkeys) = store_with_keys(3); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = RwLock::new(store); + + service.register_all(0, pubkeys.clone(), &store).await; + service.register_changed(0, &store).await; + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![3], + "nothing changed, so nothing more is sent" + ); + + settings.set_gas_limit(&pubkeys[2], 30_000_000); + service.register_changed(0, &store).await; + + let registrations = node.registrations.lock().expect("lock"); + assert_eq!(registrations.len(), 4); + assert_eq!(registrations[3].message.pubkey, pubkeys[2]); + assert_eq!(registrations[3].message.gas_limit, 30_000_000); + } +} diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index ced20d40e..6d0809408 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -5,6 +5,7 @@ //! beacon node sent: the wire representation is a transport detail, and the //! chain only ever agrees about the merkle root. +use ethlambda_types::beacon::builder::ValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants::{ DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_BUILDER, @@ -25,7 +26,7 @@ use ethlambda_types::beacon::primitives::{ // `AttestationData`, `Checkpoint`, `Fork` and `SigningData` all live in // `containers::shared`, verified against `containers/shared.rs:158,120,97,220`. use ethlambda_types::beacon::signing::{ - compute_domain, compute_epoch_at_slot, compute_signing_root, + compute_builder_domain, compute_domain, compute_epoch_at_slot, compute_signing_root, }; use crate::error::{Error, Result}; @@ -297,6 +298,24 @@ impl SigningContext { self.sign_root(store, pubkey, self.envelope_signing_root(envelope, slot)) } + /// A builder registration's signature. + /// + /// Signed under `compute_builder_domain(genesis_fork_version)`: the genesis + /// fork version and a zero validators root, per builder-specs, so a + /// registration verifies whatever fork the chain is in. Not + /// `DOMAIN_BEACON_BUILDER`, which is gloas's envelope domain. Not + /// slashable: a registration commits to nothing about the chain, and a + /// builder keeps only the newest one per validator. + pub fn sign_validator_registration( + &self, + store: &ValidatorStore, + registration: &ValidatorRegistrationV1, + ) -> Result { + let domain = compute_builder_domain(self.config.genesis_fork_version); + let root = compute_signing_root(registration.hash_tree_root(), domain); + self.sign_root(store, ®istration.pubkey, root) + } + /// Sign a payload timeliness committee vote on behalf of `pubkey`. /// /// Not slashable by the specification, and deduplicated per validator and @@ -506,6 +525,52 @@ mod tests { (store, pubkey) } + fn registration(pubkey: BlsPubkey) -> ValidatorRegistrationV1 { + ValidatorRegistrationV1 { + fee_recipient: ethlambda_types::beacon::primitives::H160([0xab; 20]), + gas_limit: 36_000_000, + timestamp: 1_700_000_000, + pubkey, + } + } + + #[test] + fn a_registration_is_signed_under_the_builder_domain() { + let (store, pubkey) = store_with_key(); + let context = context(); + let registration = registration(pubkey); + + let signature = context + .sign_validator_registration(&store, ®istration) + .expect("signs"); + + let root = compute_signing_root( + registration.hash_tree_root(), + compute_builder_domain(Config::mainnet().genesis_fork_version), + ); + assert!(verify(&pubkey, &signature, root)); + } + + /// The registration domain is not gloas's envelope domain, whose type + /// constant is similar in name only: a signature under one must not verify + /// under the other. + #[test] + fn a_registration_does_not_verify_under_the_envelope_domain() { + let (store, pubkey) = store_with_key(); + let context = context(); + let registration = registration(pubkey); + + let signature = context + .sign_validator_registration(&store, ®istration) + .expect("signs"); + + let envelope_root = compute_signing_root( + registration.hash_tree_root(), + context.domain(DOMAIN_BEACON_BUILDER, 0), + ); + assert!(!verify(&pubkey, &signature, envelope_root)); + } + #[test] fn a_randao_reveal_verifies_under_the_randao_signing_root() { let (store, pubkey) = store_with_key(); From 4d72075bee948cbdf5c853ac3c80a999f7076ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:28:11 -0300 Subject: [PATCH 05/19] feat(validator): propose builder blocks behind --builder-proposals The pieces for the builder flow (blinded blocks, registrations, publishing on the producing node) had no caller. This wires them: the proposal service sends a builder_boost_factor, signs a blinded block under the same root and guard as a full one, and skips the fee-recipient comparison since the header names the builder's address. A publish the node fails to confirm is reported as unconfirmed rather than a missed proposal, because the relay may still publish the block and a retry would be an equivocation. Registrations are sent from a spawned task at each epoch refresh and for changed keys every slot, so a slow node never delays duties. Three flags (--builder-proposals, --builder-boost-factor, --prefer-builder-proposals) configure it, the latter two requiring the first. --- bin/ethlambda/src/validator.rs | 123 +++++++++- .../rpc/src/beacon/validator_client_tests.rs | 2 + crates/validator/src/lib.rs | 73 +++++- crates/validator/src/proposal.rs | 215 +++++++++++++++++- crates/validator/src/proposer_settings.rs | 6 +- 5 files changed, 403 insertions(+), 16 deletions(-) diff --git a/bin/ethlambda/src/validator.rs b/bin/ethlambda/src/validator.rs index f97f82782..6dda48b22 100644 --- a/bin/ethlambda/src/validator.rs +++ b/bin/ethlambda/src/validator.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, H160}; -use ethlambda_validator::ValidatorConfig; use ethlambda_validator::proposer_settings::{GRAFFITI_BYTES, graffiti_from_text}; +use ethlambda_validator::{BuilderProposals, ValidatorConfig}; /// An execution address is exactly this wide. const ADDRESS_BYTES: usize = 20; @@ -78,6 +78,30 @@ pub(crate) struct ValidatorOptions { /// beyond localhost must front it with TLS. #[arg(long, default_value = "false")] pub(crate) enable_keymanager: bool, + + /// Propose builder blocks: register each validator with the beacon node's + /// builder and accept a blinded block when the builder's bid wins. Requires + /// a beacon node started with a builder (ethlambda: `--builder`). Fulu and + /// earlier only; from gloas the builder market works through proposer + /// preferences instead. + #[arg(long, default_value = "false")] + pub(crate) builder_proposals: bool, + + /// How a builder's bid is weighed against the local payload: the local + /// payload wins iff `local >= factor * bid / 100`. 100 compares them as + /// they are; 0 always builds locally. + #[arg( + long, + default_value = "100", + requires = "builder_proposals", + conflicts_with = "prefer_builder_proposals" + )] + pub(crate) builder_boost_factor: u64, + + /// Take the builder's block whenever there is a valid bid (sends + /// `builder_boost_factor = u64::MAX`). + #[arg(long, default_value = "false", requires = "builder_proposals")] + pub(crate) prefer_builder_proposals: bool, } impl ValidatorOptions { @@ -121,6 +145,21 @@ impl ValidatorOptions { Ok(Some(H160(bytes))) } + /// The builder configuration these flags describe, or `None` when builder + /// proposals are off. + /// + /// `--prefer-builder-proposals` wins over the factor by construction: + /// clap refuses the two together, so at most one is ever set. + pub(crate) fn builder(&self) -> Option { + self.builder_proposals.then_some(BuilderProposals { + boost_factor: if self.prefer_builder_proposals { + u64::MAX + } else { + self.builder_boost_factor + }, + }) + } + /// Reject a port clash before anything binds, the way the node's own /// options do. pub(crate) fn validate_ports(&self) -> eyre::Result<()> { @@ -144,6 +183,7 @@ pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> { options.validate_ports()?; let graffiti = options.graffiti()?; let suggested_fee_recipient = options.suggested_fee_recipient()?; + let builder = options.builder(); ethlambda_validator::run(ValidatorConfig { beacon_nodes: options.beacon_nodes, graffiti, @@ -154,6 +194,7 @@ pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> { keymanager: options .enable_keymanager .then(|| std::net::SocketAddr::new(options.http_address, options.keymanager_port)), + builder, }) .await .map_err(Into::into) @@ -174,7 +215,87 @@ mod tests { graffiti: graffiti.to_string(), suggested_fee_recipient: None, enable_keymanager: false, + builder_proposals: false, + builder_boost_factor: 100, + prefer_builder_proposals: false, + } + } + + /// Parse a command line the way the binary does, so `requires` and + /// `conflicts_with` are exercised rather than assumed. + fn parse(extra: &[&str]) -> Result { + use clap::Parser as _; + + #[derive(clap::Parser)] + struct Cli { + #[command(flatten)] + options: ValidatorOptions, } + let base = [ + "ethlambda", + "--beacon-nodes", + "http://localhost:5052", + "--validators-dir", + "/tmp/v", + "--secrets-dir", + "/tmp/s", + ]; + Cli::try_parse_from(base.iter().chain(extra)).map(|cli| cli.options) + } + + #[test] + fn builder_proposals_are_off_by_default() { + assert_eq!(parse(&[]).expect("parses").builder(), None); + } + + #[test] + fn builder_proposals_default_to_a_factor_of_100() { + let options = parse(&["--builder-proposals"]).expect("parses"); + assert_eq!( + options.builder(), + Some(BuilderProposals { boost_factor: 100 }) + ); + } + + #[test] + fn an_explicit_factor_is_used() { + let options = + parse(&["--builder-proposals", "--builder-boost-factor", "250"]).expect("parses"); + assert_eq!( + options.builder(), + Some(BuilderProposals { boost_factor: 250 }) + ); + } + + #[test] + fn preferring_the_builder_sends_the_maximum_factor() { + let options = + parse(&["--builder-proposals", "--prefer-builder-proposals"]).expect("parses"); + assert_eq!( + options.builder(), + Some(BuilderProposals { + boost_factor: u64::MAX + }) + ); + } + + #[test] + fn the_factor_and_the_preference_need_builder_proposals() { + parse(&["--builder-boost-factor", "250"]) + .expect_err("a factor with builder proposals off is a mistake"); + parse(&["--prefer-builder-proposals"]) + .expect_err("a preference with builder proposals off is a mistake"); + } + + #[test] + fn the_factor_and_the_preference_conflict() { + parse(&[ + "--builder-proposals", + "--builder-boost-factor", + "250", + "--prefer-builder-proposals", + ]) + .expect_err("the two say different things"); } #[test] diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 5acd928a7..21a33af99 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -1504,6 +1504,7 @@ mod gloas { None, ), ), + None, ); let duty = ProposerDutyDto { pubkey: encode_hex(&pubkey.0), @@ -1622,6 +1623,7 @@ mod gloas { None, ), ), + None, ); let duty = ProposerDutyDto { pubkey: encode_hex(&pubkey.0), diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index fc0cbdad5..40e1bce8c 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -71,6 +71,7 @@ use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; use crate::proposal::ProposalService; use crate::proposer_settings::ProposerSettings; +use crate::registration::RegistrationService; use crate::signing::SigningContext; use crate::slot_clock::SlotClock; use crate::sync_committee::SyncCommitteeService; @@ -96,6 +97,18 @@ pub struct ValidatorConfig { /// Bind address for the keymanager API. `None` when `--enable-keymanager` /// was not passed, in which case it is never spawned. pub keymanager: Option, + /// Builder proposals, or `None` when `--builder-proposals` is off. + pub builder: Option, +} + +/// Builder proposals: register each validator with the beacon node's builder +/// and accept a blinded block when the builder's bid wins. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BuilderProposals { + /// The `builder_boost_factor` sent with every pre-gloas block request: + /// 100 compares the bid and the local payload as they are; `u64::MAX` + /// takes the builder whenever there is a bid. + pub boost_factor: u64, } /// Run the validator client until the process is stopped. @@ -203,7 +216,23 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { let aggregation = AggregationService::new(beacon_node.clone(), context.clone()); let payload_attestation = PayloadAttestationService::new(beacon_node.clone(), context.clone()); let sync_committee = SyncCommitteeService::new(beacon_node.clone(), context.clone()); - let proposal = ProposalService::new(beacon_node.clone(), context.clone(), settings.clone()); + let proposal = ProposalService::new( + beacon_node.clone(), + context.clone(), + settings.clone(), + config.builder.map(|builder| builder.boost_factor), + ); + let registration = config.builder.map(|builder| { + info!( + boost_factor = builder.boost_factor, + "Builder proposals on: registering validators and accepting blinded blocks" + ); + Arc::new(RegistrationService::new( + beacon_node.clone(), + context.clone(), + settings.clone(), + )) + }); if let Some(address) = config.keymanager { let token = http_api::load_or_create_token(&config.validators_dir)?; @@ -275,6 +304,7 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &settings, &store, &context, + registration.as_ref().map(|service| (service, &store)), ); match refreshed.await { Ok(()) => last_epoch = Some(epoch), @@ -332,6 +362,15 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &store, ) .await; + + // Off the slot loop, like the epoch's full send: a keymanager change + // to a fee recipient or gas limit reaches the builder within a slot, + // and a slow node cannot delay the next slot's duties. + if let Some(registration) = ®istration { + let registration = registration.clone(); + let store = store.clone(); + tokio::spawn(async move { registration.register_changed(epoch, &store).await }); + } } } @@ -659,9 +698,20 @@ async fn propose( } } +/// What `refresh_epoch` needs to hand a spawned task the builder registration: +/// the service and the store it signs from, both shared, since the task +/// outlives the call. +type RegistrationHandles<'a, B> = ( + &'a Arc>, + &'a Arc>, +); + /// Resolve any newly activated validators, refresh the duty schedule, and +/// +/// With `registration`, also spawns the epoch's full builder registration. /// re-send subnet subscriptions if it moved. -async fn refresh_epoch( +#[allow(clippy::too_many_arguments)] +async fn refresh_epoch( beacon_node: &Arc, duties: &mut DutiesService, pubkeys: &[BlsPubkey], @@ -669,6 +719,7 @@ async fn refresh_epoch( settings: &ProposerSettings, store: &RwLock, context: &SigningContext, + registration: Option>, ) -> Result<()> { // Nothing to resolve, nothing to schedule, nothing to register. Returning // here also keeps an empty pubkey list away from `validator_indices`, whose @@ -737,6 +788,16 @@ async fn refresh_epoch( ); } + // Spawned, so a slow or hung node never delays this epoch's duties: a + // missed registration costs at most a builder block, never a duty. Only + // the validators the node resolved are registered; it would drop the rest. + if let Some((registration, store)) = registration { + let registration = registration.clone(); + let store = store.clone(); + let registered: Vec = entries.iter().map(|entry| entry.pubkey).collect(); + tokio::spawn(async move { registration.register_all(epoch, registered, &store).await }); + } + let changed = duties.refresh_around(epoch).await?; refresh_ptc(duties, epoch, context).await; refresh_sync(beacon_node, duties, epoch, context).await; @@ -895,6 +956,7 @@ mod tests { settings, &empty_store(), &context(), + None, ) .await .expect("refreshes"); @@ -993,6 +1055,7 @@ mod tests { &settings(Some(H160([0xab; 20]))), &empty_store(), &context(), + None, ) .await .expect("the refresh must survive a failed registration"); @@ -1019,6 +1082,7 @@ mod tests { &settings(Some(H160([0xab; 20]))), &empty_store(), &context(), + None, ) .await .expect("a keyless client idles rather than failing"); @@ -1101,6 +1165,7 @@ mod tests { &settings(None), &empty_store(), &gloas_context(4), + None, ) .await .expect("refreshes"); @@ -1122,6 +1187,7 @@ mod tests { &settings(None), &empty_store(), &context(), + None, ) .await .expect("refreshes"); @@ -1146,6 +1212,7 @@ mod tests { &settings(None), &empty_store(), &gloas_context(0), + None, ) .await .expect("the refresh survives it"); @@ -1253,6 +1320,7 @@ mod tests { &settings(None), &empty_store(), &altair_context(), + None, ) .await .expect("refreshes"); @@ -1294,6 +1362,7 @@ mod tests { &settings(None), &empty_store(), &altair_context(), + None, ) .await .expect("the refresh survives it"); diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 6f493ea47..2f457f860 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -39,8 +39,25 @@ //! //! The envelope is not slashable, so the guard is untouched by it. //! +//! # Blinded blocks +//! +//! With builder proposals on, the node may answer with a blinded block: the +//! payload's header in place of the payload. Its root is the full block's, so +//! the guard, the signing root and the signature are exactly what a full block +//! gets, and the same slot-level protections apply. +//! +//! The fee recipient is not compared for a blinded block. The header names the +//! builder's address, and the relay pays the validator inside the payload, so +//! a mismatch there is the flow working rather than a node paying someone else. +//! +//! The signed block goes to `blinded_blocks` on the node that produced it and +//! on no other: only that node's builder holds the payload. A publish the node +//! fails to confirm is reported as [`Published::BuilderUnconfirmed`] rather +//! than as a missed proposal, since the relay may still publish the block. +//! //! # Do not retry this call within a slot //! +//! //! For the same reason [`crate::attestation::AttestationService::attest`] must //! not be: a second call asks the node for a second block, which will differ //! from the first because the node has packed whatever arrived in between, and @@ -84,6 +101,9 @@ pub struct ProposalService { /// /// Not slashing protection. See [`ProposalGuard`]. guard: std::sync::Mutex, + /// The `builder_boost_factor` sent with each block request, or `None` when + /// builder proposals are off, in which case a blinded answer is refused. + builder_boost_factor: Option, } impl ProposalService { @@ -91,11 +111,13 @@ impl ProposalService { beacon_node: Arc, context: Arc, settings: Arc, + builder_boost_factor: Option, ) -> Self { Self { beacon_node, context, settings, + builder_boost_factor, guard: std::sync::Mutex::new(ProposalGuard::new()), } } @@ -215,7 +237,7 @@ impl ProposalService { proposer_index: duty.validator_index, randao_reveal, graffiti, - builder_boost_factor: None, + builder_boost_factor: self.builder_boost_factor, }; let mut produced = self.beacon_node.produce_block(&request).await?; @@ -253,13 +275,26 @@ impl ProposalService { ))); } - Self::check_fee_recipient(&produced, fee_recipient, slot); + // A blinded block's header names the builder's address, and the relay + // pays the validator inside the payload it reveals, so there is + // nothing here to compare. + if produced.is_blinded() { + tracing::debug!( + %slot, + "Blinded block: its header names the builder's address; the relay pays the \ + validator in the payload" + ); + } else { + Self::check_fee_recipient(&produced, fee_recipient, slot); + } let fork = produced.fork; + let blinded = produced.is_blinded(); info!( %slot, validator = duty.validator_index, fork = fork.as_str(), + blinded, blobs = produced.blob_count(), "Block produced; signing" ); @@ -303,10 +338,17 @@ impl ProposalService { let builder_index = produced.builder_index(); let body = produced.into_signed_ssz(signature); - let published = self - .publish(fork, &body, slot, duty.validator_index) - .await?; + let published = if blinded { + self.publish_blinded(slot, fork, &body, duty.validator_index) + .await? + } else { + self.publish(fork, &body, slot, duty.validator_index) + .await? + }; crate::metrics::inc_blocks_proposed(); + if published == Published::HandedToBuilder { + crate::metrics::inc_blinded_blocks_proposed(); + } // Only a self-built payload is this client's to reveal. if builder_index == Some(BUILDER_INDEX_SELF_BUILD) { @@ -401,6 +443,60 @@ impl ProposalService { Ok(()) } + /// Send a signed blinded block to the node that produced it. + /// + /// # Why an unconfirmed publish is not an error + /// + /// A 5xx or a missing answer does not say the block was not published: the + /// node may have handed it to the relay, which publishes regardless of what + /// the node then replied. So that outcome is logged loudly and counted on + /// its own series, and returned as [`Published::BuilderUnconfirmed`]. It is + /// not retried: the slot is signed, and a second block for it would be an + /// equivocation. A 4xx is a definite refusal and propagates as a failed + /// proposal. + async fn publish_blinded( + &self, + slot: Slot, + fork: ForkName, + body: &[u8], + validator: u64, + ) -> Result { + match self + .beacon_node + .publish_blinded_block(slot, fork, body) + .await + { + Ok(Published::HandedToBuilder) => { + info!(%slot, validator, bytes = body.len(), "Blinded block handed to the builder"); + Ok(Published::HandedToBuilder) + } + Ok(Published::BroadcastNotImported) => { + warn!( + %slot, + validator, + "Block was broadcast but the beacon node could not import it; \ + check that node's execution layer" + ); + crate::metrics::inc_blocks_broadcast_not_imported(); + Ok(Published::BroadcastNotImported) + } + Ok(other) => Ok(other), + Err(err @ Error::BlindedUnconfirmed { .. }) => { + tracing::error!( + %slot, + validator, + %err, + "The beacon node did not confirm the blinded block; the relay may still \ + publish it. Not retried: the slot is signed, and a second block would be \ + an equivocation" + ); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(Published::BuilderUnconfirmed) + } + Err(err) => Err(err), + } + } + /// Send the signed body and report what the node made of it. /// /// Split out so the 202 case has one place to be explained rather than @@ -482,7 +578,7 @@ mod tests { fn service(node: Arc) -> ProposalService { let settings = ProposerSettings::new(Bytes32::repeat_byte(0xab), None); - ProposalService::new(node, context(), Arc::new(settings)) + ProposalService::new(node, context(), Arc::new(settings), None) } fn service_expecting( @@ -490,7 +586,7 @@ mod tests { fee_recipient: ExecutionAddress, ) -> ProposalService { let settings = ProposerSettings::new(Bytes32::repeat_byte(0xab), Some(fee_recipient)); - ProposalService::new(node, context(), Arc::new(settings)) + ProposalService::new(node, context(), Arc::new(settings), None) } /// A slot inside mainnet's electra era. @@ -587,7 +683,7 @@ mod tests { let (store, pubkey) = store(); let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); let settings = Arc::new(ProposerSettings::new(Bytes32::repeat_byte(0xab), None)); - let service = ProposalService::new(node.clone(), context(), settings.clone()); + let service = ProposalService::new(node.clone(), context(), settings.clone(), None); settings.set_graffiti(&pubkey, Bytes32::repeat_byte(0xcd)); service @@ -675,6 +771,105 @@ mod tests { assert_eq!(published, Published::BroadcastNotImported); } + fn builder_service( + node: Arc, + fee_recipient: Option, + ) -> ProposalService { + let settings = ProposerSettings::new(Bytes32::repeat_byte(0xab), fee_recipient); + ProposalService::new(node, context(), Arc::new(settings), Some(100)) + } + + /// A blinded block takes the `blinded_blocks` route and nothing else. Its + /// header names the builder's address, so a configured fee recipient that + /// differs is not a mismatch for it. The mismatch counter is process-global + /// and shared with the test above, so it cannot be asserted on here; what + /// is asserted is that the proposal goes through unchanged. + #[tokio::test] + async fn a_blinded_block_is_published_to_the_builder_route() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + + let published = builder_service(node.clone(), Some(H160([0xfe; 20]))) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("proposes"); + assert_eq!(published, Published::HandedToBuilder); + + assert_eq!(node.block_requests()[0].builder_boost_factor, Some(100)); + assert!(node.published_blocks().is_empty()); + let sent = node.published_blinded.lock().expect("lock").clone(); + assert_eq!(sent.len(), 1); + assert_eq!((sent[0].0, sent[0].1), (slot(), ForkName::Electra)); + } + + /// A node that does not confirm the publish may still have handed the block + /// to its relay, so it is not a missed proposal, and the slot is spent: a + /// second block would be an equivocation. + #[tokio::test] + async fn an_unconfirmed_blinded_publish_is_reported_and_the_slot_is_spent() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_unconfirmed_blinded_publish("relay timed out"), + ); + let service = builder_service(node.clone(), None); + + let published = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("an unconfirmed publish is not an error"); + assert_eq!(published, Published::BuilderUnconfirmed); + + let err = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("the guard must refuse a second block for the slot"); + assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); + } + + /// Any failure that is not `BlindedUnconfirmed` is a definite refusal (a + /// 4xx): the block was not published, so the proposal failed. + #[tokio::test] + async fn a_definitely_refused_blinded_publish_is_an_error() { + let (store, pubkey) = store(); + let node = MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(); + node.failures + .lock() + .expect("lock") + .insert("publish_blinded_block", "400 bad request".to_string()); + + let result = builder_service(Arc::new(node), None) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await; + assert!(result.is_err(), "got {result:?}"); + } + + #[tokio::test] + async fn without_builder_proposals_the_request_carries_no_factor() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + + let published = service(node.clone()) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("proposes"); + assert_eq!(published, Published::Imported); + assert_eq!(node.block_requests()[0].builder_boost_factor, None); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + } + /// A duty naming a key this client does not hold must fail before anything /// is asked of the beacon node. #[tokio::test] @@ -832,7 +1027,7 @@ mod gloas_tests { ethlambda_types::beacon::primitives::Bytes32::repeat_byte(0xab), None, ); - ProposalService::new(node, context(), Arc::new(settings)) + ProposalService::new(node, context(), Arc::new(settings), None) } fn verifies(pubkey: &BlsPubkey, signature: &[u8; 96], root: Root) -> bool { @@ -909,7 +1104,7 @@ mod gloas_tests { )); let settings = Arc::new(ProposerSettings::new(Bytes32::repeat_byte(0xab), None)); settings.set_graffiti(&pubkey, Bytes32::repeat_byte(0xcd)); - ProposalService::new(node.clone(), context(), settings) + ProposalService::new(node.clone(), context(), settings, None) .propose(slot(), &duty(&pubkey), &store) .await .expect("proposes"); diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 01daf3f4d..36ef7fe8d 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -59,9 +59,9 @@ pub fn graffiti_to_text(graffiti: &Bytes32) -> String { /// The gas limit a validator with no override reports. /// -/// Lighthouse's and Nimbus's default. Nothing in this client uses it yet: it -/// matters only to builder registrations, and there is no builder flow. It is -/// held so the keymanager API can serve it as the specification requires. +/// Lighthouse's and Nimbus's default. Builder registrations carry it as the +/// gas limit the builder should target, and the keymanager API serves it as the +/// specification requires. pub const DEFAULT_GAS_LIMIT: u64 = 60_000_000; /// One validator's overrides; `None` means the default applies. From b60e645377857dbd9fbf994f954f1613874666bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:32:13 -0300 Subject: [PATCH 06/19] test(rpc): drive the builder flow end to end with the validator client Each half of the builder flow was tested alone: the node's routes against the mock relay, and the client against a mock node. Nothing checked that the two agree on the wire. These tests run the client's own HttpBeaconNode and SigningContext against this node's routes with a Builder layered over the mock relay: a registration the relay verifies, a blinded block produced under a boost factor and published, and the two cases that must stay on a full block (builder proposals off, relay down). The builder world helpers from proposal.rs are exposed to the test tree instead of copied. --- crates/net/rpc/src/beacon/proposal.rs | 22 +- .../rpc/src/beacon/validator_client_tests.rs | 208 ++++++++++++++++++ 2 files changed, 219 insertions(+), 11 deletions(-) diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index c5d0e66f8..818e11872 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -1082,7 +1082,7 @@ pub(crate) fn decimal(value: ðlambda_types::beacon::primitives::Uint256) -> S } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::test_utils::{RecordingNetwork, beacon_store_with_config}; use axum::{body::Body, http::Request}; @@ -1425,9 +1425,9 @@ mod tests { use libssz::SszDecode as _; use std::time::Duration; - const ETH: u64 = 1_000_000_000_000_000_000; + pub(crate) const ETH: u64 = 1_000_000_000_000_000_000; /// The registered gas limit the test proposer asks for. - const TARGET_GAS_LIMIT: u64 = 60_000_000; + pub(crate) const TARGET_GAS_LIMIT: u64 = 60_000_000; /// Thresholds that never trip, so a test state's history does not decide /// whether the builder is asked. @@ -1440,15 +1440,15 @@ mod tests { /// A builder world for the next slot on `fulu_store()`: a mock relay, a /// `Builder` pointing at it with the proposer registered, and the /// advanced proposal state. - struct BuilderWorld { - store: Store, - slot: Slot, - state: BeaconState, - relay: MockRelay, - builder: Arc, + pub(crate) struct BuilderWorld { + pub(crate) store: Store, + pub(crate) slot: Slot, + pub(crate) state: BeaconState, + pub(crate) relay: MockRelay, + pub(crate) builder: Arc, } - async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { + pub(crate) async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { let (store, head_slot) = fulu_store(); let slot = head_slot + 1; let config = store.config(); @@ -1488,7 +1488,7 @@ mod tests { impl BuilderWorld { /// A bid of `value` wei that passes every check for this slot, with /// `edit` applied before the relay signs it. - fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { + pub(crate) fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { let config = self.store.config(); let inputs = payload_inputs(&self.state, &config).unwrap(); let BeaconState::Fulu(fulu) = &self.state else { diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 21a33af99..d5140944b 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -1643,3 +1643,211 @@ mod gloas { assert_full_sync_aggregate(&advanced, &signed.message.body.sync_aggregate); } } + +/// The builder flow through this node: the validator client's own +/// registration, `produceBlockV3` with a boost factor, and blinded publish, +/// against this node's routes with a `Builder` layered over a mock relay. +mod builder_flow { + use super::*; + use crate::beacon::proposal::tests::{BuilderWorld, ETH, TARGET_GAS_LIMIT, builder_world}; + use ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index; + use ethlambda_types::beacon::builder::{ + SignedBlindedBeaconBlock, SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::ExecutionAddress; + use ethlambda_validator::beacon_node::Published; + use libssz::SszDecode as _; + + /// What a test drives: this node served over a socket, the proposer's key + /// and the client's signing context. + struct Harness { + world: BuilderWorld, + client: HttpBeaconNode, + proposer: u64, + keys: RwLock, + context: SigningContext, + } + + /// `world`'s routes (production, blinded publish and registration, sharing + /// one fee-recipient map as the real router does) behind a socket. + async fn harness(world: BuilderWorld, engine: ethlambda_engine::EngineClient) -> Harness { + // A block built on this head names the advanced header's root as its + // parent, which the fixture's placeholder anchor block does not have: + // keep the head state under that root too, as `serve_at` does, so the + // publish finds its parent. + let mut store = world.store.clone(); + let (_, head_state) = crate::beacon::validator::head(&store).unwrap(); + let parent_root = world.state.latest_block_header().hash_tree_root(); + store + .insert_state(parent_root, (*head_state).clone()) + .unwrap(); + + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let router = crate::beacon::proposal::routes() + .merge(crate::beacon::registrations::routes()) + .with_state(world.store.clone()) + .layer(Extension(Some(engine))) + .layer(Extension(SharedSyncCommitteePool::default())) + .layer(Extension(crate::beacon::validator::FeeRecipients::default())) + .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new( + own_version(), + )))) + .layer(Extension(network)) + .layer(Extension(world.builder.clone())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await }); + let client = HttpBeaconNode::new(format!("http://{address}")).unwrap(); + + let proposer = get_beacon_proposer_index(&world.state).unwrap(); + let keys = keys_for(&world.state, &[proposer]); + let context = SigningContext { + config: (*world.store.config()).clone(), + genesis_validators_root: world.state.genesis_validators_root(), + }; + Harness { + world, + client, + proposer, + keys, + context, + } + } + + impl Harness { + fn pubkey(&self) -> BlsPubkey { + self.world.state.validator(self.proposer).unwrap().pubkey + } + + /// The proposer's registration, signed by the client and sent through + /// this node. The relay checks the signature itself. + async fn register(&self) -> SignedValidatorRegistrationV1 { + let message = ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(0x11), + gas_limit: TARGET_GAS_LIMIT, + timestamp: 1, + pubkey: self.pubkey(), + }; + let signature = { + let keys = self.keys.read().await; + self.context + .sign_validator_registration(&keys, &message) + .unwrap() + }; + let signed = SignedValidatorRegistrationV1 { message, signature }; + self.client + .register_validators(std::slice::from_ref(&signed)) + .await + .expect("the node takes the registration"); + signed + } + + async fn produce( + &self, + builder_boost_factor: Option, + ) -> ethlambda_validator::beacon_node::block_contents::ProducedBlock { + let epoch = compute_epoch_at_slot(self.world.slot); + // Signed under the head state's own fork rather than through the + // client's schedule: this world's state is not pinned to carry + // the schedule's fulu version (see `serve_fulu_pinned`), and the + // node verifies against the state, as the other proposal tests + // here do. + let randao_domain = get_domain( + &self.world.state, + ethlambda_types::beacon::constants::DOMAIN_RANDAO, + Some(epoch), + ); + let randao_reveal = sign_for( + self.proposer as usize, + compute_signing_root(epoch.hash_tree_root(), randao_domain), + ); + let request = BlockRequest { + slot: self.world.slot, + fork: ForkName::Fulu, + proposer_index: self.proposer, + randao_reveal, + graffiti: ethlambda_types::primitives::H256([0; 32]), + builder_boost_factor, + }; + self.client.produce_block(&request).await.unwrap() + } + + async fn sign( + &self, + produced: ethlambda_validator::beacon_node::block_contents::ProducedBlock, + ) -> Vec { + let domain = get_domain( + &self.world.state, + ethlambda_types::beacon::constants::DOMAIN_BEACON_PROPOSER, + Some(compute_epoch_at_slot(self.world.slot)), + ); + let signature = sign_for( + self.proposer as usize, + compute_signing_root(produced.block_root(), domain), + ); + produced.into_signed_ssz(signature) + } + } + + #[tokio::test] + async fn the_validator_client_can_propose_a_builder_block_through_this_node() { + let world = builder_world(false, SyncStatus::Synced).await; + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let registered = h.register().await; + assert_eq!(h.world.relay.registrations(), [registered]); + + h.world.offer(ETH, |_| {}); + let produced = h.produce(Some(100)).await; + assert!(produced.is_blinded(), "a bid worth more must win"); + + let body = h.sign(produced).await; + let published = h + .client + .publish_blinded_block(h.world.slot, ForkName::Fulu, &body) + .await + .expect("the node hands the block to its builder"); + assert_eq!(published, Published::HandedToBuilder); + + let signed = SignedBlindedBeaconBlock::from_ssz_bytes(&body).unwrap(); + assert_eq!(h.world.relay.blinded_blocks(), [signed]); + } + + #[tokio::test] + async fn without_builder_proposals_the_client_gets_a_full_block() { + let world = builder_world(false, SyncStatus::Synced).await; + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + // A client with builder proposals off never registers either, which + // is what keeps the node from asking the relay at all. + h.world.offer(ETH, |_| {}); + let produced = h.produce(None).await; + assert!(!produced.is_blinded()); + assert!(h.world.relay.blinded_blocks().is_empty()); + assert_eq!( + h.world.relay.header_requests().len(), + 0, + "a client with builder proposals off must not make the node ask the relay" + ); + } + + #[tokio::test] + async fn a_relay_that_is_down_leaves_a_full_block() { + let world = builder_world(false, SyncStatus::Synced).await; + world.relay.answer_headers_with( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "application/json", + Vec::new(), + None, + ); + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + h.register().await; + h.world.offer(ETH, |_| {}); + let produced = h.produce(Some(100)).await; + assert!(!produced.is_blinded()); + } +} From da6c4e9bd635d5cf8fe11f74ef5d04522926c552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:33:01 -0300 Subject: [PATCH 07/19] docs: describe the validator client's builder proposals Operators need to know the three new flags and their constraints, the four counters that show whether registrations and blinded blocks are landing, and the three places the client departs from the builder flow (no failover for a blinded publish, no slashing record beyond the in-process guard, no registrations from gloas), since the first one decides what an unconfirmed publish means for a proposal. --- CLAUDE.md | 4 +++- docs/cli.md | 3 +++ docs/metrics.md | 4 ++++ docs/spec_deviations.md | 26 ++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index aa67c4bca..dec9283c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -862,7 +862,9 @@ transitions are in `ethlambda-types`, per the section above. Nothing above and a block at slot N packs only `(N-1, parent_root)`, replaced by the empty aggregate when `verified_sync_aggregate` fails (see `docs/spec_deviations.md`). `SlotClock` picks the - offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. It + offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. With + `--builder-proposals`, pre-gloas slots register validators with the node's builder + and may sign a blinded block, published to the producing node only. It still keeps no slashing-protection record; PTC votes have an in-memory `(validator, slot)` dedup only, and envelopes are unguarded. `get_head_node` is one bottom-up walk for every fork, with diff --git a/docs/cli.md b/docs/cli.md index b9a65aa5d..7bd8e70eb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -290,6 +290,9 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | +| `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | +| `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` always builds locally. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | +| `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid (sends `builder_boost_factor = u64::MAX`). Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | ### What it does today diff --git a/docs/metrics.md b/docs/metrics.md index 087d990f0..65dcc53d7 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -741,6 +741,10 @@ and absent is not the same as zero. | `ethlambda_validator_aggregates_published_total` | Counter | Aggregates accepted by a beacon node. Bursty rather than steady: a validator is selected a few times a day, so hours at zero are normal for a small deployment | | `ethlambda_validator_aggregation_failures_total` | Counter | Aggregation duties that ended in no published aggregate, including ones abandoned for overrunning the slot | | `ethlambda_validator_fee_recipient_mismatches_total` | Counter | Blocks paying execution rewards to an address this client did not request. Should read zero forever; a non-zero value means every proposal is paying somewhere else | +| `ethlambda_validator_builder_registrations_sent_total` | Counter | Builder registrations a beacon node accepted (`--builder-proposals`) | +| `ethlambda_validator_builder_registration_failures_total` | Counter | Registration batches no beacon node accepted. A missed registration costs at most a builder block, never a duty | +| `ethlambda_validator_blinded_blocks_proposed_total` | Counter | Blinded blocks a beacon node handed to its builder. A subset of `blocks_proposed_total` | +| `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx or no answer), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in | | `ethlambda_validator_beacon_node_available` | Gauge | 1 when a beacon node answered the last duty refresh or attestation | | `ethlambda_validator_signing_duration_seconds` | Histogram | Time to sign one slot's batch, store read lock included | diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 8e04fb262..758d01876 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -683,3 +683,29 @@ comes first. The validator client signs once, at the deadline (`SYNC_MESSAGE_DUE_BPS`, or its gloas variant), over the head root it then reads, and refuses to when that root is optimistic (`specs/bellatrix/optimistic-sync.md`). + +## The validator client's builder proposals + +With `--builder-proposals` the validator client registers its validators with +the beacon node's builder and may sign a blinded block (builder-specs and +`validator.md`'s builder flow), for fulu and earlier slots. Three departures: + +- **No slashing-protection record for a blinded block either.** It is signed + without a durable record, like every other block from this client. A blinded + block has the root of the full block it stands for, so the in-process + `ProposalGuard` covers both kinds: one slot, one signature, whichever way the + node answered. +- **A blinded block is published only on the beacon node that produced it, with + no failover.** The specification says the block goes to the node that built + it. Only that node's builder holds the payload and only that node claimed the + slot, so offering the signed block to another node would hand it to a builder + that never saw the bid. If the producing node fails, the publish fails, and + the guard has already recorded the slot, so nothing retries. A 5xx or no + answer is reported as unconfirmed (logged at error, counted on + `blinded_blocks_unconfirmed_total`) and not as a missed proposal, since the + relay may still publish the block. +- **Registrations stop at gloas.** The client sends no `register_validator` from + the gloas fork epoch on: the builder market works through proposer + preferences there, and `produceBlockV4` keeps its self-build configuration. + Validators with no fee recipient are not registered either (the field is + required), and only validators the beacon node resolved are signed for. From ff2149e9f6cf91df78f5c8bcba7dd9c6d6807a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:54:59 -0300 Subject: [PATCH 08/19] fix(validator): keep a slow relay from counting as a missed proposal The beacon node waits up to 5 s on the relay, so the duty loop's budget could fire first and count a block that was signed and handed to the builder as a failed proposal. The blinded publish now runs on its own task, which logs and counts its real outcome, and the loop treats an overrun after a blinded signing as neither a failure nor a delay sample. An unconfirmed publish no longer feeds blocks_proposed or the publication delay either. Builder registrations are signed on the blocking pool in chunks that take the store lock per chunk, are spawned only after a refresh that succeeded (so a failing refresh cannot respawn a full send every slot), and the per-slot check returns on one atomic read of a settings generation counter unless a keymanager write moved it. Removed keys are pruned from the cache. Also tightens the blinded publish's error contract (a connect error and a 503 are definite refusals), gives register_validators its own timeout, makes the tests sensitive to the boost factor and the builder domain, and moves the builder test world into a test_support module. --- bin/ethlambda/src/validator.rs | 5 +- crates/net/rpc/src/beacon/proposal.rs | 275 ++++++++++-------- .../rpc/src/beacon/validator_client_tests.rs | 62 ++-- .../src/beacon_node/block_contents.rs | 17 +- crates/validator/src/beacon_node/fallback.rs | 55 +++- crates/validator/src/beacon_node/http.rs | 157 +++++++--- crates/validator/src/beacon_node/mock.rs | 25 +- crates/validator/src/error.rs | 4 + crates/validator/src/lib.rs | 80 +++-- crates/validator/src/metrics.rs | 2 +- crates/validator/src/proposal.rs | 224 ++++++++++---- crates/validator/src/proposer_settings.rs | 16 + crates/validator/src/registration.rs | 222 +++++++++++--- crates/validator/src/signing.rs | 23 +- docs/cli.md | 21 +- docs/metrics.md | 6 +- docs/spec_deviations.md | 8 +- 17 files changed, 838 insertions(+), 364 deletions(-) diff --git a/bin/ethlambda/src/validator.rs b/bin/ethlambda/src/validator.rs index 6dda48b22..51b27a15a 100644 --- a/bin/ethlambda/src/validator.rs +++ b/bin/ethlambda/src/validator.rs @@ -89,7 +89,7 @@ pub(crate) struct ValidatorOptions { /// How a builder's bid is weighed against the local payload: the local /// payload wins iff `local >= factor * bid / 100`. 100 compares them as - /// they are; 0 always builds locally. + /// they are; 0 prefers the local payload unless it cannot be built. #[arg( long, default_value = "100", @@ -98,7 +98,8 @@ pub(crate) struct ValidatorOptions { )] pub(crate) builder_boost_factor: u64, - /// Take the builder's block whenever there is a valid bid (sends + /// Take the builder's block whenever there is a valid bid, unless the + /// execution client asks to override the builder (sends /// `builder_boost_factor = u64::MAX`). #[arg(long, default_value = "false", requires = "builder_proposals")] pub(crate) prefer_builder_proposals: bool, diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 818e11872..30d4573f7 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -1082,7 +1082,8 @@ pub(crate) fn decimal(value: ðlambda_types::beacon::primitives::Uint256) -> S } #[cfg(test)] -pub(crate) mod tests { +mod tests { + use super::test_support::*; use super::*; use crate::test_utils::{RecordingNetwork, beacon_store_with_config}; use axum::{body::Body, http::Request}; @@ -1109,28 +1110,6 @@ pub(crate) mod tests { (status, serde_json::from_slice(&body).unwrap_or_default()) } - /// A fulu head with its proposer lookahead and sync committee filled in, - /// which a block built on it needs, under a schedule that puts every slot - /// at fulu. - fn fulu_store() -> (Store, Slot) { - use ethlambda_state_transition::beacon::helpers::{ - altair::get_next_sync_committee, fulu::initialize_proposer_lookahead, - }; - let mut state = with_signing_validators_at(ForkName::Fulu, 64); - let lookahead = initialize_proposer_lookahead(&state).unwrap(); - let sync_committee = get_next_sync_committee(&state).unwrap(); - let BeaconState::Fulu(inner) = &mut state else { - unreachable!("built as fulu") - }; - inner.proposer_lookahead = lookahead.try_into().unwrap(); - inner.current_sync_committee = sync_committee.clone(); - inner.next_sync_committee = sync_committee; - let head_slot = state.slot(); - let (store, _root) = - beacon_store_with_config(state, Config::mainnet().with_fork_epoch(ForkName::Fulu, 0)); - (store, head_slot) - } - fn produce_app(store: Store, engine: Option) -> Router { routes() .with_state(store) @@ -1414,116 +1393,14 @@ pub(crate) mod tests { assert!(rejected.blocks.lock().unwrap().is_empty()); } - use crate::beacon::builder_client::mock_relay::MockRelay; - use crate::beacon::builder_client::{BuilderClient, VersionedBid}; - use crate::beacon::builder_flow::breaker::FallbackThresholds; + use crate::beacon::builder_client::BuilderClient; use ethlambda_blockchain::SyncStatusController; - use ethlambda_types::beacon::builder::{BuilderBid, ValidatorRegistrationV1}; - use ethlambda_types::beacon::containers::capella; - use ethlambda_types::beacon::containers::deneb::{ExecutionPayloadHeader, KzgCommitments}; - use ethlambda_types::beacon::containers::electra::{self, ExecutionRequests}; + use ethlambda_types::beacon::builder::ValidatorRegistrationV1; + use ethlambda_types::beacon::containers::electra; use libssz::SszDecode as _; use std::time::Duration; - pub(crate) const ETH: u64 = 1_000_000_000_000_000_000; - /// The registered gas limit the test proposer asks for. - pub(crate) const TARGET_GAS_LIMIT: u64 = 60_000_000; - - /// Thresholds that never trip, so a test state's history does not decide - /// whether the builder is asked. - const NEVER_TRIP: FallbackThresholds = FallbackThresholds { - skips: u64::MAX, - skips_per_epoch: u64::MAX, - epochs_since_finalization: u64::MAX, - }; - - /// A builder world for the next slot on `fulu_store()`: a mock relay, a - /// `Builder` pointing at it with the proposer registered, and the - /// advanced proposal state. - pub(crate) struct BuilderWorld { - pub(crate) store: Store, - pub(crate) slot: Slot, - pub(crate) state: BeaconState, - pub(crate) relay: MockRelay, - pub(crate) builder: Arc, - } - - pub(crate) async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { - let (store, head_slot) = fulu_store(); - let slot = head_slot + 1; - let config = store.config(); - let (_, head_state) = super::super::validator::head(&store).unwrap(); - let state = advance_to_slot(&head_state, slot, &config).unwrap(); - let proposer = get_beacon_proposer_index(&state).unwrap(); - let pubkey = state.validator(proposer).unwrap().pubkey; - let relay = MockRelay::start(config.genesis_fork_version).await; - let client = - BuilderClient::new(relay.url.clone(), Duration::from_secs(1), "ethlambda/test") - .unwrap(); - let builder = Arc::new(Builder::new( - client, - NEVER_TRIP, - SyncStatusController::new(sync), - )); - if registered { - let registration = ValidatorRegistrationV1 { - fee_recipient: ExecutionAddress::repeat_byte(0x11), - gas_limit: TARGET_GAS_LIMIT, - timestamp: 1, - pubkey, - }; - builder - .registrations - .record([®istration], compute_epoch_at_slot(slot)); - } - BuilderWorld { - store, - slot, - state, - relay, - builder, - } - } - impl BuilderWorld { - /// A bid of `value` wei that passes every check for this slot, with - /// `edit` applied before the relay signs it. - pub(crate) fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { - let config = self.store.config(); - let inputs = payload_inputs(&self.state, &config).unwrap(); - let BeaconState::Fulu(fulu) = &self.state else { - unreachable!("built as fulu") - }; - let parent_gas_limit = fulu.latest_execution_payload_header.gas_limit; - let step = (parent_gas_limit / 1024).saturating_sub(1); - let withdrawals: capella::Withdrawals = inputs.withdrawals.clone().try_into().unwrap(); - let mut header = - ExecutionPayloadHeader::from(&electra::BeaconBlockBody::empty().execution_payload); - header.parent_hash = inputs.parent_hash; - header.prev_randao = inputs.prev_randao; - header.timestamp = inputs.timestamp; - header.block_number = fulu.latest_execution_payload_header.block_number + 1; - header.withdrawals_root = withdrawals.hash_tree_root(); - header.gas_limit = TARGET_GAS_LIMIT.clamp( - parent_gas_limit - step, - parent_gas_limit.saturating_add(step), - ); - header.block_hash = ExecutionBlockHash::repeat_byte(0xbb); - let mut message = BuilderBid { - header, - blob_kzg_commitments: KzgCommitments::default(), - execution_requests: ExecutionRequests::default(), - value: Uint256::from(value), - pubkey: self.relay.pubkey(), - }; - edit(&mut message); - let bid = self.relay.sign_bid(message); - self.relay.offer(VersionedBid { - fork: ForkName::Fulu, - bid, - }); - } - fn app(&self, engine: Option) -> Router { produce_app(self.store.clone(), engine).layer(Extension(self.builder.clone())) } @@ -2023,3 +1900,145 @@ pub(crate) mod tests { assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); } } + +/// The builder world the proposal tests build on: a fulu head, a mock relay and +/// a `Builder` pointing at it. Shared with the validator client's end-to-end +/// tests, which drive the same routes through a real socket. +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + use crate::beacon::builder_client::BuilderClient; + use crate::beacon::builder_client::mock_relay::MockRelay; + use crate::beacon::builder_flow::breaker::FallbackThresholds; + use crate::test_utils::beacon_store_with_config; + use ethlambda_blockchain::SyncStatusController; + use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; + use ethlambda_types::beacon::builder::ValidatorRegistrationV1; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::capella; + use ethlambda_types::beacon::containers::deneb::{ExecutionPayloadHeader, KzgCommitments}; + use ethlambda_types::beacon::containers::electra; + use std::time::Duration; + + /// A fulu head with its proposer lookahead and sync committee filled in, + /// which a block built on it needs, under a schedule that puts every slot + /// at fulu. + pub(crate) fn fulu_store() -> (Store, Slot) { + use ethlambda_state_transition::beacon::helpers::{ + altair::get_next_sync_committee, fulu::initialize_proposer_lookahead, + }; + let mut state = with_signing_validators_at(ForkName::Fulu, 64); + let lookahead = initialize_proposer_lookahead(&state).unwrap(); + let sync_committee = get_next_sync_committee(&state).unwrap(); + let BeaconState::Fulu(inner) = &mut state else { + unreachable!("built as fulu") + }; + inner.proposer_lookahead = lookahead.try_into().unwrap(); + inner.current_sync_committee = sync_committee.clone(); + inner.next_sync_committee = sync_committee; + let head_slot = state.slot(); + let (store, _root) = + beacon_store_with_config(state, Config::mainnet().with_fork_epoch(ForkName::Fulu, 0)); + (store, head_slot) + } + + pub(crate) const ETH: u64 = 1_000_000_000_000_000_000; + /// The registered gas limit the test proposer asks for. + pub(crate) const TARGET_GAS_LIMIT: u64 = 60_000_000; + + /// Thresholds that never trip, so a test state's history does not decide + /// whether the builder is asked. + pub(crate) const NEVER_TRIP: FallbackThresholds = FallbackThresholds { + skips: u64::MAX, + skips_per_epoch: u64::MAX, + epochs_since_finalization: u64::MAX, + }; + + /// A builder world for the next slot on `fulu_store()`: a mock relay, a + /// `Builder` pointing at it with the proposer registered, and the + /// advanced proposal state. + pub(crate) struct BuilderWorld { + pub(crate) store: Store, + pub(crate) slot: Slot, + pub(crate) state: BeaconState, + pub(crate) relay: MockRelay, + pub(crate) builder: Arc, + } + + pub(crate) async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { + let (store, head_slot) = fulu_store(); + let slot = head_slot + 1; + let config = store.config(); + let (_, head_state) = super::super::validator::head(&store).unwrap(); + let state = advance_to_slot(&head_state, slot, &config).unwrap(); + let proposer = get_beacon_proposer_index(&state).unwrap(); + let pubkey = state.validator(proposer).unwrap().pubkey; + let relay = MockRelay::start(config.genesis_fork_version).await; + let client = + BuilderClient::new(relay.url.clone(), Duration::from_secs(1), "ethlambda/test") + .unwrap(); + let builder = Arc::new(Builder::new( + client, + NEVER_TRIP, + SyncStatusController::new(sync), + )); + if registered { + let registration = ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(0x11), + gas_limit: TARGET_GAS_LIMIT, + timestamp: 1, + pubkey, + }; + builder + .registrations + .record([®istration], compute_epoch_at_slot(slot)); + } + BuilderWorld { + store, + slot, + state, + relay, + builder, + } + } + + impl BuilderWorld { + /// A bid of `value` wei that passes every check for this slot, with + /// `edit` applied before the relay signs it. + pub(crate) fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { + let config = self.store.config(); + let inputs = payload_inputs(&self.state, &config).unwrap(); + let BeaconState::Fulu(fulu) = &self.state else { + unreachable!("built as fulu") + }; + let parent_gas_limit = fulu.latest_execution_payload_header.gas_limit; + let step = (parent_gas_limit / 1024).saturating_sub(1); + let withdrawals: capella::Withdrawals = inputs.withdrawals.clone().try_into().unwrap(); + let mut header = + ExecutionPayloadHeader::from(&electra::BeaconBlockBody::empty().execution_payload); + header.parent_hash = inputs.parent_hash; + header.prev_randao = inputs.prev_randao; + header.timestamp = inputs.timestamp; + header.block_number = fulu.latest_execution_payload_header.block_number + 1; + header.withdrawals_root = withdrawals.hash_tree_root(); + header.gas_limit = TARGET_GAS_LIMIT.clamp( + parent_gas_limit - step, + parent_gas_limit.saturating_add(step), + ); + header.block_hash = ExecutionBlockHash::repeat_byte(0xbb); + let mut message = BuilderBid { + header, + blob_kzg_commitments: KzgCommitments::default(), + execution_requests: ExecutionRequests::default(), + value: Uint256::from(value), + pubkey: self.relay.pubkey(), + }; + edit(&mut message); + let bid = self.relay.sign_bid(message); + self.relay.offer(VersionedBid { + fork: ForkName::Fulu, + bid, + }); + } + } +} diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index d5140944b..f2fd5cbca 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -131,16 +131,20 @@ async fn serve_at( .unwrap(); let parent_root = advanced.latest_block_header().hash_tree_root(); store.insert_state(parent_root, state.clone()).unwrap(); - let (client, network, _pool, address) = spawn_server(store.clone(), engine).await; + let (client, network, _pool, address) = spawn_server(store.clone(), engine, None).await; (client, state, network, address, store) } /// Serve `store` through the router the real server builds, over a socket on an /// ephemeral port, and connect this crate's client to it. Also hands back the /// payload attestation pool the router fills and the socket it listens on. +/// +/// With a `builder`, it is layered as `lib.rs` does when `--builder` is set, +/// so the builder flow's endpoints see it. async fn spawn_server( mut store: Store, engine: Option, + builder: Option>, ) -> ( HttpBeaconNode, Arc, @@ -171,6 +175,10 @@ async fn spawn_server( .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new( own_version(), )))); + let router = match builder { + Some(builder) => router.layer(Extension(builder)), + None => router, + }; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); @@ -874,7 +882,7 @@ async fn serve_fulu_pinned( head_root, slot, ); - let (client, network, _, _) = spawn_server(store, engine).await; + let (client, network, _, _) = spawn_server(store, engine, None).await; let genesis = client.genesis().await.expect("genesis"); let spec = client .spec() @@ -1175,7 +1183,8 @@ mod gloas { let head_root = header.hash_tree_root(); let store = beacon_store_with_head_block(state.clone(), config(), block, head_root, clock_slot); - let (client, network, payload_pool, _address) = spawn_server(store.clone(), engine).await; + let (client, network, payload_pool, _address) = + spawn_server(store.clone(), engine, None).await; let genesis = client.genesis().await.expect("genesis"); let spec = client .spec() @@ -1649,7 +1658,9 @@ mod gloas { /// against this node's routes with a `Builder` layered over a mock relay. mod builder_flow { use super::*; - use crate::beacon::proposal::tests::{BuilderWorld, ETH, TARGET_GAS_LIMIT, builder_world}; + use crate::beacon::proposal::test_support::{ + BuilderWorld, ETH, TARGET_GAS_LIMIT, builder_world, + }; use ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index; use ethlambda_types::beacon::builder::{ SignedBlindedBeaconBlock, SignedValidatorRegistrationV1, ValidatorRegistrationV1, @@ -1668,8 +1679,8 @@ mod builder_flow { context: SigningContext, } - /// `world`'s routes (production, blinded publish and registration, sharing - /// one fee-recipient map as the real router does) behind a socket. + /// `world`'s store served through the real beacon router, with its + /// `Builder` layered, behind a socket. async fn harness(world: BuilderWorld, engine: ethlambda_engine::EngineClient) -> Harness { // A block built on this head names the advanced header's root as its // parent, which the fixture's placeholder anchor block does not have: @@ -1682,22 +1693,8 @@ mod builder_flow { .insert_state(parent_root, (*head_state).clone()) .unwrap(); - let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); - let router = crate::beacon::proposal::routes() - .merge(crate::beacon::registrations::routes()) - .with_state(world.store.clone()) - .layer(Extension(Some(engine))) - .layer(Extension(SharedSyncCommitteePool::default())) - .layer(Extension(crate::beacon::validator::FeeRecipients::default())) - .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new( - own_version(), - )))) - .layer(Extension(network)) - .layer(Extension(world.builder.clone())); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, router).await }); - let client = HttpBeaconNode::new(format!("http://{address}")).unwrap(); + let (client, _network, _pool, _address) = + spawn_server(store, Some(engine), Some(world.builder.clone())).await; let proposer = get_beacon_proposer_index(&world.state).unwrap(); let keys = keys_for(&world.state, &[proposer]); @@ -1798,9 +1795,15 @@ mod builder_flow { let registered = h.register().await; assert_eq!(h.world.relay.registrations(), [registered]); - h.world.offer(ETH, |_| {}); - let produced = h.produce(Some(100)).await; - assert!(produced.is_blinded(), "a bid worth more must win"); + // Worth far less than the local payload (42 wei), so only the maximum + // factor makes it win: a factor that is dropped or ignored (the node's + // default is 100) leaves a full block. + h.world.offer(1, |_| {}); + let produced = h.produce(Some(u64::MAX)).await; + assert!( + produced.is_blinded(), + "the factor must reach the node and take the bid" + ); let body = h.sign(produced).await; let published = h @@ -1820,16 +1823,17 @@ mod builder_flow { let engine = fake_execution_client_paying("0x2a", false).await; let h = harness(world, engine).await; - // A client with builder proposals off never registers either, which - // is what keeps the node from asking the relay at all. + // Registered, with a bid worth far more than the local payload: only + // the factor the client sends (0 for `None`) keeps the block local. + h.register().await; h.world.offer(ETH, |_| {}); let produced = h.produce(None).await; assert!(!produced.is_blinded()); assert!(h.world.relay.blinded_blocks().is_empty()); assert_eq!( h.world.relay.header_requests().len(), - 0, - "a client with builder proposals off must not make the node ask the relay" + 1, + "the node still asks the relay at a factor of 0, as Lighthouse does" ); } diff --git a/crates/validator/src/beacon_node/block_contents.rs b/crates/validator/src/beacon_node/block_contents.rs index 040143337..9fbec02f0 100644 --- a/crates/validator/src/beacon_node/block_contents.rs +++ b/crates/validator/src/beacon_node/block_contents.rs @@ -21,7 +21,7 @@ //! //! The endpoints support it: `Accept: application/octet-stream` on //! `produceBlockV3`, `Content-Type: application/octet-stream` on -//! `publishBlockV2`. +//! `publishBlockV2` and `publishBlindedBlockV2`. //! //! # `BlockContents` is not a consensus container //! @@ -53,6 +53,8 @@ //! client has signed. No blobs travel with it, since the builder publishes //! them. Its root is the full block's, so signing and the proposal guard work //! as for any other block. Electra and fulu share one blinded container. +//! Electra blinded blocks are for third-party beacon nodes: an ethlambda node +//! serves blinded blocks for fulu only. //! //! # Gloas //! @@ -358,7 +360,15 @@ impl ProducedBlock { contents: Contents::Blinded { block }, }) } - _ => Err(Error::InconsistentResponse(format!( + // Named, as `from_ssz` names its arms, so a new fork forces a + // decision here instead of being refused by default. + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Gloas + | ForkName::Lean => Err(Error::InconsistentResponse(format!( "beacon node produced a blinded block for fork {}; blinded blocks exist for \ electra and fulu only", fork.as_str() @@ -496,7 +506,8 @@ impl ProducedBlock { } } - /// Attach `signature` and encode the body `publishBlockV2` expects. + /// Attach `signature` and encode the body `publishBlockV2` expects, or + /// `publishBlindedBlockV2` for a blinded block. /// /// Consumes the block: the proofs and blobs are moved into the published /// body rather than copied, and a blob is 128 KiB, so a six-blob block diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index f4e81cb8f..04b4652e4 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -130,7 +130,18 @@ impl FallbackBeaconNode { /// unreachable node must not stop the reachable ones being told. A failure /// is logged per node so a partially-registered client is visible rather /// than silent. - async fn try_all<'p, F>(&'p self, what: &'static str, call: F) -> Result<()> + /// + /// `node_effect` says what a refusing node costs (said once per refusal); + /// `all_failed` says what it costs when none accepts, or is `None` when the + /// caller reports that itself, so a duty that did not depend on the call is + /// not claimed missed. + async fn try_all<'p, F>( + &'p self, + what: &'static str, + node_effect: &'static str, + all_failed: Option<&'static str>, + call: F, + ) -> Result<()> where F: Fn(&'p B) -> BoxFuture<'p, Result<()>>, { @@ -140,7 +151,7 @@ impl FallbackBeaconNode { match call(node).await { Ok(()) => accepted += 1, Err(err) => { - warn!(%what, position, %err, "Beacon node did not accept; it will be unprepared for this duty"); + warn!(%what, position, %err, node_effect, "Beacon node did not accept"); last = Some(err); } } @@ -151,7 +162,9 @@ impl FallbackBeaconNode { let detail = last .map(|err| err.to_string()) .unwrap_or_else(|| "no nodes configured".to_string()); - warn!(%what, %detail, "No configured beacon node accepted; duty likely missed"); + if let Some(all_failed) = all_failed { + warn!(%what, %detail, all_failed, "No configured beacon node accepted"); + } Err(Error::AllBeaconNodesFailed(detail)) } } @@ -320,9 +333,12 @@ impl BeaconNodeApi for FallbackBeaconNode { &self, registrations: &[SignedValidatorRegistrationV1], ) -> Result<()> { - self.try_all("register_validators", |node| { - node.register_validators(registrations) - }) + self.try_all( + "register_validators", + "it will not hold these registrations", + None, + |node| node.register_validators(registrations), + ) .await } @@ -422,9 +438,12 @@ impl BeaconNodeApi for FallbackBeaconNode { /// that was never told where to pay builds a payload paying somewhere else, /// and it is the node this client falls over to that needs telling most. async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()> { - self.try_all("prepare_beacon_proposer", |node| { - node.prepare_beacon_proposer(preparations) - }) + self.try_all( + "prepare_beacon_proposer", + "it will be unprepared for this duty", + Some("duty likely missed"), + |node| node.prepare_beacon_proposer(preparations), + ) .await } @@ -433,9 +452,12 @@ impl BeaconNodeApi for FallbackBeaconNode { /// the votes an aggregate is folded from, so a node that never received one /// answers 404 for every committee this client asks about. async fn subscribe_committees(&self, subscriptions: &[CommitteeSubscriptionDto]) -> Result<()> { - self.try_all("subscribe_committees", |node| { - node.subscribe_committees(subscriptions) - }) + self.try_all( + "subscribe_committees", + "it will be unprepared for this duty", + Some("duty likely missed"), + |node| node.subscribe_committees(subscriptions), + ) .await } @@ -496,9 +518,12 @@ impl BeaconNodeApi for FallbackBeaconNode { &self, subscriptions: &[SyncCommitteeSubscriptionDto], ) -> Result<()> { - self.try_all("subscribe_sync_committees", |node| { - node.subscribe_sync_committees(subscriptions) - }) + self.try_all( + "subscribe_sync_committees", + "it will be unprepared for this duty", + Some("duty likely missed"), + |node| node.subscribe_sync_committees(subscriptions), + ) .await } } diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 60a01c50f..599119890 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -61,6 +61,12 @@ use crate::error::{BeaconNodeFailure, Error, Result}; /// a stalled request could then outlive the slot it was serving. const REQUEST_TIMEOUT: Duration = Duration::from_secs(8); +/// How long `register_validator` may take. Longer than [`REQUEST_TIMEOUT`] +/// because an ethlambda node forwards the registrations to its relay and waits +/// up to 10 s for the answer before replying; a client that gave up at 8 s +/// would report a failure for a registration the node went on to install. +const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(12); + /// What an SSZ answer carries besides its bytes: the fork, which only a header /// can say, and the two booleans the block endpoints use to say what the bytes /// are. @@ -147,12 +153,28 @@ impl HttpBeaconNode { path: &str, body: &B, consensus_version: Option<&str>, + ) -> Result<()> { + self.post_no_content_within(path, body, consensus_version, None) + .await + } + + /// [`Self::post_no_content`] with `timeout` in place of the client's + /// default, when given. + async fn post_no_content_within( + &self, + path: &str, + body: &B, + consensus_version: Option<&str>, + timeout: Option, ) -> Result<()> { let url = format!("{}{path}", self.base_url); let mut request = self.client.post(&url).json(body); if let Some(version) = consensus_version { request = request.header("Eth-Consensus-Version", version); } + if let Some(timeout) = timeout { + request = request.timeout(timeout); + } let response = request.send().await.map_err(|err| Error::BeaconNode { url: url.clone(), failure: BeaconNodeFailure::classify(&err), @@ -169,6 +191,30 @@ impl HttpBeaconNode { }) } + /// POST an SSZ body with the fork it is for, returning the URL beside the + /// outcome so each caller maps a transport failure its own way: a full + /// block's is an ordinary error, a blinded block's may be unconfirmed. + async fn post_ssz_body( + &self, + path: &str, + fork: ForkName, + body: &[u8], + ) -> ( + String, + std::result::Result, + ) { + let url = format!("{}{path}", self.base_url); + let sent = self + .client + .post(&url) + .header("Eth-Consensus-Version", fork.as_str()) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body.to_vec()) + .send() + .await; + (url, sent) + } + /// Fetch a body as SSZ, returning the fork the node named alongside it. /// /// `Accept` names SSZ alone, deliberately narrower than the specification's @@ -665,21 +711,14 @@ impl BeaconNodeApi for HttpBeaconNode { /// 202 is not an error and not a clean success, so it is neither mapped to /// `Err` nor flattened into `Ok(())`. See [`Published`]. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result { - let path = "/eth/v2/beacon/blocks"; - let url = format!("{}{path}", self.base_url); - let response = self - .client - .post(&url) - .header("Eth-Consensus-Version", fork.as_str()) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body.to_vec()) - .send() - .await - .map_err(|err| Error::BeaconNode { - url: url.clone(), - failure: BeaconNodeFailure::classify(&err), - detail: err.to_string(), - })?; + let (url, sent) = self + .post_ssz_body("/eth/v2/beacon/blocks", fork, body) + .await; + let response = sent.map_err(|err| Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + })?; let status = response.status(); if status == StatusCode::ACCEPTED { @@ -697,27 +736,37 @@ impl BeaconNodeApi for HttpBeaconNode { /// Maps the answer as the trait's contract says: 2xx is handed to the /// builder (202 is the node's broadcast-without-import, as for a full - /// block), a 4xx is a definite refusal, and a 5xx or a transport failure - /// is [`Error::BlindedUnconfirmed`], since the relay may still publish. + /// block), and a 4xx is a definite refusal. + /// + /// Two cases that look like failures are also definite, because the node + /// cannot have acted on the block: a connection error (the request never + /// left) and a 503 (the node is syncing, and refuses before it hands + /// anything to a builder). Every other 5xx, and any failure after the + /// request was sent (a timeout, a reset), is [`Error::BlindedUnconfirmed`], + /// since the relay may still publish. async fn publish_blinded_block( &self, _slot: Slot, fork: ForkName, body: &[u8], ) -> Result { - let url = format!("{}/eth/v2/beacon/blinded_blocks", self.base_url); - let response = self - .client - .post(&url) - .header("Eth-Consensus-Version", fork.as_str()) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body.to_vec()) - .send() - .await - .map_err(|err| Error::BlindedUnconfirmed { - status: None, - detail: format!("{url}: {err}"), - })?; + let (url, sent) = self + .post_ssz_body("/eth/v2/beacon/blinded_blocks", fork, body) + .await; + let response = sent.map_err(|err| { + if err.is_connect() { + Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + } + } else { + Error::BlindedUnconfirmed { + status: None, + detail: format!("{url}: {err}"), + } + } + })?; let status = response.status(); if status == StatusCode::ACCEPTED { @@ -727,7 +776,7 @@ impl BeaconNodeApi for HttpBeaconNode { return Ok(Published::HandedToBuilder); } let body = response.text().await.unwrap_or_default(); - if status.is_server_error() { + if status.is_server_error() && status != StatusCode::SERVICE_UNAVAILABLE { return Err(Error::BlindedUnconfirmed { status: Some(status.as_u16()), detail: format!("{url} answered {status}: {body}"), @@ -747,8 +796,15 @@ impl BeaconNodeApi for HttpBeaconNode { if registrations.is_empty() { return Ok(()); } - self.post_no_content("/eth/v1/validator/register_validator", ®istrations, None) - .await + // Its own timeout: the node waits up to 10 s on the relay before it + // answers, which is longer than every other request's default. + self.post_no_content_within( + "/eth/v1/validator/register_validator", + ®istrations, + None, + Some(REGISTRATION_TIMEOUT), + ) + .await } async fn execution_payload_envelope( @@ -1253,10 +1309,10 @@ mod tests { assert_eq!(value["builders"], serde_json::json!([])); } - /// A one-route Beacon API double: whatever the handler sees is pushed to - /// `seen` as (uri, body), and the answer is `status` with `headers` and - /// `reply` as the body. - type Seen = std::sync::Arc)>>>; + /// A Beacon API double that answers every route the same way: whatever + /// the handler sees is pushed to `seen` as (uri, body, request headers), + /// and the answer is `status` with `headers` and `reply` as the body. + type Seen = std::sync::Arc, axum::http::HeaderMap)>>>; type Canned = (Seen, u16, Vec<(&'static str, &'static str)>, Vec); struct Served { @@ -1276,11 +1332,14 @@ mod tests { let seen: Seen = Default::default(); let state = (seen.clone(), status, headers, reply); - let handler = |State(state): State, uri: Uri, body: Bytes| async move { + let handler = |State(state): State, + uri: Uri, + request_headers: axum::http::HeaderMap, + body: Bytes| async move { let (seen, status, headers, reply) = state; seen.lock() .expect("lock") - .push((uri.to_string(), body.to_vec())); + .push((uri.to_string(), body.to_vec(), request_headers)); let mut response = ( axum::http::StatusCode::from_u16(status).expect("a status"), reply, @@ -1426,6 +1485,8 @@ mod tests { let seen = served.seen.lock().expect("lock"); assert!(seen[0].0.ends_with("/eth/v2/beacon/blinded_blocks")); assert_eq!(seen[0].1, vec![1, 2, 3]); + assert_eq!(seen[0].2["eth-consensus-version"], "fulu"); + assert_eq!(seen[0].2["content-type"], "application/octet-stream"); } let served = serve(400, vec![], b"bad".to_vec()).await; @@ -1457,16 +1518,30 @@ mod tests { ); } + /// The request never left, so the relay cannot have the block: a definite + /// failure, not an unconfirmed one. #[tokio::test] - async fn a_blinded_publish_that_cannot_connect_is_unconfirmed() { + async fn a_blinded_publish_that_cannot_connect_is_a_definite_failure() { // Nothing listens on port 1. let node = HttpBeaconNode::new("http://127.0.0.1:1").expect("builds"); let err = node .publish_blinded_block(5, ForkName::Fulu, &[1]) .await .expect_err("no answer"); + assert!(matches!(err, Error::BeaconNode { .. }), "got {err:?}"); + } + + /// A syncing node refuses before it hands anything to a builder. + #[tokio::test] + async fn a_blinded_publish_to_a_syncing_node_is_a_definite_failure() { + let served = serve(503, vec![], b"syncing".to_vec()).await; + let err = served + .node + .publish_blinded_block(5, ForkName::Fulu, &[1]) + .await + .expect_err("refused"); assert!( - matches!(err, Error::BlindedUnconfirmed { status: None, .. }), + matches!(err, Error::BeaconNodeStatus { status: 503, .. }), "got {err:?}" ); } diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index bad9ab471..deb86bc85 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -96,6 +96,9 @@ pub struct MockBeaconNode { /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; /// `Err(detail)` is a `BlindedUnconfirmed` carrying `detail`. pub blinded_publish_outcome: Option>, + /// When set, `publish_blinded_block` answers with this status as a node + /// that definitely refused the block (a 4xx) does: `BeaconNodeStatus`. + pub blinded_publish_refusal: Option, /// What the test asserts against. pub submitted: Mutex>, @@ -289,6 +292,13 @@ impl MockBeaconNode { self } + /// Make `publish_blinded_block` fail as a node that definitely refused the + /// block, with `status`. + pub fn with_refused_blinded_publish(mut self, status: u16) -> Self { + self.blinded_publish_refusal = Some(status); + self + } + /// Answer `produce_block` at a gloas slot with a bare gloas block whose /// bid names `builder_index`, with its self-built envelope attached when /// `include_payload` is set. @@ -711,7 +721,20 @@ impl BeaconNodeApi for MockBeaconNode { fork: ForkName, body: &[u8], ) -> Result { - self.guard("publish_blinded_block")?; + // `failures` and `fail_everything` stand for a node that could not be + // reached or answered badly: over HTTP that is an unconfirmed publish, + // never a definite refusal (see `with_refused_blinded_publish`). + self.guard("publish_blinded_block") + .map_err(|err| Error::BlindedUnconfirmed { + status: None, + detail: err.to_string(), + })?; + if let Some(status) = self.blinded_publish_refusal { + return Err(Error::BeaconNodeStatus { + status, + body: "refused".to_string(), + }); + } self.published_blinded .lock() .expect("lock") diff --git a/crates/validator/src/error.rs b/crates/validator/src/error.rs index 2c0334f87..23551eda7 100644 --- a/crates/validator/src/error.rs +++ b/crates/validator/src/error.rs @@ -71,6 +71,10 @@ pub enum Error { /// publishes on receipt, so the block may well be on the network. Nothing /// can safely be retried: the slot is signed, and a second block for it /// would be an equivocation. `status` is `None` for a transport failure. + /// + /// Classified retryable like the other beacon-node errors, but that governs + /// only whether the next duty is attempted, not this block: it is never + /// sent again. #[error( "the beacon node did not confirm the blinded block (it may still be published): {detail}" )] diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 40e1bce8c..1f09d2e57 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -62,10 +62,10 @@ use tracing::{error, info, warn}; use crate::aggregation::AggregationService; use crate::attestation::AttestationService; -use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::ProposerPreparationDto; use crate::beacon_node::fallback::FallbackBeaconNode; use crate::beacon_node::http::HttpBeaconNode; +use crate::beacon_node::{BeaconNodeApi, Published}; use crate::duties::DutiesService; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; @@ -304,10 +304,21 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &settings, &store, &context, - registration.as_ref().map(|service| (service, &store)), ); match refreshed.await { - Ok(()) => last_epoch = Some(epoch), + Ok(resolved) => { + last_epoch = Some(epoch); + // Only after a refresh that got all the way through, so a + // failing one does not respawn a full send every slot, and + // spawned so a slow or hung node never delays this epoch's + // duties: a missed registration costs at most a builder + // block, never a duty. + if let Some(registration) = ®istration { + let registration = registration.clone(); + let store = store.clone(); + tokio::spawn(registration.register_all(epoch, resolved, store)); + } + } // A beacon node that is down, syncing or answering badly will // likely answer next epoch, so keep the schedule already held // and try again. Anything else will not fix itself, and looping @@ -367,9 +378,7 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // to a fee recipient or gas limit reaches the builder within a slot, // and a slow node cannot delay the next slot's duties. if let Some(registration) = ®istration { - let registration = registration.clone(); - let store = store.clone(); - tokio::spawn(async move { registration.register_changed(epoch, &store).await }); + tokio::spawn(registration.clone().register_changed(epoch, store.clone())); } } } @@ -666,7 +675,7 @@ async fn aggregate( /// Failures are logged and counted rather than propagated. A proposal is one /// slot's work; losing it must not stop the client attesting for the rest of /// the epoch. -async fn propose( +async fn propose( proposal: &ProposalService, clock: &SlotClock, slot: u64, @@ -676,6 +685,10 @@ async fn propose( let budget = clock.until_attestation(slot, SystemTime::now()); let attempt = tokio::time::timeout(budget, proposal.propose(slot, duty, store)); match attempt.await { + // Already logged at error and counted on its own series; the block + // may well be out, so it is neither a published-in-time delay nor a + // failure. + Ok(Ok(Published::BuilderUnconfirmed)) => {} Ok(Ok(_)) => { let elapsed = SystemTime::now() .duration_since(clock.start_of(slot)) @@ -686,6 +699,18 @@ async fn propose( error!(%slot, validator = duty.validator_index, %err, "Failed to propose this slot's block"); metrics::inc_block_proposal_failures(); } + // A blinded block is signed, and its publish runs on its own task: the + // beacon node waits up to 5 s for the relay, which can outlast this + // budget. That is not a missed proposal, and the task counts the real + // outcome. + Err(_) if proposal.signed_blinded_at(slot) => { + info!( + %slot, + validator = duty.validator_index, + budget_ms = budget.as_millis() as u64, + "Blinded block signed; its publish outlived the proposal budget and continues" + ); + } Err(_) => { warn!( %slot, @@ -698,20 +723,12 @@ async fn propose( } } -/// What `refresh_epoch` needs to hand a spawned task the builder registration: -/// the service and the store it signs from, both shared, since the task -/// outlives the call. -type RegistrationHandles<'a, B> = ( - &'a Arc>, - &'a Arc>, -); - /// Resolve any newly activated validators, refresh the duty schedule, and -/// -/// With `registration`, also spawns the epoch's full builder registration. /// re-send subnet subscriptions if it moved. -#[allow(clippy::too_many_arguments)] -async fn refresh_epoch( +/// +/// Returns the pubkeys the beacon node resolved to an index, which is who a +/// builder registration is signed for: the node drops the rest anyway. +async fn refresh_epoch( beacon_node: &Arc, duties: &mut DutiesService, pubkeys: &[BlsPubkey], @@ -719,13 +736,12 @@ async fn refresh_epoch( settings: &ProposerSettings, store: &RwLock, context: &SigningContext, - registration: Option>, -) -> Result<()> { +) -> Result> { // Nothing to resolve, nothing to schedule, nothing to register. Returning // here also keeps an empty pubkey list away from `validator_indices`, whose // endpoint reads one as "every validator on the chain". if pubkeys.is_empty() { - return Ok(()); + return Ok(Vec::new()); } if beacon_node.is_optimistic_or_syncing().await? { @@ -788,16 +804,6 @@ async fn refresh_epoch( ); } - // Spawned, so a slow or hung node never delays this epoch's duties: a - // missed registration costs at most a builder block, never a duty. Only - // the validators the node resolved are registered; it would drop the rest. - if let Some((registration, store)) = registration { - let registration = registration.clone(); - let store = store.clone(); - let registered: Vec = entries.iter().map(|entry| entry.pubkey).collect(); - tokio::spawn(async move { registration.register_all(epoch, registered, &store).await }); - } - let changed = duties.refresh_around(epoch).await?; refresh_ptc(duties, epoch, context).await; refresh_sync(beacon_node, duties, epoch, context).await; @@ -806,7 +812,7 @@ async fn refresh_epoch( subscriptions::subscribe(beacon_node, &duties.all(), store, context).await?; } metrics::set_beacon_node_available(true); - Ok(()) + Ok(entries.iter().map(|entry| entry.pubkey).collect()) } /// Fetch the payload timeliness committee schedule for this epoch and the next, @@ -956,7 +962,6 @@ mod tests { settings, &empty_store(), &context(), - None, ) .await .expect("refreshes"); @@ -1055,7 +1060,6 @@ mod tests { &settings(Some(H160([0xab; 20]))), &empty_store(), &context(), - None, ) .await .expect("the refresh must survive a failed registration"); @@ -1082,7 +1086,6 @@ mod tests { &settings(Some(H160([0xab; 20]))), &empty_store(), &context(), - None, ) .await .expect("a keyless client idles rather than failing"); @@ -1165,7 +1168,6 @@ mod tests { &settings(None), &empty_store(), &gloas_context(4), - None, ) .await .expect("refreshes"); @@ -1187,7 +1189,6 @@ mod tests { &settings(None), &empty_store(), &context(), - None, ) .await .expect("refreshes"); @@ -1212,7 +1213,6 @@ mod tests { &settings(None), &empty_store(), &gloas_context(0), - None, ) .await .expect("the refresh survives it"); @@ -1320,7 +1320,6 @@ mod tests { &settings(None), &empty_store(), &altair_context(), - None, ) .await .expect("refreshes"); @@ -1362,7 +1361,6 @@ mod tests { &settings(None), &empty_store(), &altair_context(), - None, ) .await .expect("the refresh survives it"); diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index 8f7a4f254..64d7ca0cc 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -114,7 +114,7 @@ static BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL: LazyLock = LazyLock::new }); /// Builder registrations a beacon node accepted, counted per registration so a -/// batch of 500 reads as 500. +/// full `REGISTRATION_BATCH` reads as that many. static BUILDER_REGISTRATIONS_SENT_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( "ethlambda_validator_builder_registrations_sent_total", diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 2f457f860..2c705e4ca 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -55,8 +55,13 @@ //! fails to confirm is reported as [`Published::BuilderUnconfirmed`] rather //! than as a missed proposal, since the relay may still publish the block. //! -//! # Do not retry this call within a slot +//! The publish runs on its own task. The beacon node waits up to 5 s for the +//! relay, which can outlast the proposal's budget (see `lib.rs`), and dropping +//! the proposal future must neither cancel the request nor lose its outcome: the +//! task logs and counts it itself, and [`ProposalService::signed_blinded_at`] +//! lets the caller tell an overrun after signing from a block never signed. //! +//! # Do not retry this call within a slot //! //! For the same reason [`crate::attestation::AttestationService::attest`] must //! not be: a second call asks the node for a second block, which will differ @@ -104,9 +109,12 @@ pub struct ProposalService { /// The `builder_boost_factor` sent with each block request, or `None` when /// builder proposals are off, in which case a blinded answer is refused. builder_boost_factor: Option, + /// The slot of the latest blinded block this service signed, set before + /// its publish starts. A `std::sync::Mutex` for the reason `guard` is one. + last_blinded_slot: std::sync::Mutex>, } -impl ProposalService { +impl ProposalService { pub fn new( beacon_node: Arc, context: Arc, @@ -118,10 +126,32 @@ impl ProposalService { context, settings, builder_boost_factor, + last_blinded_slot: std::sync::Mutex::new(None), guard: std::sync::Mutex::new(ProposalGuard::new()), } } + /// Whether this service signed a blinded block for `slot`, whose publish + /// may therefore still be in flight on its own task. + pub fn signed_blinded_at(&self, slot: Slot) -> bool { + *self + .last_blinded_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + == Some(slot) + } + + /// Whether the produced block's fee recipient is this client's to compare. + /// + /// Not for a blinded block: its header names the builder's address, and the + /// relay pays the validator inside the payload it reveals, so a difference + /// there is the flow working, not a node paying someone else. + fn compares_fee_recipient( + produced: &crate::beacon_node::block_contents::ProducedBlock, + ) -> bool { + !produced.is_blinded() + } + /// Compare the produced block's fee recipient against what was asked for, /// and complain loudly if they differ. /// @@ -275,17 +305,14 @@ impl ProposalService { ))); } - // A blinded block's header names the builder's address, and the relay - // pays the validator inside the payload it reveals, so there is - // nothing here to compare. - if produced.is_blinded() { + if Self::compares_fee_recipient(&produced) { + Self::check_fee_recipient(&produced, fee_recipient, slot); + } else { tracing::debug!( %slot, "Blinded block: its header names the builder's address; the relay pays the \ validator in the payload" ); - } else { - Self::check_fee_recipient(&produced, fee_recipient, slot); } let fork = produced.fork; @@ -328,7 +355,16 @@ impl ProposalService { } } - self.context.sign_block(&store, &pubkey, block_root, slot)? + let signature = self.context.sign_block(&store, &pubkey, block_root, slot)?; + // Before the first await after signing, so an overrun of the + // proposal's budget from here on is known to be a signed block. + if blinded { + *self + .last_blinded_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(slot); + } + signature }; // Taken out before the block is signed into its body: the envelope is @@ -338,17 +374,18 @@ impl ProposalService { let builder_index = produced.builder_index(); let body = produced.into_signed_ssz(signature); + // A blinded block counts itself on its own task (see + // `publish_blinded`); a full one is counted here. let published = if blinded { - self.publish_blinded(slot, fork, &body, duty.validator_index) + self.publish_blinded(slot, fork, body, duty.validator_index) .await? } else { - self.publish(fork, &body, slot, duty.validator_index) - .await? + let published = self + .publish(fork, &body, slot, duty.validator_index) + .await?; + crate::metrics::inc_blocks_proposed(); + published }; - crate::metrics::inc_blocks_proposed(); - if published == Published::HandedToBuilder { - crate::metrics::inc_blinded_blocks_proposed(); - } // Only a self-built payload is this client's to reveal. if builder_index == Some(BUILDER_INDEX_SELF_BUILD) { @@ -445,6 +482,13 @@ impl ProposalService { /// Send a signed blinded block to the node that produced it. /// + /// # On its own task + /// + /// The request, its logging and its counters run on a spawned task, so a + /// proposal future dropped at the duty loop's budget (the node may wait up + /// to 5 s on the relay) neither cancels the request nor loses its outcome. + /// This method only awaits the task. + /// /// # Why an unconfirmed publish is not an error /// /// A 5xx or a missing answer does not say the block was not published: the @@ -458,43 +502,66 @@ impl ProposalService { &self, slot: Slot, fork: ForkName, - body: &[u8], + body: Vec, validator: u64, ) -> Result { - match self - .beacon_node - .publish_blinded_block(slot, fork, body) - .await - { - Ok(Published::HandedToBuilder) => { - info!(%slot, validator, bytes = body.len(), "Blinded block handed to the builder"); - Ok(Published::HandedToBuilder) - } - Ok(Published::BroadcastNotImported) => { - warn!( - %slot, - validator, - "Block was broadcast but the beacon node could not import it; \ - check that node's execution layer" - ); - crate::metrics::inc_blocks_broadcast_not_imported(); - Ok(Published::BroadcastNotImported) - } - Ok(other) => Ok(other), - Err(err @ Error::BlindedUnconfirmed { .. }) => { - tracing::error!( - %slot, - validator, - %err, - "The beacon node did not confirm the blinded block; the relay may still \ - publish it. Not retried: the slot is signed, and a second block would be \ - an equivocation" - ); - crate::metrics::inc_blinded_blocks_unconfirmed(); - Ok(Published::BuilderUnconfirmed) + let beacon_node = self.beacon_node.clone(); + let task = tokio::spawn(async move { + let bytes = body.len(); + match beacon_node.publish_blinded_block(slot, fork, &body).await { + Ok(published) => { + match published { + Published::HandedToBuilder => { + info!(%slot, validator, bytes, "Blinded block handed to the builder"); + } + Published::BroadcastNotImported => { + warn!( + %slot, + validator, + "Block was broadcast but the beacon node could not import it; \ + check that node's execution layer" + ); + crate::metrics::inc_blocks_broadcast_not_imported(); + } + // Neither belongs to this path: a full block's outcome, + // and one only this service builds. Reported rather + // than assumed away. + Published::Imported | Published::BuilderUnconfirmed => warn!( + %slot, + validator, + ?published, + "Beacon node API answered a blinded publish with an outcome that \ + belongs to another call" + ), + } + if published != Published::BuilderUnconfirmed { + crate::metrics::inc_blocks_proposed(); + crate::metrics::inc_blinded_blocks_proposed(); + } + Ok(published) + } + Err(err @ Error::BlindedUnconfirmed { .. }) => { + tracing::error!( + %slot, + validator, + %err, + "The beacon node did not confirm the blinded block; the relay may still \ + publish it. Not retried: the slot is signed, and a second block would be \ + an equivocation" + ); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(Published::BuilderUnconfirmed) + } + Err(err) => Err(err), } - Err(err) => Err(err), - } + }); + // A task that panicked has no outcome to report; the block may or may + // not have gone out, which is what "unconfirmed" says. + task.await.unwrap_or_else(|err| { + tracing::error!(%slot, validator, %err, "The blinded publish task failed"); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(Published::BuilderUnconfirmed) + }) } /// Send the signed body and report what the node made of it. @@ -793,11 +860,18 @@ mod tests { .with_blinded_block(), ); - let published = builder_service(node.clone(), Some(H160([0xfe; 20]))) + let service = builder_service(node.clone(), Some(H160([0xfe; 20]))); + assert!(!service.signed_blinded_at(slot())); + let published = service .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("proposes"); assert_eq!(published, Published::HandedToBuilder); + assert!( + service.signed_blinded_at(slot()), + "the duty loop reads this to tell an overrun after signing from one before" + ); + assert!(!service.signed_blinded_at(slot() + 1)); assert_eq!(node.block_requests()[0].builder_boost_factor, Some(100)); assert!(node.published_blocks().is_empty()); @@ -806,6 +880,38 @@ mod tests { assert_eq!((sent[0].0, sent[0].1), (slot(), ForkName::Electra)); } + /// The fee-recipient comparison is for a full block only: a blinded + /// block's header names the builder's address. + #[tokio::test] + async fn only_a_full_block_has_its_fee_recipient_compared() { + use crate::beacon_node::block_contents::{Contents, ProducedBlock, empty_block_for}; + use ethlambda_types::beacon::builder::BlindedBeaconBlock; + + let blinded = ProducedBlock { + fork: ForkName::Electra, + contents: Contents::Blinded { + block: BlindedBeaconBlock::from(empty_block_for(slot(), 7)), + }, + }; + assert!(!ProposalService::::compares_fee_recipient( + &blinded + )); + + let full = MockBeaconNode::new().with_block(slot(), 7); + let request = BlockRequest { + slot: slot(), + fork: ForkName::Electra, + proposer_index: 7, + randao_reveal: Default::default(), + graffiti: Default::default(), + builder_boost_factor: None, + }; + let full = full.produce_block(&request).await.expect("produces"); + assert!(ProposalService::::compares_fee_recipient( + &full + )); + } + /// A node that does not confirm the publish may still have handed the block /// to its relay, so it is not a missed proposal, and the slot is spent: a /// second block would be an equivocation. @@ -833,23 +939,23 @@ mod tests { assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); } - /// Any failure that is not `BlindedUnconfirmed` is a definite refusal (a - /// 4xx): the block was not published, so the proposal failed. + /// A 4xx is a definite refusal: the block was not published, so the + /// proposal failed. #[tokio::test] async fn a_definitely_refused_blinded_publish_is_an_error() { let (store, pubkey) = store(); let node = MockBeaconNode::new() .with_block(slot(), 7) - .with_blinded_block(); - node.failures - .lock() - .expect("lock") - .insert("publish_blinded_block", "400 bad request".to_string()); + .with_blinded_block() + .with_refused_blinded_publish(400); let result = builder_service(Arc::new(node), None) .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await; - assert!(result.is_err(), "got {result:?}"); + assert!( + matches!(result, Err(Error::BeaconNodeStatus { status: 400, .. })), + "got {result:?}" + ); } #[tokio::test] diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 36ef7fe8d..5000779c9 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -22,6 +22,7 @@ //! signing reports as a mismatch. use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{PoisonError, RwLock}; use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, H256}; @@ -98,6 +99,10 @@ struct Inner { #[derive(Debug)] pub struct ProposerSettings { inner: RwLock, + /// Bumped on every override write or delete, so a reader that caches + /// something derived from the settings (the builder registrations) can + /// tell that nothing changed without reading every key. + generation: AtomicU64, } impl ProposerSettings { @@ -111,9 +116,16 @@ impl ProposerSettings { default_gas_limit: DEFAULT_GAS_LIMIT, overrides: HashMap::new(), }), + generation: AtomicU64::new(0), } } + /// A counter that moves whenever any override is written, cleared or + /// forgotten. Equal readings mean no per-key setting changed between them. + pub fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + fn read(&self, f: impl FnOnce(&Inner) -> T) -> T { f(&self.inner.read().unwrap_or_else(PoisonError::into_inner)) } @@ -127,6 +139,9 @@ impl ProposerSettings { if entry.is_empty() { inner.overrides.remove(pubkey); } + // Inside the write lock, so a reader that sees the new generation also + // sees the new value. + self.generation.fetch_add(1, Ordering::Release); } /// Where `pubkey`'s execution-layer rewards should go, or `None` when @@ -183,6 +198,7 @@ impl ProposerSettings { pub fn forget(&self, pubkey: &BlsPubkey) { let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner); inner.overrides.remove(pubkey); + self.generation.fetch_add(1, Ordering::Release); } } diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index b2b534e45..c9d5a717d 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -33,10 +33,27 @@ //! # Failures are never propagated //! //! A missed registration costs at most a builder block, never a duty, so a -//! failed send is logged and counted and the caller moves on. +//! failed send is logged and counted and the caller moves on. A changed +//! registration whose send fails is cached anyway, so it is not re-sent at the +//! next slot: it waits for the next epoch's full send. +//! +//! # Signing stays off the async runtime +//! +//! The first epoch signs one BLS signature per validator. That runs on the +//! blocking pool in chunks of [`SIGN_CHUNK`], taking the store's read lock +//! afresh for each chunk, so neither the runtime's threads nor a queued +//! keymanager writer wait on thousands of signatures. +//! +//! # The per-slot check is one atomic read +//! +//! Only a keymanager write changes what a registration says, so +//! [`RegistrationService::register_changed`] compares +//! [`ProposerSettings::generation`] with the one it last signed against and +//! returns before touching the store when it has not moved. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use ethlambda_types::beacon::builder::{SignedValidatorRegistrationV1, ValidatorRegistrationV1}; @@ -46,6 +63,7 @@ use tokio::sync::RwLock; use tracing::{debug, warn}; use crate::beacon_node::BeaconNodeApi; +use crate::beacon_node::dto::encode_hex; use crate::keys::ValidatorStore; use crate::proposer_settings::ProposerSettings; use crate::signing::SigningContext; @@ -55,6 +73,9 @@ use crate::signing::SigningContext; /// request stays well under a node's body limit. pub const REGISTRATION_BATCH: usize = 500; +/// Keys signed per blocking task, and so per hold of the store's read lock. +const SIGN_CHUNK: usize = 64; + /// What [`RegistrationService::current`] found for a set of validators. struct Current { /// Every validator's registration, cached or newly signed. @@ -66,6 +87,14 @@ struct Current { no_fee_recipient: usize, } +impl Current { + fn extend(&mut self, other: Current) { + self.all.extend(other.all); + self.changed.extend(other.changed); + self.no_fee_recipient += other.no_fee_recipient; + } +} + /// Signs and sends builder registrations. pub struct RegistrationService { beacon_node: Arc, @@ -75,9 +104,12 @@ pub struct RegistrationService { known: std::sync::Mutex>, /// The latest signed registration per validator. signed: std::sync::Mutex>, + /// The [`ProposerSettings::generation`] the cache was last brought up to + /// date against. + signed_generation: AtomicU64, } -impl RegistrationService { +impl RegistrationService { pub fn new( beacon_node: Arc, context: Arc, @@ -89,6 +121,7 @@ impl RegistrationService { settings, known: std::sync::Mutex::new(Vec::new()), signed: std::sync::Mutex::new(HashMap::new()), + signed_generation: AtomicU64::new(0), } } @@ -134,7 +167,7 @@ impl RegistrationService { let signature = match self.context.sign_validator_registration(store, &message) { Ok(signature) => signature, Err(err) => { - warn!(validator = ?pubkey, %err, "Could not sign a builder registration; skipping it"); + warn!(pubkey = %encode_hex(&pubkey.0), %err, "Could not sign a builder registration; skipping it"); continue; } }; @@ -157,29 +190,63 @@ impl RegistrationService { .map_or(0, |elapsed| elapsed.as_secs()) } + /// [`Self::current`] over `pubkeys`, off the async runtime, [`SIGN_CHUNK`] + /// keys per blocking task with the store's read lock taken per chunk. + async fn current_blocking( + self: &Arc, + store: &Arc>, + pubkeys: &[BlsPubkey], + ) -> Current { + let now_secs = Self::now_secs(); + let mut current = Current { + all: Vec::with_capacity(pubkeys.len()), + changed: Vec::new(), + no_fee_recipient: 0, + }; + for chunk in pubkeys.chunks(SIGN_CHUNK) { + let (service, store, chunk) = (self.clone(), store.clone(), chunk.to_vec()); + let part = tokio::task::spawn_blocking(move || { + let store = store.blocking_read(); + service.current(&store, &chunk, now_secs) + }) + .await; + match part { + Ok(part) => current.extend(part), + Err(err) => warn!(%err, "Signing builder registrations failed; skipping a chunk"), + } + } + current + } + /// Register every validator in `pubkeys`, the ones the beacon node resolved /// at this epoch's refresh. Called once per epoch. + /// + /// Also forgets any validator no longer in `pubkeys`, so a key the + /// keymanager removed stops being re-signed (and warned about) every slot. pub async fn register_all( - &self, + self: Arc, epoch: Epoch, pubkeys: Vec, - store: &RwLock, + store: Arc>, ) { if self.is_gloas(epoch) { return; } + // Read before signing, so a change that lands while this runs is seen + // by the next `register_changed` rather than lost. + let generation = self.settings.generation(); + let current_keys: HashSet = pubkeys.iter().copied().collect(); + self.signed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|pubkey, _| current_keys.contains(pubkey)); *self .known .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = pubkeys.clone(); - // Scoped away from the sends below, as `proposal.rs` does: a held read - // guard across a network call lets a queued keymanager writer block - // every reader behind it. - let current = { - let store = store.read().await; - self.current(&store, &pubkeys, Self::now_secs()) - }; + let current = self.current_blocking(&store, &pubkeys).await; + self.signed_generation.store(generation, Ordering::Release); if current.no_fee_recipient > 0 { warn!( skipped = current.no_fee_recipient, @@ -194,12 +261,23 @@ impl RegistrationService { /// [`Self::register_all`] was given. Called every slot, so a keymanager /// change reaches the builder within a slot rather than an epoch. /// + /// Returns before touching the store when no setting was written since the + /// last signing: see the module doc. + /// /// Silent about validators with no fee recipient: that is reported once /// per epoch by [`Self::register_all`], not repeated every slot. - pub async fn register_changed(&self, epoch: Epoch, store: &RwLock) { + pub async fn register_changed( + self: Arc, + epoch: Epoch, + store: Arc>, + ) { if self.is_gloas(epoch) { return; } + let generation = self.settings.generation(); + if generation == self.signed_generation.load(Ordering::Acquire) { + return; + } let known = self .known .lock() @@ -208,10 +286,8 @@ impl RegistrationService { if known.is_empty() { return; } - let current = { - let store = store.read().await; - self.current(&store, &known, Self::now_secs()) - }; + let current = self.current_blocking(&store, &known).await; + self.signed_generation.store(generation, Ordering::Release); if current.changed.is_empty() { return; } @@ -273,10 +349,17 @@ mod tests { fn service( config: Config, settings: Arc, - ) -> (RegistrationService, Arc) { + ) -> ( + Arc>, + Arc, + ) { let node = Arc::new(MockBeaconNode::new()); ( - RegistrationService::new(node.clone(), context(config), settings), + Arc::new(RegistrationService::new( + node.clone(), + context(config), + settings, + )), node, ) } @@ -346,11 +429,12 @@ mod tests { /// 501 keys is one more than a batch; signing them is a millisecond each. #[tokio::test] async fn register_all_sends_in_batches_of_500() { + assert_eq!(REGISTRATION_BATCH, 500); let (store, pubkeys) = store_with_keys(REGISTRATION_BATCH + 1); let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); - let store = RwLock::new(store); + let store = Arc::new(RwLock::new(store)); - service.register_all(0, pubkeys, &store).await; + service.register_all(0, pubkeys, store).await; assert_eq!( *node.registration_batches.lock().expect("lock"), @@ -372,6 +456,9 @@ mod tests { ); } + /// Both entry points stop at gloas, and `register_changed` is checked with + /// something to send: a service that was filled before the fork and then + /// had a setting changed. #[tokio::test] async fn nothing_is_sent_at_or_after_gloas() { let (store, pubkeys) = store_with_keys(2); @@ -379,16 +466,32 @@ mod tests { gloas_fork_epoch: 4, ..Config::mainnet() }; - let (service, node) = service(config, settings(Some(H160([0xab; 20])))); - let store = RwLock::new(store); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(config, settings.clone()); + let store = Arc::new(RwLock::new(store)); + + // Before the fork the service registers, and holds the keys. + service + .clone() + .register_all(3, pubkeys.clone(), store.clone()) + .await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2]); - service.register_all(4, pubkeys.clone(), &store).await; - service.register_changed(5, &store).await; - assert!(node.registration_batches.lock().expect("lock").is_empty()); + settings.set_gas_limit(&pubkeys[0], 30_000_000); + service + .clone() + .register_all(4, pubkeys, store.clone()) + .await; + service.clone().register_changed(5, store.clone()).await; + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2], + "a changed key is still not sent at or after the fork" + ); - // The epoch before the fork still registers. - service.register_all(3, pubkeys, &store).await; - assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2]); + // The same change is sent when the epoch is before the fork. + service.register_changed(3, store).await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 1]); } #[tokio::test] @@ -396,10 +499,13 @@ mod tests { let (store, pubkeys) = store_with_keys(3); let settings = settings(Some(H160([0xab; 20]))); let (service, node) = service(Config::mainnet(), settings.clone()); - let store = RwLock::new(store); + let store = Arc::new(RwLock::new(store)); - service.register_all(0, pubkeys.clone(), &store).await; - service.register_changed(0, &store).await; + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + service.clone().register_changed(0, store.clone()).await; assert_eq!( *node.registration_batches.lock().expect("lock"), vec![3], @@ -407,11 +513,59 @@ mod tests { ); settings.set_gas_limit(&pubkeys[2], 30_000_000); - service.register_changed(0, &store).await; + service.register_changed(0, store).await; let registrations = node.registrations.lock().expect("lock"); assert_eq!(registrations.len(), 4); assert_eq!(registrations[3].message.pubkey, pubkeys[2]); assert_eq!(registrations[3].message.gas_limit, 30_000_000); } + + /// The per-slot call is one atomic read when no setting moved: with the + /// store write-locked, so any read of it would wait forever, it still + /// returns at once, and sends nothing. + #[tokio::test] + async fn a_slot_with_no_change_makes_no_store_read_and_no_send() { + let (store, pubkeys) = store_with_keys(2); + let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys, store.clone()) + .await; + let signed_before = service.signed.lock().expect("lock").clone(); + + let _writer = store.write().await; + tokio::time::timeout( + std::time::Duration::from_millis(200), + service.clone().register_changed(0, store.clone()), + ) + .await + .expect("returns without waiting on the store"); + + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2]); + assert_eq!(*service.signed.lock().expect("lock"), signed_before); + } + + /// A key that left the resolved set is dropped from the cache, so it is not + /// re-signed (and warned about) at every change. + #[tokio::test] + async fn a_key_no_longer_resolved_is_forgotten() { + let (store, pubkeys) = store_with_keys(2); + let (service, _node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let store = Arc::new(RwLock::new(store)); + + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + service + .clone() + .register_all(0, vec![pubkeys[1]], store) + .await; + + let signed = service.signed.lock().expect("lock"); + assert_eq!(signed.keys().copied().collect::>(), vec![pubkeys[1]]); + assert_eq!(*service.known.lock().expect("lock"), vec![pubkeys[1]]); + } } diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index 6d0809408..cafa4099a 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -534,10 +534,21 @@ mod tests { } } + /// A context whose genesis validators root is not zero, so the builder + /// domain (which zeroes it) and a context-derived one cannot coincide. + fn context_with_validators_root() -> SigningContext { + SigningContext { + config: Config::mainnet(), + genesis_validators_root: Root::repeat_byte(0x42), + } + } + #[test] fn a_registration_is_signed_under_the_builder_domain() { + use ethlambda_types::beacon::constants::DOMAIN_APPLICATION_BUILDER; + let (store, pubkey) = store_with_key(); - let context = context(); + let context = context_with_validators_root(); let registration = registration(pubkey); let signature = context @@ -549,6 +560,14 @@ mod tests { compute_builder_domain(Config::mainnet().genesis_fork_version), ); assert!(verify(&pubkey, &signature, root)); + + // The same domain type under this chain's validators root is a + // different domain: a context-derived one would not verify. + let with_root = compute_signing_root( + registration.hash_tree_root(), + context.domain(DOMAIN_APPLICATION_BUILDER, 0), + ); + assert!(!verify(&pubkey, &signature, with_root)); } /// The registration domain is not gloas's envelope domain, whose type @@ -557,7 +576,7 @@ mod tests { #[test] fn a_registration_does_not_verify_under_the_envelope_domain() { let (store, pubkey) = store_with_key(); - let context = context(); + let context = context_with_validators_root(); let registration = registration(pubkey); let signature = context diff --git a/docs/cli.md b/docs/cli.md index 7bd8e70eb..a644f5b65 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -291,8 +291,8 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | | `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | -| `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` always builds locally. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | -| `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid (sends `builder_boost_factor = u64::MAX`). Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | +| `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` prefers the local payload unless it cannot be built. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | +| `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid, unless the execution client asks to override the builder (sends `builder_boost_factor = u64::MAX`). Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | ### What it does today @@ -306,6 +306,17 @@ validator's fee recipient with the beacon node. The subscription also tells the node which committees this client will aggregate for, which it has to know in advance so it can collect the votes. +With `--builder-proposals`, before gloas, it also signs a builder registration +(fee recipient and gas limit) for each validator the node resolved and sends +them to every node: the full set after each epoch's refresh, in batches, and +every slot only the registrations whose fee recipient or gas limit changed +through the keymanager API. A block request then carries a +`builder_boost_factor`, and when the node answers with a blinded block it is +signed like a full one and published to `blinded_blocks` on the node that +produced it, with no failover. A publish that node does not confirm is logged +at error and counted as unconfirmed, not as a missed proposal, since the relay +may still publish the block. + Each slot it wakes at the boundary. If one of its validators proposes that slot, it signs the RANDAO reveal, asks the beacon node for a block, checks that the block is for the slot and proposer it asked about, signs it and publishes @@ -326,7 +337,11 @@ slot. The specification defines no block-production deadline; the nearest thing it defines is the point attesters stop waiting for a block, which is the same instant. A proposal still running then has already lost most of the block's value, and every second past it comes out of this client's own attestations, so -it is abandoned and the slot's attesters still vote. +it is abandoned and the slot's attesters still vote. A blinded block that was +already signed is the exception: its publish runs on its own task, since the +beacon node may wait several seconds on the relay, so an overrun after signing +is logged at info and not counted as a failed proposal, and the task counts the +real outcome. Electra is the earliest fork it will propose under. A deneb block body has one field fewer than an electra one, so it would need a container pair of its own, diff --git a/docs/metrics.md b/docs/metrics.md index 65dcc53d7..1b63a76be 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -759,13 +759,17 @@ second points at the beacon nodes; a rise in the third points at the clock or the duty schedule and is worth investigating even though the attestation was correctly suppressed. -The proposal series split the same way, plus one that is neither a success nor +The proposal series split the same way, plus two that are neither a success nor a failure. `blocks_proposed_total` counts blocks a node accepted; `block_proposal_failures_total` counts duties that produced none; `blocks_refused_total` counts blocks deliberately not signed. Between them, `blocks_broadcast_not_imported_total` counts blocks that did reach the network but that the node answering could not import, which is a beacon-node fault rather than a validator one and is why it is not folded into either. +`blinded_blocks_unconfirmed_total` is the other: a blinded block the node did +not confirm, which the relay may still have published. It is not in +`blocks_proposed_total` nor in the publication delay histogram, and it is not a +failure either. Gloas adds two pairs. A gloas proposal ends in a block and then an envelope, and the envelope's failure does not fail the proposal: `blocks_proposed_total` diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 758d01876..84a27a23d 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -696,10 +696,10 @@ the beacon node's builder and may sign a blinded block (builder-specs and `ProposalGuard` covers both kinds: one slot, one signature, whichever way the node answered. - **A blinded block is published only on the beacon node that produced it, with - no failover.** The specification says the block goes to the node that built - it. Only that node's builder holds the payload and only that node claimed the - slot, so offering the signed block to another node would hand it to a builder - that never saw the bid. If the producing node fails, the publish fails, and + no failover.** This is this client's choice, not the specification's: only + that node's builder holds the payload and only that node claimed the slot, so + offering the signed block to another node would hand it to a builder that + never saw the bid. If the producing node fails, the publish fails, and the guard has already recorded the slot, so nothing retries. A 5xx or no answer is reported as unconfirmed (logged at error, counted on `blinded_blocks_unconfirmed_total`) and not as a missed proposal, since the From 7911c6fd1464a0446abcc9b1a2a3b2d08648c314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:00:36 -0300 Subject: [PATCH 09/19] fix(validator): report a late blinded refusal and delay from the publish task Once the duty loop's budget fires, nothing awaits the blinded publish, so a 4xx that arrived afterwards was silent and a late success never reached the publication delay histogram. The task now logs and counts a definite refusal as a failed proposal and observes the delay itself (the service takes the slot clock for this), and the loop skips both for a block it knows was signed blinded so nothing is counted twice. A test drops a proposal mid-publish and checks the block is still sent. --- .../rpc/src/beacon/validator_client_tests.rs | 2 +- crates/validator/src/beacon_node/mock.rs | 12 ++++ crates/validator/src/lib.rs | 8 ++- crates/validator/src/proposal.rs | 65 ++++++++++++++++++- docs/cli.md | 3 +- 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index f2fd5cbca..018471e06 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -1833,7 +1833,7 @@ mod builder_flow { assert_eq!( h.world.relay.header_requests().len(), 1, - "the node still asks the relay at a factor of 0, as Lighthouse does" + "the node still asks the relay at a factor of 0" ); } diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index deb86bc85..409e3aedf 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -99,6 +99,9 @@ pub struct MockBeaconNode { /// When set, `publish_blinded_block` answers with this status as a node /// that definitely refused the block (a 4xx) does: `BeaconNodeStatus`. pub blinded_publish_refusal: Option, + /// How long `publish_blinded_block` takes before it records the block and + /// answers, as a slow relay makes the node. + pub blinded_publish_delay: Option, /// What the test asserts against. pub submitted: Mutex>, @@ -292,6 +295,12 @@ impl MockBeaconNode { self } + /// Make `publish_blinded_block` take `delay` before it records the block. + pub fn with_blinded_publish_delay(mut self, delay: std::time::Duration) -> Self { + self.blinded_publish_delay = Some(delay); + self + } + /// Make `publish_blinded_block` fail as a node that definitely refused the /// block, with `status`. pub fn with_refused_blinded_publish(mut self, status: u16) -> Self { @@ -735,6 +744,9 @@ impl BeaconNodeApi for MockBeaconNode { body: "refused".to_string(), }); } + if let Some(delay) = self.blinded_publish_delay { + tokio::time::sleep(delay).await; + } self.published_blinded .lock() .expect("lock") diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 1f09d2e57..7e2732809 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -221,7 +221,8 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { context.clone(), settings.clone(), config.builder.map(|builder| builder.boost_factor), - ); + ) + .with_clock(clock.clone()); let registration = config.builder.map(|builder| { info!( boost_factor = builder.boost_factor, @@ -689,12 +690,17 @@ async fn propose( // may well be out, so it is neither a published-in-time delay nor a // failure. Ok(Ok(Published::BuilderUnconfirmed)) => {} + // A blinded publish observes its own delay, on the task that ran it. + Ok(Ok(_)) if proposal.signed_blinded_at(slot) => {} Ok(Ok(_)) => { let elapsed = SystemTime::now() .duration_since(clock.start_of(slot)) .unwrap_or_default(); metrics::observe_block_publication_delay(elapsed.as_secs_f64()); } + // Once a blinded block is signed, only its publish task reports an + // error (and counts the failure), so it is not counted twice here. + Ok(Err(_)) if proposal.signed_blinded_at(slot) => {} Ok(Err(err)) => { error!(%slot, validator = duty.validator_index, %err, "Failed to propose this slot's block"); metrics::inc_block_proposal_failures(); diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 2c705e4ca..381d25a59 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -71,6 +71,7 @@ //! dangerous, but the caller should not be relying on the guard for that. use std::sync::Arc; +use std::time::SystemTime; use ethlambda_types::beacon::constants::BUILDER_INDEX_SELF_BUILD; use ethlambda_types::beacon::fork::ForkName; @@ -87,6 +88,7 @@ use crate::keys::ValidatorStore; use crate::proposal_guard::ProposalGuard; use crate::proposer_settings::ProposerSettings; use crate::signing::SigningContext; +use crate::slot_clock::SlotClock; pub struct ProposalService { beacon_node: Arc, @@ -112,6 +114,9 @@ pub struct ProposalService { /// The slot of the latest blinded block this service signed, set before /// its publish starts. A `std::sync::Mutex` for the reason `guard` is one. last_blinded_slot: std::sync::Mutex>, + /// Where a slot starts, so a blinded publish that outlives the proposal + /// can observe its own publication delay. `None` observes nothing. + clock: Option, } impl ProposalService { @@ -127,10 +132,19 @@ impl ProposalService { settings, builder_boost_factor, last_blinded_slot: std::sync::Mutex::new(None), + clock: None, guard: std::sync::Mutex::new(ProposalGuard::new()), } } + /// Give the service the slot clock, so a blinded block's publication delay + /// is observed by the task that publishes it (which may outlive + /// [`Self::propose`]). + pub fn with_clock(mut self, clock: SlotClock) -> Self { + self.clock = Some(clock); + self + } + /// Whether this service signed a blinded block for `slot`, whose publish /// may therefore still be in flight on its own task. pub fn signed_blinded_at(&self, slot: Slot) -> bool { @@ -506,6 +520,7 @@ impl ProposalService { validator: u64, ) -> Result { let beacon_node = self.beacon_node.clone(); + let slot_start = self.clock.as_ref().map(|clock| clock.start_of(slot)); let task = tokio::spawn(async move { let bytes = body.len(); match beacon_node.publish_blinded_block(slot, fork, &body).await { @@ -537,6 +552,13 @@ impl ProposalService { if published != Published::BuilderUnconfirmed { crate::metrics::inc_blocks_proposed(); crate::metrics::inc_blinded_blocks_proposed(); + // Here rather than in the duty loop, which has stopped + // waiting if the relay was slow. + if let Some(start) = slot_start { + let elapsed = + SystemTime::now().duration_since(start).unwrap_or_default(); + crate::metrics::observe_block_publication_delay(elapsed.as_secs_f64()); + } } Ok(published) } @@ -552,7 +574,14 @@ impl ProposalService { crate::metrics::inc_blinded_blocks_unconfirmed(); Ok(Published::BuilderUnconfirmed) } - Err(err) => Err(err), + // A definite refusal. Reported here and not left to the caller, + // which may have stopped waiting at the proposal's budget: this + // task is the only one that knows the slot was missed. + Err(err) => { + tracing::error!(%slot, validator, %err, "The beacon node refused the blinded block"); + crate::metrics::inc_block_proposal_failures(); + Err(err) + } } }); // A task that panicked has no outcome to report; the block may or may @@ -939,6 +968,40 @@ mod tests { assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); } + /// The duty loop drops the proposal at its budget while the node is still + /// waiting on the relay. The publish runs on its own task, so the block is + /// still sent, and the slot is known to be signed. + #[tokio::test] + async fn a_blinded_publish_survives_a_dropped_proposal() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_blinded_publish_delay(std::time::Duration::from_millis(300)), + ); + let service = builder_service(node.clone(), None); + + let outcome = tokio::time::timeout( + std::time::Duration::from_millis(50), + service.propose(slot(), &duty(&pubkey, 7, slot()), &store), + ) + .await; + assert!( + outcome.is_err(), + "the proposal must be dropped at the budget" + ); + assert!(service.signed_blinded_at(slot())); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + + tokio::time::sleep(std::time::Duration::from_millis(600)).await; + assert_eq!( + node.published_blinded.lock().expect("lock").len(), + 1, + "the publish must complete although the proposal future is gone" + ); + } + /// A 4xx is a definite refusal: the block was not published, so the /// proposal failed. #[tokio::test] diff --git a/docs/cli.md b/docs/cli.md index a644f5b65..e0978f6bb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -341,7 +341,8 @@ it is abandoned and the slot's attesters still vote. A blinded block that was already signed is the exception: its publish runs on its own task, since the beacon node may wait several seconds on the relay, so an overrun after signing is logged at info and not counted as a failed proposal, and the task counts the -real outcome. +real outcome (a refusal as a failed proposal, success with its publication +delay). Electra is the earliest fork it will propose under. A deneb block body has one field fewer than an electra one, so it would need a container pair of its own, From e07123fc1d38980f041a3227d004a80cf7bf83cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:48:56 -0300 Subject: [PATCH 10/19] docs: say what the keymanager's gas limit and fee recipient now reach The keymanager table still called `gas_limit` "stored and served only" because "there is no builder flow"; with `--builder-proposals` a changed value goes out in the next slot's builder registration, and so does a changed fee recipient, which an ethlambda node then applies. The README's validator flag table, rewritten on main by #716, gains `--builder-proposals`. --- README.md | 1 + docs/cli.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9cc433a6..e111d22f4 100644 --- a/README.md +++ b/README.md @@ -365,6 +365,7 @@ the standard Beacon API, so it works against `ethlambda beacon` (run with the | `--secrets-dir` | One password file per keystore, named after the validator's public key | | `--suggested-fee-recipient` | Execution address for block rewards, `0x`-prefixed. Optional, but without it the beacon node picks an address that is not yours | | `--graffiti` | Text for proposed blocks, at most 32 bytes | +| `--builder-proposals` | Propose builder blocks before gloas: register each validator with the beacon node's builder (`--builder` on an ethlambda node) and sign a blinded block when the builder's bid wins. Off by default | | `--enable-keymanager` | Serve the keymanager API on `--keymanager-port` (default `5062`). Off by default, since it mutates key material | Metrics are served on `--metrics-port` (default `5064`). See diff --git a/docs/cli.md b/docs/cli.md index e0978f6bb..998f1e8fd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -435,9 +435,9 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch | +| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next slot's builder registration carries it too, and an ethlambda beacon node takes it from there | | `graffiti` | `--graffiti` | The next proposal | -| `gas_limit` | `DEFAULT_GAS_LIMIT` | Stored and served only: it matters to builder registrations, and there is no builder flow | +| `gas_limit` | `DEFAULT_GAS_LIMIT` | With `--builder-proposals`, the next slot's builder registration, which the builder builds toward. Otherwise stored and served only | **These are held in memory only.** A restart puts every validator back on the defaults, so an operator driving these endpoints must set them again after one. From ee5fed8e64ff0921acba1ba21901e8aaca74a204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:11:13 -0300 Subject: [PATCH 11/19] fix(validator): address the builder proposals review Registrations: - Run one epoch send and one per-slot send at a time (`in_flight`), so a slow node no longer stacks up sends that each re-send the whole set. - Send to every beacon node at once rather than in turn, as Lighthouse broadcasts: a call takes as long as its slowest node, not the sum. - Sign each re-signed registration strictly newer than the one it replaces, max(now, last + 1), so a relay keeps it after two changes in a second or a clock stepped back. - Read a validator's fee recipient and gas limit under one lock. - Say at error when --suggested-fee-recipient is the zero address. Accepted, as every other client's flag accepts it. Blinded blocks: - Refuse a blinded block before signing when builder proposals are off, besides the HTTP client's own refusal. - Record the producing node with the block's root as well as its slot, check it at publish, and clear the record when no node produced. HTTP client: - Read a failed answer's body up to 1 MiB and carry at most 1 KiB of it into an error, with line breaks written out. Docs: the status codes the blinded publish reads the wrong way, the signed-but-undelivered window, and why a refused batch is not split. Tests: the timestamp rule, overlapping epoch sends, concurrent fan-out, a blinded answer with builder proposals off, the signature over the full block's root, wrong slot/proposer/fork on a blinded answer, a retry after the dropped proposal, the root check, a failed production clearing the record, error bodies, and end to end through this node: relay refuses, relay fails to confirm, and the node refusing a second block. Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- Cargo.lock | 1 + .../rpc/src/beacon/validator_client_tests.rs | 102 ++++++++++++- crates/validator/Cargo.toml | 2 + crates/validator/src/beacon_node/fallback.rs | 133 +++++++++++++---- crates/validator/src/beacon_node/http.rs | 108 ++++++++++++-- crates/validator/src/beacon_node/mock.rs | 29 +++- crates/validator/src/beacon_node/mod.rs | 6 +- crates/validator/src/in_flight.rs | 57 ++++++++ crates/validator/src/lib.rs | 13 ++ crates/validator/src/proposal.rs | 135 +++++++++++++++++- crates/validator/src/proposer_settings.rs | 45 ++++++ crates/validator/src/registration.rs | 130 ++++++++++++++++- docs/spec_deviations.md | 15 +- 13 files changed, 721 insertions(+), 55 deletions(-) create mode 100644 crates/validator/src/in_flight.rs diff --git a/Cargo.lock b/Cargo.lock index 405f49c0f..3080e4952 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2274,6 +2274,7 @@ dependencies = [ "ctr", "ethlambda-metrics", "ethlambda-types", + "futures-util", "hex", "hmac", "http-body-util", diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 8bca0f3f9..b7701fbab 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -1836,6 +1836,105 @@ mod builder_flow { ); produced.into_signed_ssz(signature) } + + /// Registered, a bid taken at the maximum factor, and the blinded + /// block signed: its root and the body to publish. + async fn signed_blinded(&self) -> (ethlambda_types::primitives::H256, Vec) { + self.register().await; + self.world.offer(1, |_| {}); + let produced = self.produce(Some(u64::MAX)).await; + assert!(produced.is_blinded()); + let block_root = produced.block_root(); + (block_root, self.sign(produced).await) + } + + async fn publish_blinded( + &self, + block_root: ethlambda_types::primitives::H256, + body: &[u8], + ) -> ethlambda_validator::Result { + self.client + .publish_blinded_block(self.world.slot, block_root, ForkName::Fulu, body) + .await + } + } + + /// A relay refusing the block is a definite refusal at the client: the + /// node passes the builder's 400 through. + #[tokio::test] + async fn a_relay_refusing_the_blinded_block_is_a_definite_refusal() { + let world = builder_world(false, SyncStatus::Synced).await; + world + .relay + .answer_submissions_with(axum::http::StatusCode::BAD_REQUEST); + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let (block_root, body) = h.signed_blinded().await; + let err = h + .publish_blinded(block_root, &body) + .await + .expect_err("the relay refused it"); + assert!( + matches!(err, Error::BeaconNodeStatus { status: 400, .. }), + "got {err:?}" + ); + } + + /// A relay failing without saying whether it published is unconfirmed at + /// the client, not a missed proposal: the relay may still publish. + #[tokio::test] + async fn a_relay_failing_on_the_blinded_block_leaves_it_unconfirmed() { + let world = builder_world(false, SyncStatus::Synced).await; + world + .relay + .answer_submissions_with(axum::http::StatusCode::INTERNAL_SERVER_ERROR); + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let (block_root, body) = h.signed_blinded().await; + let err = h + .publish_blinded(block_root, &body) + .await + .expect_err("the relay did not confirm it"); + assert!( + matches!( + err, + Error::BlindedUnconfirmed { + status: Some(500), + .. + } + ), + "got {err:?}" + ); + } + + /// Once the node handed a blinded block to its builder, it refuses a + /// different block for the slot from the same client, as a second + /// signature for it would be an equivocation. + #[tokio::test] + async fn the_node_refuses_a_second_block_after_a_blinded_one() { + let world = builder_world(false, SyncStatus::Synced).await; + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let (block_root, body) = h.signed_blinded().await; + let published = h.publish_blinded(block_root, &body).await.expect("handed"); + assert_eq!(published, Published::HandedToBuilder); + + let full = h.produce(None).await; + assert!(!full.is_blinded()); + let full = h.sign(full).await; + let err = h + .client + .publish_block(ForkName::Fulu, &full) + .await + .expect_err("the slot is claimed by the blinded block"); + assert!( + matches!(&err, Error::BeaconNodeStatus { status: 400, body } + if body.contains("already published a different block")), + "got {err:?}" + ); } #[tokio::test] @@ -1857,10 +1956,11 @@ mod builder_flow { "the factor must reach the node and take the bid" ); + let block_root = produced.block_root(); let body = h.sign(produced).await; let published = h .client - .publish_blinded_block(h.world.slot, ForkName::Fulu, &body) + .publish_blinded_block(h.world.slot, block_root, ForkName::Fulu, &body) .await .expect("the node hands the block to its builder"); assert_eq!(published, Published::HandedToBuilder); diff --git a/crates/validator/Cargo.toml b/crates/validator/Cargo.toml index c227e995f..87f16f082 100644 --- a/crates/validator/Cargo.toml +++ b/crates/validator/Cargo.toml @@ -41,6 +41,8 @@ blst = "0.3.16" # the `impl Future + Send` bounds at every method is noise this trait does not # need. async-trait = "0.1" +# `join_all`, so the calls every beacon node must hear reach them all at once. +futures-util = "0.3" # EIP-2335 keystore decryption. scrypt = { version = "0.11", default-features = false } diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 04b4652e4..01a41fb2f 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -40,10 +40,20 @@ type BoxFuture<'a, T> = Pin + Send + 'a>>; /// first one that succeeds. pub struct FallbackBeaconNode { nodes: Vec, - /// The node that produced the latest blinded block, by slot. A blinded - /// block can only be published where it was produced: that node's builder - /// holds the payload, and that node claimed the slot. - blinded_producer: std::sync::Mutex>, + /// The latest blinded block produced, and the node that produced it. A + /// blinded block can only be published where it was produced: that node's + /// builder holds the payload, and that node claimed the slot. + blinded_producer: std::sync::Mutex>, +} + +/// Which node produced which blinded block. +#[derive(Debug, Clone, Copy)] +struct BlindedProducer { + slot: Slot, + /// Checked at publish, so the record answers for the block being + /// published rather than for whatever was last produced at its slot. + block_root: Root, + position: usize, } impl FallbackBeaconNode { @@ -131,6 +141,11 @@ impl FallbackBeaconNode { /// is logged per node so a partially-registered client is visible rather /// than silent. /// + /// The nodes are called at once rather than in turn, as Lighthouse + /// broadcasts, so the call takes as long as its slowest node rather than + /// the sum of every node's timeout: a hung node no longer holds back the + /// healthy ones behind it in the list. + /// /// `node_effect` says what a refusing node costs (said once per refusal); /// `all_failed` says what it costs when none accepts, or is `None` when the /// caller reports that itself, so a duty that did not depend on the call is @@ -145,10 +160,11 @@ impl FallbackBeaconNode { where F: Fn(&'p B) -> BoxFuture<'p, Result<()>>, { + let answers = futures_util::future::join_all(self.nodes.iter().map(&call)).await; let mut accepted = 0; let mut last = None; - for (position, node) in self.nodes.iter().enumerate() { - match call(node).await { + for (position, answer) in answers.into_iter().enumerate() { + match answer { Ok(()) => accepted += 1, Err(err) => { warn!(%what, position, %err, node_effect, "Beacon node did not accept"); @@ -262,18 +278,26 @@ impl BeaconNodeApi for FallbackBeaconNode { } /// Remembers which node answered when the block is blinded, and forgets - /// it when it is not: the record is for the latest block only, so a full - /// block never leaves a stale claim that a blinded publish could match. + /// it otherwise, a production that failed on every node included: the + /// record is for the latest block only, so nothing leaves a stale claim + /// that a blinded publish could match. async fn produce_block(&self, request: &BlockRequest) -> Result { - let (position, block) = self + let produced = self .try_each_at("produce_block", |node| node.produce_block(request)) - .await?; - let record = block.is_blinded().then(|| (block.slot(), position)); + .await; + let record = match &produced { + Ok((position, block)) if block.is_blinded() => Some(BlindedProducer { + slot: block.slot(), + block_root: block.block_root(), + position: *position, + }), + _ => None, + }; *self .blinded_producer .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = record; - Ok(block) + produced.map(|(_, block)| block) } /// First node that accepts it wins, rather than a broadcast to every node. @@ -307,6 +331,7 @@ impl BeaconNodeApi for FallbackBeaconNode { async fn publish_blinded_block( &self, slot: Slot, + block_root: Root, fork: ForkName, body: &[u8], ) -> Result { @@ -315,13 +340,14 @@ impl BeaconNodeApi for FallbackBeaconNode { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); match record { - Some((produced_slot, position)) if produced_slot == slot => { - self.nodes[position] - .publish_blinded_block(slot, fork, body) + Some(producer) if producer.slot == slot && producer.block_root == block_root => { + self.nodes[producer.position] + .publish_blinded_block(slot, block_root, fork, body) .await } _ => Err(Error::InconsistentResponse(format!( - "no beacon node produced a blinded block for slot {slot}" + "no beacon node produced blinded block {} for slot {slot}", + crate::beacon_node::dto::encode_hex(&block_root.0) ))), } } @@ -975,7 +1001,7 @@ mod tests { assert!(produced.is_blinded()); let published = fallback - .publish_blinded_block(5, ForkName::Electra, &[1, 2, 3]) + .publish_blinded_block(5, produced.block_root(), ForkName::Electra, &[1, 2, 3]) .await .expect("published"); assert_eq!(published, Published::HandedToBuilder); @@ -997,12 +1023,12 @@ mod tests { let first = MockBeaconNode::new().failing_call("produce_block", "down"); let fallback = FallbackBeaconNode::new(vec![first, second]); - fallback + let produced = fallback .produce_block(&blinded_request(5, 7)) .await .expect("node 1 produces"); let err = fallback - .publish_blinded_block(5, ForkName::Electra, &[9]) + .publish_blinded_block(5, produced.block_root(), ForkName::Electra, &[9]) .await .expect_err("node 1 failed to confirm"); assert!( @@ -1019,19 +1045,27 @@ mod tests { ); } + /// The record answers for one block: another slot, or another root at the + /// same slot, is refused without reaching any node. #[tokio::test] - async fn a_blinded_publish_for_another_slot_is_refused() { + async fn a_blinded_publish_for_another_slot_or_block_is_refused() { let fallback = FallbackBeaconNode::new(vec![blinded_node()]); - fallback + let produced = fallback .produce_block(&blinded_request(5, 7)) .await .expect("produces"); - - let err = fallback - .publish_blinded_block(6, ForkName::Electra, &[1]) - .await - .expect_err("nothing was produced for slot 6"); - assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + let root = produced.block_root(); + + for (slot, block_root, what) in [ + (6, root, "nothing was produced for slot 6"), + (5, Root::repeat_byte(0xee), "that block was not produced"), + ] { + let err = fallback + .publish_blinded_block(slot, block_root, ForkName::Electra, &[1]) + .await + .expect_err(what); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + } assert!( fallback.nodes[0] .published_blinded @@ -1041,12 +1075,32 @@ mod tests { ); } + /// A production that fails on every node clears the record too, so the + /// earlier block's claim does not outlive the attempt that replaced it. + #[tokio::test] + async fn a_failed_production_clears_the_blinded_record() { + let fallback = FallbackBeaconNode::new(vec![blinded_node()]); + let produced = fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("blinded"); + fallback + .produce_block(&blinded_request(6, 7)) + .await + .expect_err("the node holds no block for slot 6"); + + fallback + .publish_blinded_block(5, produced.block_root(), ForkName::Electra, &[1]) + .await + .expect_err("the record is gone"); + } + /// A full block produced afterwards clears the claim, so an old blinded /// block's slot cannot be published against a node that no longer holds it. #[tokio::test] async fn a_full_block_clears_the_blinded_record() { let fallback = FallbackBeaconNode::new(vec![blinded_node()]); - fallback + let blinded = fallback .produce_block(&blinded_request(5, 7)) .await .expect("blinded"); @@ -1056,7 +1110,7 @@ mod tests { .expect("full, since no factor was sent"); fallback - .publish_blinded_block(5, ForkName::Electra, &[1]) + .publish_blinded_block(5, blinded.block_root(), ForkName::Electra, &[1]) .await .expect_err("the record is gone"); } @@ -1103,4 +1157,25 @@ mod tests { .is_empty() ); } + + /// Every node is called at once: two slow nodes cost one delay, not the + /// sum, so a hung node does not hold back the ones behind it. + #[tokio::test] + async fn every_node_is_called_at_once() { + let delay = std::time::Duration::from_millis(300); + let slow = || MockBeaconNode::new().with_registration_delay(delay); + let fallback = FallbackBeaconNode::new(vec![slow(), slow()]); + + let started = std::time::Instant::now(); + fallback + .register_validators(&[]) + .await + .expect("both accept"); + let elapsed = started.elapsed(); + + assert!(elapsed < delay * 2, "took {elapsed:?}"); + for node in &fallback.nodes { + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![0]); + } + } } diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 599119890..2087d59f5 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -67,6 +67,48 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(8); /// would report a failure for a registration the node went on to install. const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(12); +/// How much of a failed answer's body is read. Enough for a pool submission's +/// 400, which lists each refused entry and is parsed for them. +const ERROR_BODY_READ_LIMIT: usize = 1024 * 1024; + +/// How much of a failed answer's body an error carries, and so a log line. +const ERROR_TEXT_LIMIT: usize = 1024; + +/// A failed answer's body, read up to [`ERROR_BODY_READ_LIMIT`] bytes, so a +/// node answering with an endless body cannot hold the client on it. +async fn read_error_body(mut response: reqwest::Response) -> String { + let mut bytes = Vec::new(); + while bytes.len() < ERROR_BODY_READ_LIMIT { + match response.chunk().await { + Ok(Some(chunk)) => bytes.extend_from_slice(&chunk), + _ => break, + } + } + bytes.truncate(ERROR_BODY_READ_LIMIT); + String::from_utf8_lossy(&bytes).into_owned() +} + +/// A body as an error shows it: line breaks written out as `\n` and `\r`, so +/// the log line it lands in stays one line, and cut to [`ERROR_TEXT_LIMIT`] +/// bytes. +fn error_text(body: String) -> String { + let mut text = body.replace('\n', "\\n").replace('\r', "\\r"); + if text.len() > ERROR_TEXT_LIMIT { + let mut end = ERROR_TEXT_LIMIT; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + text.push_str("..."); + } + text +} + +/// [`read_error_body`] as [`error_text`] shows it. +async fn error_body(response: reqwest::Response) -> String { + error_text(read_error_body(response).await) +} + /// What an SSZ answer carries besides its bytes: the fork, which only a header /// can say, and the two booleans the block endpoints use to say what the bytes /// are. @@ -184,7 +226,7 @@ impl HttpBeaconNode { if status.is_success() { return Ok(()); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -283,7 +325,7 @@ impl HttpBeaconNode { return Err(Error::BeaconNodeSyncing); } if !status.is_success() { - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; return Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -493,7 +535,7 @@ impl HttpBeaconNode { Err(Error::BeaconNodeStatus { status: status.as_u16(), - body, + body: error_text(body), }) } @@ -503,7 +545,7 @@ impl HttpBeaconNode { return Err(Error::BeaconNodeSyncing); } if !status.is_success() { - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; return Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -727,7 +769,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(Published::Imported); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -744,9 +786,20 @@ impl BeaconNodeApi for HttpBeaconNode { /// anything to a builder). Every other 5xx, and any failure after the /// request was sent (a timeout, a reset), is [`Error::BlindedUnconfirmed`], /// since the relay may still publish. + /// + /// The status alone cannot always tell, and two edges are knowingly read + /// the wrong way. Some of ethlambda's own 500s are definite (no builder + /// configured, the blinded block failing the node's checks) and are read as + /// unconfirmed. A 503 from a proxy that had already forwarded the request + /// is read as definite. Neither changes what happens next: nothing is + /// retried and nothing is signed again either way. Only the log line and + /// the counter it lands on are wrong. async fn publish_blinded_block( &self, + // For the fallback, which picks the node that produced this block; + // one node has nothing to choose. _slot: Slot, + _block_root: Root, fork: ForkName, body: &[u8], ) -> Result { @@ -775,7 +828,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(Published::HandedToBuilder); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; if status.is_server_error() && status != StatusCode::SERVICE_UNAVAILABLE { return Err(Error::BlindedUnconfirmed { status: Some(status.as_u16()), @@ -864,7 +917,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(()); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -933,7 +986,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(messages.len()); } - let body = response.text().await.unwrap_or_default(); + let body = read_error_body(response).await; Self::handle_pool_submission(status, body, messages.len()) } @@ -963,7 +1016,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(attestations.len()); } - let body = response.text().await.unwrap_or_default(); + let body = read_error_body(response).await; Self::handle_pool_submission(status, body, attestations.len()) } @@ -1128,7 +1181,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(messages.len()); } - let body = response.text().await.unwrap_or_default(); + let body = read_error_body(response).await; Self::handle_pool_submission(status, body, messages.len()) } @@ -1469,6 +1522,31 @@ mod tests { assert!(served.seen.lock().expect("lock").is_empty()); } + /// A node's error body reaches the logs on one line and cut short: a body + /// with line breaks cannot forge log lines, and a long one cannot flood + /// them. + #[tokio::test] + async fn an_error_body_is_one_line_and_capped() { + let reply = "first line\nsecond line\r\n".repeat(200).into_bytes(); + let served = serve(400, vec![], reply).await; + let err = served.node.genesis().await.expect_err("a 400"); + let Error::BeaconNodeStatus { status, body } = err else { + panic!("got {err:?}"); + }; + assert_eq!(status, 400); + assert!(body.starts_with("first line\\nsecond line\\r\\nfirst line")); + assert!(!body.contains('\n') && !body.contains('\r')); + assert_eq!(body.len(), ERROR_TEXT_LIMIT + "...".len()); + } + + #[test] + fn error_text_cuts_on_a_character_boundary() { + let text = error_text("é".repeat(ERROR_TEXT_LIMIT)); + assert!(text.ends_with("...")); + assert!(text.len() <= ERROR_TEXT_LIMIT + "...".len()); + assert_eq!(error_text("short".to_string()), "short"); + } + #[tokio::test] async fn a_blinded_publish_maps_each_status() { for (status, expected) in [ @@ -1478,7 +1556,7 @@ mod tests { let served = serve(status, vec![], vec![]).await; let published = served .node - .publish_blinded_block(5, ForkName::Fulu, &[1, 2, 3]) + .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1, 2, 3]) .await .expect("accepted"); assert_eq!(published, expected, "status {status}"); @@ -1492,7 +1570,7 @@ mod tests { let served = serve(400, vec![], b"bad".to_vec()).await; let err = served .node - .publish_blinded_block(5, ForkName::Fulu, &[1]) + .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) .await .expect_err("a refusal"); assert!( @@ -1503,7 +1581,7 @@ mod tests { let served = serve(500, vec![], b"boom".to_vec()).await; let err = served .node - .publish_blinded_block(5, ForkName::Fulu, &[1]) + .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) .await .expect_err("unconfirmed"); assert!( @@ -1525,7 +1603,7 @@ mod tests { // Nothing listens on port 1. let node = HttpBeaconNode::new("http://127.0.0.1:1").expect("builds"); let err = node - .publish_blinded_block(5, ForkName::Fulu, &[1]) + .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) .await .expect_err("no answer"); assert!(matches!(err, Error::BeaconNode { .. }), "got {err:?}"); @@ -1537,7 +1615,7 @@ mod tests { let served = serve(503, vec![], b"syncing".to_vec()).await; let err = served .node - .publish_blinded_block(5, ForkName::Fulu, &[1]) + .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) .await .expect_err("refused"); assert!( diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 409e3aedf..f91284a83 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -86,11 +86,18 @@ pub struct MockBeaconNode { /// request with no factor still gets a full block, as the contract on /// `BeaconNodeApi::produce_block` requires. pub produces_blinded: bool, + /// With [`Self::produces_blinded`], answer with the blinded block even + /// when the request carries no factor: a node breaking the contract, so a + /// test can see what the caller does with one. + pub blinded_regardless: bool, /// Every registration `register_validators` received, in call order. pub registrations: Mutex>, /// The size of each `register_validators` call, in call order, so a test /// can see how a send was batched. pub registration_batches: Mutex>, + /// How long `register_validators` takes before it records the batch, as + /// a slow node would. + pub registration_delay: Option, /// Every blinded block published, as (slot, fork, body). pub published_blinded: Mutex)>>, /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; @@ -288,6 +295,14 @@ impl MockBeaconNode { self } + /// Answer every pre-gloas `produce_block` with a blinded block, factor or + /// not, as a node breaking the contract on `BeaconNodeApi::produce_block`. + pub fn with_blinded_block_regardless(mut self) -> Self { + self.produces_blinded = true; + self.blinded_regardless = true; + self + } + /// Make `publish_blinded_block` fail as a node that did not confirm the /// block, carrying `detail`. pub fn with_unconfirmed_blinded_publish(mut self, detail: &str) -> Self { @@ -295,6 +310,12 @@ impl MockBeaconNode { self } + /// Make `register_validators` take `delay` before it records the batch. + pub fn with_registration_delay(mut self, delay: std::time::Duration) -> Self { + self.registration_delay = Some(delay); + self + } + /// Make `publish_blinded_block` take `delay` before it records the block. pub fn with_blinded_publish_delay(mut self, delay: std::time::Duration) -> Self { self.blinded_publish_delay = Some(delay); @@ -690,7 +711,9 @@ impl BeaconNodeApi for MockBeaconNode { failure: BeaconNodeFailure::Request, detail: "no block set".into(), })?; - if self.produces_blinded && request.builder_boost_factor.is_some() { + if self.produces_blinded + && (request.builder_boost_factor.is_some() || self.blinded_regardless) + { let block = ProducedBlock { fork: ForkName::Electra, contents: Contents::Blinded { @@ -727,6 +750,7 @@ impl BeaconNodeApi for MockBeaconNode { async fn publish_blinded_block( &self, slot: Slot, + _block_root: Root, fork: ForkName, body: &[u8], ) -> Result { @@ -766,6 +790,9 @@ impl BeaconNodeApi for MockBeaconNode { registrations: &[SignedValidatorRegistrationV1], ) -> Result<()> { self.guard("register_validators")?; + if let Some(delay) = self.registration_delay { + tokio::time::sleep(delay).await; + } self.registration_batches .lock() .expect("lock") diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index ed292c5f4..23666c85c 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -387,8 +387,9 @@ pub trait BeaconNodeApi: Send + Sync { /// proposal, whether or not the first one succeeded. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result; - /// Publish a signed blinded block, produced at `slot` under `fork`, to - /// `POST /eth/v2/beacon/blinded_blocks`. The node hands it to its builder. + /// Publish a signed blinded block, the one with root `block_root` produced + /// at `slot` under `fork`, to `POST /eth/v2/beacon/blinded_blocks`. The + /// node hands it to its builder. /// /// # Contract: the node that produced it, and no other /// @@ -404,6 +405,7 @@ pub trait BeaconNodeApi: Send + Sync { async fn publish_blinded_block( &self, slot: Slot, + block_root: Root, fork: ForkName, body: &[u8], ) -> Result; diff --git a/crates/validator/src/in_flight.rs b/crates/validator/src/in_flight.rs new file mode 100644 index 000000000..62ab3225e --- /dev/null +++ b/crates/validator/src/in_flight.rs @@ -0,0 +1,57 @@ +//! One run at a time for work spawned off the slot loop. +//! +//! The slot loop spawns its sends to beacon nodes rather than awaiting them, so +//! a slow node cannot delay the next slot's duties. The cost is that nothing +//! stops a new run starting while the last one still waits on that node. Each +//! run holds its own copy of the work and sends it again, so the runs pile up +//! for as long as the node stays slow. A run that finds the previous one still +//! going skips instead, as Teku's registration round does. + +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Whether a run is going. +#[derive(Debug, Default)] +pub(crate) struct InFlight(AtomicBool); + +impl InFlight { + /// The guard for a new run, or `None` when one is already going. + pub(crate) fn enter(&self) -> Option> { + let running = self.0.swap(true, Ordering::AcqRel); + (!running).then_some(InFlightGuard(&self.0)) + } +} + +/// Ends the run on every exit from it, a panic included, so one failed run +/// cannot stop every later one. +pub(crate) struct InFlightGuard<'a>(&'a AtomicBool); + +impl Drop for InFlightGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_second_run_is_refused_until_the_first_ends() { + let in_flight = InFlight::default(); + let first = in_flight.enter().expect("nothing is running"); + assert!(in_flight.enter().is_none()); + drop(first); + assert!(in_flight.enter().is_some()); + } + + #[test] + fn a_run_that_panics_still_ends() { + let in_flight = InFlight::default(); + let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _run = in_flight.enter().expect("nothing is running"); + panic!("the run fails"); + })); + assert!(unwound.is_err()); + assert!(in_flight.enter().is_some()); + } +} diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 7e2732809..c2c138cb9 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -37,6 +37,7 @@ pub mod beacon_node; pub mod duties; pub mod error; pub mod http_api; +pub(crate) mod in_flight; pub mod keys; pub mod metrics; pub mod payload_attestation; @@ -144,6 +145,18 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { keymanager API." ); } + // Accepted, as every other client's flag accepts it, but said loudly: + // nobody holds the zero address's key, so what it is paid is burned, and + // a builder registration or proposer preference signs that into a + // commitment the builder honours. Only the default can be zero, since the + // keymanager API refuses it for a single validator. + if config.suggested_fee_recipient == Some(ExecutionAddress::ZERO) { + error!( + "--suggested-fee-recipient is the zero address. Every validator without its own \ + address burns its execution-layer rewards, and builder registrations and proposer \ + preferences sign that address for the builder to pay." + ); + } let settings = Arc::new(ProposerSettings::new( config.graffiti, config.suggested_fee_recipient, diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 381d25a59..ab185e76f 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -299,6 +299,17 @@ impl ProposalService { // a slot already proposed. validate_produced_block(&request, &produced)?; + // Also the implementation's contract, and checked again for the same + // reason. With builder proposals off this client never publishes a + // blinded block, so signing one would record the slot in the guard for + // a block that can never be sent. + if produced.is_blinded() && self.builder_boost_factor.is_none() { + return Err(Error::InconsistentResponse(format!( + "node produced a blinded block for slot {slot}; builder proposals are off, so \ + this client cannot publish one" + ))); + } + // The node's fork and this client's must agree, or the signature is // computed under a fork version the network does not accept. // @@ -391,7 +402,7 @@ impl ProposalService { // A blinded block counts itself on its own task (see // `publish_blinded`); a full one is counted here. let published = if blinded { - self.publish_blinded(slot, fork, body, duty.validator_index) + self.publish_blinded(slot, block_root, fork, body, duty.validator_index) .await? } else { let published = self @@ -515,6 +526,7 @@ impl ProposalService { async fn publish_blinded( &self, slot: Slot, + block_root: Root, fork: ForkName, body: Vec, validator: u64, @@ -523,7 +535,10 @@ impl ProposalService { let slot_start = self.clock.as_ref().map(|clock| clock.start_of(slot)); let task = tokio::spawn(async move { let bytes = body.len(); - match beacon_node.publish_blinded_block(slot, fork, &body).await { + match beacon_node + .publish_blinded_block(slot, block_root, fork, &body) + .await + { Ok(published) => { match published { Published::HandedToBuilder => { @@ -1000,6 +1015,15 @@ mod tests { 1, "the publish must complete although the proposal future is gone" ); + + // The guard recorded the slot before the publish started, so a retry + // after the dropped future is refused without asking for a block. + let err = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("the slot is signed"); + assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); + assert_eq!(node.block_requests().len(), 1); } /// A 4xx is a definite refusal: the block was not published, so the @@ -1039,6 +1063,113 @@ mod tests { assert!(node.published_blinded.lock().expect("lock").is_empty()); } + /// A node breaking its contract with a blinded answer to a request with no + /// factor is refused before signing. With builder proposals off this client + /// never publishes a blinded block, so signing would burn the slot for + /// nothing. The second attempt is refused for the same reason rather than + /// by the guard, which shows nothing was recorded. + #[tokio::test] + async fn a_blinded_block_is_refused_before_signing_with_builder_proposals_off() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block_regardless(), + ); + let service = service(node.clone()); + + for attempt in 0..2 { + let err = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("a blinded block must not be signed"); + assert!( + matches!(err, Error::InconsistentResponse(_)), + "attempt {attempt}: got {err:?}" + ); + } + assert!(!service.signed_blinded_at(slot())); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + assert!(node.published_blocks().is_empty()); + } + + /// The signature in the published blinded block verifies over the full + /// block's root. The blinded container's root equals it, which is what lets + /// the relay put the payload back under a signature over a block the + /// validator never saw whole. + #[tokio::test] + async fn a_blinded_block_is_signed_over_the_full_blocks_root() { + use crate::beacon_node::block_contents::empty_block_for; + use blst::min_pk::{PublicKey, Signature}; + use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; + + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + builder_service(node.clone(), None) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("proposes"); + + let (_, _, body) = node.published_blinded.lock().expect("lock").remove(0); + let published = SignedBlindedBeaconBlock::from_ssz_bytes(&body).expect("decodes"); + let full_root = empty_block_for(slot(), 7).hash_tree_root(); + assert_eq!(published.message.hash_tree_root(), full_root); + + let root = context().block_signing_root(full_root, slot()); + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(&published.signature.0).expect("valid signature"); + assert_eq!( + sig.verify( + true, + root.as_slice(), + b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_", + &[], + &pk, + true + ), + blst::BLST_ERROR::BLST_SUCCESS + ); + } + + /// A blinded answer is held to what a full one is: the slot and proposer + /// asked for, and the fork this client's schedule puts the slot in. Each + /// is refused before signing, so none spends the slot. + #[tokio::test] + async fn a_blinded_block_for_another_slot_proposer_or_fork_is_refused() { + let (store, pubkey) = store(); + // The mock always names electra, and slot 96 is phase0 on mainnet. + let cases = [ + ("slot", (slot() + 1, 7), slot()), + ("proposer", (slot(), 8), slot()), + ("fork", (96, 7), 96), + ]; + for (what, (block_slot, proposer), requested) in cases { + let node = Arc::new( + MockBeaconNode::new() + .with_block(block_slot, proposer) + .with_blinded_block(), + ); + let service = builder_service(node.clone(), None); + let err = service + .propose(requested, &duty(&pubkey, 7, requested), &store) + .await + .expect_err(what); + assert!( + matches!(err, Error::InconsistentResponse(_)), + "{what}: got {err:?}" + ); + assert!(!service.signed_blinded_at(requested), "{what}"); + assert!( + node.published_blinded.lock().expect("lock").is_empty(), + "{what}" + ); + } + } + /// A duty naming a key this client does not hold must fail before anything /// is asked of the beacon node. #[tokio::test] diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 5000779c9..2d73f4397 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -153,6 +153,29 @@ impl ProposerSettings { }) } + /// `pubkey`'s fee recipient and gas limit read together, under one lock, or + /// `None` when it has no fee recipient. + /// + /// For a signed message that carries both (a builder registration, proposer + /// preferences): two separate reads could straddle a keymanager write and + /// pair an old address with a new gas limit, a combination the operator + /// never configured, which the builder then keeps. + pub fn fee_recipient_and_gas_limit( + &self, + pubkey: &BlsPubkey, + ) -> Option<(ExecutionAddress, u64)> { + self.read(|inner| { + let overrides = inner.overrides.get(pubkey); + let fee_recipient = overrides + .and_then(|o| o.fee_recipient) + .or(inner.default_fee_recipient)?; + let gas_limit = overrides + .and_then(|o| o.gas_limit) + .unwrap_or(inner.default_gas_limit); + Some((fee_recipient, gas_limit)) + }) + } + /// The graffiti `pubkey` proposes with. pub fn graffiti(&self, pubkey: &BlsPubkey) -> Bytes32 { self.read(|inner| { @@ -239,6 +262,28 @@ mod tests { assert_eq!(settings.gas_limit(&key(2)), DEFAULT_GAS_LIMIT); } + /// The pair a signed registration or preference carries agrees with the + /// two separate reads, and is `None` exactly when there is no address. + #[test] + fn the_fee_recipient_and_gas_limit_read_together_match_the_single_reads() { + let settings = ProposerSettings::new(Bytes32::ZERO, None); + assert_eq!(settings.fee_recipient_and_gas_limit(&key(1)), None); + + settings.set_gas_limit(&key(1), 30_000_000); + assert_eq!(settings.fee_recipient_and_gas_limit(&key(1)), None); + settings.set_fee_recipient(&key(1), H160([0xbb; 20])); + assert_eq!( + settings.fee_recipient_and_gas_limit(&key(1)), + Some((H160([0xbb; 20]), 30_000_000)) + ); + + let defaults = self::settings(); + assert_eq!( + defaults.fee_recipient_and_gas_limit(&key(2)), + Some((H160([0xaa; 20]), DEFAULT_GAS_LIMIT)) + ); + } + /// Clearing one field leaves the others overridden. #[test] fn clearing_an_override_restores_that_default_only() { diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index c9d5a717d..a33cf5298 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -37,6 +37,21 @@ //! registration whose send fails is cached anyway, so it is not re-sent at the //! next slot: it waits for the next epoch's full send. //! +//! A batch a node refuses with a 400 is not split to find the entry it +//! objects to: every entry is signed here by keys this client holds, so a +//! refusal says more about the node or the relay than about one entry, and no +//! other client splits either. +//! +//! # One send of each kind at a time +//! +//! Both sends are spawned off the slot loop, so nothing else would stop a new +//! one starting while the last still waits on a slow node: a full send every +//! epoch, each holding the whole registration set and sending it again to the +//! healthy nodes. A send that finds the previous one of its kind still going +//! is skipped (see [`crate::in_flight`]). A skipped full send waits for the +//! next epoch, and a skipped per-slot one for the next slot, since it has not +//! marked its change as sent. +//! //! # Signing stays off the async runtime //! //! The first epoch signs one BLS signature per validator. That runs on the @@ -64,6 +79,7 @@ use tracing::{debug, warn}; use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::encode_hex; +use crate::in_flight::InFlight; use crate::keys::ValidatorStore; use crate::proposer_settings::ProposerSettings; use crate::signing::SigningContext; @@ -95,6 +111,27 @@ impl Current { } } +/// The timestamp for a registration replacing `held`: `now_secs`, or one +/// second past `held`'s when the clock does not say later. +/// +/// A relay keeps a validator's registration with the latest timestamp and +/// drops one that is not newer. Two settings changes within one second, or a +/// wall clock stepped back between them, would otherwise sign the new settings +/// under a timestamp the relay ignores. The cached registration is then re-sent +/// every epoch while the builder keeps the old fee recipient or gas limit. +/// +/// Two changes within a second put it one second ahead of the clock, which no +/// relay refuses as being in the future. After a larger backward step it is +/// further ahead: a relay may refuse it until its own clock catches up, and the +/// epoch's full re-send then delivers it. With the clock's own reading +/// instead, the relay would drop it as older on every re-send, since the +/// cached signature keeps that timestamp until the settings change again. +fn newer_than(held: Option<&SignedValidatorRegistrationV1>, now_secs: u64) -> u64 { + held.map_or(now_secs, |held| { + now_secs.max(held.message.timestamp.saturating_add(1)) + }) +} + /// Signs and sends builder registrations. pub struct RegistrationService { beacon_node: Arc, @@ -107,6 +144,10 @@ pub struct RegistrationService { /// The [`ProposerSettings::generation`] the cache was last brought up to /// date against. signed_generation: AtomicU64, + /// The epoch's full send, while it runs. + full_send: InFlight, + /// The per-slot send of changed keys, while it runs. + changed_send: InFlight, } impl RegistrationService { @@ -122,6 +163,8 @@ impl RegistrationService { known: std::sync::Mutex::new(Vec::new()), signed: std::sync::Mutex::new(HashMap::new()), signed_generation: AtomicU64::new(0), + full_send: InFlight::default(), + changed_send: InFlight::default(), } } @@ -142,13 +185,15 @@ impl RegistrationService { no_fee_recipient: 0, }; for pubkey in pubkeys { - let Some(fee_recipient) = self.settings.fee_recipient(pubkey) else { + let Some((fee_recipient, gas_limit)) = + self.settings.fee_recipient_and_gas_limit(pubkey) + else { current.no_fee_recipient += 1; continue; }; - let gas_limit = self.settings.gas_limit(pubkey); - if let Some(held) = signed.get(pubkey) + let held = signed.get(pubkey); + if let Some(held) = held && held.message.fee_recipient == fee_recipient && held.message.gas_limit == gas_limit { @@ -159,7 +204,7 @@ impl RegistrationService { let message = ValidatorRegistrationV1 { fee_recipient, gas_limit, - timestamp: now_secs, + timestamp: newer_than(held, now_secs), pubkey: *pubkey, }; // A key the keymanager removed since the epoch refresh is the one @@ -232,6 +277,13 @@ impl RegistrationService { if self.is_gloas(epoch) { return; } + let Some(_running) = self.full_send.enter() else { + warn!( + epoch, + "The previous builder registration send is still running; skipping this epoch's" + ); + return; + }; // Read before signing, so a change that lands while this runs is seen // by the next `register_changed` rather than lost. let generation = self.settings.generation(); @@ -274,6 +326,10 @@ impl RegistrationService { if self.is_gloas(epoch) { return; } + // Silently: the change stays unsent, so the next slot picks it up. + let Some(_running) = self.changed_send.enter() else { + return; + }; let generation = self.settings.generation(); if generation == self.signed_generation.load(Ordering::Acquire) { return; @@ -400,6 +456,31 @@ mod tests { assert_eq!(second.all[2], first.all[2], "the third is untouched"); } + /// A relay drops a registration that is not newer than the one it holds, + /// so a change signed within the same second, or after the clock stepped + /// back, still gets a later timestamp than the registration it replaces. + #[test] + fn a_re_signed_registration_is_always_newer_than_the_last() { + let (store, pubkeys) = store_with_keys(1); + let settings = settings(Some(H160([0xab; 20]))); + let (service, _) = service(Config::mainnet(), settings.clone()); + let first = service.current(&store, &pubkeys, 2_000); + assert_eq!(first.all[0].message.timestamp, 2_000); + + settings.set_gas_limit(&pubkeys[0], 30_000_000); + let same_second = service.current(&store, &pubkeys, 2_000); + assert_eq!(same_second.changed[0].message.timestamp, 2_001); + + settings.set_gas_limit(&pubkeys[0], 31_000_000); + let stepped_back = service.current(&store, &pubkeys, 1_000); + assert_eq!(stepped_back.changed[0].message.timestamp, 2_002); + + // Once the clock is ahead again, its reading is used. + settings.set_gas_limit(&pubkeys[0], 32_000_000); + let caught_up = service.current(&store, &pubkeys, 5_000); + assert_eq!(caught_up.changed[0].message.timestamp, 5_000); + } + #[test] fn a_validator_without_a_fee_recipient_is_not_registered() { let (store, pubkeys) = store_with_keys(2); @@ -442,6 +523,47 @@ mod tests { ); } + /// A full send that is still waiting on a slow node makes the next one + /// skip, rather than both holding the set and sending it again; once it + /// ends, the next one runs. + #[tokio::test] + async fn an_epoch_send_still_running_makes_the_next_one_skip() { + let (store, pubkeys) = store_with_keys(2); + let node = Arc::new( + MockBeaconNode::new().with_registration_delay(std::time::Duration::from_millis(300)), + ); + let service = Arc::new(RegistrationService::new( + node.clone(), + context(Config::mainnet()), + settings(Some(H160([0xab; 20]))), + )); + let store = Arc::new(RwLock::new(store)); + + let slow = tokio::spawn( + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()), + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + tokio::time::timeout( + std::time::Duration::from_millis(100), + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()), + ) + .await + .expect("skips at once rather than waiting on the node"); + slow.await.expect("the first send ends"); + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2], + "only the first send reached the node" + ); + + service.register_all(2, pubkeys, store).await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 2]); + } + #[tokio::test] async fn send_chunks_by_the_size_it_is_given() { let (store, pubkeys) = store_with_keys(5); diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index b36b77b80..c917a8867 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -747,7 +747,20 @@ the beacon node's builder and may sign a blinded block (builder-specs and the guard has already recorded the slot, so nothing retries. A 5xx or no answer is reported as unconfirmed (logged at error, counted on `blinded_blocks_unconfirmed_total`) and not as a missed proposal, since the - relay may still publish the block. + relay may still publish the block. The producing node is recorded with the + block's slot and root, and a publish for any other block is refused, as + Teku does; Lighthouse, Lodestar and Nimbus fail over or broadcast instead. + Two status codes are read the wrong way: some of ethlambda's own 500s are + definite (no builder configured, the block failing the node's checks) and + count as unconfirmed, and a proxy's 503 sent after it forwarded the request + counts as definite. Neither leads to a retry or a second signature. +- **A blinded block stays signed but undelivered for longer.** The node waits + up to 5 s on the relay and the client up to 8 s on the node, against well + under a second for a full block. The publish runs on its own task so the + proposal budget cannot cut it off, but the process handles no signals: a + SIGINT in that window ends the task with the request. The record of the + latest blinded block (`signed_blinded_at`) is one per process, not per + validator, since one slot has one proposer. - **Registrations stop at gloas.** The client sends no `register_validator` from the gloas fork epoch on: the builder market works through proposer preferences there, and `produceBlockV4` keeps its self-build configuration. From e91965561c0838376f2a26634fc106635732e20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:15:07 -0300 Subject: [PATCH 12/19] docs(validator): say a zero --suggested-fee-recipient is reported Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index 56536cd6a..908fc1b4a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -288,7 +288,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--http-address` | `127.0.0.1` | Bind address for the metrics and keymanager servers | | `--metrics-port` | `5064` | Prometheus metrics port | | `--keymanager-port` | `5062` | Keymanager API port. Only bound with `--enable-keymanager` | -| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours | +| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours. The zero address is accepted, as every other client accepts it, but reported at error at startup: it burns the rewards, and builder registrations and proposer preferences sign it for the builder to pay | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | | `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | From 7eced44991ec7c598e0112a9502bf17e351a7f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:24:07 -0300 Subject: [PATCH 13/19] fix(validator): write out every control character in a node's error text Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- crates/validator/src/beacon_node/http.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 2087d59f5..b10c3ed1b 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -88,11 +88,21 @@ async fn read_error_body(mut response: reqwest::Response) -> String { String::from_utf8_lossy(&bytes).into_owned() } -/// A body as an error shows it: line breaks written out as `\n` and `\r`, so -/// the log line it lands in stays one line, and cut to [`ERROR_TEXT_LIMIT`] -/// bytes. +/// A body as an error shows it: control characters written out (`\n`, `\r`, +/// `\u{1b}`, ...), so the log line it lands in stays one line and carries no +/// terminal escapes, and cut to [`ERROR_TEXT_LIMIT`] bytes. fn error_text(body: String) -> String { - let mut text = body.replace('\n', "\\n").replace('\r', "\\r"); + let mut text = String::with_capacity(body.len()); + for c in body.chars() { + // Every control character, and the two Unicode line separators, is + // written out, so neither a line break nor a terminal escape sequence + // reaches the log as itself. + if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { + text.extend(c.escape_default()); + } else { + text.push(c); + } + } if text.len() > ERROR_TEXT_LIMIT { let mut end = ERROR_TEXT_LIMIT; while !text.is_char_boundary(end) { @@ -1547,6 +1557,12 @@ mod tests { assert_eq!(error_text("short".to_string()), "short"); } + #[test] + fn error_text_writes_out_every_control_character() { + let text = error_text("a\u{1b}[31mb\tc\u{2028}d".to_string()); + assert_eq!(text, "a\\u{1b}[31mb\\tc\\u{2028}d"); + } + #[tokio::test] async fn a_blinded_publish_maps_each_status() { for (status, expected) in [ From 25f3ca5dca835c51502034b8197a526a406b6f14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:37:08 -0300 Subject: [PATCH 14/19] fix(validator): keep a running send's flag set when another is refused `then_some` built the guard on the refused path too, and dropping it cleared the flag of the send still running, so a third send could start beside it. Claude-Session: https://claude.ai/code/session_011vghNneoZ1Ta9W6JNqzjxm --- crates/validator/src/in_flight.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/validator/src/in_flight.rs b/crates/validator/src/in_flight.rs index 62ab3225e..ffbac8b21 100644 --- a/crates/validator/src/in_flight.rs +++ b/crates/validator/src/in_flight.rs @@ -17,7 +17,14 @@ impl InFlight { /// The guard for a new run, or `None` when one is already going. pub(crate) fn enter(&self) -> Option> { let running = self.0.swap(true, Ordering::AcqRel); - (!running).then_some(InFlightGuard(&self.0)) + // The guard is built only for a run that starts: `then_some` would + // build it either way and drop it on the refused path, and dropping + // it clears the flag of the run that is still going. + if running { + None + } else { + Some(InFlightGuard(&self.0)) + } } } @@ -40,6 +47,10 @@ mod tests { let in_flight = InFlight::default(); let first = in_flight.enter().expect("nothing is running"); assert!(in_flight.enter().is_none()); + assert!( + in_flight.enter().is_none(), + "a refused run leaves the running one's flag set" + ); drop(first); assert!(in_flight.enter().is_some()); } From ad59936180cc75078f4cc2c3e738f56489fc957a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:52:46 -0300 Subject: [PATCH 15/19] fix(validator): address the second builder proposals review Registrations: - One send lock shared by the epoch's full send and the per-slot send, taken before the cache is read and held until the last batch is answered, so a change is never overtaken at a node by an older copy a full send read before it. A per-slot send that finds it taken skips to the next slot; a full send waits for a per-slot one. The cache is only touched under it, so register_all never waits on a signing chunk. - Batching moved below the fallback's fan-out (BeaconNodeApi::register_validators_in_batches): each node works through its own batches beside the others, and one that cannot be reached is sent no more that round, so a hung node costs one timeout and holds back nobody else. A batch counts as sent when any node took it, as before. Blinded publish: - A 3 s connect timeout inside the 8 s total, so an unreachable producing node fails as a connect error (definite) rather than the total timeout (unconfirmed). - One rule for definite versus unconfirmed, stated in the trait, the error, Published, metrics.md and spec_deviations.md; the mock's unreachable node is now definite like the HTTP one. Error text: one implementation, ethlambda_types::text::one_line, used by the builder client and the validator client; it stops at the cap and never cuts an escape in two. A partial pool rejection's per-entry messages go through it too. Tests: the dropped-proposal test and the registration concurrency tests wait on gates in the mock rather than on wall-clock delays. --- crates/common/types/src/lib.rs | 1 + crates/common/types/src/text.rs | 70 +++++ .../net/rpc/src/beacon/builder_client/mod.rs | 34 +-- crates/validator/src/beacon_node/fallback.rs | 198 +++++++++++++- crates/validator/src/beacon_node/http.rs | 146 +++++++--- crates/validator/src/beacon_node/mock.rs | 82 ++++-- crates/validator/src/beacon_node/mod.rs | 84 +++++- crates/validator/src/error.rs | 8 +- crates/validator/src/proposal.rs | 56 ++-- crates/validator/src/registration.rs | 257 ++++++++++++++---- docs/cli.md | 4 +- docs/metrics.md | 2 +- docs/spec_deviations.md | 11 +- 13 files changed, 746 insertions(+), 207 deletions(-) create mode 100644 crates/common/types/src/text.rs diff --git a/crates/common/types/src/lib.rs b/crates/common/types/src/lib.rs index ecf92456e..aae73aab8 100644 --- a/crates/common/types/src/lib.rs +++ b/crates/common/types/src/lib.rs @@ -9,6 +9,7 @@ pub mod enr; pub mod genesis; pub mod primitives; pub mod state; +pub mod text; pub mod time; /// Display helper for truncated root hashes (8 hex chars) diff --git a/crates/common/types/src/text.rs b/crates/common/types/src/text.rs new file mode 100644 index 000000000..b67c32ad1 --- /dev/null +++ b/crates/common/types/src/text.rs @@ -0,0 +1,70 @@ +//! Text from a remote peer, made safe to put in a log line. + +/// `text` with every character that could break or restyle a line written +/// out, cut to at most `max_bytes` bytes. +/// +/// Text a remote server sends (an error body, a relay's message) reaches this +/// node's logs and error messages. There a raw line break would start what +/// reads as a log line of its own, and an ESC would start an ANSI sequence that +/// recolors, moves the cursor in or clears a terminal. So every control +/// character (`char::is_control`: C0, DEL and C1) and the Unicode line and +/// paragraph separators U+2028/U+2029, which some viewers break lines at, are +/// written as Rust escapes: `\n`, `\r` and `\t`, and `\u{1b}` for the rest. +/// Everything else passes through unchanged, quotes and backslashes included. +/// +/// The cap is applied to the escaped text, and never cuts a character or an +/// escape in two. The walk stops at the cap, so a large body costs no more +/// than the text it keeps. +pub fn one_line(text: &str, max_bytes: usize) -> String { + let mut written = String::with_capacity(text.len().min(max_bytes)); + for c in text.chars() { + if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { + let escaped = c.escape_default(); + if written.len() + escaped.len() > max_bytes { + break; + } + written.extend(escaped); + } else { + if written.len() + c.len_utf8() > max_bytes { + break; + } + written.push(c); + } + } + written +} + +#[cfg(test)] +mod tests { + use super::one_line; + + #[test] + fn control_characters_and_line_separators_are_written_out() { + let text = "\x1b[31mred\x1b[0m\ttab\x07bell\u{85}nel\u{2028}ls\u{2029}ps \"q\" \\ ok\r\n"; + assert_eq!( + one_line(text, 1024), + r#"\u{1b}[31mred\u{1b}[0m\ttab\u{7}bell\u{85}nel\u{2028}ls\u{2029}ps "q" \ ok\r\n"# + ); + } + + #[test] + fn the_cap_never_cuts_an_escape_in_two() { + // 1020 bytes, then a six-byte escape that would end past the cap. + let text = format!("{}\x1b", "x".repeat(1020)); + assert_eq!(one_line(&text, 1024), "x".repeat(1020)); + let text = format!("{}\x1b", "x".repeat(1018)); + assert_eq!( + one_line(&text, 1024), + format!("{}\\u{{1b}}", "x".repeat(1018)) + ); + } + + #[test] + fn the_cap_never_cuts_a_character_in_two() { + // 'é' is two bytes, so 1024 bytes hold 512 of them and no half. + let text = "é".repeat(1024); + let written = one_line(&text, 1023); + assert_eq!(written, "é".repeat(511)); + assert_eq!(one_line("short", 1024), "short"); + } +} diff --git a/crates/net/rpc/src/beacon/builder_client/mod.rs b/crates/net/rpc/src/beacon/builder_client/mod.rs index fd8ecc5b9..cbfdcded2 100644 --- a/crates/net/rpc/src/beacon/builder_client/mod.rs +++ b/crates/net/rpc/src/beacon/builder_client/mod.rs @@ -467,37 +467,15 @@ pub(super) async fn read_body( Ok(body) } -/// `text` with every character that could break or restyle a line written -/// out, cut to at most [`MAX_ERROR_TEXT`] bytes. +/// `text` as [`ethlambda_types::text::one_line`] writes it, cut to at most +/// [`MAX_ERROR_TEXT`] bytes. /// /// A builder's text reaches this node's logs and a validator client's error -/// message. There a raw line break would start what reads as a log line of -/// its own, and an ESC would start an ANSI sequence that recolors, moves the -/// cursor in or clears a terminal. So every control character -/// (`char::is_control`: C0, DEL and C1) and the Unicode line and paragraph -/// separators U+2028/U+2029, which some viewers break lines at, are written -/// as Rust escapes: `\n`, `\r` and `\t`, and `\u{1b}` for the rest. Everything -/// else passes through unchanged, quotes and backslashes included. -/// -/// The cap is applied to the escaped text, and never cuts a character or an -/// escape in two. +/// message, so no line break or terminal escape in it may reach either as +/// itself. The validator client writes a beacon node's error text through the +/// same function. pub(super) fn error_text(text: String) -> String { - let mut written = String::with_capacity(text.len().min(MAX_ERROR_TEXT)); - for c in text.chars() { - if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { - let escaped = c.escape_default(); - if written.len() + escaped.len() > MAX_ERROR_TEXT { - break; - } - written.extend(escaped); - } else { - if written.len() + c.len_utf8() > MAX_ERROR_TEXT { - break; - } - written.push(c); - } - } - written + ethlambda_types::text::one_line(&text, MAX_ERROR_TEXT) } /// The fork `version` names, if its bids have the shape [`SignedBuilderBid`] diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 7bebcc6df..87dfee51d 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -368,6 +368,68 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Every node works through its own batches at once, so a node that hangs + /// holds back only its own sequence: the healthy ones have every batch + /// while it is still timing out on its first. A batch counts as sent when + /// any node accepted it, as [`Self::register_validators`] does for one. + /// + /// Each node that refused anything is logged once, with how many batches, + /// rather than once per batch. + async fn register_validators_in_batches( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Vec> { + let per_node = futures_util::future::join_all( + self.nodes + .iter() + .map(|node| node.register_validators_in_batches(registrations)), + ) + .await; + + let batches = registrations.len().div_ceil(super::REGISTRATION_BATCH); + let mut accepted = vec![false; batches]; + let mut last = vec![None; batches]; + for (position, outcomes) in per_node.into_iter().enumerate() { + let mut refused = 0; + let mut first_refusal = None; + for (batch, outcome) in outcomes.into_iter().enumerate() { + match outcome { + Ok(()) => accepted[batch] = true, + Err(err) => { + refused += 1; + let detail = err.to_string(); + first_refusal.get_or_insert_with(|| detail.clone()); + last[batch] = Some(detail); + } + } + } + if let Some(err) = first_refusal { + warn!( + what = "register_validators", + position, + refused, + batches, + %err, + node_effect = "it will not hold these registrations", + "Beacon node did not accept" + ); + } + } + accepted + .into_iter() + .zip(last) + .map(|(accepted, last)| { + if accepted { + Ok(()) + } else { + Err(Error::AllBeaconNodesFailed( + last.unwrap_or_else(|| "no nodes configured".to_string()), + )) + } + }) + .collect() + } + /// Fails over like any query: the envelope the node cached is only on the /// node that produced the block, and one that did not answers 404. async fn execution_payload_envelope( @@ -557,8 +619,10 @@ impl BeaconNodeApi for FallbackBeaconNode { #[cfg(test)] mod tests { use super::*; + use crate::beacon_node::REGISTRATION_BATCH; use crate::beacon_node::mock::MockBeaconNode; use ethlambda_types::beacon::primitives::Root; + use std::sync::Arc; fn healthy() -> MockBeaconNode { let mut node = MockBeaconNode::new(); @@ -1158,24 +1222,134 @@ mod tests { ); } - /// Every node is called at once: two slow nodes cost one delay, not the - /// sum, so a hung node does not hold back the ones behind it. + /// Every node is called at once: the second is called while the first + /// has not answered yet, so a hung node does not hold back the ones behind + /// it. Called in turn, the second would wait for a permit that is only + /// given once both have been called. #[tokio::test] async fn every_node_is_called_at_once() { - let delay = std::time::Duration::from_millis(300); - let slow = || MockBeaconNode::new().with_registration_delay(delay); - let fallback = FallbackBeaconNode::new(vec![slow(), slow()]); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let held = || MockBeaconNode::new().with_registration_gate(gate.clone()); + let fallback = Arc::new(FallbackBeaconNode::new(vec![held(), held()])); - let started = std::time::Instant::now(); - fallback - .register_validators(&[]) - .await - .expect("both accept"); - let elapsed = started.elapsed(); + let call = tokio::spawn({ + let fallback = fallback.clone(); + async move { fallback.register_validators(&[]).await } + }); + for node in &fallback.nodes { + node.wait_for_registration_calls(1).await; + } + gate.add_permits(2); + call.await.expect("the call ends").expect("both accept"); - assert!(elapsed < delay * 2, "took {elapsed:?}"); for node in &fallback.nodes { assert_eq!(*node.registration_batches.lock().expect("lock"), vec![0]); } } + + /// `count` registrations with distinct keys. Unsigned: no mock checks. + fn registrations( + count: usize, + ) -> Vec { + use ethlambda_types::beacon::builder::{ + SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature, H160}; + + (0..count) + .map(|index| { + let mut pubkey = [0u8; 48]; + pubkey[..8].copy_from_slice(&(index as u64).to_be_bytes()); + SignedValidatorRegistrationV1 { + message: ValidatorRegistrationV1 { + fee_recipient: H160([1; 20]), + gas_limit: 30_000_000, + timestamp: 5, + pubkey: BlsPubkey(pubkey), + }, + signature: BlsSignature([3; 96]), + } + }) + .collect() + } + + /// Each node works through the batches on its own: while one node holds + /// its first batch, the other has been sent all of them. In step, the + /// healthy node would have its second batch only once the hung one + /// answered its first. + #[tokio::test] + async fn a_hung_node_holds_back_only_its_own_batches() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let hung = MockBeaconNode::new().with_registration_gate(gate.clone()); + let fallback = Arc::new(FallbackBeaconNode::new(vec![hung, MockBeaconNode::new()])); + let sent = registrations(2 * REGISTRATION_BATCH + 1); + + let send = tokio::spawn({ + let (fallback, sent) = (fallback.clone(), sent.clone()); + async move { fallback.register_validators_in_batches(&sent).await } + }); + let healthy_has_every_batch = async { + while fallback.nodes[1] + .registration_batches + .lock() + .expect("lock") + .len() + < 3 + { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), healthy_has_every_batch) + .await + .expect("the healthy node is not held back by the hung one"); + assert_eq!( + fallback.nodes[0] + .registration_attempts + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "the hung node is still on its first batch" + ); + + gate.add_permits(3); + let outcomes = send.await.expect("the send ends"); + assert_eq!(outcomes.len(), 3); + assert!(outcomes.iter().all(Result::is_ok)); + assert_eq!( + *fallback.nodes[0].registration_batches.lock().expect("lock"), + vec![REGISTRATION_BATCH, REGISTRATION_BATCH, 1] + ); + } + + /// A node that cannot be reached is sent no more batches: each would cost + /// another timeout. A batch counts as sent when any node took it, and as + /// failed when none did. + #[tokio::test] + async fn an_unreachable_node_is_sent_no_more_batches() { + let sent = registrations(REGISTRATION_BATCH + 1); + + let fallback = + FallbackBeaconNode::new(vec![MockBeaconNode::failing("down"), MockBeaconNode::new()]); + let outcomes = fallback.register_validators_in_batches(&sent).await; + assert!(outcomes.iter().all(Result::is_ok), "node 1 took both"); + let attempts = |position: usize| { + fallback.nodes[position] + .registration_attempts + .load(std::sync::atomic::Ordering::SeqCst) + }; + assert_eq!(attempts(0), 1, "stopped after its first batch"); + assert_eq!(attempts(1), 2); + + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::failing("down"), + MockBeaconNode::failing("also down"), + ]); + let outcomes = fallback.register_validators_in_batches(&sent).await; + assert_eq!(outcomes.len(), 2); + assert!( + outcomes + .iter() + .all(|outcome| matches!(outcome, Err(Error::AllBeaconNodesFailed(_)))), + "got {outcomes:?}" + ); + } } diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 4c8abe531..a623d4b3c 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -61,6 +61,22 @@ use crate::error::{BeaconNodeFailure, Error, Result}; /// a stalled request could then outlive the slot it was serving. const REQUEST_TIMEOUT: Duration = Duration::from_secs(8); +/// How long establishing a connection (TCP, and TLS for `https`) may take, +/// within [`REQUEST_TIMEOUT`]. +/// +/// Its own bound so that a node that cannot be reached fails as a connect +/// error, which says the request never left, rather than as the total +/// timeout, which cannot say whether it did. That difference decides how a +/// blinded publish is reported: a block that never reached the node is a +/// missed proposal, while one the node may have handed to its relay is +/// unconfirmed (see [`BeaconNodeApi::publish_blinded_block`]). Without it, an +/// unreachable producing node counted a slot that was certainly missed as one +/// the relay might still publish. +/// +/// Three seconds is far beyond a healthy handshake, local or across a +/// continent, and leaves most of the request budget to the answer. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(3); + /// How long `register_validator` may take. Longer than [`REQUEST_TIMEOUT`] /// because an ethlambda node forwards the registrations to its relay and waits /// up to 10 s for the answer before replying; a client that gave up at 8 s @@ -88,35 +104,20 @@ async fn read_error_body(mut response: reqwest::Response) -> String { String::from_utf8_lossy(&bytes).into_owned() } -/// A body as an error shows it: control characters written out (`\n`, `\r`, -/// `\u{1b}`, ...), so the log line it lands in stays one line and carries no -/// terminal escapes, and cut to [`ERROR_TEXT_LIMIT`] bytes. -fn error_text(body: String) -> String { - let mut text = String::with_capacity(body.len()); - for c in body.chars() { - // Every control character, and the two Unicode line separators, is - // written out, so neither a line break nor a terminal escape sequence - // reaches the log as itself. - if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { - text.extend(c.escape_default()); - } else { - text.push(c); - } - } - if text.len() > ERROR_TEXT_LIMIT { - let mut end = ERROR_TEXT_LIMIT; - while !text.is_char_boundary(end) { - end -= 1; - } - text.truncate(end); - text.push_str("..."); - } - text +/// Text a node sent, as an error or a log line shows it: control characters +/// and line separators written out (`\n`, `\u{1b}`, ...), so the line it lands +/// in stays one line and carries no terminal escapes, and cut to +/// [`ERROR_TEXT_LIMIT`] bytes without splitting a character or an escape. +/// +/// The same function the beacon node's builder client writes a relay's text +/// through, [`ethlambda_types::text::one_line`]. +fn error_text(text: &str) -> String { + ethlambda_types::text::one_line(text, ERROR_TEXT_LIMIT) } /// [`read_error_body`] as [`error_text`] shows it. async fn error_body(response: reqwest::Response) -> String { - error_text(read_error_body(response).await) + error_text(&read_error_body(response).await) } /// What an SSZ answer carries besides its bytes: the fork, which only a header @@ -146,9 +147,19 @@ impl HttpBeaconNode { /// Builds a client for the node at `base_url`, stripping any trailing /// slash so callers may pass either form. pub fn new(base_url: impl Into) -> Result { + Self::with_connect_timeout(base_url, CONNECT_TIMEOUT) + } + + /// [`Self::new`] with `connect_timeout` in place of [`CONNECT_TIMEOUT`], + /// so a test can see a connect timeout without waiting three seconds. + fn with_connect_timeout( + base_url: impl Into, + connect_timeout: Duration, + ) -> Result { let base_url = base_url.into().trim_end_matches('/').to_string(); let client = Client::builder() .timeout(REQUEST_TIMEOUT) + .connect_timeout(connect_timeout) .build() .map_err(|err| Error::BeaconNode { url: base_url.clone(), @@ -533,9 +544,11 @@ impl HttpBeaconNode { let failed = error.failures.len(); if failed > 0 && failed < submitted { for failure in &error.failures { + // The node's text, so written out and capped like a whole + // error body: one line per failure, whatever it sent. warn!( submission_index = failure.index, - reason = %failure.message, + reason = %error_text(&failure.message), "One attestation in this slot's batch was rejected" ); } @@ -545,7 +558,7 @@ impl HttpBeaconNode { Err(Error::BeaconNodeStatus { status: status.as_u16(), - body: error_text(body), + body: error_text(&body), }) } @@ -788,14 +801,13 @@ impl BeaconNodeApi for HttpBeaconNode { /// Maps the answer as the trait's contract says: 2xx is handed to the /// builder (202 is the node's broadcast-without-import, as for a full - /// block), and a 4xx is a definite refusal. + /// block); a 4xx, a 503 and a connection that was never established are + /// definite; every other 5xx, and any failure once connected, is + /// [`Error::BlindedUnconfirmed`]. /// - /// Two cases that look like failures are also definite, because the node - /// cannot have acted on the block: a connection error (the request never - /// left) and a 503 (the node is syncing, and refuses before it hands - /// anything to a builder). Every other 5xx, and any failure after the - /// request was sent (a timeout, a reset), is [`Error::BlindedUnconfirmed`], - /// since the relay may still publish. + /// A connect error covers a connection refused and one that timed out + /// ([`CONNECT_TIMEOUT`]); reqwest reports the latter as both a connect + /// error and a timeout, and the connect check comes first. /// /// The status alone cannot always tell, and two edges are knowingly read /// the wrong way. Some of ethlambda's own 500s are definite (no builder @@ -1546,20 +1558,27 @@ mod tests { assert_eq!(status, 400); assert!(body.starts_with("first line\\nsecond line\\r\\nfirst line")); assert!(!body.contains('\n') && !body.contains('\r')); - assert_eq!(body.len(), ERROR_TEXT_LIMIT + "...".len()); + assert!(body.len() <= ERROR_TEXT_LIMIT, "{} bytes", body.len()); + assert!( + body.len() > ERROR_TEXT_LIMIT - "\\r".len(), + "filled to the cap" + ); } + /// The cap applies to the written-out text and never splits a character + /// or an escape: a body of escapes ends on a whole one. #[test] - fn error_text_cuts_on_a_character_boundary() { - let text = error_text("é".repeat(ERROR_TEXT_LIMIT)); - assert!(text.ends_with("...")); - assert!(text.len() <= ERROR_TEXT_LIMIT + "...".len()); - assert_eq!(error_text("short".to_string()), "short"); + fn error_text_cuts_on_a_character_or_escape_boundary() { + let text = error_text(&"é".repeat(ERROR_TEXT_LIMIT)); + assert_eq!(text, "é".repeat(ERROR_TEXT_LIMIT / 2)); + let text = error_text(&"\u{1b}".repeat(ERROR_TEXT_LIMIT)); + assert_eq!(text, "\\u{1b}".repeat(ERROR_TEXT_LIMIT / "\\u{1b}".len())); + assert_eq!(error_text("short"), "short"); } #[test] fn error_text_writes_out_every_control_character() { - let text = error_text("a\u{1b}[31mb\tc\u{2028}d".to_string()); + let text = error_text("a\u{1b}[31mb\tc\u{2028}d"); assert_eq!(text, "a\\u{1b}[31mb\\tc\\u{2028}d"); } @@ -1625,6 +1644,51 @@ mod tests { assert!(matches!(err, Error::BeaconNode { .. }), "got {err:?}"); } + /// A connection that never completes is a connect error too, not the + /// total timeout, so it is definite: nothing was sent. The listener + /// accepts TCP but never answers the TLS handshake, which is part of + /// connecting, so the connect timeout fires on every platform and with no + /// unroutable address involved. + #[tokio::test] + async fn a_blinded_publish_whose_connection_times_out_is_a_definite_failure() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("binds"); + let address = listener.local_addr().expect("address"); + let held = tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + let connect_timeout = Duration::from_millis(200); + let node = + HttpBeaconNode::with_connect_timeout(format!("https://{address}"), connect_timeout) + .expect("builds"); + + let started = std::time::Instant::now(); + let err = node + .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) + .await + .expect_err("never connects"); + held.abort(); + + assert!( + matches!( + err, + Error::BeaconNode { + failure: BeaconNodeFailure::Timeout, + .. + } + ), + "got {err:?}" + ); + assert!( + started.elapsed() < REQUEST_TIMEOUT, + "the connect timeout, not the request's, ended it" + ); + } + /// A syncing node refuses before it hands anything to a builder. #[tokio::test] async fn a_blinded_publish_to_a_syncing_node_is_a_definite_failure() { diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index f91284a83..94cf99294 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -2,7 +2,7 @@ //! service tests. Compiled only under `cfg(test)`. use std::collections::HashMap; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; use ethlambda_types::beacon::builder::{BlindedBeaconBlock, SignedValidatorRegistrationV1}; @@ -14,6 +14,7 @@ use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::containers::shared::Checkpoint; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; +use tokio::sync::{Notify, Semaphore}; use crate::beacon_node::block_contents::{ Contents, GloasPayload, ProducedBlock, empty_block_for, empty_gloas_block_for, @@ -95,9 +96,13 @@ pub struct MockBeaconNode { /// The size of each `register_validators` call, in call order, so a test /// can see how a send was batched. pub registration_batches: Mutex>, - /// How long `register_validators` takes before it records the batch, as - /// a slow node would. - pub registration_delay: Option, + /// Every `register_validators` call, refused and still-waiting ones + /// included, so a test can tell a call reached the node. + pub registration_attempts: std::sync::atomic::AtomicUsize, + /// When set, each `register_validators` call takes a permit before it + /// records its batch, so a test releases calls one at a time instead of + /// guessing how long a delay must be. + pub registration_gate: Option>, /// Every blinded block published, as (slot, fork, body). pub published_blinded: Mutex)>>, /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; @@ -106,9 +111,14 @@ pub struct MockBeaconNode { /// When set, `publish_blinded_block` answers with this status as a node /// that definitely refused the block (a 4xx) does: `BeaconNodeStatus`. pub blinded_publish_refusal: Option, - /// How long `publish_blinded_block` takes before it records the block and - /// answers, as a slow relay makes the node. - pub blinded_publish_delay: Option, + /// When set, `publish_blinded_block` takes a permit before it records the + /// block and answers, as a node waiting on a slow relay does, for as long + /// as the test holds the permit back. + pub blinded_publish_gate: Option>, + /// Notified when `publish_blinded_block` is called, before the gate. A + /// test that publishes once can await it whether or not the call came + /// first, since `notify_one` keeps the permit for a later waiter. + pub blinded_publish_entered: Arc, /// What the test asserts against. pub submitted: Mutex>, @@ -310,15 +320,34 @@ impl MockBeaconNode { self } - /// Make `register_validators` take `delay` before it records the batch. - pub fn with_registration_delay(mut self, delay: std::time::Duration) -> Self { - self.registration_delay = Some(delay); + /// Wait, up to five seconds, until `register_validators` has been called + /// `calls` times, counting calls still held at the gate. + pub async fn wait_for_registration_calls(&self, calls: usize) { + let reached = async { + while self + .registration_attempts + .load(std::sync::atomic::Ordering::SeqCst) + < calls + { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), reached) + .await + .unwrap_or_else(|_| panic!("register_validators was not called {calls} times")); + } + + /// Make each `register_validators` call wait for a permit from `gate` + /// before it records its batch. + pub fn with_registration_gate(mut self, gate: Arc) -> Self { + self.registration_gate = Some(gate); self } - /// Make `publish_blinded_block` take `delay` before it records the block. - pub fn with_blinded_publish_delay(mut self, delay: std::time::Duration) -> Self { - self.blinded_publish_delay = Some(delay); + /// Make `publish_blinded_block` wait for a permit from `gate` before it + /// records the block and answers. + pub fn with_blinded_publish_gate(mut self, gate: Arc) -> Self { + self.blinded_publish_gate = Some(gate); self } @@ -755,21 +784,21 @@ impl BeaconNodeApi for MockBeaconNode { body: &[u8], ) -> Result { // `failures` and `fail_everything` stand for a node that could not be - // reached or answered badly: over HTTP that is an unconfirmed publish, - // never a definite refusal (see `with_refused_blinded_publish`). - self.guard("publish_blinded_block") - .map_err(|err| Error::BlindedUnconfirmed { - status: None, - detail: err.to_string(), - })?; + // reached, which over HTTP is a definite failure: the request never + // left. An unconfirmed publish is `with_unconfirmed_blinded_publish`. + self.guard("publish_blinded_block")?; if let Some(status) = self.blinded_publish_refusal { return Err(Error::BeaconNodeStatus { status, body: "refused".to_string(), }); } - if let Some(delay) = self.blinded_publish_delay { - tokio::time::sleep(delay).await; + self.blinded_publish_entered.notify_one(); + if let Some(gate) = &self.blinded_publish_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); } self.published_blinded .lock() @@ -789,9 +818,14 @@ impl BeaconNodeApi for MockBeaconNode { &self, registrations: &[SignedValidatorRegistrationV1], ) -> Result<()> { + self.registration_attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); self.guard("register_validators")?; - if let Some(delay) = self.registration_delay { - tokio::time::sleep(delay).await; + if let Some(gate) = &self.registration_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); } self.registration_batches .lock() diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 860f033c0..2c176de87 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -20,7 +20,7 @@ use crate::beacon_node::dto::{ AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, PtcDutyDto, SingleAttestationDto, SyncCommitteeSubscriptionDto, SyncDutyDto, }; -use crate::error::Result; +use crate::error::{BeaconNodeFailure, Error, Result}; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::electra; use ethlambda_types::beacon::containers::gloas; @@ -33,6 +33,11 @@ pub mod http; #[cfg(test)] pub mod mock; +/// Builder registrations per request. Lighthouse's batch size: large enough +/// that a few thousand validators take a handful of requests, small enough +/// that one request stays well under a node's body limit. +pub const REGISTRATION_BATCH: usize = 500; + /// Check that `data` is an answer about `slot`, as the contract on /// [`BeaconNodeApi::attestation_data`] requires. /// @@ -167,9 +172,10 @@ pub enum Published { /// its builder, which publishes the block and reveals the payload. Nothing /// says the node imported it. HandedToBuilder, - /// The node failed to confirm the blinded block (5xx or no answer). The - /// relay may still have published it; see `Error::BlindedUnconfirmed`. - /// Produced only by `ProposalService`, never by an implementation. + /// The node failed to confirm the blinded block (a 5xx other than 503, or + /// no answer once connected). The relay may still have published it; see + /// `Error::BlindedUnconfirmed`. Produced only by `ProposalService`, never + /// by an implementation. BuilderUnconfirmed, } @@ -411,10 +417,18 @@ pub trait BeaconNodeApi: Send + Sync { /// /// # Contract: which error means what /// - /// A definite refusal (4xx) is an `Err` other than - /// `Error::BlindedUnconfirmed`: the block was not published. A 5xx or a - /// transport failure is `Error::BlindedUnconfirmed`, because the relay may - /// still publish the block. + /// The line is whether the node can have handed the block to its builder. + /// + /// - **Definite**, an `Err` other than `Error::BlindedUnconfirmed`: the + /// block was not published. A 4xx; a 503, which a syncing node answers + /// before it hands anything to a builder; and a connection that was never + /// established, refused or timed out, since the request never left. + /// - **Unconfirmed**, `Error::BlindedUnconfirmed`: every other 5xx, and any + /// failure once the connection was up (no answer in time, a reset). The + /// relay may still publish the block. + /// + /// The status cannot always tell, and the HTTP implementation documents the + /// two edges it knowingly reads the wrong way. async fn publish_blinded_block( &self, slot: Slot, @@ -423,14 +437,62 @@ pub trait BeaconNodeApi: Send + Sync { body: &[u8], ) -> Result; - /// Send signed builder registrations to the node, which forwards them to its - /// builder. JSON, since a node may forward the body unchanged to MEV-Boost. - /// State installed on the node's builder, so failover sends it to every node. + /// Send signed builder registrations to the node in one request, which it + /// forwards to its builder. JSON, since a node may forward the body + /// unchanged to MEV-Boost. State installed on the node's builder, so + /// failover sends it to every node. async fn register_validators( &self, registrations: &[SignedValidatorRegistrationV1], ) -> Result<()>; + /// Send `registrations` in requests of at most [`REGISTRATION_BATCH`], one + /// after another, and say what became of each batch: one outcome per + /// batch, in order, `Ok` when the node accepted it. + /// + /// Batched here, below the fallback's fan-out, so that each node works + /// through its own sequence of batches beside the others: a hung node + /// holds back only its own. Batched above it, every batch would wait for + /// every node before the next one started. + /// + /// Once the node cannot be reached ([`Error::BeaconNode`]) the batches left + /// are not sent, and fail with that error: each would cost another + /// timeout, and the send waiting on them holds back the next keymanager + /// change (see `crate::registration`). A node that answered with an error + /// status is up, so it is still given the rest. + /// + /// [`Error::BeaconNode`]: crate::error::Error::BeaconNode + async fn register_validators_in_batches( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Vec> { + let mut outcomes = Vec::new(); + let mut unreachable: Option<(String, BeaconNodeFailure, String)> = None; + for batch in registrations.chunks(REGISTRATION_BATCH) { + if let Some((url, failure, detail)) = &unreachable { + outcomes.push(Err(Error::BeaconNode { + url: url.clone(), + failure: *failure, + detail: format!( + "not sent, an earlier batch could not reach the node: {detail}" + ), + })); + continue; + } + let outcome = self.register_validators(batch).await; + if let Err(Error::BeaconNode { + url, + failure, + detail, + }) = &outcome + { + unreachable = Some((url.clone(), *failure, detail.clone())); + } + outcomes.push(outcome); + } + outcomes + } + /// The envelope the node cached for the self-built payload of the block /// whose root is `block_root`, proposed at `slot`. /// diff --git a/crates/validator/src/error.rs b/crates/validator/src/error.rs index 050651239..75f26ce7d 100644 --- a/crates/validator/src/error.rs +++ b/crates/validator/src/error.rs @@ -63,10 +63,12 @@ pub enum Error { #[error("beacon node is syncing")] BeaconNodeSyncing, /// A signed blinded block was sent and the node's answer does not say - /// whether it went out: a 5xx, or no answer at all. + /// whether it went out: a 5xx other than 503, or no answer once connected. /// - /// Distinct from [`Self::BeaconNodeStatus`], which for this call means a - /// definite refusal (a 4xx: the block was not published). Here the node + /// Distinct from [`Self::BeaconNodeStatus`] and [`Self::BeaconNode`], + /// which for this call mean a definite failure (a 4xx or a 503, or a + /// connection never established: the block was not published; see + /// `BeaconNodeApi::publish_blinded_block`). Here the node /// may have handed the block to its builder before failing, and the relay /// publishes on receipt, so the block may well be on the network. Nothing /// can safely be retried: the slot is signed, and a second block for it diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 3a7f0931d..370fc64ef 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -483,12 +483,14 @@ impl ProposalService { /// /// # Why an unconfirmed publish is not an error /// - /// A 5xx or a missing answer does not say the block was not published: the - /// node may have handed it to the relay, which publishes regardless of what - /// the node then replied. So that outcome is logged loudly and counted on - /// its own series, and returned as [`Published::BuilderUnconfirmed`]. It is - /// not retried: the slot is signed, and a second block for it would be an - /// equivocation. A 4xx is a definite refusal and propagates as a failed + /// A 5xx or a missing answer once connected does not say the block was not + /// published: the node may have handed it to the relay, which publishes + /// regardless of what the node then replied. So that outcome is logged + /// loudly and counted on its own series, and returned as + /// [`Published::BuilderUnconfirmed`]. It is not retried: the slot is + /// signed, and a second block for it would be an equivocation. A definite + /// failure (a 4xx, a 503, a connection never made; see + /// [`BeaconNodeApi::publish_blinded_block`]) propagates as a failed /// proposal. async fn publish_blinded( &self, @@ -1078,35 +1080,45 @@ mod tests { /// The duty loop drops the proposal at its budget while the node is still /// waiting on the relay. The publish runs on its own task, so the block is /// still sent, and the slot is known to be signed. + /// + /// The proposal is dropped once the publish has reached the node, which + /// holds it until the test lets it answer, rather than after a fixed + /// delay: the slashing-protection write and the signatures come first, and + /// on a loaded machine they can take longer than any delay picked here. #[tokio::test] async fn a_blinded_publish_survives_a_dropped_proposal() { let (store, pubkey) = store(); + let relay_answers = Arc::new(tokio::sync::Semaphore::new(0)); let node = Arc::new( MockBeaconNode::new() .with_block(slot(), 7) .with_blinded_block() - .with_blinded_publish_delay(std::time::Duration::from_millis(300)), + .with_blinded_publish_gate(relay_answers.clone()), ); let service = builder_service(node.clone(), None); - let outcome = tokio::time::timeout( - std::time::Duration::from_millis(50), - service.propose(slot(), &duty(&pubkey, 7, slot()), &store), - ) - .await; - assert!( - outcome.is_err(), - "the proposal must be dropped at the budget" - ); + let at_the_node = node.blinded_publish_entered.clone(); + let this_duty = duty(&pubkey, 7, slot()); + tokio::select! { + outcome = service.propose(slot(), &this_duty, &store) => { + panic!("the node is still waiting on the relay, yet the proposal ended: {outcome:?}") + } + () = at_the_node.notified() => {} + } + // The proposal future is dropped here, as the budget drops it. assert!(service.signed_blinded_at(slot())); assert!(node.published_blinded.lock().expect("lock").is_empty()); - tokio::time::sleep(std::time::Duration::from_millis(600)).await; - assert_eq!( - node.published_blinded.lock().expect("lock").len(), - 1, - "the publish must complete although the proposal future is gone" - ); + relay_answers.add_permits(1); + let published = async { + while node.published_blinded.lock().expect("lock").is_empty() { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), published) + .await + .expect("the publish must complete although the proposal future is gone"); + assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); // Slashing protection recorded the slot before the publish started, so a retry // after the dropped future is refused without asking for a block. diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index a33cf5298..6fd5d8560 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -42,22 +42,44 @@ //! refusal says more about the node or the relay than about one entry, and no //! other client splits either. //! -//! # One send of each kind at a time +//! # Each node works through the batches on its own //! -//! Both sends are spawned off the slot loop, so nothing else would stop a new -//! one starting while the last still waits on a slow node: a full send every -//! epoch, each holding the whole registration set and sending it again to the -//! healthy nodes. A send that finds the previous one of its kind still going -//! is skipped (see [`crate::in_flight`]). A skipped full send waits for the -//! next epoch, and a skipped per-slot one for the next slot, since it has not -//! marked its change as sent. +//! The full set goes out in batches of [`REGISTRATION_BATCH`], and each node +//! is sent its batches in turn beside the other nodes rather than in step with +//! them (see [`BeaconNodeApi::register_validators_in_batches`]). A node that +//! hangs costs only its own batches, and only one timeout: what it has not +//! been sent by then is dropped until the next epoch. +//! +//! # One send at a time, and the newest registration last +//! +//! An ethlambda beacon node keeps the fee recipient of every registration it +//! is sent, newest timestamp or not, so the order sends reach a node in is the +//! order its fee recipients end up in. Every send therefore takes one lock, +//! [`RegistrationService::sending`], before it reads the cache and holds it +//! until the last batch is answered. A per-slot send can then never be +//! overtaken by an older registration from a full send that read the cache +//! before the change, and no batch of either is still in flight when the next +//! begins. The one window left is a request this client gave up on at its +//! timeout that the node still processes afterwards; the next epoch's full +//! send, which carries the newest registration, closes it. +//! +//! A per-slot send that finds the lock taken skips: it has not marked its +//! change as sent, so a later slot picks it up. While a full send runs, that +//! is the first slot after it ends, which against a node that is slow to +//! answer each batch can be several slots on. A full send waits for it +//! instead, since a per-slot send is short and skipping would cost an epoch. +//! A full send that finds the previous full send still running, or waiting, +//! skips to the next epoch (see [`crate::in_flight`]), so a hung node cannot +//! queue up one waiting send per epoch. //! //! # Signing stays off the async runtime //! //! The first epoch signs one BLS signature per validator. That runs on the //! blocking pool in chunks of [`SIGN_CHUNK`], taking the store's read lock //! afresh for each chunk, so neither the runtime's threads nor a queued -//! keymanager writer wait on thousands of signatures. +//! keymanager writer wait on thousands of signatures. The cache's own lock is +//! only ever taken under [`RegistrationService::sending`], so nothing on the +//! runtime waits on a chunk that holds it. //! //! # The per-slot check is one atomic read //! @@ -74,21 +96,16 @@ use std::time::{SystemTime, UNIX_EPOCH}; use ethlambda_types::beacon::builder::{SignedValidatorRegistrationV1, ValidatorRegistrationV1}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch}; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock}; use tracing::{debug, warn}; -use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::encode_hex; +use crate::beacon_node::{BeaconNodeApi, REGISTRATION_BATCH}; use crate::in_flight::InFlight; use crate::keys::ValidatorStore; use crate::proposer_settings::ProposerSettings; use crate::signing::SigningContext; -/// Registrations per request. Lighthouse's batch size: large enough that a few -/// thousand validators take a handful of requests, small enough that one -/// request stays well under a node's body limit. -pub const REGISTRATION_BATCH: usize = 500; - /// Keys signed per blocking task, and so per hold of the store's read lock. const SIGN_CHUNK: usize = 64; @@ -138,16 +155,21 @@ pub struct RegistrationService { context: Arc, settings: Arc, /// The validators the beacon node resolved at the last epoch refresh. + /// Taken under [`Self::sending`] only. known: std::sync::Mutex>, - /// The latest signed registration per validator. + /// The latest signed registration per validator. Taken under + /// [`Self::sending`] only, so it is never contended: a `std` mutex is + /// enough, and a signing chunk holding it keeps no one else waiting. signed: std::sync::Mutex>, /// The [`ProposerSettings::generation`] the cache was last brought up to /// date against. signed_generation: AtomicU64, - /// The epoch's full send, while it runs. + /// Held by a send from before it reads the cache until its last batch is + /// answered, so sends reach every node in the order they read the cache. + /// See the module doc. + sending: Mutex<()>, + /// The epoch's full send, while it runs or waits for [`Self::sending`]. full_send: InFlight, - /// The per-slot send of changed keys, while it runs. - changed_send: InFlight, } impl RegistrationService { @@ -163,8 +185,8 @@ impl RegistrationService { known: std::sync::Mutex::new(Vec::new()), signed: std::sync::Mutex::new(HashMap::new()), signed_generation: AtomicU64::new(0), + sending: Mutex::new(()), full_send: InFlight::default(), - changed_send: InFlight::default(), } } @@ -284,6 +306,9 @@ impl RegistrationService { ); return; }; + // Waits out a per-slot send rather than skipping: that one is short, + // and the full set is sent once an epoch. + let _sending = self.sending.lock().await; // Read before signing, so a change that lands while this runs is seen // by the next `register_changed` rather than lost. let generation = self.settings.generation(); @@ -305,7 +330,7 @@ impl RegistrationService { "Validators without a fee recipient are not registered with the builder" ); } - self.send(current.all, REGISTRATION_BATCH).await; + self.send(¤t.all).await; } /// Send only the registrations whose fee recipient or gas limit changed @@ -326,10 +351,16 @@ impl RegistrationService { if self.is_gloas(epoch) { return; } - // Silently: the change stays unsent, so the next slot picks it up. - let Some(_running) = self.changed_send.enter() else { + if self.settings.generation() == self.signed_generation.load(Ordering::Acquire) { + return; + } + // Silently: another send is running, and the change stays unsent, so + // the next slot picks it up. + let Ok(_sending) = self.sending.try_lock() else { return; }; + // Again, under the lock: a full send that just ended may have signed + // this change already. let generation = self.settings.generation(); if generation == self.signed_generation.load(Ordering::Acquire) { return; @@ -347,25 +378,28 @@ impl RegistrationService { if current.changed.is_empty() { return; } - self.send(current.changed, REGISTRATION_BATCH).await; + self.send(¤t.changed).await; } - /// Send `registrations` to the beacon node in chunks of `batch_size`. - /// - /// The size is a parameter so a test can see the chunking without signing - /// hundreds of keys. - async fn send(&self, registrations: Vec, batch_size: usize) { - for chunk in registrations.chunks(batch_size) { - match self.beacon_node.register_validators(chunk).await { - Ok(()) => { - debug!(count = chunk.len(), "Builder registrations sent"); - crate::metrics::inc_builder_registrations_sent(chunk.len()); - } - Err(err) => { - warn!(count = chunk.len(), %err, "No beacon node accepted the builder registrations"); + /// Send `registrations` to the beacon node in batches, counting each batch + /// as sent when a node accepted it and as a failure when none did. + async fn send(&self, registrations: &[SignedValidatorRegistrationV1]) { + let outcomes = self + .beacon_node + .register_validators_in_batches(registrations) + .await; + for (batch, outcome) in registrations.chunks(REGISTRATION_BATCH).zip(outcomes) { + let count = batch.len(); + // Only logged and counted: see "Failures are never propagated". + let _ = outcome + .inspect(|()| { + debug!(count, "Builder registrations sent"); + crate::metrics::inc_builder_registrations_sent(count); + }) + .inspect_err(|err| { + warn!(count, %err, "No beacon node accepted the builder registrations"); crate::metrics::inc_builder_registration_failures(); - } - } + }); } } } @@ -376,6 +410,7 @@ mod tests { use crate::beacon_node::mock::MockBeaconNode; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::primitives::{Bytes32, H160, Root}; + use tokio::sync::Semaphore; fn context(config: Config) -> Arc { Arc::new(SigningContext { @@ -523,20 +558,41 @@ mod tests { ); } + /// A service over a node whose `register_validators` calls each wait for + /// a permit from the returned gate, which starts with `permits`. + fn gated_service( + settings: Arc, + permits: usize, + ) -> ( + Arc>, + Arc, + Arc, + ) { + let gate = Arc::new(Semaphore::new(permits)); + let node = Arc::new(MockBeaconNode::new().with_registration_gate(gate.clone())); + let service = Arc::new(RegistrationService::new( + node.clone(), + context(Config::mainnet()), + settings, + )); + (service, node, gate) + } + + /// Bounds a call that must return on its own, so a regression fails the + /// test rather than hanging it. + async fn promptly(what: &str, call: F) -> F::Output { + tokio::time::timeout(std::time::Duration::from_secs(5), call) + .await + .unwrap_or_else(|_| panic!("{what}")) + } + /// A full send that is still waiting on a slow node makes the next one /// skip, rather than both holding the set and sending it again; once it /// ends, the next one runs. #[tokio::test] async fn an_epoch_send_still_running_makes_the_next_one_skip() { let (store, pubkeys) = store_with_keys(2); - let node = Arc::new( - MockBeaconNode::new().with_registration_delay(std::time::Duration::from_millis(300)), - ); - let service = Arc::new(RegistrationService::new( - node.clone(), - context(Config::mainnet()), - settings(Some(H160([0xab; 20]))), - )); + let (service, node, gate) = gated_service(settings(Some(H160([0xab; 20]))), 0); let store = Arc::new(RwLock::new(store)); let slow = tokio::spawn( @@ -544,15 +600,15 @@ mod tests { .clone() .register_all(0, pubkeys.clone(), store.clone()), ); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - tokio::time::timeout( - std::time::Duration::from_millis(100), + node.wait_for_registration_calls(1).await; + promptly( + "skips rather than waiting on the node", service .clone() .register_all(1, pubkeys.clone(), store.clone()), ) - .await - .expect("skips at once rather than waiting on the node"); + .await; + gate.add_permits(1); slow.await.expect("the first send ends"); assert_eq!( *node.registration_batches.lock().expect("lock"), @@ -560,21 +616,104 @@ mod tests { "only the first send reached the node" ); + gate.add_permits(1); service.register_all(2, pubkeys, store).await; assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 2]); } + /// The reviewer's race: a full send holds the registrations it read + /// before a change, and a per-slot send must not deliver the change while + /// that older copy is still to reach the node, or the older one lands last + /// and the node keeps its fee recipient. The per-slot send skips instead, + /// and the next slot's delivers the change after the full send. #[tokio::test] - async fn send_chunks_by_the_size_it_is_given() { - let (store, pubkeys) = store_with_keys(5); - let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); - let current = service.current(&store, &pubkeys, 1_000); + async fn a_change_is_never_overtaken_by_a_full_send_that_read_before_it() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node, gate) = gated_service(settings.clone(), 0); + let store = Arc::new(RwLock::new(store)); - service.send(current.all, 2).await; + // The full send has read the cache and is at the node. + let full = tokio::spawn( + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()), + ); + node.wait_for_registration_calls(1).await; + + settings.set_fee_recipient(&pubkeys[0], H160([0xcd; 20])); + promptly( + "the per-slot send skips rather than waiting on the node", + service.clone().register_changed(0, store.clone()), + ) + .await; + assert_eq!( + node.registration_attempts + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "nothing reached the node beside the full send" + ); + + gate.add_permits(1); + full.await.expect("the full send ends"); + gate.add_permits(1); + service.register_changed(0, store).await; + + let registrations = node.registrations.lock().expect("lock"); + let latest_for_first = registrations + .iter() + .rfind(|registration| registration.message.pubkey == pubkeys[0]) + .expect("registered"); + assert_eq!( + latest_for_first.message.fee_recipient, + H160([0xcd; 20]), + "the change reaches the node last" + ); + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 1]); + } + + /// The other way round, the full send waits for a running per-slot send + /// rather than skipping its epoch, and then sends the change too, since it + /// reads the cache only once the per-slot send is done. + #[tokio::test] + async fn a_full_send_waits_for_a_running_per_slot_send() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node, gate) = gated_service(settings.clone(), 1); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + settings.set_gas_limit(&pubkeys[1], 30_000_000); + let changed = tokio::spawn(service.clone().register_changed(0, store.clone())); + node.wait_for_registration_calls(2).await; + let full = tokio::spawn( + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()), + ); + // Long enough for a full send that skipped to have returned. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !full.is_finished(), + "the full send waits rather than skipping" + ); + + gate.add_permits(2); + changed.await.expect("the per-slot send ends"); + full.await.expect("the full send ends"); assert_eq!( *node.registration_batches.lock().expect("lock"), - vec![2, 2, 1] + vec![2, 1, 2] + ); + let registrations = node.registrations.lock().expect("lock"); + assert_eq!(registrations[2].message.gas_limit, 30_000_000); + assert_eq!( + registrations[4], registrations[2], + "the full send carries the change it waited for" ); } diff --git a/docs/cli.md b/docs/cli.md index 9c7d9f2e3..6b8c81247 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -446,9 +446,9 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next slot's builder registration carries it too, and an ethlambda beacon node takes it from there | +| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running), and an ethlambda beacon node takes it from there | | `graffiti` | `--graffiti` | The next proposal | -| `gas_limit` | `DEFAULT_GAS_LIMIT` | With `--builder-proposals`, the next slot's builder registration, which the builder builds toward. Otherwise stored and served only | +| `gas_limit` | `DEFAULT_GAS_LIMIT` | With `--builder-proposals`, the next builder registration, which the builder builds toward: usually at the next slot, later while the epoch's full registration send is still running. Otherwise stored and served only | **These are held in memory only.** A restart puts every validator back on the defaults, so an operator driving these endpoints must set them again after one. diff --git a/docs/metrics.md b/docs/metrics.md index 10b6ae097..646f2f913 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -751,7 +751,7 @@ and absent is not the same as zero. | `ethlambda_validator_builder_registrations_sent_total` | Counter | Builder registrations a beacon node accepted (`--builder-proposals`) | | `ethlambda_validator_builder_registration_failures_total` | Counter | Registration batches no beacon node accepted. A missed registration costs at most a builder block, never a duty | | `ethlambda_validator_blinded_blocks_proposed_total` | Counter | Blinded blocks a beacon node handed to its builder. A subset of `blocks_proposed_total` | -| `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx or no answer), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | +| `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx other than 503, or no answer once connected), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in | | `ethlambda_validator_beacon_node_available` | Gauge | 1 when a beacon node answered the last duty refresh or attestation | | `ethlambda_validator_signing_duration_seconds` | Histogram | Time to sign one slot's batch, store read lock included | diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 24107cfbe..fc6593929 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -775,10 +775,13 @@ signature, whichever way the node answered. Three departures: that node's builder holds the payload and only that node claimed the slot, so offering the signed block to another node would hand it to a builder that never saw the bid. If the producing node fails, the publish fails, and - slashing protection has already recorded the slot, so nothing retries. A 5xx or no - answer is reported as unconfirmed (logged at error, counted on - `blinded_blocks_unconfirmed_total`) and not as a missed proposal, since the - relay may still publish the block. The producing node is recorded with the + slashing protection has already recorded the slot, so nothing retries. A 5xx + other than 503, or no answer once connected, is reported as unconfirmed + (logged at error, counted on `blinded_blocks_unconfirmed_total`) and not as a + missed proposal, since the relay may still publish the block. A 4xx, a 503 or + a connection never made (refused, or past the client's 3 s connect timeout) + is a missed proposal: the node cannot have handed the block on. The + producing node is recorded with the block's slot and root, and a publish for any other block is refused, as Teku does; Lighthouse, Lodestar and Nimbus fail over or broadcast instead. Two status codes are read the wrong way: some of ethlambda's own 500s are From 2e4ec290a18b0d275f5df3d896b40f1d4c35dcf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:03:02 -0300 Subject: [PATCH 16/19] refactor(validator): carry the producer with the block and the deadline into propose Review points 10 to 12 of the second builder proposals review. - Producer travels with the block: ProducedBlock gains a Producer the fallback fills in at produce_block, and publish_blinded_block(producer, fork, body) goes back to that node. The fallback's single-slot record, its mutex, the root-mismatch path and the HTTP client's unused slot and root parameters are gone. The proposal path takes the producer from the block it signed, so the pairing the record checked at runtime holds there without a check; a position the fallback never handed out is refused. - The proposal budget moved into ProposalService::propose_by(.., deadline). Production, signing and a full block's publish end at it with Error::ProposalDeadline; a signed blinded block's publish goes on, and past the deadline comes back as BlindedOutcome::StillPublishing. The publish task hands its outcome back over a oneshot: the duty loop reports it when still waiting, the task when not, and close() plus try_recv() at the deadline decides which for a caller that awaits to the end, as the duty loop does. signed_blinded_at and its shared state are gone, and the duty loop matches on the outcome instead of three guarded arms. - Published::BuilderUnconfirmed is gone: Published is what a node can answer, and propose returns its own Proposed outcome. Tests: propose_by within its deadline (full, blinded, a refusal), and the deadline cases, which pause and advance the clock once the mock holds the call (new produce gate), so neither depends on timing. --- .../rpc/src/beacon/validator_client_tests.rs | 29 +- .../src/beacon_node/block_contents.rs | 31 + crates/validator/src/beacon_node/fallback.rs | 155 +--- crates/validator/src/beacon_node/http.rs | 17 +- crates/validator/src/beacon_node/mock.rs | 32 +- crates/validator/src/beacon_node/mod.rs | 19 +- crates/validator/src/error.rs | 11 + crates/validator/src/lib.rs | 42 +- crates/validator/src/proposal.rs | 714 +++++++++++++----- docs/spec_deviations.md | 17 +- 10 files changed, 687 insertions(+), 380 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 88c8a7951..94f1bb741 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -1750,6 +1750,7 @@ mod builder_flow { }; use ethlambda_types::beacon::primitives::ExecutionAddress; use ethlambda_validator::beacon_node::Published; + use ethlambda_validator::beacon_node::block_contents::Producer; use libssz::SszDecode as _; /// What a test drives: this node served over a socket, the proposer's key @@ -1873,23 +1874,23 @@ mod builder_flow { } /// Registered, a bid taken at the maximum factor, and the blinded - /// block signed: its root and the body to publish. - async fn signed_blinded(&self) -> (ethlambda_types::primitives::H256, Vec) { + /// block signed: the node that produced it and the body to publish. + async fn signed_blinded(&self) -> (Producer, Vec) { self.register().await; self.world.offer(1, |_| {}); let produced = self.produce(Some(u64::MAX)).await; assert!(produced.is_blinded()); - let block_root = produced.block_root(); - (block_root, self.sign(produced).await) + let producer = produced.producer; + (producer, self.sign(produced).await) } async fn publish_blinded( &self, - block_root: ethlambda_types::primitives::H256, + producer: Producer, body: &[u8], ) -> ethlambda_validator::Result { self.client - .publish_blinded_block(self.world.slot, block_root, ForkName::Fulu, body) + .publish_blinded_block(producer, ForkName::Fulu, body) .await } } @@ -1905,9 +1906,9 @@ mod builder_flow { let engine = fake_execution_client_paying("0x2a", false).await; let h = harness(world, engine).await; - let (block_root, body) = h.signed_blinded().await; + let (producer, body) = h.signed_blinded().await; let err = h - .publish_blinded(block_root, &body) + .publish_blinded(producer, &body) .await .expect_err("the relay refused it"); assert!( @@ -1927,9 +1928,9 @@ mod builder_flow { let engine = fake_execution_client_paying("0x2a", false).await; let h = harness(world, engine).await; - let (block_root, body) = h.signed_blinded().await; + let (producer, body) = h.signed_blinded().await; let err = h - .publish_blinded(block_root, &body) + .publish_blinded(producer, &body) .await .expect_err("the relay did not confirm it"); assert!( @@ -1953,8 +1954,8 @@ mod builder_flow { let engine = fake_execution_client_paying("0x2a", false).await; let h = harness(world, engine).await; - let (block_root, body) = h.signed_blinded().await; - let published = h.publish_blinded(block_root, &body).await.expect("handed"); + let (producer, body) = h.signed_blinded().await; + let published = h.publish_blinded(producer, &body).await.expect("handed"); assert_eq!(published, Published::HandedToBuilder); let full = h.produce(None).await; @@ -1991,11 +1992,11 @@ mod builder_flow { "the factor must reach the node and take the bid" ); - let block_root = produced.block_root(); + let producer = produced.producer; let body = h.sign(produced).await; let published = h .client - .publish_blinded_block(h.world.slot, block_root, ForkName::Fulu, &body) + .publish_blinded_block(producer, ForkName::Fulu, &body) .await .expect("the node hands the block to its builder"); assert_eq!(published, Published::HandedToBuilder); diff --git a/crates/validator/src/beacon_node/block_contents.rs b/crates/validator/src/beacon_node/block_contents.rs index c46341a2c..f39d349a5 100644 --- a/crates/validator/src/beacon_node/block_contents.rs +++ b/crates/validator/src/beacon_node/block_contents.rs @@ -246,6 +246,33 @@ pub struct ProducedBlock { /// told which. pub fork: ForkName, pub contents: Contents, + /// Which configured node produced it, for a blinded block's publish to go + /// back to. See [`Producer`]. + pub producer: Producer, +} + +/// Which of the configured beacon nodes produced a block. +/// +/// A blinded block can only be published where it was produced: that node's +/// builder holds the payload, and that node claimed the slot. The fallback +/// fills this in when it returns a block and reads it when the signed block +/// is published, so the block carries its own way back: the proposal path +/// takes the producer from the very block it signed. A single node has +/// nothing to choose and leaves it at the default. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Producer(usize); + +impl Producer { + /// The node at `position` in the fallback's list. + pub(crate) fn at(position: usize) -> Self { + Self(position) + } + + /// The position in the fallback's list of the node that produced the + /// block. + pub(crate) fn position(self) -> usize { + self.0 + } } /// The block and the blob material that came with it. @@ -309,6 +336,7 @@ impl ProducedBlock { kzg_proofs: contents.kzg_proofs, blobs: contents.blobs, }, + producer: Producer::default(), }) } ForkName::Fulu => { @@ -321,6 +349,7 @@ impl ProducedBlock { kzg_proofs: contents.kzg_proofs, blobs: contents.blobs, }, + producer: Producer::default(), }) } // Gloas needs one more fact than the fork: whether the node @@ -358,6 +387,7 @@ impl ProducedBlock { Ok(Self { fork, contents: Contents::Blinded { block }, + producer: Producer::default(), }) } // Named, as `from_ssz` names its arms, so a new fork forces a @@ -411,6 +441,7 @@ impl ProducedBlock { Ok(Self { fork: ForkName::Gloas, contents: Contents::Gloas { block, payload }, + producer: Producer::default(), }) } diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 87dfee51d..de09561ab 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -17,7 +17,7 @@ use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; use tracing::{debug, warn}; -use crate::beacon_node::block_contents::ProducedBlock; +use crate::beacon_node::block_contents::{ProducedBlock, Producer}; use crate::beacon_node::dto::{ CommitteeSubscriptionDto, ProposerPreparationDto, SingleAttestationDto, SyncCommitteeSubscriptionDto, SyncDutyDto, @@ -40,29 +40,12 @@ type BoxFuture<'a, T> = Pin + Send + 'a>>; /// first one that succeeds. pub struct FallbackBeaconNode { nodes: Vec, - /// The latest blinded block produced, and the node that produced it. A - /// blinded block can only be published where it was produced: that node's - /// builder holds the payload, and that node claimed the slot. - blinded_producer: std::sync::Mutex>, -} - -/// Which node produced which blinded block. -#[derive(Debug, Clone, Copy)] -struct BlindedProducer { - slot: Slot, - /// Checked at publish, so the record answers for the block being - /// published rather than for whatever was last produced at its slot. - block_root: Root, - position: usize, } impl FallbackBeaconNode { pub fn new(nodes: Vec) -> Self { assert!(!nodes.is_empty(), "at least one beacon node is required"); - Self { - nodes, - blinded_producer: std::sync::Mutex::new(None), - } + Self { nodes } } /// Try each node in order, returning the first success. @@ -277,27 +260,14 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } - /// Remembers which node answered when the block is blinded, and forgets - /// it otherwise, a production that failed on every node included: the - /// record is for the latest block only, so nothing leaves a stale claim - /// that a blinded publish could match. + /// Marks the block with the node that produced it, which is where a + /// blinded block's publish has to go back to (see [`Producer`]). async fn produce_block(&self, request: &BlockRequest) -> Result { - let produced = self + let (position, mut block) = self .try_each_at("produce_block", |node| node.produce_block(request)) - .await; - let record = match &produced { - Ok((position, block)) if block.is_blinded() => Some(BlindedProducer { - slot: block.slot(), - block_root: block.block_root(), - position: *position, - }), - _ => None, - }; - *self - .blinded_producer - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = record; - produced.map(|(_, block)| block) + .await?; + block.producer = Producer::at(position); + Ok(block) } /// First node that accepts it wins, rather than a broadcast to every node. @@ -330,26 +300,22 @@ impl BeaconNodeApi for FallbackBeaconNode { /// protection has already recorded the slot, so nothing retries it. async fn publish_blinded_block( &self, - slot: Slot, - block_root: Root, + producer: Producer, fork: ForkName, body: &[u8], ) -> Result { - let record = *self - .blinded_producer - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - match record { - Some(producer) if producer.slot == slot && producer.block_root == block_root => { - self.nodes[producer.position] - .publish_blinded_block(slot, block_root, fork, body) - .await - } - _ => Err(Error::InconsistentResponse(format!( - "no beacon node produced blinded block {} for slot {slot}", - crate::beacon_node::dto::encode_hex(&block_root.0) - ))), - } + // A position this fallback never handed out: there is no node the + // block could go back to. + let node = self.nodes.get(producer.position()).ok_or_else(|| { + Error::InconsistentResponse(format!( + "a blinded block names beacon node {} as its producer, but {} are configured", + producer.position(), + self.nodes.len() + )) + })?; + // The node behind this one is a single node, with nothing to choose. + node.publish_blinded_block(Producer::default(), fork, body) + .await } /// To every node, like the other calls that install state on a node: a @@ -1051,8 +1017,8 @@ mod tests { MockBeaconNode::new().with_block(5, 7).with_blinded_block() } - /// Node 0 cannot produce, node 1 can and answers blinded: the block must go - /// back to node 1, and node 0 must never see it. + /// Node 0 cannot produce, node 1 can and answers blinded: the block names + /// node 1 as its producer, goes back to node 1, and node 0 never sees it. #[tokio::test] async fn a_blinded_block_is_published_where_it_was_produced() { let first = MockBeaconNode::new().failing_call("produce_block", "down"); @@ -1063,9 +1029,10 @@ mod tests { .await .expect("node 1 produces"); assert!(produced.is_blinded()); + assert_eq!(produced.producer, Producer::at(1)); let published = fallback - .publish_blinded_block(5, produced.block_root(), ForkName::Electra, &[1, 2, 3]) + .publish_blinded_block(produced.producer, ForkName::Electra, &[1, 2, 3]) .await .expect("published"); assert_eq!(published, Published::HandedToBuilder); @@ -1073,10 +1040,7 @@ mod tests { let first_published = fallback.nodes[0].published_blinded.lock().expect("lock"); let second_published = fallback.nodes[1].published_blinded.lock().expect("lock"); assert!(first_published.is_empty(), "node 0 never produced it"); - assert_eq!( - *second_published, - vec![(5, ForkName::Electra, vec![1, 2, 3])] - ); + assert_eq!(*second_published, vec![(ForkName::Electra, vec![1, 2, 3])]); } /// The producing node failing the publish is the end of it: another node's @@ -1092,7 +1056,7 @@ mod tests { .await .expect("node 1 produces"); let err = fallback - .publish_blinded_block(5, produced.block_root(), ForkName::Electra, &[9]) + .publish_blinded_block(produced.producer, ForkName::Electra, &[9]) .await .expect_err("node 1 failed to confirm"); assert!( @@ -1109,27 +1073,16 @@ mod tests { ); } - /// The record answers for one block: another slot, or another root at the - /// same slot, is refused without reaching any node. + /// A producer this fallback never handed out has no node to go back to, + /// and is refused without reaching any node. #[tokio::test] - async fn a_blinded_publish_for_another_slot_or_block_is_refused() { + async fn a_blinded_publish_naming_an_unknown_producer_is_refused() { let fallback = FallbackBeaconNode::new(vec![blinded_node()]); - let produced = fallback - .produce_block(&blinded_request(5, 7)) + let err = fallback + .publish_blinded_block(Producer::at(3), ForkName::Electra, &[1]) .await - .expect("produces"); - let root = produced.block_root(); - - for (slot, block_root, what) in [ - (6, root, "nothing was produced for slot 6"), - (5, Root::repeat_byte(0xee), "that block was not produced"), - ] { - let err = fallback - .publish_blinded_block(slot, block_root, ForkName::Electra, &[1]) - .await - .expect_err(what); - assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); - } + .expect_err("there is no node 3"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); assert!( fallback.nodes[0] .published_blinded @@ -1139,46 +1092,6 @@ mod tests { ); } - /// A production that fails on every node clears the record too, so the - /// earlier block's claim does not outlive the attempt that replaced it. - #[tokio::test] - async fn a_failed_production_clears_the_blinded_record() { - let fallback = FallbackBeaconNode::new(vec![blinded_node()]); - let produced = fallback - .produce_block(&blinded_request(5, 7)) - .await - .expect("blinded"); - fallback - .produce_block(&blinded_request(6, 7)) - .await - .expect_err("the node holds no block for slot 6"); - - fallback - .publish_blinded_block(5, produced.block_root(), ForkName::Electra, &[1]) - .await - .expect_err("the record is gone"); - } - - /// A full block produced afterwards clears the claim, so an old blinded - /// block's slot cannot be published against a node that no longer holds it. - #[tokio::test] - async fn a_full_block_clears_the_blinded_record() { - let fallback = FallbackBeaconNode::new(vec![blinded_node()]); - let blinded = fallback - .produce_block(&blinded_request(5, 7)) - .await - .expect("blinded"); - fallback - .produce_block(&request(5, 7)) - .await - .expect("full, since no factor was sent"); - - fallback - .publish_blinded_block(5, blinded.block_root(), ForkName::Electra, &[1]) - .await - .expect_err("the record is gone"); - } - #[tokio::test] async fn registrations_reach_every_node() { use ethlambda_types::beacon::builder::{ diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index a623d4b3c..ca4425cf9 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -37,7 +37,7 @@ use tracing::{debug, warn}; use libssz::SszDecode as _; -use crate::beacon_node::block_contents::ProducedBlock; +use crate::beacon_node::block_contents::{ProducedBlock, Producer}; use crate::beacon_node::dto::{ AttestationDataDto, AttestationDto, AttesterDutyDto, BlockRootResponse, CommitteeSubscriptionDto, DataResponse, DutiesResponse, GenesisDto, IndexedErrorResponse, @@ -820,8 +820,7 @@ impl BeaconNodeApi for HttpBeaconNode { &self, // For the fallback, which picks the node that produced this block; // one node has nothing to choose. - _slot: Slot, - _block_root: Root, + _producer: Producer, fork: ForkName, body: &[u8], ) -> Result { @@ -1591,7 +1590,7 @@ mod tests { let served = serve(status, vec![], vec![]).await; let published = served .node - .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1, 2, 3]) + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1, 2, 3]) .await .expect("accepted"); assert_eq!(published, expected, "status {status}"); @@ -1605,7 +1604,7 @@ mod tests { let served = serve(400, vec![], b"bad".to_vec()).await; let err = served .node - .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) .await .expect_err("a refusal"); assert!( @@ -1616,7 +1615,7 @@ mod tests { let served = serve(500, vec![], b"boom".to_vec()).await; let err = served .node - .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) .await .expect_err("unconfirmed"); assert!( @@ -1638,7 +1637,7 @@ mod tests { // Nothing listens on port 1. let node = HttpBeaconNode::new("http://127.0.0.1:1").expect("builds"); let err = node - .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) .await .expect_err("no answer"); assert!(matches!(err, Error::BeaconNode { .. }), "got {err:?}"); @@ -1668,7 +1667,7 @@ mod tests { let started = std::time::Instant::now(); let err = node - .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) .await .expect_err("never connects"); held.abort(); @@ -1695,7 +1694,7 @@ mod tests { let served = serve(503, vec![], b"syncing".to_vec()).await; let err = served .node - .publish_blinded_block(5, Root::ZERO, ForkName::Fulu, &[1]) + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) .await .expect_err("refused"); assert!( diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 94cf99294..4d6af0aa0 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -17,7 +17,7 @@ use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, Validato use tokio::sync::{Notify, Semaphore}; use crate::beacon_node::block_contents::{ - Contents, GloasPayload, ProducedBlock, empty_block_for, empty_gloas_block_for, + Contents, GloasPayload, ProducedBlock, Producer, empty_block_for, empty_gloas_block_for, empty_gloas_envelope_for, }; use crate::beacon_node::dto::{ @@ -103,8 +103,8 @@ pub struct MockBeaconNode { /// records its batch, so a test releases calls one at a time instead of /// guessing how long a delay must be. pub registration_gate: Option>, - /// Every blinded block published, as (slot, fork, body). - pub published_blinded: Mutex)>>, + /// Every blinded block published, as (fork, body). + pub published_blinded: Mutex)>>, /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; /// `Err(detail)` is a `BlindedUnconfirmed` carrying `detail`. pub blinded_publish_outcome: Option>, @@ -119,6 +119,11 @@ pub struct MockBeaconNode { /// test that publishes once can await it whether or not the call came /// first, since `notify_one` keeps the permit for a later waiter. pub blinded_publish_entered: Arc, + /// When set, `produce_block` takes a permit before it answers, as a node + /// slow to build a block does, for as long as the test holds it back. + pub produce_gate: Option>, + /// Notified when `produce_block` is called, before the gate. + pub produce_entered: Arc, /// What the test asserts against. pub submitted: Mutex>, @@ -344,6 +349,12 @@ impl MockBeaconNode { self } + /// Make `produce_block` wait for a permit from `gate` before it answers. + pub fn with_produce_gate(mut self, gate: Arc) -> Self { + self.produce_gate = Some(gate); + self + } + /// Make `publish_blinded_block` wait for a permit from `gate` before it /// records the block and answers. pub fn with_blinded_publish_gate(mut self, gate: Arc) -> Self { @@ -709,6 +720,13 @@ impl BeaconNodeApi for MockBeaconNode { /// failover moves past it. async fn produce_block(&self, request: &BlockRequest) -> Result { self.guard("produce_block")?; + self.produce_entered.notify_one(); + if let Some(gate) = &self.produce_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } self.block_requests .lock() .expect("lock") @@ -731,6 +749,7 @@ impl BeaconNodeApi for MockBeaconNode { let block = ProducedBlock { fork: ForkName::Gloas, contents: Contents::Gloas { block, payload }, + producer: Default::default(), }; crate::beacon_node::validate_produced_block(request, &block)?; return Ok(block); @@ -748,6 +767,7 @@ impl BeaconNodeApi for MockBeaconNode { contents: Contents::Blinded { block: BlindedBeaconBlock::from(empty_block_for(slot, proposer_index)), }, + producer: Default::default(), }; crate::beacon_node::validate_produced_block(request, &block)?; return Ok(block); @@ -762,6 +782,7 @@ impl BeaconNodeApi for MockBeaconNode { kzg_proofs: Default::default(), blobs: Default::default(), }, + producer: Default::default(), }; crate::beacon_node::validate_produced_block(request, &block)?; Ok(block) @@ -778,8 +799,7 @@ impl BeaconNodeApi for MockBeaconNode { async fn publish_blinded_block( &self, - slot: Slot, - _block_root: Root, + _producer: Producer, fork: ForkName, body: &[u8], ) -> Result { @@ -803,7 +823,7 @@ impl BeaconNodeApi for MockBeaconNode { self.published_blinded .lock() .expect("lock") - .push((slot, fork, body.to_vec())); + .push((fork, body.to_vec())); match &self.blinded_publish_outcome { None => Ok(Published::HandedToBuilder), Some(Ok(published)) => Ok(*published), diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 2c176de87..7a1706ab3 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -15,7 +15,7 @@ use ethlambda_types::beacon::primitives::{ }; use ethlambda_types::beacon::signing::compute_epoch_at_slot; -use crate::beacon_node::block_contents::ProducedBlock; +use crate::beacon_node::block_contents::{ProducedBlock, Producer}; use crate::beacon_node::dto::{ AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, PtcDutyDto, SingleAttestationDto, SyncCommitteeSubscriptionDto, SyncDutyDto, @@ -172,11 +172,6 @@ pub enum Published { /// its builder, which publishes the block and reveals the payload. Nothing /// says the node imported it. HandedToBuilder, - /// The node failed to confirm the blinded block (a 5xx other than 503, or - /// no answer once connected). The relay may still have published it; see - /// `Error::BlindedUnconfirmed`. Produced only by `ProposalService`, never - /// by an implementation. - BuilderUnconfirmed, } /// Check that `block` is the block that was asked for. @@ -406,14 +401,15 @@ pub trait BeaconNodeApi: Send + Sync { /// proposal, whether or not the first one succeeded. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result; - /// Publish a signed blinded block, the one with root `block_root` produced - /// at `slot` under `fork`, to `POST /eth/v2/beacon/blinded_blocks`. The - /// node hands it to its builder. + /// Publish a signed blinded block, produced under `fork` by `producer` + /// (the [`ProducedBlock::producer`] of the block it was signed from), to + /// `POST /eth/v2/beacon/blinded_blocks`. The node hands it to its builder. /// /// # Contract: the node that produced it, and no other /// /// Only the producing node's builder holds the payload, and only that node - /// claimed the slot. Failover must not offer it elsewhere. + /// claimed the slot. Failover must not offer it elsewhere: the fallback + /// sends it to `producer` alone. /// /// # Contract: which error means what /// @@ -431,8 +427,7 @@ pub trait BeaconNodeApi: Send + Sync { /// two edges it knowingly reads the wrong way. async fn publish_blinded_block( &self, - slot: Slot, - block_root: Root, + producer: Producer, fork: ForkName, body: &[u8], ) -> Result; diff --git a/crates/validator/src/error.rs b/crates/validator/src/error.rs index 75f26ce7d..b2d10aa37 100644 --- a/crates/validator/src/error.rs +++ b/crates/validator/src/error.rs @@ -91,6 +91,16 @@ pub enum Error { /// abandoned attestation itself is never retried, by design. #[error("attestation duty for slot {slot} ran past the end of its slot")] AttestationDeadline { slot: u64 }, + /// The slot's proposal ran past its deadline, the point attesters stop + /// waiting for a block, before its block was published, and was + /// abandoned. + /// + /// The proposal counterpart of [`Self::AttestationDeadline`], and + /// retryable in the same sense only. Never raised once a blinded block is + /// signed: its publish goes on past the deadline (see + /// `ProposalService::propose_by`). + #[error("proposal for slot {slot} ran past the point attesters stop waiting for its block")] + ProposalDeadline { slot: u64 }, /// Slashing protection refused to sign a block for this slot, or could not /// record it. /// @@ -183,6 +193,7 @@ impl Error { | Self::BlindedUnconfirmed { .. } | Self::InconsistentResponse(_) | Self::AttestationDeadline { .. } + | Self::ProposalDeadline { .. } | Self::Decode(_) => true, // A refusal repeats itself, and a record that could not be written // needs an operator: retrying within the slot helps neither. diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 4017122e9..67e135053 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -57,14 +57,14 @@ use tracing::{error, info, warn}; use crate::aggregation::AggregationService; use crate::attestation::AttestationService; +use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::ProposerPreparationDto; use crate::beacon_node::fallback::FallbackBeaconNode; use crate::beacon_node::http::HttpBeaconNode; -use crate::beacon_node::{BeaconNodeApi, Published}; use crate::duties::DutiesService; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; -use crate::proposal::ProposalService; +use crate::proposal::{BlindedOutcome, ProposalService, Proposed}; use crate::proposer_settings::ProposerSettings; use crate::registration::RegistrationService; use crate::signing::SigningContext; @@ -707,32 +707,22 @@ async fn propose( store: &RwLock, ) { let budget = clock.until_attestation(slot, SystemTime::now()); - let attempt = tokio::time::timeout(budget, proposal.propose(slot, duty, store)); - match attempt.await { - // Already logged at error and counted on its own series; the block - // may well be out, so it is neither a published-in-time delay nor a - // failure. - Ok(Ok(Published::BuilderUnconfirmed)) => {} - // A blinded publish observes its own delay, on the task that ran it. - Ok(Ok(_)) if proposal.signed_blinded_at(slot) => {} - Ok(Ok(_)) => { + let deadline = tokio::time::Instant::now() + budget; + match proposal.propose_by(slot, duty, store, deadline).await { + Ok(Proposed::Full(_) | Proposed::Blinded(BlindedOutcome::Published(_))) => { let elapsed = SystemTime::now() .duration_since(clock.start_of(slot)) .unwrap_or_default(); metrics::observe_block_publication_delay(elapsed.as_secs_f64()); } - // Once a blinded block is signed, only its publish task reports an - // error (and counts the failure), so it is not counted twice here. - Ok(Err(_)) if proposal.signed_blinded_at(slot) => {} - Ok(Err(err)) => { - error!(%slot, validator = duty.validator_index, %err, "Failed to propose this slot's block"); - metrics::inc_block_proposal_failures(); - } - // A blinded block is signed, and its publish runs on its own task: the - // beacon node waits up to 5 s for the relay, which can outlast this - // budget. That is not a missed proposal, and the task counts the real - // outcome. - Err(_) if proposal.signed_blinded_at(slot) => { + // Already logged at error and counted on its own series; the block + // may well be out, so it is neither a published-in-time delay nor a + // failure. + Ok(Proposed::Blinded(BlindedOutcome::Unconfirmed)) => {} + // The beacon node waits up to 5 s for the relay, which can outlast + // this budget. Not a missed proposal: the publish goes on, on its own + // task, which reports and counts the real outcome. + Ok(Proposed::Blinded(BlindedOutcome::StillPublishing)) => { info!( %slot, validator = duty.validator_index, @@ -740,7 +730,7 @@ async fn propose( "Blinded block signed; its publish outlived the proposal budget and continues" ); } - Err(_) => { + Err(Error::ProposalDeadline { .. }) => { warn!( %slot, validator = duty.validator_index, @@ -749,6 +739,10 @@ async fn propose( ); metrics::inc_block_proposal_failures(); } + Err(err) => { + error!(%slot, validator = duty.validator_index, %err, "Failed to propose this slot's block"); + metrics::inc_block_proposal_failures(); + } } } diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 370fc64ef..efab3af75 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -54,15 +54,21 @@ //! a mismatch there is the flow working rather than a node paying someone else. //! //! The signed block goes to `blinded_blocks` on the node that produced it and -//! on no other: only that node's builder holds the payload. A publish the node -//! fails to confirm is reported as [`Published::BuilderUnconfirmed`] rather -//! than as a missed proposal, since the relay may still publish the block. +//! on no other (the block's [`Producer`]): only that node's builder holds the +//! payload. A publish the node fails to confirm is reported as +//! [`BlindedOutcome::Unconfirmed`] rather than as a missed proposal, since the +//! relay may still publish the block. //! -//! The publish runs on its own task. The beacon node waits up to 5 s for the -//! relay, which can outlast the proposal's budget (see `lib.rs`), and dropping -//! the proposal future must neither cancel the request nor lose its outcome: the -//! task logs and counts it itself, and [`ProposalService::signed_blinded_at`] -//! lets the caller tell an overrun after signing from a block never signed. +//! # The deadline +//! +//! The duty loop gives a proposal until the point attesters stop waiting for +//! the block ([`ProposalService::propose_by`]). Everything up to a full +//! block's publish ends there. A signed blinded block's publish does not: it +//! runs on its own task, since the beacon node waits up to 5 s for the relay, +//! and past the deadline the proposal returns +//! [`BlindedOutcome::StillPublishing`] while the task reports the outcome. +//! The deadline is enforced here rather than by the caller dropping the +//! future, because only here is it known whether a blinded block was signed. //! //! # Do not retry this call within a slot //! @@ -81,9 +87,10 @@ use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress, Root, Slot}; use ethlambda_types::beacon::signing::compute_epoch_at_slot; use tokio::sync::RwLock; +use tokio::time::Instant; use tracing::{error, info, warn}; -use crate::beacon_node::block_contents::{GloasPayload, signed_envelope_ssz}; +use crate::beacon_node::block_contents::{GloasPayload, Producer, signed_envelope_ssz}; use crate::beacon_node::dto::{ProposerDutyDto, encode_hex, parse_pubkey}; use crate::beacon_node::{BeaconNodeApi, BlockRequest, Published, validate_produced_block}; use crate::error::{Error, Result}; @@ -93,6 +100,70 @@ use crate::signing::SigningContext; use crate::slashing_protection::{Refusal, SharedProtection}; use crate::slot_clock::SlotClock; +/// What became of a proposal, as the duty loop needs it. +/// +/// Every outcome here has already been logged and its block counted. What the +/// loop still owes is the publication delay of a block the node confirmed +/// while it was waiting: [`Proposed::Full`] and +/// [`BlindedOutcome::Published`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Proposed { + /// A full block, and what the node made of it. + Full(Published), + /// A blinded block, signed and sent to the node that produced it. + Blinded(BlindedOutcome), +} + +/// What became of a signed blinded block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlindedOutcome { + /// The node confirmed it: handed to its builder, or broadcast but not + /// imported (a 202). + Published(Published), + /// The node did not confirm it, so the relay may still publish it (see + /// [`Error::BlindedUnconfirmed`]). Logged at error and counted on its own + /// series: neither a published block nor a failed proposal. Not retried, + /// since the slot is signed and a second block would be an equivocation. + Unconfirmed, + /// The publish outlived the deadline and goes on, on its own task, which + /// logs and counts the outcome when it comes: a refusal as a failed + /// proposal, a confirmed block with its publication delay. + StillPublishing, +} + +/// A block signed and ready to publish: what [`ProposalService::produce_and_sign`] +/// hands the publish step. +struct SignedBlock { + pubkey: BlsPubkey, + fork: ForkName, + block_root: Root, + blinded: bool, + producer: Producer, + /// The self-built payload's envelope, taken out before the block was + /// signed into its body (gloas only). + payload: Option, + builder_index: Option, + body: Vec, +} + +/// `future`'s result, or [`Error::ProposalDeadline`] once `deadline` passes. +/// With no deadline, `future` runs to the end. +/// +/// A future that completes is never discarded, even at the deadline: the +/// timeout is checked only after the future has been polled. +async fn before( + deadline: Option, + slot: Slot, + future: impl std::future::Future>, +) -> Result { + match deadline { + Some(deadline) => tokio::time::timeout_at(deadline, future) + .await + .unwrap_or(Err(Error::ProposalDeadline { slot })), + None => future.await, + } +} + pub struct ProposalService { beacon_node: Arc, context: Arc, @@ -109,11 +180,7 @@ pub struct ProposalService { /// The `builder_boost_factor` sent with each block request, or `None` when /// builder proposals are off, in which case a blinded answer is refused. builder_boost_factor: Option, - /// The slot of the latest blinded block this service signed, set before - /// its publish starts. A `std::sync::Mutex` because the critical section - /// is a single store with no await inside it. - last_blinded_slot: std::sync::Mutex>, - /// Where a slot starts, so a blinded publish that outlives the proposal + /// Where a slot starts, so a blinded publish that outlives the deadline /// can observe its own publication delay. `None` observes nothing. clock: Option, } @@ -132,29 +199,18 @@ impl ProposalService { settings, protection, builder_boost_factor, - last_blinded_slot: std::sync::Mutex::new(None), clock: None, } } /// Give the service the slot clock, so a blinded block's publication delay - /// is observed by the task that publishes it (which may outlive - /// [`Self::propose`]). + /// is observed by the task that publishes it when that outlives + /// [`Self::propose_by`]'s deadline. pub fn with_clock(mut self, clock: SlotClock) -> Self { self.clock = Some(clock); self } - /// Whether this service signed a blinded block for `slot`, whose publish - /// may therefore still be in flight on its own task. - pub fn signed_blinded_at(&self, slot: Slot) -> bool { - *self - .last_blinded_slot - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - == Some(slot) - } - /// Whether the produced block's fee recipient is this client's to compare. /// /// Not for a blinded block: its header names the builder's address, and the @@ -216,16 +272,89 @@ impl ProposalService { } } - /// Produce, sign and publish the block for `slot`. + /// Produce, sign and publish the block for `slot`, however long it takes. /// - /// Returns what became of it, which is not always a clean success: see - /// [`Published`]. + /// [`Self::propose_by`] with no deadline, so a blinded block's publish is + /// always waited for and never comes back as + /// [`BlindedOutcome::StillPublishing`]. pub async fn propose( &self, slot: Slot, duty: &ProposerDutyDto, store: &RwLock, - ) -> Result { + ) -> Result { + self.propose_within(slot, duty, store, None).await + } + + /// Produce, sign and publish the block for `slot`, giving up at + /// `deadline`. + /// + /// # What the deadline cuts + /// + /// Production and signing, and a full block's publish (with its envelope + /// from gloas), end at the deadline with [`Error::ProposalDeadline`]: a + /// full block that late has lost most of its value, and every further + /// second comes out of the slot's attestations. + /// + /// A signed blinded block's publish is not cut: the beacon node may wait + /// several seconds on the relay, and the block may already be out. It runs + /// on its own task, and at the deadline this returns + /// [`BlindedOutcome::StillPublishing`] while the task reports the outcome. + /// Whichever of the two ends up holding the outcome reports it, so a + /// caller that awaits this to the end, as the duty loop does, sees each + /// outcome reported once (see [`Self::publish_blinded`] for the edges). + pub async fn propose_by( + &self, + slot: Slot, + duty: &ProposerDutyDto, + store: &RwLock, + deadline: Instant, + ) -> Result { + self.propose_within(slot, duty, store, Some(deadline)).await + } + + async fn propose_within( + &self, + slot: Slot, + duty: &ProposerDutyDto, + store: &RwLock, + deadline: Option, + ) -> Result { + // Nothing awaits between signing and the end of `produce_and_sign`, + // and a completed future is never discarded at the deadline, so a + // signed block always reaches the publish step below. + let signed = before(deadline, slot, self.produce_and_sign(slot, duty, store)).await?; + let validator = duty.validator_index; + if signed.blinded { + return self + .publish_blinded( + slot, + validator, + signed.producer, + signed.fork, + signed.body, + deadline, + ) + .await; + } + before( + deadline, + slot, + self.publish_full(slot, validator, store, signed), + ) + .await + .map(Proposed::Full) + } + + /// Everything up to and including the block signature: the slashing + /// protection checks, the RANDAO reveal, the block request and the checks + /// on what came back. + async fn produce_and_sign( + &self, + slot: Slot, + duty: &ProposerDutyDto, + store: &RwLock, + ) -> Result { let pubkey = parse_pubkey(&duty.pubkey)?; let epoch = compute_epoch_at_slot(slot); @@ -347,37 +476,49 @@ impl ProposalService { let signature = { let store = store.read().await; - let signature = self.context.sign_block(&store, &pubkey, block_root, slot)?; - // Before the first await after signing, so an overrun of the - // proposal's budget from here on is known to be a signed block. - if blinded { - *self - .last_blinded_slot - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(slot); - } - signature + self.context.sign_block(&store, &pubkey, block_root, slot)? }; + // Nothing below awaits: see `propose_within`. + // // Taken out before the block is signed into its body: the envelope is // published on its own, after the block, and the bare gloas block body // has no place for it. let payload = produced.take_gloas_payload(); let builder_index = produced.builder_index(); + let producer = produced.producer; + Ok(SignedBlock { + pubkey, + fork, + block_root, + blinded, + producer, + payload, + builder_index, + body: produced.into_signed_ssz(signature), + }) + } - let body = produced.into_signed_ssz(signature); - // A blinded block counts itself on its own task (see - // `publish_blinded`); a full one is counted here. - let published = if blinded { - self.publish_blinded(slot, block_root, fork, body, duty.validator_index) - .await? - } else { - let published = self - .publish(fork, &body, slot, duty.validator_index) - .await?; - crate::metrics::inc_blocks_proposed(); - published - }; + /// Publish a signed full block, then, for a self-built gloas payload, its + /// envelope. + async fn publish_full( + &self, + slot: Slot, + validator: u64, + store: &RwLock, + signed: SignedBlock, + ) -> Result { + let SignedBlock { + pubkey, + fork, + block_root, + payload, + builder_index, + body, + .. + } = signed; + let published = self.publish(fork, &body, slot, validator).await?; + crate::metrics::inc_blocks_proposed(); // Only a self-built payload is this client's to reveal. if builder_index == Some(BUILDER_INDEX_SELF_BUILD) { @@ -385,20 +526,13 @@ impl ProposalService { // the proposal did happen; what failed is the reveal, which has its // own series because it is the failure that costs the payload. let _ = self - .publish_envelope( - slot, - duty.validator_index, - &pubkey, - block_root, - payload, - store, - ) + .publish_envelope(slot, validator, &pubkey, block_root, payload, store) .await .inspect(|()| crate::metrics::inc_envelopes_published()) .inspect_err(|err| { tracing::error!( %slot, - validator = duty.validator_index, + validator, block_root = %ethlambda_types::ShortRoot(&block_root.0), %err, "Block published but its envelope was not; the slot's payload is withheld" @@ -474,12 +608,22 @@ impl ProposalService { /// Send a signed blinded block to the node that produced it. /// - /// # On its own task + /// # On its own task, reported by whoever holds the outcome + /// + /// The request runs on a spawned task, so neither the deadline nor a + /// dropped proposal cancels it: the beacon node may wait up to 5 s on the + /// relay, and the block may already be out. The task logs and counts the + /// node's answer ([`blinded_outcome`]) and hands the outcome back over a + /// channel. Whoever holds it then does the rest: the duty loop when this + /// returns it in time, and the task itself when this has stopped waiting + /// ([`report_unawaited`]). For a caller that awaits this to the end, the + /// channel decides which, so it is never both and never neither. /// - /// The request, its logging and its counters run on a spawned task, so a - /// proposal future dropped at the duty loop's budget (the node may wait up - /// to 5 s on the relay) neither cancels the request nor loses its outcome. - /// This method only awaits the task. + /// Two edges are not covered. A task that panics after the deadline is + /// reported by nobody. A caller that drops the proposal after the task + /// has sent its outcome takes the rest of the reporting (the delay sample, + /// or a refusal's failure count) with it. The node's answer itself is + /// logged and counted by the task either way. /// /// # Why an unconfirmed publish is not an error /// @@ -487,94 +631,62 @@ impl ProposalService { /// published: the node may have handed it to the relay, which publishes /// regardless of what the node then replied. So that outcome is logged /// loudly and counted on its own series, and returned as - /// [`Published::BuilderUnconfirmed`]. It is not retried: the slot is - /// signed, and a second block for it would be an equivocation. A definite - /// failure (a 4xx, a 503, a connection never made; see - /// [`BeaconNodeApi::publish_blinded_block`]) propagates as a failed + /// [`BlindedOutcome::Unconfirmed`]. It is not retried: the slot is signed, + /// and a second block for it would be an equivocation. A definite failure + /// (a 4xx, a 503, a connection never made; see + /// [`BeaconNodeApi::publish_blinded_block`]) is an error, a failed /// proposal. async fn publish_blinded( &self, slot: Slot, - block_root: Root, + validator: u64, + producer: Producer, fork: ForkName, body: Vec, - validator: u64, - ) -> Result { + deadline: Option, + ) -> Result { let beacon_node = self.beacon_node.clone(); let slot_start = self.clock.as_ref().map(|clock| clock.start_of(slot)); - let task = tokio::spawn(async move { + let (handoff, mut outcome) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { let bytes = body.len(); - match beacon_node - .publish_blinded_block(slot, block_root, fork, &body) - .await - { - Ok(published) => { - match published { - Published::HandedToBuilder => { - info!(%slot, validator, bytes, "Blinded block handed to the builder"); - } - Published::BroadcastNotImported => { - warn!( - %slot, - validator, - "Block was broadcast but the beacon node could not import it; \ - check that node's execution layer" - ); - crate::metrics::inc_blocks_broadcast_not_imported(); - } - // Neither belongs to this path: a full block's outcome, - // and one only this service builds. Reported rather - // than assumed away. - Published::Imported | Published::BuilderUnconfirmed => warn!( - %slot, - validator, - ?published, - "Beacon node API answered a blinded publish with an outcome that \ - belongs to another call" - ), - } - if published != Published::BuilderUnconfirmed { - crate::metrics::inc_blocks_proposed(); - crate::metrics::inc_blinded_blocks_proposed(); - // Here rather than in the duty loop, which has stopped - // waiting if the relay was slow. - if let Some(start) = slot_start { - let elapsed = - SystemTime::now().duration_since(start).unwrap_or_default(); - crate::metrics::observe_block_publication_delay(elapsed.as_secs_f64()); - } + let answer = beacon_node + .publish_blinded_block(producer, fork, &body) + .await; + let outcome = blinded_outcome(slot, validator, bytes, answer); + // Fails once the receiving side has stopped waiting, which hands + // the outcome back here to report. + if let Err(outcome) = handoff.send(outcome) { + report_unawaited(slot, validator, slot_start, outcome); + } + }); + + let received = match deadline { + None => (&mut outcome).await, + Some(deadline) => match tokio::time::timeout_at(deadline, &mut outcome).await { + Ok(received) => received, + Err(_) => { + // Stop waiting. After `close`, the task's send fails and + // the task reports; an outcome sent before it is still + // here, and is taken. + outcome.close(); + match outcome.try_recv() { + Ok(sent) => Ok(sent), + Err(_) => return Ok(Proposed::Blinded(BlindedOutcome::StillPublishing)), } - Ok(published) - } - Err(err @ Error::BlindedUnconfirmed { .. }) => { - tracing::error!( - %slot, - validator, - %err, - "The beacon node did not confirm the blinded block; the relay may still \ - publish it. Not retried: the slot is signed, and a second block would be \ - an equivocation" - ); - crate::metrics::inc_blinded_blocks_unconfirmed(); - Ok(Published::BuilderUnconfirmed) - } - // A definite refusal. Reported here and not left to the caller, - // which may have stopped waiting at the proposal's budget: this - // task is the only one that knows the slot was missed. - Err(err) => { - tracing::error!(%slot, validator, %err, "The beacon node refused the blinded block"); - crate::metrics::inc_block_proposal_failures(); - Err(err) } + }, + }; + match received { + Ok(outcome) => outcome.map(Proposed::Blinded), + // The task ended without sending, so it panicked. The block may or + // may not have gone out, which is what "unconfirmed" says. + Err(_) => { + tracing::error!(%slot, validator, "The blinded publish task failed"); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(Proposed::Blinded(BlindedOutcome::Unconfirmed)) } - }); - // A task that panicked has no outcome to report; the block may or may - // not have gone out, which is what "unconfirmed" says. - task.await.unwrap_or_else(|err| { - tracing::error!(%slot, validator, %err, "The blinded publish task failed"); - crate::metrics::inc_blinded_blocks_unconfirmed(); - Ok(Published::BuilderUnconfirmed) - }) + } } /// Send the signed body and report what the node made of it. @@ -608,15 +720,97 @@ impl ProposalService { ); crate::metrics::inc_blocks_broadcast_not_imported(); } - // `publish_block` never answers with these: a blinded block goes - // through `publish_blinded_block`, and `BuilderUnconfirmed` is - // built by this service, not by an implementation. - Published::HandedToBuilder | Published::BuilderUnconfirmed => {} + // `publish_block` never answers with this: a blinded block goes + // through `publish_blinded_block`. + Published::HandedToBuilder => {} } Ok(published) } } +/// Log and count a blinded publish's answer, as [`ProposalService::publish`] +/// does a full block's, and turn an unconfirmed send into an outcome. A +/// definite refusal stays an error, for whoever holds the outcome to report. +fn blinded_outcome( + slot: Slot, + validator: u64, + bytes: usize, + answer: Result, +) -> Result { + match answer { + Ok(published) => { + match published { + Published::HandedToBuilder => { + info!(%slot, validator, bytes, "Blinded block handed to the builder"); + } + Published::BroadcastNotImported => { + warn!( + %slot, + validator, + "Block was broadcast but the beacon node could not import it; \ + check that node's execution layer" + ); + crate::metrics::inc_blocks_broadcast_not_imported(); + } + // A full block's outcome. Reported rather than assumed away; + // the block did go out, so it is still counted. + Published::Imported => warn!( + %slot, + validator, + ?published, + "Beacon node API answered a blinded publish with an outcome that belongs to \ + another call" + ), + } + crate::metrics::inc_blocks_proposed(); + crate::metrics::inc_blinded_blocks_proposed(); + Ok(BlindedOutcome::Published(published)) + } + Err(err @ Error::BlindedUnconfirmed { .. }) => { + tracing::error!( + %slot, + validator, + %err, + "The beacon node did not confirm the blinded block; the relay may still publish \ + it. Not retried: the slot is signed, and a second block would be an equivocation" + ); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(BlindedOutcome::Unconfirmed) + } + Err(err) => Err(err), + } +} + +/// What the duty loop does with a blinded outcome it receives in time, done by +/// the publish task when nobody is waiting for it any more (past the deadline, +/// or a dropped proposal): the publication delay of a confirmed block, and a +/// refusal counted as a failed proposal. +fn report_unawaited( + slot: Slot, + validator: u64, + slot_start: Option, + outcome: Result, +) { + match outcome { + Ok(BlindedOutcome::Published(_)) => { + if let Some(start) = slot_start { + let elapsed = SystemTime::now().duration_since(start).unwrap_or_default(); + crate::metrics::observe_block_publication_delay(elapsed.as_secs_f64()); + } + } + Ok(BlindedOutcome::Unconfirmed | BlindedOutcome::StillPublishing) => {} + Err(err) => { + tracing::error!( + %slot, + validator, + %err, + "The beacon node refused the blinded block, after the proposal stopped waiting" + ); + crate::metrics::inc_block_proposal_failures(); + } + } +} + /// Slashing protection's answer about a block, as the duty's outcome: a judged /// refusal, or a record that could not be consulted or written (`what` names /// which), is a counted [`Error::ProposalRefused`]. @@ -738,7 +932,7 @@ mod tests { .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("proposes"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); let sent = node.published_blocks(); assert_eq!(sent.len(), 1); @@ -967,7 +1161,7 @@ mod tests { .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("202 is not an error"); - assert_eq!(published, Published::BroadcastNotImported); + assert_eq!(published, Proposed::Full(Published::BroadcastNotImported)); } fn builder_service( @@ -991,6 +1185,8 @@ mod tests { /// is asserted is that the proposal goes through unchanged. #[tokio::test] async fn a_blinded_block_is_published_to_the_builder_route() { + use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; + let (store, pubkey) = store(); let node = Arc::new( MockBeaconNode::new() @@ -999,23 +1195,22 @@ mod tests { ); let service = builder_service(node.clone(), Some(H160([0xfe; 20]))); - assert!(!service.signed_blinded_at(slot())); let published = service .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("proposes"); - assert_eq!(published, Published::HandedToBuilder); - assert!( - service.signed_blinded_at(slot()), - "the duty loop reads this to tell an overrun after signing from one before" + assert_eq!( + published, + Proposed::Blinded(BlindedOutcome::Published(Published::HandedToBuilder)) ); - assert!(!service.signed_blinded_at(slot() + 1)); assert_eq!(node.block_requests()[0].builder_boost_factor, Some(100)); assert!(node.published_blocks().is_empty()); let sent = node.published_blinded.lock().expect("lock").clone(); assert_eq!(sent.len(), 1); - assert_eq!((sent[0].0, sent[0].1), (slot(), ForkName::Electra)); + assert_eq!(sent[0].0, ForkName::Electra); + let block = SignedBlindedBeaconBlock::from_ssz_bytes(&sent[0].1).expect("decodes"); + assert_eq!(block.message.slot, slot()); } /// The fee-recipient comparison is for a full block only: a blinded @@ -1030,6 +1225,7 @@ mod tests { contents: Contents::Blinded { block: BlindedBeaconBlock::from(empty_block_for(slot(), 7)), }, + producer: Default::default(), }; assert!(!ProposalService::::compares_fee_recipient( &blinded @@ -1068,7 +1264,7 @@ mod tests { .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("an unconfirmed publish is not an error"); - assert_eq!(published, Published::BuilderUnconfirmed); + assert_eq!(published, Proposed::Blinded(BlindedOutcome::Unconfirmed)); let err = service .propose(slot(), &duty(&pubkey, 7, slot()), &store) @@ -1077,16 +1273,29 @@ mod tests { assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); } - /// The duty loop drops the proposal at its budget while the node is still - /// waiting on the relay. The publish runs on its own task, so the block is - /// still sent, and the slot is known to be signed. + /// Wait, up to five seconds, until `node` has recorded a blinded block. + async fn until_published_blinded(node: &MockBeaconNode) { + let published = async { + while node.published_blinded.lock().expect("lock").is_empty() { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), published) + .await + .expect("the publish completes on its own task"); + } + + /// The deadline passes while the node waits on the relay. The proposal + /// returns `StillPublishing` rather than an error, and the publish, on its + /// own task, still goes out. /// - /// The proposal is dropped once the publish has reached the node, which - /// holds it until the test lets it answer, rather than after a fixed - /// delay: the slashing-protection write and the signatures come first, and - /// on a loaded machine they can take longer than any delay picked here. + /// The deadline is moved past by pausing the clock and advancing it once + /// the publish is at the node, which holds it until the test lets it + /// answer, rather than by picking a deadline and hoping the signing came + /// first: the slashing-protection write runs on the blocking pool and can + /// take longer than any margin on a loaded machine. #[tokio::test] - async fn a_blinded_publish_survives_a_dropped_proposal() { + async fn a_blinded_publish_past_the_deadline_still_goes_out() { let (store, pubkey) = store(); let relay_answers = Arc::new(tokio::sync::Semaphore::new(0)); let node = Arc::new( @@ -1099,29 +1308,30 @@ mod tests { let at_the_node = node.blinded_publish_entered.clone(); let this_duty = duty(&pubkey, 7, slot()); + let hour = std::time::Duration::from_secs(3600); + let proposal = service.propose_by(slot(), &this_duty, &store, Instant::now() + hour); + tokio::pin!(proposal); tokio::select! { - outcome = service.propose(slot(), &this_duty, &store) => { + outcome = &mut proposal => { panic!("the node is still waiting on the relay, yet the proposal ended: {outcome:?}") } () = at_the_node.notified() => {} } - // The proposal future is dropped here, as the budget drops it. - assert!(service.signed_blinded_at(slot())); + tokio::time::pause(); + tokio::time::advance(hour * 2).await; + let outcome = proposal + .await + .expect("past the deadline is not an error once signed"); + tokio::time::resume(); + assert_eq!(outcome, Proposed::Blinded(BlindedOutcome::StillPublishing)); assert!(node.published_blinded.lock().expect("lock").is_empty()); relay_answers.add_permits(1); - let published = async { - while node.published_blinded.lock().expect("lock").is_empty() { - tokio::time::sleep(std::time::Duration::from_millis(1)).await; - } - }; - tokio::time::timeout(std::time::Duration::from_secs(5), published) - .await - .expect("the publish must complete although the proposal future is gone"); + until_published_blinded(&node).await; assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); - // Slashing protection recorded the slot before the publish started, so a retry - // after the dropped future is refused without asking for a block. + // Slashing protection recorded the slot before the publish started, so + // a retry is refused without asking for a block. let err = service .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await @@ -1130,6 +1340,140 @@ mod tests { assert_eq!(node.block_requests().len(), 1); } + /// The duty loop's entry point, with the deadline far off: a full block + /// and a blinded one both come back confirmed, as from `propose`. + #[tokio::test] + async fn a_proposal_within_its_deadline_returns_what_the_node_said() { + let hour = std::time::Duration::from_secs(3600); + + let (full_store, full_key) = store(); + let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); + let full = builder_service(node.clone(), None) + .propose_by( + slot(), + &duty(&full_key, 7, slot()), + &full_store, + Instant::now() + hour, + ) + .await + .expect("proposes"); + assert_eq!(full, Proposed::Full(Published::Imported)); + assert_eq!(node.published_blocks().len(), 1); + + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + let blinded = builder_service(node.clone(), None) + .propose_by( + slot(), + &duty(&pubkey, 7, slot()), + &store, + Instant::now() + hour, + ) + .await + .expect("proposes"); + assert_eq!( + blinded, + Proposed::Blinded(BlindedOutcome::Published(Published::HandedToBuilder)) + ); + assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); + } + + /// A definite refusal that arrives within the deadline is the proposal's + /// error, for the duty loop to report, not something the task kept. + #[tokio::test] + async fn a_blinded_refusal_within_the_deadline_is_the_proposals_error() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_refused_blinded_publish(400), + ); + let hour = std::time::Duration::from_secs(3600); + let result = builder_service(node, None) + .propose_by( + slot(), + &duty(&pubkey, 7, slot()), + &store, + Instant::now() + hour, + ) + .await; + assert!( + matches!(result, Err(Error::BeaconNodeStatus { status: 400, .. })), + "got {result:?}" + ); + } + + /// A deadline that passes before anything is signed abandons the proposal: + /// an error, and nothing published. The node is still building the block + /// when the clock is moved past the deadline. + #[tokio::test] + async fn a_proposal_past_its_deadline_before_signing_is_abandoned() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_produce_gate(Arc::new(tokio::sync::Semaphore::new(0))), + ); + let service = builder_service(node.clone(), None); + + let building = node.produce_entered.clone(); + let this_duty = duty(&pubkey, 7, slot()); + let hour = std::time::Duration::from_secs(3600); + let proposal = service.propose_by(slot(), &this_duty, &store, Instant::now() + hour); + tokio::pin!(proposal); + tokio::select! { + outcome = &mut proposal => panic!("the node is still building, yet: {outcome:?}"), + () = building.notified() => {} + } + tokio::time::pause(); + tokio::time::advance(hour * 2).await; + let err = proposal.await.expect_err("abandoned at the deadline"); + tokio::time::resume(); + assert!( + matches!(err, Error::ProposalDeadline { slot: at } if at == slot()), + "got {err:?}" + ); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + assert!(node.published_blocks().is_empty()); + } + + /// A caller that drops the proposal while the node waits on the relay does + /// not cancel the publish: it runs on its own task, which reports the + /// outcome itself once nobody is waiting for it. + #[tokio::test] + async fn a_blinded_publish_survives_a_dropped_proposal() { + let (store, pubkey) = store(); + let relay_answers = Arc::new(tokio::sync::Semaphore::new(0)); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_blinded_publish_gate(relay_answers.clone()), + ); + let service = builder_service(node.clone(), None); + + let at_the_node = node.blinded_publish_entered.clone(); + let this_duty = duty(&pubkey, 7, slot()); + tokio::select! { + outcome = service.propose(slot(), &this_duty, &store) => { + panic!("the node is still waiting on the relay, yet the proposal ended: {outcome:?}") + } + () = at_the_node.notified() => {} + } + // The proposal future is dropped here. + assert!(node.published_blinded.lock().expect("lock").is_empty()); + + relay_answers.add_permits(1); + until_published_blinded(&node).await; + assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); + } + /// A 4xx is a definite refusal: the block was not published, so the /// proposal failed. #[tokio::test] @@ -1162,7 +1506,7 @@ mod tests { .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("proposes"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); assert_eq!(node.block_requests()[0].builder_boost_factor, None); assert!(node.published_blinded.lock().expect("lock").is_empty()); } @@ -1192,7 +1536,6 @@ mod tests { "attempt {attempt}: got {err:?}" ); } - assert!(!service.signed_blinded_at(slot())); assert!(node.published_blinded.lock().expect("lock").is_empty()); assert!(node.published_blocks().is_empty()); } @@ -1218,7 +1561,7 @@ mod tests { .await .expect("proposes"); - let (_, _, body) = node.published_blinded.lock().expect("lock").remove(0); + let (_, body) = node.published_blinded.lock().expect("lock").remove(0); let published = SignedBlindedBeaconBlock::from_ssz_bytes(&body).expect("decodes"); let full_root = empty_block_for(slot(), 7).hash_tree_root(); assert_eq!(published.message.hash_tree_root(), full_root); @@ -1266,7 +1609,6 @@ mod tests { matches!(err, Error::InconsistentResponse(_)), "{what}: got {err:?}" ); - assert!(!service.signed_blinded_at(requested), "{what}"); assert!( node.published_blinded.lock().expect("lock").is_empty(), "{what}" @@ -1304,7 +1646,7 @@ mod tests { .await .expect("a wrong fee recipient must not stop the proposal"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); assert_eq!(node.published_blocks().len(), 1); } @@ -1465,7 +1807,7 @@ mod gloas_tests { .propose(slot(), &duty(&pubkey), &store) .await .expect("proposes"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); let request = node.block_requests().remove(0); assert_eq!(request.fork, ForkName::Gloas); @@ -1663,7 +2005,7 @@ mod gloas_tests { .propose(slot(), &duty(&pubkey), &store) .await .expect("the block was published"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); assert!(node.published_envelopes().is_empty()); } } diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index fc6593929..5c49fdc4f 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -780,10 +780,11 @@ signature, whichever way the node answered. Three departures: (logged at error, counted on `blinded_blocks_unconfirmed_total`) and not as a missed proposal, since the relay may still publish the block. A 4xx, a 503 or a connection never made (refused, or past the client's 3 s connect timeout) - is a missed proposal: the node cannot have handed the block on. The - producing node is recorded with the - block's slot and root, and a publish for any other block is refused, as - Teku does; Lighthouse, Lodestar and Nimbus fail over or broadcast instead. + is a missed proposal: the node cannot have handed the block on. The block + carries the node that produced it (`Producer`, filled in by the fallback), + and the proposal path sends the signed block back to it; Teku pins a blinded + block to its producer too, in a cache keyed by slot and root, while + Lighthouse, Lodestar and Nimbus fail over or broadcast instead. Two status codes are read the wrong way: some of ethlambda's own 500s are definite (no builder configured, the block failing the node's checks) and count as unconfirmed, and a proxy's 503 sent after it forwarded the request @@ -791,10 +792,10 @@ signature, whichever way the node answered. Three departures: - **A blinded block stays signed but undelivered for longer.** The node waits up to 5 s on the relay and the client up to 8 s on the node, against well under a second for a full block. The publish runs on its own task so the - proposal budget cannot cut it off, but the process handles no signals: a - SIGINT in that window ends the task with the request. The record of the - latest blinded block (`signed_blinded_at`) is one per process, not per - validator, since one slot has one proposer. + proposal budget cannot cut it off: past the budget the proposal returns + "still publishing" and the task reports the outcome itself. The process + handles no signals, though: a SIGINT in that window ends the task with the + request. - **Registrations stop at gloas.** The client sends no `register_validator` from the gloas fork epoch on: the builder market works through proposer preferences there, and `produceBlockV4` keeps its self-build configuration. From 285f3ac6f425ba4265f09f6f544035e865bfb44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:52:04 -0300 Subject: [PATCH 17/19] fix(validator): mark a cut error text; give a delayed registration a number From the third review of the builder proposals flow. `one_line` ends a text it cuts in `...`, counted within the cap, so a truncated error body no longer reads as all the node or relay said. A text that fits exactly stays unmarked, and the cut still never splits a character or an escape: it backs up to the last whole one that leaves the marker room. The validator's own `error_text` had the marker before the shared function replaced it; the beacon node's builder client gains it too. The registration module doc said a per-slot send can wait "several slots" behind a full send. It now says how long: at most 12 s per `REGISTRATION_BATCH` validators, and about 100 s (eight slots) for 5,000 validators against an ethlambda node whose relay does not answer, which replies to each batch with a 500 after its own 10 s and is still sent the rest. --- crates/common/types/src/text.rs | 71 ++++++++++++++----- .../rpc/src/beacon/builder_client/tests.rs | 12 ++-- crates/validator/src/beacon_node/http.rs | 28 ++++---- crates/validator/src/registration.rs | 20 ++++-- 4 files changed, 88 insertions(+), 43 deletions(-) diff --git a/crates/common/types/src/text.rs b/crates/common/types/src/text.rs index b67c32ad1..8328a88f9 100644 --- a/crates/common/types/src/text.rs +++ b/crates/common/types/src/text.rs @@ -1,5 +1,8 @@ //! Text from a remote peer, made safe to put in a log line. +/// What a cut text ends in. +const CUT: &str = "..."; + /// `text` with every character that could break or restyle a line written /// out, cut to at most `max_bytes` bytes. /// @@ -13,22 +16,34 @@ /// Everything else passes through unchanged, quotes and backslashes included. /// /// The cap is applied to the escaped text, and never cuts a character or an -/// escape in two. The walk stops at the cap, so a large body costs no more +/// escape in two. A text that does not fit ends in `...`, within the cap, so +/// a cut one does not read as all that was sent; one that fits exactly is +/// left unmarked. The walk stops at the cap, so a large body costs no more /// than the text it keeps. pub fn one_line(text: &str, max_bytes: usize) -> String { let mut written = String::with_capacity(text.len().min(max_bytes)); + // What `written` is cut back to if the text does not fit: the longest + // part of it that leaves room for the marker. + let mut kept_if_cut = 0; for c in text.chars() { - if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { - let escaped = c.escape_default(); - if written.len() + escaped.len() > max_bytes { - break; - } - written.extend(escaped); - } else { - if written.len() + c.len_utf8() > max_bytes { - break; - } - written.push(c); + let breaks_the_line = c.is_control() || matches!(c, '\u{2028}' | '\u{2029}'); + let escaped = breaks_the_line.then(|| c.escape_default()); + let len = escaped + .as_ref() + .map_or(c.len_utf8(), ExactSizeIterator::len); + if written.len() + len > max_bytes { + written.truncate(kept_if_cut); + // All of the marker, unless the cap is smaller than the marker. + let room = max_bytes - kept_if_cut; + written.push_str(&CUT[..room.min(CUT.len())]); + break; + } + match escaped { + Some(escaped) => written.extend(escaped), + None => written.push(c), + } + if written.len() + CUT.len() <= max_bytes { + kept_if_cut = written.len(); } } written @@ -47,24 +62,42 @@ mod tests { ); } + /// A cut text ends in the marker, counted within the cap; a text that + /// fits exactly is not marked. + #[test] + fn a_cut_text_ends_in_the_marker() { + let text = "x".repeat(1024); + assert_eq!(one_line(&text, 1024), text); + let text = "x".repeat(1025); + assert_eq!(one_line(&text, 1024), format!("{}...", "x".repeat(1021))); + assert_eq!(one_line("short", 1024), "short"); + // A cap smaller than the marker keeps as much of it as fits. + assert_eq!(one_line("abcdef", 2), ".."); + } + #[test] fn the_cap_never_cuts_an_escape_in_two() { - // 1020 bytes, then a six-byte escape that would end past the cap. - let text = format!("{}\x1b", "x".repeat(1020)); - assert_eq!(one_line(&text, 1024), "x".repeat(1020)); + // 1018 bytes, then a six-byte escape that ends right at the cap. let text = format!("{}\x1b", "x".repeat(1018)); assert_eq!( one_line(&text, 1024), format!("{}\\u{{1b}}", "x".repeat(1018)) ); + // A byte more and the escape does not fit: it is left out whole, not + // cut to `\u{` ahead of the marker. + let text = format!("{}\x1b", "x".repeat(1019)); + assert_eq!(one_line(&text, 1024), format!("{}...", "x".repeat(1019))); + // The escape fits but leaves the marker no room once the next + // character does not: the cut backs up over the whole escape. + let text = format!("{}\x1by", "x".repeat(1018)); + assert_eq!(one_line(&text, 1024), format!("{}...", "x".repeat(1018))); } #[test] fn the_cap_never_cuts_a_character_in_two() { - // 'é' is two bytes, so 1024 bytes hold 512 of them and no half. + // 'é' is two bytes. 1023 bytes hold 510 of them and the marker, and + // never half of a 511th. let text = "é".repeat(1024); - let written = one_line(&text, 1023); - assert_eq!(written, "é".repeat(511)); - assert_eq!(one_line("short", 1024), "short"); + assert_eq!(one_line(&text, 1023), format!("{}...", "é".repeat(510))); } } diff --git a/crates/net/rpc/src/beacon/builder_client/tests.rs b/crates/net/rpc/src/beacon/builder_client/tests.rs index b5c26e94a..6c006ec26 100644 --- a/crates/net/rpc/src/beacon/builder_client/tests.rs +++ b/crates/net/rpc/src/beacon/builder_client/tests.rs @@ -480,11 +480,12 @@ async fn a_long_error_is_cut_short() { let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { panic!("expected a bad request"); }; - assert_eq!(message.len(), 1024); + assert_eq!(message, format!("{}...", "x".repeat(1021))); } /// Line breaks are written out, so a builder's text cannot start a log line of -/// its own, and the escaped text is what the length cap applies to. +/// its own, and the escaped text is what the length cap applies to: 510 of +/// the two-byte escapes, then the marker. #[tokio::test] async fn line_breaks_in_an_error_are_written_out() { let relay = MockRelay::start(GENESIS_FORK_VERSION).await; @@ -500,7 +501,7 @@ async fn line_breaks_in_an_error_are_written_out() { let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { panic!("expected a bad request"); }; - assert_eq!(message.len(), 1024); + assert_eq!(message, format!("{}...", r"\n".repeat(510))); } /// Every other control character is written out too (an ESC would start an @@ -534,9 +535,10 @@ async fn control_characters_in_an_error_are_written_out() { /// The cap never leaves half an escape behind. #[test] fn the_cap_never_cuts_an_escape_in_two() { - // 1020 bytes, then a six-byte escape that would end past the cap. + // 1020 bytes, then a six-byte escape that would end past the cap: it is + // left out whole, and the marker ends the text. let text = format!("{}\x1b", "x".repeat(1020)); - assert_eq!(super::error_text(text), "x".repeat(1020)); + assert_eq!(super::error_text(text), format!("{}...", "x".repeat(1020))); let text = format!("{}\x1b", "x".repeat(1018)); assert_eq!( super::error_text(text), diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index ca4425cf9..d5dbc4c4f 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -107,7 +107,8 @@ async fn read_error_body(mut response: reqwest::Response) -> String { /// Text a node sent, as an error or a log line shows it: control characters /// and line separators written out (`\n`, `\u{1b}`, ...), so the line it lands /// in stays one line and carries no terminal escapes, and cut to -/// [`ERROR_TEXT_LIMIT`] bytes without splitting a character or an escape. +/// [`ERROR_TEXT_LIMIT`] bytes without splitting a character or an escape, +/// ending in `...` when it is cut. /// /// The same function the beacon node's builder client writes a relay's text /// through, [`ethlambda_types::text::one_line`]. @@ -1543,9 +1544,9 @@ mod tests { assert!(served.seen.lock().expect("lock").is_empty()); } - /// A node's error body reaches the logs on one line and cut short: a body - /// with line breaks cannot forge log lines, and a long one cannot flood - /// them. + /// A node's error body reaches the logs on one line and cut short, saying + /// so: a body with line breaks cannot forge log lines, a long one cannot + /// flood them, and a cut one does not read as all the node said. #[tokio::test] async fn an_error_body_is_one_line_and_capped() { let reply = "first line\nsecond line\r\n".repeat(200).into_bytes(); @@ -1555,23 +1556,24 @@ mod tests { panic!("got {err:?}"); }; assert_eq!(status, 400); - assert!(body.starts_with("first line\\nsecond line\\r\\nfirst line")); assert!(!body.contains('\n') && !body.contains('\r')); - assert!(body.len() <= ERROR_TEXT_LIMIT, "{} bytes", body.len()); - assert!( - body.len() > ERROR_TEXT_LIMIT - "\\r".len(), - "filled to the cap" - ); + // Written out, then cut to the cap less the marker's three bytes, + // which here falls inside a word. + let written = "first line\\nsecond line\\r\\n".repeat(200); + let kept = ERROR_TEXT_LIMIT - "...".len(); + assert_eq!(body, format!("{}...", &written[..kept])); } /// The cap applies to the written-out text and never splits a character - /// or an escape: a body of escapes ends on a whole one. + /// or an escape: a body of escapes keeps whole ones, then the marker. #[test] fn error_text_cuts_on_a_character_or_escape_boundary() { let text = error_text(&"é".repeat(ERROR_TEXT_LIMIT)); - assert_eq!(text, "é".repeat(ERROR_TEXT_LIMIT / 2)); + let room = ERROR_TEXT_LIMIT - "...".len(); + assert_eq!(text, format!("{}...", "é".repeat(room / 2))); let text = error_text(&"\u{1b}".repeat(ERROR_TEXT_LIMIT)); - assert_eq!(text, "\\u{1b}".repeat(ERROR_TEXT_LIMIT / "\\u{1b}".len())); + let escapes = "\\u{1b}".repeat(room / "\\u{1b}".len()); + assert_eq!(text, format!("{escapes}...")); assert_eq!(error_text("short"), "short"); } diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index 6fd5d8560..7bca9f2ed 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -65,12 +65,20 @@ //! //! A per-slot send that finds the lock taken skips: it has not marked its //! change as sent, so a later slot picks it up. While a full send runs, that -//! is the first slot after it ends, which against a node that is slow to -//! answer each batch can be several slots on. A full send waits for it -//! instead, since a per-slot send is short and skipping would cost an epoch. -//! A full send that finds the previous full send still running, or waiting, -//! skips to the next epoch (see [`crate::in_flight`]), so a hung node cannot -//! queue up one waiting send per epoch. +//! is the first slot after it ends, and a full send lasts as long as its +//! slowest node takes over every batch. A node has 12 s to answer each one +//! (one that takes longer is treated as unreachable and sent no more), so the +//! wait is at most 12 s per [`REGISTRATION_BATCH`] validators. An ethlambda +//! node whose relay does not answer comes close: it replies to each batch +//! with a 500 once its own 10 s wait on the relay runs out, and is still sent +//! the rest. With 5,000 validators, ten batches, a keymanager change then +//! waits about 100 s, eight slots, and two minutes at most. +//! +//! A full send waits for a running per-slot send instead, since that send is +//! short and skipping would cost an epoch. A full send that finds the +//! previous full send still running, or waiting, skips to the next epoch (see +//! [`crate::in_flight`]), so a hung node cannot queue up one waiting send per +//! epoch. //! //! # Signing stays off the async runtime //! From 99f899ea4514595fc98d515d6961147085d7c2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:52:14 -0300 Subject: [PATCH 18/19] fix(validator): follow-ups from the bots' third review - A signing chunk whose blocking task fails no longer lets `signed_generation` advance. The keys in it were left out of the send, so a keymanager change among them waited for the next epoch; the generation now stays behind and the next slot's per-slot send signs them. - `read_error_body` copies only what fits under its limit out of each chunk, instead of extending by the whole chunk and truncating after. - `--prefer-builder-proposals`' help said the execution client sends `builder_boost_factor = u64::MAX`. This client sends it; the execution client's say is `shouldOverrideBuilder`, which the beacon node still honours. --- bin/ethlambda/src/validator.rs | 7 ++++--- crates/validator/src/beacon_node/http.rs | 15 +++++++------- crates/validator/src/registration.rs | 25 +++++++++++++++++++++--- docs/cli.md | 2 +- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/bin/ethlambda/src/validator.rs b/bin/ethlambda/src/validator.rs index 3efbfc03b..9fb1d05e7 100644 --- a/bin/ethlambda/src/validator.rs +++ b/bin/ethlambda/src/validator.rs @@ -98,9 +98,10 @@ pub(crate) struct ValidatorOptions { )] pub(crate) builder_boost_factor: u64, - /// Take the builder's block whenever there is a valid bid, unless the - /// execution client asks to override the builder (sends - /// `builder_boost_factor = u64::MAX`). + /// Take the builder's block whenever there is a valid bid: this client + /// sends `builder_boost_factor = u64::MAX`. The beacon node still keeps + /// its local payload when its execution client sets + /// `shouldOverrideBuilder`. #[arg(long, default_value = "false", requires = "builder_proposals")] pub(crate) prefer_builder_proposals: bool, diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index d5dbc4c4f..91f5609cb 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -91,16 +91,17 @@ const ERROR_BODY_READ_LIMIT: usize = 1024 * 1024; const ERROR_TEXT_LIMIT: usize = 1024; /// A failed answer's body, read up to [`ERROR_BODY_READ_LIMIT`] bytes, so a -/// node answering with an endless body cannot hold the client on it. +/// node answering with an endless body cannot hold the client on it. Only +/// what fits is copied out of each chunk. async fn read_error_body(mut response: reqwest::Response) -> String { let mut bytes = Vec::new(); - while bytes.len() < ERROR_BODY_READ_LIMIT { - match response.chunk().await { - Ok(Some(chunk)) => bytes.extend_from_slice(&chunk), - _ => break, + while let Ok(Some(chunk)) = response.chunk().await { + let room = ERROR_BODY_READ_LIMIT - bytes.len(); + bytes.extend_from_slice(&chunk[..chunk.len().min(room)]); + if bytes.len() == ERROR_BODY_READ_LIMIT { + break; } } - bytes.truncate(ERROR_BODY_READ_LIMIT); String::from_utf8_lossy(&bytes).into_owned() } @@ -1568,8 +1569,8 @@ mod tests { /// or an escape: a body of escapes keeps whole ones, then the marker. #[test] fn error_text_cuts_on_a_character_or_escape_boundary() { - let text = error_text(&"é".repeat(ERROR_TEXT_LIMIT)); let room = ERROR_TEXT_LIMIT - "...".len(); + let text = error_text(&"é".repeat(ERROR_TEXT_LIMIT)); assert_eq!(text, format!("{}...", "é".repeat(room / 2))); let text = error_text(&"\u{1b}".repeat(ERROR_TEXT_LIMIT)); let escapes = "\\u{1b}".repeat(room / "\\u{1b}".len()); diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index 7bca9f2ed..35ddf8803 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -126,6 +126,9 @@ struct Current { changed: Vec, /// Validators skipped because no fee recipient is configured for them. no_fee_recipient: usize, + /// Chunks whose signing task failed, so whose validators are in neither + /// list. + failed_chunks: usize, } impl Current { @@ -133,6 +136,7 @@ impl Current { self.all.extend(other.all); self.changed.extend(other.changed); self.no_fee_recipient += other.no_fee_recipient; + self.failed_chunks += other.failed_chunks; } } @@ -213,6 +217,7 @@ impl RegistrationService { all: Vec::with_capacity(pubkeys.len()), changed: Vec::new(), no_fee_recipient: 0, + failed_chunks: 0, }; for pubkey in pubkeys { let Some((fee_recipient, gas_limit)) = @@ -277,6 +282,7 @@ impl RegistrationService { all: Vec::with_capacity(pubkeys.len()), changed: Vec::new(), no_fee_recipient: 0, + failed_chunks: 0, }; for chunk in pubkeys.chunks(SIGN_CHUNK) { let (service, store, chunk) = (self.clone(), store.clone(), chunk.to_vec()); @@ -287,12 +293,25 @@ impl RegistrationService { .await; match part { Ok(part) => current.extend(part), - Err(err) => warn!(%err, "Signing builder registrations failed; skipping a chunk"), + Err(err) => { + warn!(%err, "Signing builder registrations failed; skipping a chunk"); + current.failed_chunks += 1; + } } } current } + /// Record that every setting up to `generation` is signed, unless a chunk + /// of `current` failed to sign. Then the generation stays behind, so the + /// next slot's [`Self::register_changed`] signs what that chunk held + /// rather than leaving a keymanager change in it unsent for an epoch. + fn mark_signed(&self, generation: u64, current: &Current) { + if current.failed_chunks == 0 { + self.signed_generation.store(generation, Ordering::Release); + } + } + /// Register every validator in `pubkeys`, the ones the beacon node resolved /// at this epoch's refresh. Called once per epoch. /// @@ -331,7 +350,7 @@ impl RegistrationService { .unwrap_or_else(std::sync::PoisonError::into_inner) = pubkeys.clone(); let current = self.current_blocking(&store, &pubkeys).await; - self.signed_generation.store(generation, Ordering::Release); + self.mark_signed(generation, ¤t); if current.no_fee_recipient > 0 { warn!( skipped = current.no_fee_recipient, @@ -382,7 +401,7 @@ impl RegistrationService { return; } let current = self.current_blocking(&store, &known).await; - self.signed_generation.store(generation, Ordering::Release); + self.mark_signed(generation, ¤t); if current.changed.is_empty() { return; } diff --git a/docs/cli.md b/docs/cli.md index 6b8c81247..d0077fc18 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -302,7 +302,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | | `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | | `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` prefers the local payload unless it cannot be built. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | -| `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid, unless the execution client asks to override the builder (sends `builder_boost_factor = u64::MAX`). Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | +| `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid: this client sends `builder_boost_factor = u64::MAX`. The beacon node still keeps its local payload when its execution client sets `shouldOverrideBuilder`. Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | | `--init-slashing-protection` | off | Register every loaded key with no slashing-protection record as never having signed, instead of refusing to start. Dangerous: only for keys that have signed nothing anywhere for at least two epochs, and even then it rules out a double vote only, not a surround of an older vote; a key with a history should bring it instead. Never lowers an existing record; remove it once the keys are registered, and startup warns while it is set for nothing | ### What it does today From 6fd0fa5d96eb566f6a3e2eb34798fac29c09b35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:07:09 -0300 Subject: [PATCH 19/19] fix(validator): re-send a refused per-slot registration every slot From the other review comments on the builder proposals flow. A per-slot send carries a keymanager fee-recipient or gas-limit change. When no node accepted it, the change waited for the next epoch's full send, and blinded blocks meanwhile paid the old address. The keys in a refused batch are now kept in `unsent` and their registrations sent again, as signed, at every slot until a node takes them: nothing is re-signed, so timestamps stay where they were. `signed_generation` stays behind while any are waiting, so the per-slot check is still one atomic read. Every full send ends the retry, delivered or not. It carries those keys too, and one it could not deliver waits for the next epoch's full send with the rest of the set, so a node that keeps refusing costs a warning a slot for an epoch at most. A failed full send still waits for the next epoch, since re-sending every key each slot would load a node that is already failing. Also: - A blinded block's debug line names the header's fee recipient and the configured one, so a log shows afterwards who built the block and where this client asked to be paid. - `docs/cli.md`: a keymanager `DELETE` that leaves a validator with no fee recipient sends nothing, so the builder keeps the last registration it was given. --- crates/validator/src/proposal.rs | 9 ++ crates/validator/src/registration.rs | 207 ++++++++++++++++++++++++--- docs/cli.md | 4 +- 3 files changed, 198 insertions(+), 22 deletions(-) diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index efab3af75..00dcfb21f 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -440,8 +440,17 @@ impl ProposalService { if Self::compares_fee_recipient(&produced) { Self::check_fee_recipient(&produced, fee_recipient, slot); } else { + // Both addresses, so a log can say afterwards who built the block + // and where this client had asked to be paid. + let header = crate::beacon_node::dto::encode_hex(&produced.fee_recipient().0); + let configured = fee_recipient.map_or_else( + || "none".to_string(), + |address| crate::beacon_node::dto::encode_hex(&address.0), + ); tracing::debug!( %slot, + header_fee_recipient = %header, + configured_fee_recipient = %configured, "Blinded block: its header names the builder's address; the relay pays the \ validator in the payload" ); diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index 35ddf8803..3f5b2cc46 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -33,9 +33,18 @@ //! # Failures are never propagated //! //! A missed registration costs at most a builder block, never a duty, so a -//! failed send is logged and counted and the caller moves on. A changed -//! registration whose send fails is cached anyway, so it is not re-sent at the -//! next slot: it waits for the next epoch's full send. +//! failed send is logged and counted and the caller moves on. +//! +//! A per-slot send that no node accepts is sent again at every slot until one +//! does: it carries a keymanager change, and until a node takes it the builder +//! keeps paying the old fee recipient. The registrations are re-sent as they +//! were signed, so nothing is re-signed and their timestamps stay put, and +//! [`RegistrationService::signed_generation`] stays behind meanwhile, which +//! keeps the per-slot check below one atomic read. A full send that fails +//! waits for the next epoch's instead: re-sending every key each slot would +//! load a node that is already failing, and the next full send carries them +//! all anyway. A full send ends every per-slot retry, delivered or not, so a +//! node that keeps refusing costs a warning a slot for an epoch at most. //! //! A batch a node refuses with a 400 is not split to find the entry it //! objects to: every entry is signed here by keys this client holds, so a @@ -174,8 +183,14 @@ pub struct RegistrationService { /// enough, and a signing chunk holding it keeps no one else waiting. signed: std::sync::Mutex>, /// The [`ProposerSettings::generation`] the cache was last brought up to - /// date against. + /// date against. Held behind while [`Self::unsent`] is not empty, so the + /// next per-slot call retries it. signed_generation: AtomicU64, + /// Validators whose newest registration a per-slot send could not get any + /// node to accept: sent again, as signed, at every slot until one does or + /// the next full send, which carries them anyway, ends the retry. Taken + /// under [`Self::sending`] only. + unsent: std::sync::Mutex>, /// Held by a send from before it reads the cache until its last batch is /// answered, so sends reach every node in the order they read the cache. /// See the module doc. @@ -197,6 +212,7 @@ impl RegistrationService { known: std::sync::Mutex::new(Vec::new()), signed: std::sync::Mutex::new(HashMap::new()), signed_generation: AtomicU64::new(0), + unsent: std::sync::Mutex::new(HashSet::new()), sending: Mutex::new(()), full_send: InFlight::default(), } @@ -302,16 +318,26 @@ impl RegistrationService { current } - /// Record that every setting up to `generation` is signed, unless a chunk - /// of `current` failed to sign. Then the generation stays behind, so the - /// next slot's [`Self::register_changed`] signs what that chunk held - /// rather than leaving a keymanager change in it unsent for an epoch. - fn mark_signed(&self, generation: u64, current: &Current) { - if current.failed_chunks == 0 { + /// Record that every setting up to `generation` is signed and sent, + /// unless a chunk of the signing failed (`failed_chunks`) or a registration + /// is waiting to be sent again ([`Self::unsent`]). Then the generation + /// stays behind, so the next slot's [`Self::register_changed`] signs what + /// that chunk held, or re-sends what no node took, rather than leaving a + /// keymanager change unsent for an epoch. + fn mark_signed(&self, generation: u64, failed_chunks: usize) { + if failed_chunks == 0 && self.unsent().is_empty() { self.signed_generation.store(generation, Ordering::Release); } } + /// [`Self::unsent`], locked. A poisoned lock is recovered: every write is + /// one set operation. + fn unsent(&self) -> std::sync::MutexGuard<'_, HashSet> { + self.unsent + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + /// Register every validator in `pubkeys`, the ones the beacon node resolved /// at this epoch's refresh. Called once per epoch. /// @@ -350,7 +376,6 @@ impl RegistrationService { .unwrap_or_else(std::sync::PoisonError::into_inner) = pubkeys.clone(); let current = self.current_blocking(&store, &pubkeys).await; - self.mark_signed(generation, ¤t); if current.no_fee_recipient > 0 { warn!( skipped = current.no_fee_recipient, @@ -358,12 +383,19 @@ impl RegistrationService { ); } self.send(¤t.all).await; + // This send carried every key, so it ends each per-slot retry: a key it + // delivered needs no more, and one it did not waits for the next + // epoch's full send with the rest of its batch. + self.unsent().clear(); + self.mark_signed(generation, current.failed_chunks); } /// Send only the registrations whose fee recipient or gas limit changed /// since they were last signed, over the validators the last - /// [`Self::register_all`] was given. Called every slot, so a keymanager - /// change reaches the builder within a slot rather than an epoch. + /// [`Self::register_all`] was given, plus any an earlier slot could not + /// get a node to accept ([`Self::unsent`]). Called every slot, so a + /// keymanager change reaches the builder within a slot rather than an + /// epoch. /// /// Returns before touching the store when no setting was written since the /// last signing: see the module doc. @@ -401,24 +433,44 @@ impl RegistrationService { return; } let current = self.current_blocking(&store, &known).await; - self.mark_signed(generation, ¤t); - if current.changed.is_empty() { - return; + // What changed, and what an earlier slot could not get any node to + // take. Both are read off `all`, the newest registration per key, so a + // key that changed again since its failed send goes out once, as now. + let retry = std::mem::take(&mut *self.unsent()); + let changed: HashSet = current + .changed + .iter() + .map(|registration| registration.message.pubkey) + .collect(); + let outgoing: Vec = current + .all + .into_iter() + .filter(|registration| { + let pubkey = ®istration.message.pubkey; + changed.contains(pubkey) || retry.contains(pubkey) + }) + .collect(); + if !outgoing.is_empty() { + let failed = self.send(&outgoing).await; + self.unsent().extend(failed); } - self.send(¤t.changed).await; + self.mark_signed(generation, current.failed_chunks); } /// Send `registrations` to the beacon node in batches, counting each batch /// as sent when a node accepted it and as a failure when none did. - async fn send(&self, registrations: &[SignedValidatorRegistrationV1]) { + /// + /// Returns the validators in the batches no node accepted. + async fn send(&self, registrations: &[SignedValidatorRegistrationV1]) -> HashSet { let outcomes = self .beacon_node .register_validators_in_batches(registrations) .await; + let mut failed = HashSet::new(); for (batch, outcome) in registrations.chunks(REGISTRATION_BATCH).zip(outcomes) { let count = batch.len(); // Only logged and counted: see "Failures are never propagated". - let _ = outcome + let refused = outcome .inspect(|()| { debug!(count, "Builder registrations sent"); crate::metrics::inc_builder_registrations_sent(count); @@ -426,8 +478,13 @@ impl RegistrationService { .inspect_err(|err| { warn!(count, %err, "No beacon node accepted the builder registrations"); crate::metrics::inc_builder_registration_failures(); - }); + }) + .is_err(); + if refused { + failed.extend(batch.iter().map(|registration| registration.message.pubkey)); + } } + failed } } @@ -809,6 +866,116 @@ mod tests { assert_eq!(registrations[3].message.gas_limit, 30_000_000); } + fn refuse_registrations(node: &MockBeaconNode) { + node.failures + .lock() + .expect("lock") + .insert("register_validators", "relay down".to_string()); + } + + fn registration_calls(node: &MockBeaconNode) -> usize { + node.registration_attempts + .load(std::sync::atomic::Ordering::SeqCst) + } + + /// A change no node accepted goes out again at every slot, as it was + /// signed, until a node takes it; the slot after that sends nothing. + #[tokio::test] + async fn a_change_no_node_accepted_is_sent_again_until_one_does() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + refuse_registrations(&node); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + service.clone().register_changed(0, store.clone()).await; + service.clone().register_changed(0, store.clone()).await; + assert_eq!(registration_calls(&node), 3, "refused, then tried again"); + + node.stop_failing("register_validators"); + service.clone().register_changed(0, store.clone()).await; + let signed = service.signed.lock().expect("lock")[&pubkeys[1]].clone(); + { + let registrations = node.registrations.lock().expect("lock"); + assert_eq!( + registrations.len(), + 3, + "the full send's two, then the change" + ); + assert_eq!(registrations[2].message.gas_limit, 30_000_000); + assert_eq!( + registrations[2], signed, + "re-sent as signed, not signed again" + ); + } + + service.clone().register_changed(0, store.clone()).await; + assert_eq!(registration_calls(&node), 4, "delivered, so nothing more"); + } + + /// A full send that delivers a key waiting for its retry ends the retry, + /// and the slots after it go back to sending nothing. + #[tokio::test] + async fn a_full_send_that_delivers_a_waiting_change_ends_its_retry() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + refuse_registrations(&node); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + service.clone().register_changed(0, store.clone()).await; + + node.stop_failing("register_validators"); + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()) + .await; + service.clone().register_changed(1, store.clone()).await; + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2, 2], + "the two full sends, and no retry after the second" + ); + } + + /// A full send ends the retry even when it is refused too: the change then + /// waits for the next epoch's full send with the rest of the set, rather + /// than costing a refused send at every slot for as long as the node + /// refuses. + #[tokio::test] + async fn a_refused_full_send_ends_the_retry_too() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + refuse_registrations(&node); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + service.clone().register_changed(0, store.clone()).await; + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()) + .await; + assert_eq!(registration_calls(&node), 3, "full, per-slot, full"); + + service.clone().register_changed(1, store.clone()).await; + assert_eq!(registration_calls(&node), 3, "no retry after the full send"); + } + /// The per-slot call is one atomic read when no setting moved: with the /// store write-locked, so any read of it would wait forever, it still /// returns at once, and sends nothing. diff --git a/docs/cli.md b/docs/cli.md index d0077fc18..b01b9243c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -446,9 +446,9 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running), and an ethlambda beacon node takes it from there | +| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no registration, so the builder keeps paying the address it was last given | | `graffiti` | `--graffiti` | The next proposal | -| `gas_limit` | `DEFAULT_GAS_LIMIT` | With `--builder-proposals`, the next builder registration, which the builder builds toward: usually at the next slot, later while the epoch's full registration send is still running. Otherwise stored and served only | +| `gas_limit` | `DEFAULT_GAS_LIMIT` | With `--builder-proposals`, the next builder registration, which the builder builds toward: usually at the next slot, later while the epoch's full registration send is still running, and again every slot until a node accepts it. Otherwise stored and served only | **These are held in memory only.** A restart puts every validator back on the defaults, so an operator driving these endpoints must set them again after one.