diff --git a/CLAUDE.md b/CLAUDE.md index 482ec53d2..58798eb3a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -862,11 +862,14 @@ transitions are in `ethlambda-types`, per the section above. Nothing above and a block at slot N packs only `(N-1, parent_root)`, replaced by the empty aggregate when `verified_sync_aggregate` fails (see `docs/spec_deviations.md`). `SlotClock` picks the - offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. It + offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. With + `--builder-proposals`, pre-gloas slots register validators with the node's builder + and may sign a blinded block, published to the producing node only. It keeps EIP-3076 minimal-watermark slashing protection in `--validators-dir/slashing_protection.json` (`crate::slashing_protection`), - durable before every block and attestation signature; PTC votes have an - in-memory `(validator, slot)` dedup only, and envelopes are unguarded. + durable before every block (blinded ones included) and attestation signature; + PTC votes have an in-memory `(validator, slot)` dedup only, and envelopes are + unguarded. `get_head_node` is one bottom-up walk for every fork, with the current slot's fork selecting the payload rules and the boost gate, so a follower on any network that schedules gloas (Sepolia does) pays work diff --git a/Cargo.lock b/Cargo.lock index 405f49c0f..3080e4952 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2274,6 +2274,7 @@ dependencies = [ "ctr", "ethlambda-metrics", "ethlambda-types", + "futures-util", "hex", "hmac", "http-body-util", diff --git a/README.md b/README.md index 961de5fbe..729dd1c44 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,7 @@ the standard Beacon API, so it works against `ethlambda beacon` (run with the | `--secrets-dir` | One password file per keystore, named after the validator's public key | | `--suggested-fee-recipient` | Execution address for block rewards, `0x`-prefixed. Optional, but without it the beacon node picks an address that is not yours | | `--graffiti` | Text for proposed blocks, at most 32 bytes | +| `--builder-proposals` | Propose builder blocks before gloas: register each validator with the beacon node's builder (`--builder` on an ethlambda node) and sign a blinded block when the builder's bid wins. Off by default | | `--enable-keymanager` | Serve the keymanager API on `--keymanager-port` (default `5062`). Off by default, since it mutates key material | Metrics are served on `--metrics-port` (default `5064`). See diff --git a/bin/ethlambda/src/validator.rs b/bin/ethlambda/src/validator.rs index 968244963..9fb1d05e7 100644 --- a/bin/ethlambda/src/validator.rs +++ b/bin/ethlambda/src/validator.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, H160}; -use ethlambda_validator::ValidatorConfig; use ethlambda_validator::proposer_settings::{GRAFFITI_BYTES, graffiti_from_text}; +use ethlambda_validator::{BuilderProposals, ValidatorConfig}; /// An execution address is exactly this wide. const ADDRESS_BYTES: usize = 20; @@ -79,6 +79,32 @@ pub(crate) struct ValidatorOptions { #[arg(long, default_value = "false")] pub(crate) enable_keymanager: bool, + /// Propose builder blocks: register each validator with the beacon node's + /// builder and accept a blinded block when the builder's bid wins. Requires + /// a beacon node started with a builder (ethlambda: `--builder`). Fulu and + /// earlier only; from gloas the builder market works through proposer + /// preferences instead. + #[arg(long, default_value = "false")] + pub(crate) builder_proposals: bool, + + /// How a builder's bid is weighed against the local payload: the local + /// payload wins iff `local >= factor * bid / 100`. 100 compares them as + /// they are; 0 prefers the local payload unless it cannot be built. + #[arg( + long, + default_value = "100", + requires = "builder_proposals", + conflicts_with = "prefer_builder_proposals" + )] + pub(crate) builder_boost_factor: u64, + + /// Take the builder's block whenever there is a valid bid: this client + /// sends `builder_boost_factor = u64::MAX`. The beacon node still keeps + /// its local payload when its execution client sets + /// `shouldOverrideBuilder`. + #[arg(long, default_value = "false", requires = "builder_proposals")] + pub(crate) prefer_builder_proposals: bool, + /// Register every loaded key that has no slashing-protection record, /// instead of refusing to start. /// @@ -134,6 +160,21 @@ impl ValidatorOptions { Ok(Some(H160(bytes))) } + /// The builder configuration these flags describe, or `None` when builder + /// proposals are off. + /// + /// `--prefer-builder-proposals` wins over the factor by construction: + /// clap refuses the two together, so at most one is ever set. + pub(crate) fn builder(&self) -> Option { + self.builder_proposals.then_some(BuilderProposals { + boost_factor: if self.prefer_builder_proposals { + u64::MAX + } else { + self.builder_boost_factor + }, + }) + } + /// Reject a port clash before anything binds, the way the node's own /// options do. pub(crate) fn validate_ports(&self) -> eyre::Result<()> { @@ -157,6 +198,7 @@ pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> { options.validate_ports()?; let graffiti = options.graffiti()?; let suggested_fee_recipient = options.suggested_fee_recipient()?; + let builder = options.builder(); ethlambda_validator::run(ValidatorConfig { beacon_nodes: options.beacon_nodes, graffiti, @@ -167,6 +209,7 @@ pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> { keymanager: options .enable_keymanager .then(|| std::net::SocketAddr::new(options.http_address, options.keymanager_port)), + builder, init_slashing_protection: options.init_slashing_protection, }) .await @@ -188,10 +231,90 @@ mod tests { graffiti: graffiti.to_string(), suggested_fee_recipient: None, enable_keymanager: false, + builder_proposals: false, + builder_boost_factor: 100, + prefer_builder_proposals: false, init_slashing_protection: false, } } + /// Parse a command line the way the binary does, so `requires` and + /// `conflicts_with` are exercised rather than assumed. + fn parse(extra: &[&str]) -> Result { + use clap::Parser as _; + + #[derive(clap::Parser)] + struct Cli { + #[command(flatten)] + options: ValidatorOptions, + } + let base = [ + "ethlambda", + "--beacon-nodes", + "http://localhost:5052", + "--validators-dir", + "/tmp/v", + "--secrets-dir", + "/tmp/s", + ]; + Cli::try_parse_from(base.iter().chain(extra)).map(|cli| cli.options) + } + + #[test] + fn builder_proposals_are_off_by_default() { + assert_eq!(parse(&[]).expect("parses").builder(), None); + } + + #[test] + fn builder_proposals_default_to_a_factor_of_100() { + let options = parse(&["--builder-proposals"]).expect("parses"); + assert_eq!( + options.builder(), + Some(BuilderProposals { boost_factor: 100 }) + ); + } + + #[test] + fn an_explicit_factor_is_used() { + let options = + parse(&["--builder-proposals", "--builder-boost-factor", "250"]).expect("parses"); + assert_eq!( + options.builder(), + Some(BuilderProposals { boost_factor: 250 }) + ); + } + + #[test] + fn preferring_the_builder_sends_the_maximum_factor() { + let options = + parse(&["--builder-proposals", "--prefer-builder-proposals"]).expect("parses"); + assert_eq!( + options.builder(), + Some(BuilderProposals { + boost_factor: u64::MAX + }) + ); + } + + #[test] + fn the_factor_and_the_preference_need_builder_proposals() { + parse(&["--builder-boost-factor", "250"]) + .expect_err("a factor with builder proposals off is a mistake"); + parse(&["--prefer-builder-proposals"]) + .expect_err("a preference with builder proposals off is a mistake"); + } + + #[test] + fn the_factor_and_the_preference_conflict() { + parse(&[ + "--builder-proposals", + "--builder-boost-factor", + "250", + "--prefer-builder-proposals", + ]) + .expect_err("the two say different things"); + } + #[test] fn the_default_graffiti_is_empty() { let bytes = options("").graffiti().expect("valid"); diff --git a/crates/common/types/src/lib.rs b/crates/common/types/src/lib.rs index ecf92456e..aae73aab8 100644 --- a/crates/common/types/src/lib.rs +++ b/crates/common/types/src/lib.rs @@ -9,6 +9,7 @@ pub mod enr; pub mod genesis; pub mod primitives; pub mod state; +pub mod text; pub mod time; /// Display helper for truncated root hashes (8 hex chars) diff --git a/crates/common/types/src/text.rs b/crates/common/types/src/text.rs new file mode 100644 index 000000000..8328a88f9 --- /dev/null +++ b/crates/common/types/src/text.rs @@ -0,0 +1,103 @@ +//! Text from a remote peer, made safe to put in a log line. + +/// What a cut text ends in. +const CUT: &str = "..."; + +/// `text` with every character that could break or restyle a line written +/// out, cut to at most `max_bytes` bytes. +/// +/// Text a remote server sends (an error body, a relay's message) reaches this +/// node's logs and error messages. There a raw line break would start what +/// reads as a log line of its own, and an ESC would start an ANSI sequence that +/// recolors, moves the cursor in or clears a terminal. So every control +/// character (`char::is_control`: C0, DEL and C1) and the Unicode line and +/// paragraph separators U+2028/U+2029, which some viewers break lines at, are +/// written as Rust escapes: `\n`, `\r` and `\t`, and `\u{1b}` for the rest. +/// Everything else passes through unchanged, quotes and backslashes included. +/// +/// The cap is applied to the escaped text, and never cuts a character or an +/// escape in two. A text that does not fit ends in `...`, within the cap, so +/// a cut one does not read as all that was sent; one that fits exactly is +/// left unmarked. The walk stops at the cap, so a large body costs no more +/// than the text it keeps. +pub fn one_line(text: &str, max_bytes: usize) -> String { + let mut written = String::with_capacity(text.len().min(max_bytes)); + // What `written` is cut back to if the text does not fit: the longest + // part of it that leaves room for the marker. + let mut kept_if_cut = 0; + for c in text.chars() { + let breaks_the_line = c.is_control() || matches!(c, '\u{2028}' | '\u{2029}'); + let escaped = breaks_the_line.then(|| c.escape_default()); + let len = escaped + .as_ref() + .map_or(c.len_utf8(), ExactSizeIterator::len); + if written.len() + len > max_bytes { + written.truncate(kept_if_cut); + // All of the marker, unless the cap is smaller than the marker. + let room = max_bytes - kept_if_cut; + written.push_str(&CUT[..room.min(CUT.len())]); + break; + } + match escaped { + Some(escaped) => written.extend(escaped), + None => written.push(c), + } + if written.len() + CUT.len() <= max_bytes { + kept_if_cut = written.len(); + } + } + written +} + +#[cfg(test)] +mod tests { + use super::one_line; + + #[test] + fn control_characters_and_line_separators_are_written_out() { + let text = "\x1b[31mred\x1b[0m\ttab\x07bell\u{85}nel\u{2028}ls\u{2029}ps \"q\" \\ ok\r\n"; + assert_eq!( + one_line(text, 1024), + r#"\u{1b}[31mred\u{1b}[0m\ttab\u{7}bell\u{85}nel\u{2028}ls\u{2029}ps "q" \ ok\r\n"# + ); + } + + /// A cut text ends in the marker, counted within the cap; a text that + /// fits exactly is not marked. + #[test] + fn a_cut_text_ends_in_the_marker() { + let text = "x".repeat(1024); + assert_eq!(one_line(&text, 1024), text); + let text = "x".repeat(1025); + assert_eq!(one_line(&text, 1024), format!("{}...", "x".repeat(1021))); + assert_eq!(one_line("short", 1024), "short"); + // A cap smaller than the marker keeps as much of it as fits. + assert_eq!(one_line("abcdef", 2), ".."); + } + + #[test] + fn the_cap_never_cuts_an_escape_in_two() { + // 1018 bytes, then a six-byte escape that ends right at the cap. + let text = format!("{}\x1b", "x".repeat(1018)); + assert_eq!( + one_line(&text, 1024), + format!("{}\\u{{1b}}", "x".repeat(1018)) + ); + // A byte more and the escape does not fit: it is left out whole, not + // cut to `\u{` ahead of the marker. + let text = format!("{}\x1b", "x".repeat(1019)); + assert_eq!(one_line(&text, 1024), format!("{}...", "x".repeat(1019))); + // The escape fits but leaves the marker no room once the next + // character does not: the cut backs up over the whole escape. + let text = format!("{}\x1by", "x".repeat(1018)); + assert_eq!(one_line(&text, 1024), format!("{}...", "x".repeat(1018))); + } + + #[test] + fn the_cap_never_cuts_a_character_in_two() { + // 'é' is two bytes. 1023 bytes hold 510 of them and the marker, and + // never half of a 511th. + let text = "é".repeat(1024); + assert_eq!(one_line(&text, 1023), format!("{}...", "é".repeat(510))); + } +} diff --git a/crates/net/rpc/src/beacon/builder_client/mod.rs b/crates/net/rpc/src/beacon/builder_client/mod.rs index fd8ecc5b9..cbfdcded2 100644 --- a/crates/net/rpc/src/beacon/builder_client/mod.rs +++ b/crates/net/rpc/src/beacon/builder_client/mod.rs @@ -467,37 +467,15 @@ pub(super) async fn read_body( Ok(body) } -/// `text` with every character that could break or restyle a line written -/// out, cut to at most [`MAX_ERROR_TEXT`] bytes. +/// `text` as [`ethlambda_types::text::one_line`] writes it, cut to at most +/// [`MAX_ERROR_TEXT`] bytes. /// /// A builder's text reaches this node's logs and a validator client's error -/// message. There a raw line break would start what reads as a log line of -/// its own, and an ESC would start an ANSI sequence that recolors, moves the -/// cursor in or clears a terminal. So every control character -/// (`char::is_control`: C0, DEL and C1) and the Unicode line and paragraph -/// separators U+2028/U+2029, which some viewers break lines at, are written -/// as Rust escapes: `\n`, `\r` and `\t`, and `\u{1b}` for the rest. Everything -/// else passes through unchanged, quotes and backslashes included. -/// -/// The cap is applied to the escaped text, and never cuts a character or an -/// escape in two. +/// message, so no line break or terminal escape in it may reach either as +/// itself. The validator client writes a beacon node's error text through the +/// same function. pub(super) fn error_text(text: String) -> String { - let mut written = String::with_capacity(text.len().min(MAX_ERROR_TEXT)); - for c in text.chars() { - if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { - let escaped = c.escape_default(); - if written.len() + escaped.len() > MAX_ERROR_TEXT { - break; - } - written.extend(escaped); - } else { - if written.len() + c.len_utf8() > MAX_ERROR_TEXT { - break; - } - written.push(c); - } - } - written + ethlambda_types::text::one_line(&text, MAX_ERROR_TEXT) } /// The fork `version` names, if its bids have the shape [`SignedBuilderBid`] diff --git a/crates/net/rpc/src/beacon/builder_client/tests.rs b/crates/net/rpc/src/beacon/builder_client/tests.rs index b5c26e94a..6c006ec26 100644 --- a/crates/net/rpc/src/beacon/builder_client/tests.rs +++ b/crates/net/rpc/src/beacon/builder_client/tests.rs @@ -480,11 +480,12 @@ async fn a_long_error_is_cut_short() { let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { panic!("expected a bad request"); }; - assert_eq!(message.len(), 1024); + assert_eq!(message, format!("{}...", "x".repeat(1021))); } /// Line breaks are written out, so a builder's text cannot start a log line of -/// its own, and the escaped text is what the length cap applies to. +/// its own, and the escaped text is what the length cap applies to: 510 of +/// the two-byte escapes, then the marker. #[tokio::test] async fn line_breaks_in_an_error_are_written_out() { let relay = MockRelay::start(GENESIS_FORK_VERSION).await; @@ -500,7 +501,7 @@ async fn line_breaks_in_an_error_are_written_out() { let Err(BuilderError::BadRequest(message)) = get_header(&client_for(&relay)).await else { panic!("expected a bad request"); }; - assert_eq!(message.len(), 1024); + assert_eq!(message, format!("{}...", r"\n".repeat(510))); } /// Every other control character is written out too (an ESC would start an @@ -534,9 +535,10 @@ async fn control_characters_in_an_error_are_written_out() { /// The cap never leaves half an escape behind. #[test] fn the_cap_never_cuts_an_escape_in_two() { - // 1020 bytes, then a six-byte escape that would end past the cap. + // 1020 bytes, then a six-byte escape that would end past the cap: it is + // left out whole, and the marker ends the text. let text = format!("{}\x1b", "x".repeat(1020)); - assert_eq!(super::error_text(text), "x".repeat(1020)); + assert_eq!(super::error_text(text), format!("{}...", "x".repeat(1020))); let text = format!("{}\x1b", "x".repeat(1018)); assert_eq!( super::error_text(text), diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index d8d3e3423..db0a765b3 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -1272,6 +1272,7 @@ pub(crate) fn decimal(value: ðlambda_types::beacon::primitives::Uint256) -> S #[cfg(test)] mod tests { + use super::test_support::*; use super::*; use crate::test_utils::{RecordingNetwork, beacon_store_with_config}; use axum::{body::Body, http::Request}; @@ -1298,28 +1299,6 @@ mod tests { (status, serde_json::from_slice(&body).unwrap_or_default()) } - /// A fulu head with its proposer lookahead and sync committee filled in, - /// which a block built on it needs, under a schedule that puts every slot - /// at fulu. - fn fulu_store() -> (Store, Slot) { - use ethlambda_state_transition::beacon::helpers::{ - altair::get_next_sync_committee, fulu::initialize_proposer_lookahead, - }; - let mut state = with_signing_validators_at(ForkName::Fulu, 64); - let lookahead = initialize_proposer_lookahead(&state).unwrap(); - let sync_committee = get_next_sync_committee(&state).unwrap(); - let BeaconState::Fulu(inner) = &mut state else { - unreachable!("built as fulu") - }; - inner.proposer_lookahead = lookahead.try_into().unwrap(); - inner.current_sync_committee = sync_committee.clone(); - inner.next_sync_committee = sync_committee; - let head_slot = state.slot(); - let (store, _root) = - beacon_store_with_config(state, Config::mainnet().with_fork_epoch(ForkName::Fulu, 0)); - (store, head_slot) - } - fn produce_app(store: Store, engine: Option) -> Router { produce_app_with_sync(store, engine, SyncStatusController::new(SyncStatus::Synced)) } @@ -1712,116 +1691,14 @@ mod tests { assert_eq!(network.blocks.lock().unwrap().len(), 2); } - use crate::beacon::builder_client::mock_relay::MockRelay; - use crate::beacon::builder_client::{BuilderClient, VersionedBid}; - use crate::beacon::builder_flow::breaker::FallbackThresholds; + use crate::beacon::builder_client::BuilderClient; use ethlambda_blockchain::SyncStatusController; - use ethlambda_types::beacon::builder::{BuilderBid, ValidatorRegistrationV1}; - use ethlambda_types::beacon::containers::capella; - use ethlambda_types::beacon::containers::deneb::{ExecutionPayloadHeader, KzgCommitments}; - use ethlambda_types::beacon::containers::electra::{self, ExecutionRequests}; + use ethlambda_types::beacon::builder::ValidatorRegistrationV1; + use ethlambda_types::beacon::containers::electra; use libssz::SszDecode as _; use std::time::Duration; - const ETH: u64 = 1_000_000_000_000_000_000; - /// The registered gas limit the test proposer asks for. - const TARGET_GAS_LIMIT: u64 = 60_000_000; - - /// Thresholds that never trip, so a test state's history does not decide - /// whether the builder is asked. - const NEVER_TRIP: FallbackThresholds = FallbackThresholds { - skips: u64::MAX, - skips_per_epoch: u64::MAX, - epochs_since_finalization: u64::MAX, - }; - - /// A builder world for the next slot on `fulu_store()`: a mock relay, a - /// `Builder` pointing at it with the proposer registered, and the - /// advanced proposal state. - struct BuilderWorld { - store: Store, - slot: Slot, - state: BeaconState, - relay: MockRelay, - builder: Arc, - } - - async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { - let (store, head_slot) = fulu_store(); - let slot = head_slot + 1; - let config = store.config(); - let (_, head_state) = super::super::validator::head(&store).unwrap(); - let state = advance_to_slot(&head_state, slot, &config).unwrap(); - let proposer = get_beacon_proposer_index(&state).unwrap(); - let pubkey = state.validator(proposer).unwrap().pubkey; - let relay = MockRelay::start(config.genesis_fork_version).await; - let client = - BuilderClient::new(relay.url.clone(), Duration::from_secs(1), "ethlambda/test") - .unwrap(); - let builder = Arc::new(Builder::new( - client, - NEVER_TRIP, - SyncStatusController::new(sync), - )); - if registered { - let registration = ValidatorRegistrationV1 { - fee_recipient: ExecutionAddress::repeat_byte(0x11), - gas_limit: TARGET_GAS_LIMIT, - timestamp: 1, - pubkey, - }; - builder - .registrations - .record([®istration], compute_epoch_at_slot(slot)); - } - BuilderWorld { - store, - slot, - state, - relay, - builder, - } - } - impl BuilderWorld { - /// A bid of `value` wei that passes every check for this slot, with - /// `edit` applied before the relay signs it. - fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { - let config = self.store.config(); - let inputs = payload_inputs(&self.state, &config).unwrap(); - let BeaconState::Fulu(fulu) = &self.state else { - unreachable!("built as fulu") - }; - let parent_gas_limit = fulu.latest_execution_payload_header.gas_limit; - let step = (parent_gas_limit / 1024).saturating_sub(1); - let withdrawals: capella::Withdrawals = inputs.withdrawals.clone().try_into().unwrap(); - let mut header = - ExecutionPayloadHeader::from(&electra::BeaconBlockBody::empty().execution_payload); - header.parent_hash = inputs.parent_hash; - header.prev_randao = inputs.prev_randao; - header.timestamp = inputs.timestamp; - header.block_number = fulu.latest_execution_payload_header.block_number + 1; - header.withdrawals_root = withdrawals.hash_tree_root(); - header.gas_limit = TARGET_GAS_LIMIT.clamp( - parent_gas_limit - step, - parent_gas_limit.saturating_add(step), - ); - header.block_hash = ExecutionBlockHash::repeat_byte(0xbb); - let mut message = BuilderBid { - header, - blob_kzg_commitments: KzgCommitments::default(), - execution_requests: ExecutionRequests::default(), - value: Uint256::from(value), - pubkey: self.relay.pubkey(), - }; - edit(&mut message); - let bid = self.relay.sign_bid(message); - self.relay.offer(VersionedBid { - fork: ForkName::Fulu, - bid, - }); - } - /// The production wiring: the router and the builder read one sync /// status. fn app(&self, engine: Option) -> Router { @@ -2653,3 +2530,145 @@ mod tests { assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); } } + +/// The builder world the proposal tests build on: a fulu head, a mock relay and +/// a `Builder` pointing at it. Shared with the validator client's end-to-end +/// tests, which drive the same routes through a real socket. +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + use crate::beacon::builder_client::BuilderClient; + use crate::beacon::builder_client::mock_relay::MockRelay; + use crate::beacon::builder_flow::breaker::FallbackThresholds; + use crate::test_utils::beacon_store_with_config; + use ethlambda_blockchain::SyncStatusController; + use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; + use ethlambda_types::beacon::builder::ValidatorRegistrationV1; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::capella; + use ethlambda_types::beacon::containers::deneb::{ExecutionPayloadHeader, KzgCommitments}; + use ethlambda_types::beacon::containers::electra; + use std::time::Duration; + + /// A fulu head with its proposer lookahead and sync committee filled in, + /// which a block built on it needs, under a schedule that puts every slot + /// at fulu. + pub(crate) fn fulu_store() -> (Store, Slot) { + use ethlambda_state_transition::beacon::helpers::{ + altair::get_next_sync_committee, fulu::initialize_proposer_lookahead, + }; + let mut state = with_signing_validators_at(ForkName::Fulu, 64); + let lookahead = initialize_proposer_lookahead(&state).unwrap(); + let sync_committee = get_next_sync_committee(&state).unwrap(); + let BeaconState::Fulu(inner) = &mut state else { + unreachable!("built as fulu") + }; + inner.proposer_lookahead = lookahead.try_into().unwrap(); + inner.current_sync_committee = sync_committee.clone(); + inner.next_sync_committee = sync_committee; + let head_slot = state.slot(); + let (store, _root) = + beacon_store_with_config(state, Config::mainnet().with_fork_epoch(ForkName::Fulu, 0)); + (store, head_slot) + } + + pub(crate) const ETH: u64 = 1_000_000_000_000_000_000; + /// The registered gas limit the test proposer asks for. + pub(crate) const TARGET_GAS_LIMIT: u64 = 60_000_000; + + /// Thresholds that never trip, so a test state's history does not decide + /// whether the builder is asked. + pub(crate) const NEVER_TRIP: FallbackThresholds = FallbackThresholds { + skips: u64::MAX, + skips_per_epoch: u64::MAX, + epochs_since_finalization: u64::MAX, + }; + + /// A builder world for the next slot on `fulu_store()`: a mock relay, a + /// `Builder` pointing at it with the proposer registered, and the + /// advanced proposal state. + pub(crate) struct BuilderWorld { + pub(crate) store: Store, + pub(crate) slot: Slot, + pub(crate) state: BeaconState, + pub(crate) relay: MockRelay, + pub(crate) builder: Arc, + } + + pub(crate) async fn builder_world(registered: bool, sync: SyncStatus) -> BuilderWorld { + let (store, head_slot) = fulu_store(); + let slot = head_slot + 1; + let config = store.config(); + let (_, head_state) = super::super::validator::head(&store).unwrap(); + let state = advance_to_slot(&head_state, slot, &config).unwrap(); + let proposer = get_beacon_proposer_index(&state).unwrap(); + let pubkey = state.validator(proposer).unwrap().pubkey; + let relay = MockRelay::start(config.genesis_fork_version).await; + let client = + BuilderClient::new(relay.url.clone(), Duration::from_secs(1), "ethlambda/test") + .unwrap(); + let builder = Arc::new(Builder::new( + client, + NEVER_TRIP, + SyncStatusController::new(sync), + )); + if registered { + let registration = ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(0x11), + gas_limit: TARGET_GAS_LIMIT, + timestamp: 1, + pubkey, + }; + builder + .registrations + .record([®istration], compute_epoch_at_slot(slot)); + } + BuilderWorld { + store, + slot, + state, + relay, + builder, + } + } + + impl BuilderWorld { + /// A bid of `value` wei that passes every check for this slot, with + /// `edit` applied before the relay signs it. + pub(crate) fn offer(&self, value: u64, edit: impl FnOnce(&mut BuilderBid)) { + let config = self.store.config(); + let inputs = payload_inputs(&self.state, &config).unwrap(); + let BeaconState::Fulu(fulu) = &self.state else { + unreachable!("built as fulu") + }; + let parent_gas_limit = fulu.latest_execution_payload_header.gas_limit; + let step = (parent_gas_limit / 1024).saturating_sub(1); + let withdrawals: capella::Withdrawals = inputs.withdrawals.clone().try_into().unwrap(); + let mut header = + ExecutionPayloadHeader::from(&electra::BeaconBlockBody::empty().execution_payload); + header.parent_hash = inputs.parent_hash; + header.prev_randao = inputs.prev_randao; + header.timestamp = inputs.timestamp; + header.block_number = fulu.latest_execution_payload_header.block_number + 1; + header.withdrawals_root = withdrawals.hash_tree_root(); + header.gas_limit = TARGET_GAS_LIMIT.clamp( + parent_gas_limit - step, + parent_gas_limit.saturating_add(step), + ); + header.block_hash = ExecutionBlockHash::repeat_byte(0xbb); + let mut message = BuilderBid { + header, + blob_kzg_commitments: KzgCommitments::default(), + execution_requests: ExecutionRequests::default(), + value: Uint256::from(value), + pubkey: self.relay.pubkey(), + }; + edit(&mut message); + let bid = self.relay.sign_bid(message); + self.relay.offer(VersionedBid { + fork: ForkName::Fulu, + bid, + }); + } + } +} diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 56ffc9b06..94f1bb741 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -129,7 +129,7 @@ async fn serve_at( // keep the state under that root too, as a real anchor would be. let parent_root = real_head_root(&state); store.insert_state(parent_root, state.clone()).unwrap(); - let (client, network, _pool, address) = spawn_server(store.clone(), engine).await; + let (client, network, _pool, address) = spawn_server(store.clone(), engine, None).await; (client, state, network, address, store) } @@ -148,16 +148,21 @@ fn real_head_root(state: &BeaconState) -> ethlambda_types::primitives::H256 { /// Serve `store` through the router the real server builds, over a socket on an /// ephemeral port, and connect this crate's client to it. Also hands back the /// payload attestation pool the router fills and the socket it listens on. +/// +/// With a `builder`, it is layered as `lib.rs` does when `--builder` is set, +/// so the builder flow's endpoints see it. async fn spawn_server( store: Store, engine: Option, + builder: Option>, ) -> ( HttpBeaconNode, Arc, SharedPayloadAttestationPool, std::net::SocketAddr, ) { - spawn_server_sharing(store, engine, SharedSyncCommitteePool::default()).await + let sync_pool = SharedSyncCommitteePool::default(); + spawn_server_sharing(store, engine, sync_pool, builder).await } /// [`spawn_server`] pooling sync committee messages in `sync_pool`, so a second @@ -168,6 +173,7 @@ async fn spawn_server_sharing( mut store: Store, engine: Option, sync_pool: SharedSyncCommitteePool, + builder: Option>, ) -> ( HttpBeaconNode, Arc, @@ -198,6 +204,10 @@ async fn spawn_server_sharing( .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new( own_version(), )))); + let router = match builder { + Some(builder) => router.layer(Extension(builder)), + None => router, + }; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); @@ -420,6 +430,7 @@ async fn block_production_without_an_execution_client_is_retryable() { proposer_index: 0, randao_reveal: Default::default(), graffiti: Default::default(), + builder_boost_factor: None, }; let err = client.produce_block(&request).await.unwrap_err(); assert!(matches!(err, Error::BeaconNodeSyncing), "got {err:?}"); @@ -660,6 +671,7 @@ async fn produce_with_pool( proposer_index: proposer, randao_reveal, graffiti: ethlambda_types::primitives::H256(graffiti), + builder_boost_factor: None, }; let produced = client.produce_block(&request).await.unwrap(); assert_eq!(produced.slot(), slot); @@ -694,6 +706,8 @@ fn fulu_block( match &produced.contents { Contents::WithCellProofs { block, .. } | Contents::WithBlobProofs { block, .. } => block, Contents::Gloas { .. } => unreachable!("a fulu proposal"), + // These tests ask with no `builder_boost_factor`, so no blinded answer. + Contents::Blinded { .. } => unreachable!("builder proposals are off"), } } @@ -924,7 +938,7 @@ async fn serve_fulu_pinned( head_root, slot + clock_ahead, ); - let (client, network, _, _) = spawn_server_sharing(store, engine, sync_pool).await; + let (client, network, _, _) = spawn_server_sharing(store, engine, sync_pool, None).await; let genesis = client.genesis().await.expect("genesis"); let spec = client .spec() @@ -1119,6 +1133,7 @@ async fn the_validator_client_can_serve_on_the_sync_committee() { proposer_index: proposer, randao_reveal, graffiti: Default::default(), + builder_boost_factor: None, }; let produced = client.produce_block(&request).await.unwrap(); let block_domain = get_domain(&advanced, DOMAIN_BEACON_PROPOSER, Some(epoch)); @@ -1240,7 +1255,7 @@ mod gloas { let store = beacon_store_with_head_block(state.clone(), config(), block, head_root, clock_slot); let (client, network, payload_pool, _address) = - spawn_server_sharing(store.clone(), engine, sync_pool).await; + spawn_server_sharing(store.clone(), engine, sync_pool, None).await; let genesis = client.genesis().await.expect("genesis"); let spec = client .spec() @@ -1570,6 +1585,7 @@ mod gloas { ), ), ethlambda_validator::slashing_protection::SlashingProtection::in_memory(), + None, ); let duty = ProposerDutyDto { pubkey: encode_hex(&pubkey.0), @@ -1699,6 +1715,7 @@ mod gloas { ), ), ethlambda_validator::slashing_protection::SlashingProtection::in_memory(), + None, ); let duty = ProposerDutyDto { pubkey: encode_hex(&pubkey.0), @@ -1718,3 +1735,311 @@ mod gloas { assert_full_sync_aggregate(&advanced, &signed.message.body.sync_aggregate); } } + +/// The builder flow through this node: the validator client's own +/// registration, `produceBlockV3` with a boost factor, and blinded publish, +/// against this node's routes with a `Builder` layered over a mock relay. +mod builder_flow { + use super::*; + use crate::beacon::proposal::test_support::{ + BuilderWorld, ETH, TARGET_GAS_LIMIT, builder_world, + }; + use ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index; + use ethlambda_types::beacon::builder::{ + SignedBlindedBeaconBlock, SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::ExecutionAddress; + use ethlambda_validator::beacon_node::Published; + use ethlambda_validator::beacon_node::block_contents::Producer; + use libssz::SszDecode as _; + + /// What a test drives: this node served over a socket, the proposer's key + /// and the client's signing context. + struct Harness { + world: BuilderWorld, + client: HttpBeaconNode, + proposer: u64, + keys: RwLock, + context: SigningContext, + } + + /// `world`'s store served through the real beacon router, with its + /// `Builder` layered, behind a socket. + async fn harness(world: BuilderWorld, engine: ethlambda_engine::EngineClient) -> Harness { + // A block built on this head names the advanced header's root as its + // parent, which the fixture's placeholder anchor block does not have: + // keep the head state under that root too, as `serve_at` does, so the + // publish finds its parent. + let mut store = world.store.clone(); + let (anchor_root, head_state) = crate::beacon::validator::head(&store).unwrap(); + let parent_root = world.state.latest_block_header().hash_tree_root(); + store + .insert_state(parent_root, (*head_state).clone()) + .unwrap(); + // The publish endpoints refuse a parent at the finalized slot other + // than the finalized block, so finalize the root the block names, as a + // real anchor's checkpoint does. + crate::test_utils::finalize_at(&mut store, anchor_root, parent_root, head_state.slot()); + + let (client, _network, _pool, _address) = + spawn_server(store, Some(engine), Some(world.builder.clone())).await; + + let proposer = get_beacon_proposer_index(&world.state).unwrap(); + let keys = keys_for(&world.state, &[proposer]); + let context = SigningContext { + config: (*world.store.config()).clone(), + genesis_validators_root: world.state.genesis_validators_root(), + }; + Harness { + world, + client, + proposer, + keys, + context, + } + } + + impl Harness { + fn pubkey(&self) -> BlsPubkey { + self.world.state.validator(self.proposer).unwrap().pubkey + } + + /// The proposer's registration, signed by the client and sent through + /// this node. The relay checks the signature itself. + async fn register(&self) -> SignedValidatorRegistrationV1 { + let message = ValidatorRegistrationV1 { + fee_recipient: ExecutionAddress::repeat_byte(0x11), + gas_limit: TARGET_GAS_LIMIT, + timestamp: 1, + pubkey: self.pubkey(), + }; + let signature = { + let keys = self.keys.read().await; + self.context + .sign_validator_registration(&keys, &message) + .unwrap() + }; + let signed = SignedValidatorRegistrationV1 { message, signature }; + self.client + .register_validators(std::slice::from_ref(&signed)) + .await + .expect("the node takes the registration"); + signed + } + + async fn produce( + &self, + builder_boost_factor: Option, + ) -> ethlambda_validator::beacon_node::block_contents::ProducedBlock { + let epoch = compute_epoch_at_slot(self.world.slot); + // Signed under the head state's own fork rather than through the + // client's schedule: this world's state is not pinned to carry + // the schedule's fulu version (see `serve_fulu_pinned`), and the + // node verifies against the state, as the other proposal tests + // here do. + let randao_domain = get_domain( + &self.world.state, + ethlambda_types::beacon::constants::DOMAIN_RANDAO, + Some(epoch), + ); + let randao_reveal = sign_for( + self.proposer as usize, + compute_signing_root(epoch.hash_tree_root(), randao_domain), + ); + let request = BlockRequest { + slot: self.world.slot, + fork: ForkName::Fulu, + proposer_index: self.proposer, + randao_reveal, + graffiti: ethlambda_types::primitives::H256([0; 32]), + builder_boost_factor, + }; + self.client.produce_block(&request).await.unwrap() + } + + async fn sign( + &self, + produced: ethlambda_validator::beacon_node::block_contents::ProducedBlock, + ) -> Vec { + let domain = get_domain( + &self.world.state, + ethlambda_types::beacon::constants::DOMAIN_BEACON_PROPOSER, + Some(compute_epoch_at_slot(self.world.slot)), + ); + let signature = sign_for( + self.proposer as usize, + compute_signing_root(produced.block_root(), domain), + ); + produced.into_signed_ssz(signature) + } + + /// Registered, a bid taken at the maximum factor, and the blinded + /// block signed: the node that produced it and the body to publish. + async fn signed_blinded(&self) -> (Producer, Vec) { + self.register().await; + self.world.offer(1, |_| {}); + let produced = self.produce(Some(u64::MAX)).await; + assert!(produced.is_blinded()); + let producer = produced.producer; + (producer, self.sign(produced).await) + } + + async fn publish_blinded( + &self, + producer: Producer, + body: &[u8], + ) -> ethlambda_validator::Result { + self.client + .publish_blinded_block(producer, ForkName::Fulu, body) + .await + } + } + + /// A relay refusing the block is a definite refusal at the client: the + /// node passes the builder's 400 through. + #[tokio::test] + async fn a_relay_refusing_the_blinded_block_is_a_definite_refusal() { + let world = builder_world(false, SyncStatus::Synced).await; + world + .relay + .answer_submissions_with(axum::http::StatusCode::BAD_REQUEST); + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let (producer, body) = h.signed_blinded().await; + let err = h + .publish_blinded(producer, &body) + .await + .expect_err("the relay refused it"); + assert!( + matches!(err, Error::BeaconNodeStatus { status: 400, .. }), + "got {err:?}" + ); + } + + /// A relay failing without saying whether it published is unconfirmed at + /// the client, not a missed proposal: the relay may still publish. + #[tokio::test] + async fn a_relay_failing_on_the_blinded_block_leaves_it_unconfirmed() { + let world = builder_world(false, SyncStatus::Synced).await; + world + .relay + .answer_submissions_with(axum::http::StatusCode::INTERNAL_SERVER_ERROR); + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let (producer, body) = h.signed_blinded().await; + let err = h + .publish_blinded(producer, &body) + .await + .expect_err("the relay did not confirm it"); + assert!( + matches!( + err, + Error::BlindedUnconfirmed { + status: Some(500), + .. + } + ), + "got {err:?}" + ); + } + + /// Once the node handed a blinded block to its builder, it refuses a + /// different block for the slot from the same client, as a second + /// signature for it would be an equivocation. + #[tokio::test] + async fn the_node_refuses_a_second_block_after_a_blinded_one() { + let world = builder_world(false, SyncStatus::Synced).await; + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let (producer, body) = h.signed_blinded().await; + let published = h.publish_blinded(producer, &body).await.expect("handed"); + assert_eq!(published, Published::HandedToBuilder); + + let full = h.produce(None).await; + assert!(!full.is_blinded()); + let full = h.sign(full).await; + let err = h + .client + .publish_block(ForkName::Fulu, &full) + .await + .expect_err("the slot is claimed by the blinded block"); + assert!( + matches!(&err, Error::BeaconNodeStatus { status: 400, body } + if body.contains("already published a different block")), + "got {err:?}" + ); + } + + #[tokio::test] + async fn the_validator_client_can_propose_a_builder_block_through_this_node() { + let world = builder_world(false, SyncStatus::Synced).await; + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + let registered = h.register().await; + assert_eq!(h.world.relay.registrations(), [registered]); + + // Worth far less than the local payload (42 wei), so only the maximum + // factor makes it win: a factor that is dropped or ignored (the node's + // default is 100) leaves a full block. + h.world.offer(1, |_| {}); + let produced = h.produce(Some(u64::MAX)).await; + assert!( + produced.is_blinded(), + "the factor must reach the node and take the bid" + ); + + let producer = produced.producer; + let body = h.sign(produced).await; + let published = h + .client + .publish_blinded_block(producer, ForkName::Fulu, &body) + .await + .expect("the node hands the block to its builder"); + assert_eq!(published, Published::HandedToBuilder); + + let signed = SignedBlindedBeaconBlock::from_ssz_bytes(&body).unwrap(); + assert_eq!(h.world.relay.blinded_blocks(), [signed]); + } + + #[tokio::test] + async fn without_builder_proposals_the_client_gets_a_full_block() { + let world = builder_world(false, SyncStatus::Synced).await; + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + // Registered, with a bid worth far more than the local payload: only + // the factor the client sends (0 for `None`) keeps the block local. + h.register().await; + h.world.offer(ETH, |_| {}); + let produced = h.produce(None).await; + assert!(!produced.is_blinded()); + assert!(h.world.relay.blinded_blocks().is_empty()); + assert_eq!( + h.world.relay.header_requests().len(), + 1, + "the node still asks the relay at a factor of 0" + ); + } + + #[tokio::test] + async fn a_relay_that_is_down_leaves_a_full_block() { + let world = builder_world(false, SyncStatus::Synced).await; + world.relay.answer_headers_with( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "application/json", + Vec::new(), + None, + ); + let engine = fake_execution_client_paying("0x2a", false).await; + let h = harness(world, engine).await; + + h.register().await; + h.world.offer(ETH, |_| {}); + let produced = h.produce(Some(100)).await; + assert!(!produced.is_blinded()); + } +} diff --git a/crates/validator/Cargo.toml b/crates/validator/Cargo.toml index b34e72488..56b24478c 100644 --- a/crates/validator/Cargo.toml +++ b/crates/validator/Cargo.toml @@ -42,6 +42,8 @@ blst = "0.3.16" # the `impl Future + Send` bounds at every method is noise this trait does not # need. async-trait = "0.1" +# `join_all`, so the calls every beacon node must hear reach them all at once. +futures-util = "0.3" # EIP-2335 keystore decryption. scrypt = { version = "0.11", default-features = false } diff --git a/crates/validator/src/beacon_node/block_contents.rs b/crates/validator/src/beacon_node/block_contents.rs index c4d2806ca..f39d349a5 100644 --- a/crates/validator/src/beacon_node/block_contents.rs +++ b/crates/validator/src/beacon_node/block_contents.rs @@ -21,7 +21,7 @@ //! //! The endpoints support it: `Accept: application/octet-stream` on //! `produceBlockV3`, `Content-Type: application/octet-stream` on -//! `publishBlockV2`. +//! `publishBlockV2` and `publishBlindedBlockV2`. //! //! # `BlockContents` is not a consensus container //! @@ -42,8 +42,20 @@ //! | Fork | Produced | Published | //! |---|---|---| //! | Deneb, Electra, Fulu | `BlockContents` | `SignedBlockContents` | +//! | Blinded (electra, fulu) | `BlindedBeaconBlock` | `SignedBlindedBeaconBlock` | //! | Gloas and later | `GloasBlockContents` (block and self-built envelope) or a bare block | bare signed block, then the envelope separately | //! +//! # Blinded blocks +//! +//! When the beacon node's builder wins the slot, `produceBlockV3` answers with +//! a [`BlindedBeaconBlock`] (`Eth-Execution-Payload-Blinded: true`): the block +//! carries the payload's header, and the builder reveals the payload once this +//! client has signed. No blobs travel with it, since the builder publishes +//! them. Its root is the full block's, so signing and slashing protection work +//! as for any other block. Electra and fulu share one blinded container. +//! Electra blinded blocks are for third-party beacon nodes: an ethlambda node +//! serves blinded blocks for fulu only. +//! //! # Gloas //! //! The block no longer carries the payload (EIP-7732): it commits to a bid, and @@ -85,6 +97,7 @@ //! thirty-two blobs would fail to decode, in a client that had been correct for //! months. +use ethlambda_types::beacon::builder::{BlindedBeaconBlock, SignedBlindedBeaconBlock}; use ethlambda_types::beacon::containers::deneb::Blob; use ethlambda_types::beacon::containers::electra::{BeaconBlock, SignedBeaconBlock}; use ethlambda_types::beacon::containers::gloas; @@ -233,6 +246,33 @@ pub struct ProducedBlock { /// told which. pub fork: ForkName, pub contents: Contents, + /// Which configured node produced it, for a blinded block's publish to go + /// back to. See [`Producer`]. + pub producer: Producer, +} + +/// Which of the configured beacon nodes produced a block. +/// +/// A blinded block can only be published where it was produced: that node's +/// builder holds the payload, and that node claimed the slot. The fallback +/// fills this in when it returns a block and reads it when the signed block +/// is published, so the block carries its own way back: the proposal path +/// takes the producer from the very block it signed. A single node has +/// nothing to choose and leaves it at the default. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Producer(usize); + +impl Producer { + /// The node at `position` in the fallback's list. + pub(crate) fn at(position: usize) -> Self { + Self(position) + } + + /// The position in the fallback's list of the node that produced the + /// block. + pub(crate) fn position(self) -> usize { + self.0 + } } /// The block and the blob material that came with it. @@ -256,6 +296,10 @@ pub enum Contents { block: gloas::BeaconBlock, payload: Option>, }, + /// Electra or fulu, when a builder supplied the payload: the block carries + /// the payload's header, and the builder reveals the payload after this + /// client signs. No blobs travel with it; the builder publishes them. + Blinded { block: BlindedBeaconBlock }, } impl ProducedBlock { @@ -292,6 +336,7 @@ impl ProducedBlock { kzg_proofs: contents.kzg_proofs, blobs: contents.blobs, }, + producer: Producer::default(), }) } ForkName::Fulu => { @@ -304,6 +349,7 @@ impl ProducedBlock { kzg_proofs: contents.kzg_proofs, blobs: contents.blobs, }, + producer: Producer::default(), }) } // Gloas needs one more fact than the fork: whether the node @@ -326,6 +372,46 @@ impl ProducedBlock { } } + /// Decode a blinded `produceBlockV3` body (`Eth-Execution-Payload-Blinded: + /// true`). Electra and fulu share one blinded container; every other fork is + /// refused, as [`Self::from_ssz`] refuses them. + pub fn from_blinded_ssz(fork: ForkName, bytes: &[u8]) -> crate::error::Result { + match fork { + ForkName::Electra | ForkName::Fulu => { + let block = BlindedBeaconBlock::from_ssz_bytes(bytes).map_err(|err| { + Error::Decode(format!( + "produced blinded block for fork {} did not decode: {err:?}", + fork.as_str() + )) + })?; + Ok(Self { + fork, + contents: Contents::Blinded { block }, + producer: Producer::default(), + }) + } + // Named, as `from_ssz` names its arms, so a new fork forces a + // decision here instead of being refused by default. + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Gloas + | ForkName::Lean => Err(Error::InconsistentResponse(format!( + "beacon node produced a blinded block for fork {}; blinded blocks exist for \ + electra and fulu only", + fork.as_str() + ))), + } + } + + /// Whether a builder supplied the payload, so the block is published to + /// `blinded_blocks` and its fee recipient is the builder's. + pub fn is_blinded(&self) -> bool { + matches!(self.contents, Contents::Blinded { .. }) + } + /// Decode a gloas `produceBlockV4` body. `payload_included` is the node's /// `Eth-Execution-Payload-Included`, which selects between /// [`GloasBlockContents`] and a bare block: SSZ carries no tag, so a wrong @@ -355,6 +441,7 @@ impl ProducedBlock { Ok(Self { fork: ForkName::Gloas, contents: Contents::Gloas { block, payload }, + producer: Producer::default(), }) } @@ -365,6 +452,7 @@ impl ProducedBlock { block.slot } Contents::Gloas { block, .. } => block.slot, + Contents::Blinded { block } => block.slot, } } @@ -375,6 +463,7 @@ impl ProducedBlock { block.proposer_index } Contents::Gloas { block, .. } => block.proposer_index, + Contents::Blinded { block } => block.proposer_index, } } @@ -387,11 +476,13 @@ impl ProducedBlock { block.hash_tree_root() } Contents::Gloas { block, .. } => block.hash_tree_root(), + Contents::Blinded { block } => block.hash_tree_root(), } } /// Where the block pays its execution-layer rewards: the payload's fee - /// recipient before gloas, the bid's from it on. + /// recipient before gloas, the bid's from it on. For a blinded block, the + /// header's, which is the builder's address rather than the validator's. pub fn fee_recipient(&self) -> ethlambda_types::beacon::primitives::ExecutionAddress { match &self.contents { Contents::WithBlobProofs { block, .. } | Contents::WithCellProofs { block, .. } => { @@ -404,6 +495,7 @@ impl ProducedBlock { .message .fee_recipient } + Contents::Blinded { block } => block.body.execution_payload_header.fee_recipient, } } @@ -441,10 +533,12 @@ impl ProducedBlock { Contents::Gloas { payload, .. } => { payload.as_ref().map_or(0, |payload| payload.blobs.len()) } + Contents::Blinded { block } => block.body.blob_kzg_commitments.len(), } } - /// Attach `signature` and encode the body `publishBlockV2` expects. + /// Attach `signature` and encode the body `publishBlockV2` expects, or + /// `publishBlindedBlockV2` for a blinded block. /// /// Consumes the block: the proofs and blobs are moved into the published /// body rather than copied, and a blob is 128 KiB, so a six-blob block @@ -485,6 +579,11 @@ impl ProducedBlock { signature, } .to_ssz(), + Contents::Blinded { block } => SignedBlindedBeaconBlock { + message: block, + signature, + } + .to_ssz(), } } } @@ -775,6 +874,46 @@ mod tests { } } + #[test] + fn a_blinded_block_decodes_and_keeps_the_full_blocks_root() { + let full = empty_block_for(5, 7); + let blinded = BlindedBeaconBlock::from(full.clone()); + + let produced = + ProducedBlock::from_blinded_ssz(ForkName::Fulu, &blinded.to_ssz()).expect("decodes"); + + assert!(produced.is_blinded()); + assert_eq!(produced.slot(), 5); + assert_eq!(produced.proposer_index(), 7); + assert_eq!(produced.block_root(), full.hash_tree_root()); + assert_eq!(produced.blob_count(), 0); + } + + #[test] + fn a_signed_blinded_block_encodes_as_signed_blinded_beacon_block() { + let blinded = BlindedBeaconBlock::from(empty_block_for(5, 7)); + let produced = + ProducedBlock::from_blinded_ssz(ForkName::Electra, &blinded.to_ssz()).expect("ok"); + + let signed = produced.into_signed_ssz(BlsSignature([6; 96])); + let decoded = SignedBlindedBeaconBlock::from_ssz_bytes(&signed).expect("decodes"); + assert_eq!(decoded.message, blinded); + assert_eq!(decoded.signature, BlsSignature([6; 96])); + } + + #[test] + fn a_blinded_block_for_gloas_or_deneb_is_refused() { + let bytes = BlindedBeaconBlock::from(empty_block_for(5, 7)).to_ssz(); + for fork in [ForkName::Gloas, ForkName::Deneb] { + let err = ProducedBlock::from_blinded_ssz(fork, &bytes).expect_err("must refuse"); + assert!( + matches!(err, Error::InconsistentResponse(_)), + "{} gave {err:?}", + fork.as_str() + ); + } + } + #[test] fn a_truncated_body_is_a_decode_error_not_a_panic() { let bytes = contents_bytes(1, 1); diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index f05107ab0..de09561ab 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -8,6 +8,7 @@ use std::future::Future; use std::pin::Pin; use async_trait::async_trait; +use ethlambda_types::beacon::builder::SignedValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::gloas; @@ -16,7 +17,7 @@ use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; use tracing::{debug, warn}; -use crate::beacon_node::block_contents::ProducedBlock; +use crate::beacon_node::block_contents::{ProducedBlock, Producer}; use crate::beacon_node::dto::{ CommitteeSubscriptionDto, ProposerPreparationDto, SingleAttestationDto, SyncCommitteeSubscriptionDto, SyncDutyDto, @@ -68,13 +69,23 @@ impl FallbackBeaconNode { /// both `&self` and the future to one named `'p` lets the compiler unify /// it with whatever the call site's shortest borrow actually is. async fn try_each<'p, T, F>(&'p self, what: &'static str, call: F) -> Result + where + F: Fn(&'p B) -> BoxFuture<'p, Result>, + { + self.try_each_at(what, call).await.map(|(_, value)| value) + } + + /// [`Self::try_each`], also returning the position of the node that + /// answered, for the calls whose follow-up must go to the same node. The + /// logging and the final `warn!` live here so the two cannot drift. + async fn try_each_at<'p, T, F>(&'p self, what: &'static str, call: F) -> Result<(usize, T)> where F: Fn(&'p B) -> BoxFuture<'p, Result>, { let mut last = None; for (position, node) in self.nodes.iter().enumerate() { match call(node).await { - Ok(value) => return Ok(value), + Ok(value) => return Ok((position, value)), Err(err) => { // Debug, not warn: with more than one node configured, an // outage on one of them fires this every slot for as long @@ -112,17 +123,34 @@ impl FallbackBeaconNode { /// unreachable node must not stop the reachable ones being told. A failure /// is logged per node so a partially-registered client is visible rather /// than silent. - async fn try_all<'p, F>(&'p self, what: &'static str, call: F) -> Result<()> + /// + /// The nodes are called at once rather than in turn, as Lighthouse + /// broadcasts, so the call takes as long as its slowest node rather than + /// the sum of every node's timeout: a hung node no longer holds back the + /// healthy ones behind it in the list. + /// + /// `node_effect` says what a refusing node costs (said once per refusal); + /// `all_failed` says what it costs when none accepts, or is `None` when the + /// caller reports that itself, so a duty that did not depend on the call is + /// not claimed missed. + async fn try_all<'p, F>( + &'p self, + what: &'static str, + node_effect: &'static str, + all_failed: Option<&'static str>, + call: F, + ) -> Result<()> where F: Fn(&'p B) -> BoxFuture<'p, Result<()>>, { + let answers = futures_util::future::join_all(self.nodes.iter().map(&call)).await; let mut accepted = 0; let mut last = None; - for (position, node) in self.nodes.iter().enumerate() { - match call(node).await { + for (position, answer) in answers.into_iter().enumerate() { + match answer { Ok(()) => accepted += 1, Err(err) => { - warn!(%what, position, %err, "Beacon node did not accept; it will be unprepared for this duty"); + warn!(%what, position, %err, node_effect, "Beacon node did not accept"); last = Some(err); } } @@ -133,7 +161,9 @@ impl FallbackBeaconNode { let detail = last .map(|err| err.to_string()) .unwrap_or_else(|| "no nodes configured".to_string()); - warn!(%what, %detail, "No configured beacon node accepted; duty likely missed"); + if let Some(all_failed) = all_failed { + warn!(%what, %detail, all_failed, "No configured beacon node accepted"); + } Err(Error::AllBeaconNodesFailed(detail)) } } @@ -230,9 +260,14 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Marks the block with the node that produced it, which is where a + /// blinded block's publish has to go back to (see [`Producer`]). async fn produce_block(&self, request: &BlockRequest) -> Result { - self.try_each("produce_block", |node| node.produce_block(request)) - .await + let (position, mut block) = self + .try_each_at("produce_block", |node| node.produce_block(request)) + .await?; + block.producer = Producer::at(position); + Ok(block) } /// First node that accepts it wins, rather than a broadcast to every node. @@ -253,6 +288,114 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Only on the node that produced the block, with no failover. + /// + /// Unlike [`Self::publish_block`], offering the block to another node + /// would not reach the network: only the producing node's builder holds + /// the payload the header commits to, and only that node claimed the slot + /// with it. A second node would at best refuse it and at worst hand a + /// signed block to a builder that never saw the bid. So if the producing + /// node fails, the publish fails, and the answer is returned unchanged so + /// the caller can tell a refusal from an unconfirmed send. Slashing + /// protection has already recorded the slot, so nothing retries it. + async fn publish_blinded_block( + &self, + producer: Producer, + fork: ForkName, + body: &[u8], + ) -> Result { + // A position this fallback never handed out: there is no node the + // block could go back to. + let node = self.nodes.get(producer.position()).ok_or_else(|| { + Error::InconsistentResponse(format!( + "a blinded block names beacon node {} as its producer, but {} are configured", + producer.position(), + self.nodes.len() + )) + })?; + // The node behind this one is a single node, with nothing to choose. + node.publish_blinded_block(Producer::default(), fork, body) + .await + } + + /// To every node, like the other calls that install state on a node: a + /// node that never heard the registrations cannot serve a builder bid for + /// these validators after a failover. + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + self.try_all( + "register_validators", + "it will not hold these registrations", + None, + |node| node.register_validators(registrations), + ) + .await + } + + /// Every node works through its own batches at once, so a node that hangs + /// holds back only its own sequence: the healthy ones have every batch + /// while it is still timing out on its first. A batch counts as sent when + /// any node accepted it, as [`Self::register_validators`] does for one. + /// + /// Each node that refused anything is logged once, with how many batches, + /// rather than once per batch. + async fn register_validators_in_batches( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Vec> { + let per_node = futures_util::future::join_all( + self.nodes + .iter() + .map(|node| node.register_validators_in_batches(registrations)), + ) + .await; + + let batches = registrations.len().div_ceil(super::REGISTRATION_BATCH); + let mut accepted = vec![false; batches]; + let mut last = vec![None; batches]; + for (position, outcomes) in per_node.into_iter().enumerate() { + let mut refused = 0; + let mut first_refusal = None; + for (batch, outcome) in outcomes.into_iter().enumerate() { + match outcome { + Ok(()) => accepted[batch] = true, + Err(err) => { + refused += 1; + let detail = err.to_string(); + first_refusal.get_or_insert_with(|| detail.clone()); + last[batch] = Some(detail); + } + } + } + if let Some(err) = first_refusal { + warn!( + what = "register_validators", + position, + refused, + batches, + %err, + node_effect = "it will not hold these registrations", + "Beacon node did not accept" + ); + } + } + accepted + .into_iter() + .zip(last) + .map(|(accepted, last)| { + if accepted { + Ok(()) + } else { + Err(Error::AllBeaconNodesFailed( + last.unwrap_or_else(|| "no nodes configured".to_string()), + )) + } + }) + .collect() + } + /// Fails over like any query: the envelope the node cached is only on the /// node that produced the block, and one that did not answers 404. async fn execution_payload_envelope( @@ -349,9 +492,12 @@ impl BeaconNodeApi for FallbackBeaconNode { /// that was never told where to pay builds a payload paying somewhere else, /// and it is the node this client falls over to that needs telling most. async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()> { - self.try_all("prepare_beacon_proposer", |node| { - node.prepare_beacon_proposer(preparations) - }) + self.try_all( + "prepare_beacon_proposer", + "it will be unprepared for this duty", + Some("duty likely missed"), + |node| node.prepare_beacon_proposer(preparations), + ) .await } @@ -360,9 +506,12 @@ impl BeaconNodeApi for FallbackBeaconNode { /// the votes an aggregate is folded from, so a node that never received one /// answers 404 for every committee this client asks about. async fn subscribe_committees(&self, subscriptions: &[CommitteeSubscriptionDto]) -> Result<()> { - self.try_all("subscribe_committees", |node| { - node.subscribe_committees(subscriptions) - }) + self.try_all( + "subscribe_committees", + "it will be unprepared for this duty", + Some("duty likely missed"), + |node| node.subscribe_committees(subscriptions), + ) .await } @@ -423,9 +572,12 @@ impl BeaconNodeApi for FallbackBeaconNode { &self, subscriptions: &[SyncCommitteeSubscriptionDto], ) -> Result<()> { - self.try_all("subscribe_sync_committees", |node| { - node.subscribe_sync_committees(subscriptions) - }) + self.try_all( + "subscribe_sync_committees", + "it will be unprepared for this duty", + Some("duty likely missed"), + |node| node.subscribe_sync_committees(subscriptions), + ) .await } } @@ -433,8 +585,10 @@ impl BeaconNodeApi for FallbackBeaconNode { #[cfg(test)] mod tests { use super::*; + use crate::beacon_node::REGISTRATION_BATCH; use crate::beacon_node::mock::MockBeaconNode; use ethlambda_types::beacon::primitives::Root; + use std::sync::Arc; fn healthy() -> MockBeaconNode { let mut node = MockBeaconNode::new(); @@ -601,6 +755,7 @@ mod tests { proposer_index, randao_reveal: BlsSignature([0; 96]), graffiti: Bytes32::default(), + builder_boost_factor: None, } } @@ -850,4 +1005,264 @@ mod tests { "the healthy second node must be found" ); } + + fn blinded_request(slot: Slot, proposer_index: ValidatorIndex) -> BlockRequest { + BlockRequest { + builder_boost_factor: Some(100), + ..request(slot, proposer_index) + } + } + + fn blinded_node() -> MockBeaconNode { + MockBeaconNode::new().with_block(5, 7).with_blinded_block() + } + + /// Node 0 cannot produce, node 1 can and answers blinded: the block names + /// node 1 as its producer, goes back to node 1, and node 0 never sees it. + #[tokio::test] + async fn a_blinded_block_is_published_where_it_was_produced() { + let first = MockBeaconNode::new().failing_call("produce_block", "down"); + let fallback = FallbackBeaconNode::new(vec![first, blinded_node()]); + + let produced = fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("node 1 produces"); + assert!(produced.is_blinded()); + assert_eq!(produced.producer, Producer::at(1)); + + let published = fallback + .publish_blinded_block(produced.producer, ForkName::Electra, &[1, 2, 3]) + .await + .expect("published"); + assert_eq!(published, Published::HandedToBuilder); + + let first_published = fallback.nodes[0].published_blinded.lock().expect("lock"); + let second_published = fallback.nodes[1].published_blinded.lock().expect("lock"); + assert!(first_published.is_empty(), "node 0 never produced it"); + assert_eq!(*second_published, vec![(ForkName::Electra, vec![1, 2, 3])]); + } + + /// The producing node failing the publish is the end of it: another node's + /// builder never saw the bid, so it must not be offered the block. + #[tokio::test] + async fn a_blinded_publish_does_not_fail_over() { + let second = blinded_node().with_unconfirmed_blinded_publish("relay timed out"); + let first = MockBeaconNode::new().failing_call("produce_block", "down"); + let fallback = FallbackBeaconNode::new(vec![first, second]); + + let produced = fallback + .produce_block(&blinded_request(5, 7)) + .await + .expect("node 1 produces"); + let err = fallback + .publish_blinded_block(produced.producer, ForkName::Electra, &[9]) + .await + .expect_err("node 1 failed to confirm"); + assert!( + matches!(err, Error::BlindedUnconfirmed { .. }), + "got {err:?}" + ); + assert!( + fallback.nodes[0] + .published_blinded + .lock() + .expect("lock") + .is_empty(), + "no failover to node 0" + ); + } + + /// A producer this fallback never handed out has no node to go back to, + /// and is refused without reaching any node. + #[tokio::test] + async fn a_blinded_publish_naming_an_unknown_producer_is_refused() { + let fallback = FallbackBeaconNode::new(vec![blinded_node()]); + let err = fallback + .publish_blinded_block(Producer::at(3), ForkName::Electra, &[1]) + .await + .expect_err("there is no node 3"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + assert!( + fallback.nodes[0] + .published_blinded + .lock() + .expect("lock") + .is_empty() + ); + } + + #[tokio::test] + async fn registrations_reach_every_node() { + use ethlambda_types::beacon::builder::{ + SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature, H160}; + + let registration = SignedValidatorRegistrationV1 { + message: ValidatorRegistrationV1 { + fee_recipient: H160([1; 20]), + gas_limit: 30_000_000, + timestamp: 5, + pubkey: BlsPubkey([2; 48]), + }, + signature: BlsSignature([3; 96]), + }; + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::new(), + MockBeaconNode::failing("middle down"), + MockBeaconNode::new(), + ]); + + fallback + .register_validators(std::slice::from_ref(®istration)) + .await + .expect("two of three accepted"); + + for position in [0, 2] { + assert_eq!( + *fallback.nodes[position].registrations.lock().expect("lock"), + vec![registration.clone()], + "node {position}" + ); + } + assert!( + fallback.nodes[1] + .registrations + .lock() + .expect("lock") + .is_empty() + ); + } + + /// Every node is called at once: the second is called while the first + /// has not answered yet, so a hung node does not hold back the ones behind + /// it. Called in turn, the second would wait for a permit that is only + /// given once both have been called. + #[tokio::test] + async fn every_node_is_called_at_once() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let held = || MockBeaconNode::new().with_registration_gate(gate.clone()); + let fallback = Arc::new(FallbackBeaconNode::new(vec![held(), held()])); + + let call = tokio::spawn({ + let fallback = fallback.clone(); + async move { fallback.register_validators(&[]).await } + }); + for node in &fallback.nodes { + node.wait_for_registration_calls(1).await; + } + gate.add_permits(2); + call.await.expect("the call ends").expect("both accept"); + + for node in &fallback.nodes { + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![0]); + } + } + + /// `count` registrations with distinct keys. Unsigned: no mock checks. + fn registrations( + count: usize, + ) -> Vec { + use ethlambda_types::beacon::builder::{ + SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature, H160}; + + (0..count) + .map(|index| { + let mut pubkey = [0u8; 48]; + pubkey[..8].copy_from_slice(&(index as u64).to_be_bytes()); + SignedValidatorRegistrationV1 { + message: ValidatorRegistrationV1 { + fee_recipient: H160([1; 20]), + gas_limit: 30_000_000, + timestamp: 5, + pubkey: BlsPubkey(pubkey), + }, + signature: BlsSignature([3; 96]), + } + }) + .collect() + } + + /// Each node works through the batches on its own: while one node holds + /// its first batch, the other has been sent all of them. In step, the + /// healthy node would have its second batch only once the hung one + /// answered its first. + #[tokio::test] + async fn a_hung_node_holds_back_only_its_own_batches() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let hung = MockBeaconNode::new().with_registration_gate(gate.clone()); + let fallback = Arc::new(FallbackBeaconNode::new(vec![hung, MockBeaconNode::new()])); + let sent = registrations(2 * REGISTRATION_BATCH + 1); + + let send = tokio::spawn({ + let (fallback, sent) = (fallback.clone(), sent.clone()); + async move { fallback.register_validators_in_batches(&sent).await } + }); + let healthy_has_every_batch = async { + while fallback.nodes[1] + .registration_batches + .lock() + .expect("lock") + .len() + < 3 + { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), healthy_has_every_batch) + .await + .expect("the healthy node is not held back by the hung one"); + assert_eq!( + fallback.nodes[0] + .registration_attempts + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "the hung node is still on its first batch" + ); + + gate.add_permits(3); + let outcomes = send.await.expect("the send ends"); + assert_eq!(outcomes.len(), 3); + assert!(outcomes.iter().all(Result::is_ok)); + assert_eq!( + *fallback.nodes[0].registration_batches.lock().expect("lock"), + vec![REGISTRATION_BATCH, REGISTRATION_BATCH, 1] + ); + } + + /// A node that cannot be reached is sent no more batches: each would cost + /// another timeout. A batch counts as sent when any node took it, and as + /// failed when none did. + #[tokio::test] + async fn an_unreachable_node_is_sent_no_more_batches() { + let sent = registrations(REGISTRATION_BATCH + 1); + + let fallback = + FallbackBeaconNode::new(vec![MockBeaconNode::failing("down"), MockBeaconNode::new()]); + let outcomes = fallback.register_validators_in_batches(&sent).await; + assert!(outcomes.iter().all(Result::is_ok), "node 1 took both"); + let attempts = |position: usize| { + fallback.nodes[position] + .registration_attempts + .load(std::sync::atomic::Ordering::SeqCst) + }; + assert_eq!(attempts(0), 1, "stopped after its first batch"); + assert_eq!(attempts(1), 2); + + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::failing("down"), + MockBeaconNode::failing("also down"), + ]); + let outcomes = fallback.register_validators_in_batches(&sent).await; + assert_eq!(outcomes.len(), 2); + assert!( + outcomes + .iter() + .all(|outcome| matches!(outcome, Err(Error::AllBeaconNodesFailed(_)))), + "got {outcomes:?}" + ); + } } diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 013964c74..91f5609cb 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -23,6 +23,7 @@ use std::time::Duration; use async_trait::async_trait; +use ethlambda_types::beacon::builder::SignedValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::electra::Attestation; @@ -36,7 +37,7 @@ use tracing::{debug, warn}; use libssz::SszDecode as _; -use crate::beacon_node::block_contents::ProducedBlock; +use crate::beacon_node::block_contents::{ProducedBlock, Producer}; use crate::beacon_node::dto::{ AttestationDataDto, AttestationDto, AttesterDutyDto, BlockRootResponse, CommitteeSubscriptionDto, DataResponse, DutiesResponse, GenesisDto, IndexedErrorResponse, @@ -60,6 +61,67 @@ use crate::error::{BeaconNodeFailure, Error, Result}; /// a stalled request could then outlive the slot it was serving. const REQUEST_TIMEOUT: Duration = Duration::from_secs(8); +/// How long establishing a connection (TCP, and TLS for `https`) may take, +/// within [`REQUEST_TIMEOUT`]. +/// +/// Its own bound so that a node that cannot be reached fails as a connect +/// error, which says the request never left, rather than as the total +/// timeout, which cannot say whether it did. That difference decides how a +/// blinded publish is reported: a block that never reached the node is a +/// missed proposal, while one the node may have handed to its relay is +/// unconfirmed (see [`BeaconNodeApi::publish_blinded_block`]). Without it, an +/// unreachable producing node counted a slot that was certainly missed as one +/// the relay might still publish. +/// +/// Three seconds is far beyond a healthy handshake, local or across a +/// continent, and leaves most of the request budget to the answer. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(3); + +/// How long `register_validator` may take. Longer than [`REQUEST_TIMEOUT`] +/// because an ethlambda node forwards the registrations to its relay and waits +/// up to 10 s for the answer before replying; a client that gave up at 8 s +/// would report a failure for a registration the node went on to install. +const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(12); + +/// How much of a failed answer's body is read. Enough for a pool submission's +/// 400, which lists each refused entry and is parsed for them. +const ERROR_BODY_READ_LIMIT: usize = 1024 * 1024; + +/// How much of a failed answer's body an error carries, and so a log line. +const ERROR_TEXT_LIMIT: usize = 1024; + +/// A failed answer's body, read up to [`ERROR_BODY_READ_LIMIT`] bytes, so a +/// node answering with an endless body cannot hold the client on it. Only +/// what fits is copied out of each chunk. +async fn read_error_body(mut response: reqwest::Response) -> String { + let mut bytes = Vec::new(); + while let Ok(Some(chunk)) = response.chunk().await { + let room = ERROR_BODY_READ_LIMIT - bytes.len(); + bytes.extend_from_slice(&chunk[..chunk.len().min(room)]); + if bytes.len() == ERROR_BODY_READ_LIMIT { + break; + } + } + String::from_utf8_lossy(&bytes).into_owned() +} + +/// Text a node sent, as an error or a log line shows it: control characters +/// and line separators written out (`\n`, `\u{1b}`, ...), so the line it lands +/// in stays one line and carries no terminal escapes, and cut to +/// [`ERROR_TEXT_LIMIT`] bytes without splitting a character or an escape, +/// ending in `...` when it is cut. +/// +/// The same function the beacon node's builder client writes a relay's text +/// through, [`ethlambda_types::text::one_line`]. +fn error_text(text: &str) -> String { + ethlambda_types::text::one_line(text, ERROR_TEXT_LIMIT) +} + +/// [`read_error_body`] as [`error_text`] shows it. +async fn error_body(response: reqwest::Response) -> String { + error_text(&read_error_body(response).await) +} + /// What an SSZ answer carries besides its bytes: the fork, which only a header /// can say, and the two booleans the block endpoints use to say what the bytes /// are. @@ -87,9 +149,19 @@ impl HttpBeaconNode { /// Builds a client for the node at `base_url`, stripping any trailing /// slash so callers may pass either form. pub fn new(base_url: impl Into) -> Result { + Self::with_connect_timeout(base_url, CONNECT_TIMEOUT) + } + + /// [`Self::new`] with `connect_timeout` in place of [`CONNECT_TIMEOUT`], + /// so a test can see a connect timeout without waiting three seconds. + fn with_connect_timeout( + base_url: impl Into, + connect_timeout: Duration, + ) -> Result { let base_url = base_url.into().trim_end_matches('/').to_string(); let client = Client::builder() .timeout(REQUEST_TIMEOUT) + .connect_timeout(connect_timeout) .build() .map_err(|err| Error::BeaconNode { url: base_url.clone(), @@ -146,12 +218,28 @@ impl HttpBeaconNode { path: &str, body: &B, consensus_version: Option<&str>, + ) -> Result<()> { + self.post_no_content_within(path, body, consensus_version, None) + .await + } + + /// [`Self::post_no_content`] with `timeout` in place of the client's + /// default, when given. + async fn post_no_content_within( + &self, + path: &str, + body: &B, + consensus_version: Option<&str>, + timeout: Option, ) -> Result<()> { let url = format!("{}{path}", self.base_url); let mut request = self.client.post(&url).json(body); if let Some(version) = consensus_version { request = request.header("Eth-Consensus-Version", version); } + if let Some(timeout) = timeout { + request = request.timeout(timeout); + } let response = request.send().await.map_err(|err| Error::BeaconNode { url: url.clone(), failure: BeaconNodeFailure::classify(&err), @@ -161,13 +249,37 @@ impl HttpBeaconNode { if status.is_success() { return Ok(()); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; Err(Error::BeaconNodeStatus { status: status.as_u16(), body, }) } + /// POST an SSZ body with the fork it is for, returning the URL beside the + /// outcome so each caller maps a transport failure its own way: a full + /// block's is an ordinary error, a blinded block's may be unconfirmed. + async fn post_ssz_body( + &self, + path: &str, + fork: ForkName, + body: &[u8], + ) -> ( + String, + std::result::Result, + ) { + let url = format!("{}{path}", self.base_url); + let sent = self + .client + .post(&url) + .header("Eth-Consensus-Version", fork.as_str()) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body.to_vec()) + .send() + .await; + (url, sent) + } + /// Fetch a body as SSZ, returning the fork the node named alongside it. /// /// `Accept` names SSZ alone, deliberately narrower than the specification's @@ -236,7 +348,7 @@ impl HttpBeaconNode { return Err(Error::BeaconNodeSyncing); } if !status.is_success() { - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; return Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -434,9 +546,11 @@ impl HttpBeaconNode { let failed = error.failures.len(); if failed > 0 && failed < submitted { for failure in &error.failures { + // The node's text, so written out and capped like a whole + // error body: one line per failure, whatever it sent. warn!( submission_index = failure.index, - reason = %failure.message, + reason = %error_text(&failure.message), "One attestation in this slot's batch was rejected" ); } @@ -446,7 +560,7 @@ impl HttpBeaconNode { Err(Error::BeaconNodeStatus { status: status.as_u16(), - body, + body: error_text(&body), }) } @@ -456,7 +570,7 @@ impl HttpBeaconNode { return Err(Error::BeaconNodeSyncing); } if !status.is_success() { - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; return Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -593,8 +707,9 @@ impl BeaconNodeApi for HttpBeaconNode { Ok(data) } - /// `builder_boost_factor=0` is how this client says it does not do the - /// builder flow. + /// `builder_boost_factor` is how this client says whether it does the + /// builder flow: the request's factor when builder proposals are on, `0` + /// when they are off (`None`). /// /// The parameter is a bid comparison: the node returns the local execution /// payload when its value is at least `factor / 100` of the builder's, so @@ -607,19 +722,21 @@ impl BeaconNodeApi for HttpBeaconNode { /// answer with a blinded block. The one unconditional guarantee is the /// other half: a node with no builder configured MUST return a full block. /// - /// A blinded answer is therefore rejected below rather than assumed away. - /// This client cannot publish what it cannot unblind, so signing one would - /// burn the slot's proposal guard entry for a block that can never be - /// sent. + /// A blinded answer is therefore never assumed away. With builder proposals + /// off (`None`) it is rejected: this client would not publish it through + /// `blinded_blocks`, so signing one would burn the slot's slashing-protection + /// record for a block that can never be sent. With them on it is decoded as + /// a blinded block, which `publish_blinded_block` sends. async fn produce_block(&self, request: &BlockRequest) -> Result { if request.fork >= ForkName::Gloas { return self.produce_gloas_block(request).await; } let path = format!( - "/eth/v3/validator/blocks/{}?randao_reveal={}&graffiti={}&builder_boost_factor=0", + "/eth/v3/validator/blocks/{}?randao_reveal={}&graffiti={}&builder_boost_factor={}", request.slot, encode_hex(&request.randao_reveal.0), encode_hex(&request.graffiti.0), + request.builder_boost_factor.unwrap_or(0), ); let SszResponse { fork, @@ -639,15 +756,19 @@ impl BeaconNodeApi for HttpBeaconNode { .to_string(), ) })?; - if blinded { + if blinded && request.builder_boost_factor.is_none() { return Err(Error::InconsistentResponse(format!( - "node produced a blinded block for slot {}; this client does not implement the \ - builder flow and cannot publish one", + "node produced a blinded block for slot {}; builder proposals are off, so this \ + client cannot publish one", request.slot ))); } - let block = ProducedBlock::from_ssz(fork, &body)?; + let block = if blinded { + ProducedBlock::from_blinded_ssz(fork, &body)? + } else { + ProducedBlock::from_ssz(fork, &body)? + }; // Enforced here rather than at the call site so failover works: see the // contract on `BeaconNodeApi::produce_block`. validate_produced_block(request, &block)?; @@ -657,21 +778,14 @@ impl BeaconNodeApi for HttpBeaconNode { /// 202 is not an error and not a clean success, so it is neither mapped to /// `Err` nor flattened into `Ok(())`. See [`Published`]. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result { - let path = "/eth/v2/beacon/blocks"; - let url = format!("{}{path}", self.base_url); - let response = self - .client - .post(&url) - .header("Eth-Consensus-Version", fork.as_str()) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body.to_vec()) - .send() - .await - .map_err(|err| Error::BeaconNode { - url: url.clone(), - failure: BeaconNodeFailure::classify(&err), - detail: err.to_string(), - })?; + let (url, sent) = self + .post_ssz_body("/eth/v2/beacon/blocks", fork, body) + .await; + let response = sent.map_err(|err| Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + })?; let status = response.status(); if status == StatusCode::ACCEPTED { @@ -680,13 +794,95 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(Published::Imported); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; Err(Error::BeaconNodeStatus { status: status.as_u16(), body, }) } + /// Maps the answer as the trait's contract says: 2xx is handed to the + /// builder (202 is the node's broadcast-without-import, as for a full + /// block); a 4xx, a 503 and a connection that was never established are + /// definite; every other 5xx, and any failure once connected, is + /// [`Error::BlindedUnconfirmed`]. + /// + /// A connect error covers a connection refused and one that timed out + /// ([`CONNECT_TIMEOUT`]); reqwest reports the latter as both a connect + /// error and a timeout, and the connect check comes first. + /// + /// The status alone cannot always tell, and two edges are knowingly read + /// the wrong way. Some of ethlambda's own 500s are definite (no builder + /// configured, the blinded block failing the node's checks) and are read as + /// unconfirmed. A 503 from a proxy that had already forwarded the request + /// is read as definite. Neither changes what happens next: nothing is + /// retried and nothing is signed again either way. Only the log line and + /// the counter it lands on are wrong. + async fn publish_blinded_block( + &self, + // For the fallback, which picks the node that produced this block; + // one node has nothing to choose. + _producer: Producer, + fork: ForkName, + body: &[u8], + ) -> Result { + let (url, sent) = self + .post_ssz_body("/eth/v2/beacon/blinded_blocks", fork, body) + .await; + let response = sent.map_err(|err| { + if err.is_connect() { + Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + } + } else { + Error::BlindedUnconfirmed { + status: None, + detail: format!("{url}: {err}"), + } + } + })?; + + let status = response.status(); + if status == StatusCode::ACCEPTED { + return Ok(Published::BroadcastNotImported); + } + if status.is_success() { + return Ok(Published::HandedToBuilder); + } + let body = error_body(response).await; + if status.is_server_error() && status != StatusCode::SERVICE_UNAVAILABLE { + return Err(Error::BlindedUnconfirmed { + status: Some(status.as_u16()), + detail: format!("{url} answered {status}: {body}"), + }); + } + Err(Error::BeaconNodeStatus { + status: status.as_u16(), + body, + }) + } + + /// JSON, an empty list never reaching the wire. + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + if registrations.is_empty() { + return Ok(()); + } + // Its own timeout: the node waits up to 10 s on the relay before it + // answers, which is longer than every other request's default. + self.post_no_content_within( + "/eth/v1/validator/register_validator", + ®istrations, + None, + Some(REGISTRATION_TIMEOUT), + ) + .await + } + async fn execution_payload_envelope( &self, slot: Slot, @@ -744,7 +940,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(()); } - let body = response.text().await.unwrap_or_default(); + let body = error_body(response).await; Err(Error::BeaconNodeStatus { status: status.as_u16(), body, @@ -813,7 +1009,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(messages.len()); } - let body = response.text().await.unwrap_or_default(); + let body = read_error_body(response).await; Self::handle_pool_submission(status, body, messages.len()) } @@ -843,7 +1039,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(attestations.len()); } - let body = response.text().await.unwrap_or_default(); + let body = read_error_body(response).await; Self::handle_pool_submission(status, body, attestations.len()) } @@ -1008,7 +1204,7 @@ impl BeaconNodeApi for HttpBeaconNode { if status.is_success() { return Ok(messages.len()); } - let body = response.text().await.unwrap_or_default(); + let body = read_error_body(response).await; Self::handle_pool_submission(status, body, messages.len()) } @@ -1189,6 +1385,327 @@ mod tests { assert_eq!(value["builders"], serde_json::json!([])); } + /// A Beacon API double that answers every route the same way: whatever + /// the handler sees is pushed to `seen` as (uri, body, request headers), + /// and the answer is `status` with `headers` and `reply` as the body. + type Seen = std::sync::Arc, axum::http::HeaderMap)>>>; + type Canned = (Seen, u16, Vec<(&'static str, &'static str)>, Vec); + + struct Served { + node: HttpBeaconNode, + seen: Seen, + } + + async fn serve( + status: u16, + headers: Vec<(&'static str, &'static str)>, + reply: Vec, + ) -> Served { + use axum::body::Bytes; + use axum::extract::State; + use axum::http::Uri; + use axum::response::IntoResponse; + + let seen: Seen = Default::default(); + let state = (seen.clone(), status, headers, reply); + let handler = |State(state): State, + uri: Uri, + request_headers: axum::http::HeaderMap, + body: Bytes| async move { + let (seen, status, headers, reply) = state; + seen.lock() + .expect("lock") + .push((uri.to_string(), body.to_vec(), request_headers)); + let mut response = ( + axum::http::StatusCode::from_u16(status).expect("a status"), + reply, + ) + .into_response(); + for (name, value) in headers { + response + .headers_mut() + .insert(name, axum::http::HeaderValue::from_static(value)); + } + response + }; + let app = axum::Router::new().fallback(handler).with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("binds"); + let address = listener.local_addr().expect("address"); + tokio::spawn(async move { axum::serve(listener, app).await }); + Served { + node: HttpBeaconNode::new(format!("http://{address}")).expect("builds"), + seen, + } + } + + fn block_request(factor: Option) -> BlockRequest { + BlockRequest { + slot: 5, + fork: ForkName::Fulu, + proposer_index: 7, + randao_reveal: Default::default(), + graffiti: Default::default(), + builder_boost_factor: factor, + } + } + + const SSZ_HEADERS: [(&str, &str); 3] = [ + ("content-type", "application/octet-stream"), + ("eth-consensus-version", "fulu"), + ("eth-execution-payload-blinded", "true"), + ]; + + fn blinded_bytes() -> Vec { + use libssz::SszEncode as _; + ethlambda_types::beacon::builder::BlindedBeaconBlock::from( + crate::beacon_node::block_contents::empty_block_for(5, 7), + ) + .to_ssz() + } + + #[tokio::test] + async fn the_boost_factor_is_zero_without_builder_proposals_and_the_factor_with() { + for (factor, expected) in [ + (None, "builder_boost_factor=0"), + (Some(250), "builder_boost_factor=250"), + ] { + let served = serve(500, vec![], vec![]).await; + let _ = served.node.produce_block(&block_request(factor)).await; + let seen = served.seen.lock().expect("lock"); + assert_eq!(seen.len(), 1); + assert!( + seen[0].0.contains(expected), + "{} lacks {expected}", + seen[0].0 + ); + } + } + + #[tokio::test] + async fn a_blinded_answer_is_refused_without_builder_proposals() { + let served = serve(200, SSZ_HEADERS.to_vec(), blinded_bytes()).await; + let err = served + .node + .produce_block(&block_request(None)) + .await + .expect_err("a blinded block cannot be published here"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + } + + #[tokio::test] + async fn a_blinded_answer_decodes_as_blinded_with_builder_proposals() { + let served = serve(200, SSZ_HEADERS.to_vec(), blinded_bytes()).await; + let produced = served + .node + .produce_block(&block_request(Some(100))) + .await + .expect("decodes"); + assert!(produced.is_blinded()); + assert_eq!(produced.slot(), 5); + assert_eq!(produced.proposer_index(), 7); + } + + #[tokio::test] + async fn registrations_are_posted_as_json_that_round_trips() { + use ethlambda_types::beacon::builder::{ + SignedValidatorRegistrationV1, ValidatorRegistrationV1, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature, H160}; + + let registrations = vec![SignedValidatorRegistrationV1 { + message: ValidatorRegistrationV1 { + fee_recipient: H160([0xab; 20]), + gas_limit: 36_000_000, + timestamp: 1_700_000_000, + pubkey: BlsPubkey([3; 48]), + }, + signature: BlsSignature([4; 96]), + }]; + let served = serve(200, vec![], vec![]).await; + served + .node + .register_validators(®istrations) + .await + .expect("accepted"); + + let seen = served.seen.lock().expect("lock"); + assert_eq!(seen.len(), 1); + assert!(seen[0].0.ends_with("/eth/v1/validator/register_validator")); + let decoded: Vec = + serde_json::from_slice(&seen[0].1).expect("the body is the JSON a relay takes"); + assert_eq!(decoded, registrations); + } + + #[tokio::test] + async fn an_empty_registration_list_never_reaches_the_wire() { + let served = serve(200, vec![], vec![]).await; + served.node.register_validators(&[]).await.expect("a no-op"); + assert!(served.seen.lock().expect("lock").is_empty()); + } + + /// A node's error body reaches the logs on one line and cut short, saying + /// so: a body with line breaks cannot forge log lines, a long one cannot + /// flood them, and a cut one does not read as all the node said. + #[tokio::test] + async fn an_error_body_is_one_line_and_capped() { + let reply = "first line\nsecond line\r\n".repeat(200).into_bytes(); + let served = serve(400, vec![], reply).await; + let err = served.node.genesis().await.expect_err("a 400"); + let Error::BeaconNodeStatus { status, body } = err else { + panic!("got {err:?}"); + }; + assert_eq!(status, 400); + assert!(!body.contains('\n') && !body.contains('\r')); + // Written out, then cut to the cap less the marker's three bytes, + // which here falls inside a word. + let written = "first line\\nsecond line\\r\\n".repeat(200); + let kept = ERROR_TEXT_LIMIT - "...".len(); + assert_eq!(body, format!("{}...", &written[..kept])); + } + + /// The cap applies to the written-out text and never splits a character + /// or an escape: a body of escapes keeps whole ones, then the marker. + #[test] + fn error_text_cuts_on_a_character_or_escape_boundary() { + let room = ERROR_TEXT_LIMIT - "...".len(); + let text = error_text(&"é".repeat(ERROR_TEXT_LIMIT)); + assert_eq!(text, format!("{}...", "é".repeat(room / 2))); + let text = error_text(&"\u{1b}".repeat(ERROR_TEXT_LIMIT)); + let escapes = "\\u{1b}".repeat(room / "\\u{1b}".len()); + assert_eq!(text, format!("{escapes}...")); + assert_eq!(error_text("short"), "short"); + } + + #[test] + fn error_text_writes_out_every_control_character() { + let text = error_text("a\u{1b}[31mb\tc\u{2028}d"); + assert_eq!(text, "a\\u{1b}[31mb\\tc\\u{2028}d"); + } + + #[tokio::test] + async fn a_blinded_publish_maps_each_status() { + for (status, expected) in [ + (200, Published::HandedToBuilder), + (202, Published::BroadcastNotImported), + ] { + let served = serve(status, vec![], vec![]).await; + let published = served + .node + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1, 2, 3]) + .await + .expect("accepted"); + assert_eq!(published, expected, "status {status}"); + let seen = served.seen.lock().expect("lock"); + assert!(seen[0].0.ends_with("/eth/v2/beacon/blinded_blocks")); + assert_eq!(seen[0].1, vec![1, 2, 3]); + assert_eq!(seen[0].2["eth-consensus-version"], "fulu"); + assert_eq!(seen[0].2["content-type"], "application/octet-stream"); + } + + let served = serve(400, vec![], b"bad".to_vec()).await; + let err = served + .node + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) + .await + .expect_err("a refusal"); + assert!( + matches!(err, Error::BeaconNodeStatus { status: 400, .. }), + "got {err:?}" + ); + + let served = serve(500, vec![], b"boom".to_vec()).await; + let err = served + .node + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) + .await + .expect_err("unconfirmed"); + assert!( + matches!( + err, + Error::BlindedUnconfirmed { + status: Some(500), + .. + } + ), + "got {err:?}" + ); + } + + /// The request never left, so the relay cannot have the block: a definite + /// failure, not an unconfirmed one. + #[tokio::test] + async fn a_blinded_publish_that_cannot_connect_is_a_definite_failure() { + // Nothing listens on port 1. + let node = HttpBeaconNode::new("http://127.0.0.1:1").expect("builds"); + let err = node + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) + .await + .expect_err("no answer"); + assert!(matches!(err, Error::BeaconNode { .. }), "got {err:?}"); + } + + /// A connection that never completes is a connect error too, not the + /// total timeout, so it is definite: nothing was sent. The listener + /// accepts TCP but never answers the TLS handshake, which is part of + /// connecting, so the connect timeout fires on every platform and with no + /// unroutable address involved. + #[tokio::test] + async fn a_blinded_publish_whose_connection_times_out_is_a_definite_failure() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("binds"); + let address = listener.local_addr().expect("address"); + let held = tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + let connect_timeout = Duration::from_millis(200); + let node = + HttpBeaconNode::with_connect_timeout(format!("https://{address}"), connect_timeout) + .expect("builds"); + + let started = std::time::Instant::now(); + let err = node + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) + .await + .expect_err("never connects"); + held.abort(); + + assert!( + matches!( + err, + Error::BeaconNode { + failure: BeaconNodeFailure::Timeout, + .. + } + ), + "got {err:?}" + ); + assert!( + started.elapsed() < REQUEST_TIMEOUT, + "the connect timeout, not the request's, ended it" + ); + } + + /// A syncing node refuses before it hands anything to a builder. + #[tokio::test] + async fn a_blinded_publish_to_a_syncing_node_is_a_definite_failure() { + let served = serve(503, vec![], b"syncing".to_vec()).await; + let err = served + .node + .publish_blinded_block(Producer::default(), ForkName::Fulu, &[1]) + .await + .expect_err("refused"); + assert!( + matches!(err, Error::BeaconNodeStatus { status: 503, .. }), + "got {err:?}" + ); + } + #[test] fn a_boolean_header_is_read_strictly() { assert_eq!( diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index fe4a952b9..4d6af0aa0 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -2,9 +2,10 @@ //! service tests. Compiled only under `cfg(test)`. use std::collections::HashMap; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; +use ethlambda_types::beacon::builder::{BlindedBeaconBlock, SignedValidatorRegistrationV1}; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::electra::{Attestation, SignedAggregateAndProof}; @@ -13,9 +14,10 @@ use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::containers::shared::Checkpoint; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; +use tokio::sync::{Notify, Semaphore}; use crate::beacon_node::block_contents::{ - Contents, GloasPayload, ProducedBlock, empty_block_for, empty_gloas_block_for, + Contents, GloasPayload, ProducedBlock, Producer, empty_block_for, empty_gloas_block_for, empty_gloas_envelope_for, }; use crate::beacon_node::dto::{ @@ -79,6 +81,49 @@ pub struct MockBeaconNode { /// What `publish_block` reports. Defaults to `Imported`; a test that cares /// about the 202 path sets it. pub publish_outcome: Option, + /// When set, `produce_block` answers a pre-gloas request that carries a + /// `builder_boost_factor` with a blinded block (for the slot and proposer + /// `produces_block` names), as a node whose builder won the slot does. A + /// request with no factor still gets a full block, as the contract on + /// `BeaconNodeApi::produce_block` requires. + pub produces_blinded: bool, + /// With [`Self::produces_blinded`], answer with the blinded block even + /// when the request carries no factor: a node breaking the contract, so a + /// test can see what the caller does with one. + pub blinded_regardless: bool, + /// Every registration `register_validators` received, in call order. + pub registrations: Mutex>, + /// The size of each `register_validators` call, in call order, so a test + /// can see how a send was batched. + pub registration_batches: Mutex>, + /// Every `register_validators` call, refused and still-waiting ones + /// included, so a test can tell a call reached the node. + pub registration_attempts: std::sync::atomic::AtomicUsize, + /// When set, each `register_validators` call takes a permit before it + /// records its batch, so a test releases calls one at a time instead of + /// guessing how long a delay must be. + pub registration_gate: Option>, + /// Every blinded block published, as (fork, body). + pub published_blinded: Mutex)>>, + /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; + /// `Err(detail)` is a `BlindedUnconfirmed` carrying `detail`. + pub blinded_publish_outcome: Option>, + /// When set, `publish_blinded_block` answers with this status as a node + /// that definitely refused the block (a 4xx) does: `BeaconNodeStatus`. + pub blinded_publish_refusal: Option, + /// When set, `publish_blinded_block` takes a permit before it records the + /// block and answers, as a node waiting on a slow relay does, for as long + /// as the test holds the permit back. + pub blinded_publish_gate: Option>, + /// Notified when `publish_blinded_block` is called, before the gate. A + /// test that publishes once can await it whether or not the call came + /// first, since `notify_one` keeps the permit for a later waiter. + pub blinded_publish_entered: Arc, + /// When set, `produce_block` takes a permit before it answers, as a node + /// slow to build a block does, for as long as the test holds it back. + pub produce_gate: Option>, + /// Notified when `produce_block` is called, before the gate. + pub produce_entered: Arc, /// What the test asserts against. pub submitted: Mutex>, @@ -258,6 +303,72 @@ impl MockBeaconNode { self } + /// Answer a `produce_block` that carries a `builder_boost_factor` with a + /// blinded block, for the slot and proposer `with_block` names. + pub fn with_blinded_block(mut self) -> Self { + self.produces_blinded = true; + self + } + + /// Answer every pre-gloas `produce_block` with a blinded block, factor or + /// not, as a node breaking the contract on `BeaconNodeApi::produce_block`. + pub fn with_blinded_block_regardless(mut self) -> Self { + self.produces_blinded = true; + self.blinded_regardless = true; + self + } + + /// Make `publish_blinded_block` fail as a node that did not confirm the + /// block, carrying `detail`. + pub fn with_unconfirmed_blinded_publish(mut self, detail: &str) -> Self { + self.blinded_publish_outcome = Some(Err(detail.to_string())); + self + } + + /// Wait, up to five seconds, until `register_validators` has been called + /// `calls` times, counting calls still held at the gate. + pub async fn wait_for_registration_calls(&self, calls: usize) { + let reached = async { + while self + .registration_attempts + .load(std::sync::atomic::Ordering::SeqCst) + < calls + { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), reached) + .await + .unwrap_or_else(|_| panic!("register_validators was not called {calls} times")); + } + + /// Make each `register_validators` call wait for a permit from `gate` + /// before it records its batch. + pub fn with_registration_gate(mut self, gate: Arc) -> Self { + self.registration_gate = Some(gate); + self + } + + /// Make `produce_block` wait for a permit from `gate` before it answers. + pub fn with_produce_gate(mut self, gate: Arc) -> Self { + self.produce_gate = Some(gate); + self + } + + /// Make `publish_blinded_block` wait for a permit from `gate` before it + /// records the block and answers. + pub fn with_blinded_publish_gate(mut self, gate: Arc) -> Self { + self.blinded_publish_gate = Some(gate); + self + } + + /// Make `publish_blinded_block` fail as a node that definitely refused the + /// block, with `status`. + pub fn with_refused_blinded_publish(mut self, status: u16) -> Self { + self.blinded_publish_refusal = Some(status); + self + } + /// Answer `produce_block` at a gloas slot with a bare gloas block whose /// bid names `builder_index`, with its self-built envelope attached when /// `include_payload` is set. @@ -609,6 +720,13 @@ impl BeaconNodeApi for MockBeaconNode { /// failover moves past it. async fn produce_block(&self, request: &BlockRequest) -> Result { self.guard("produce_block")?; + self.produce_entered.notify_one(); + if let Some(gate) = &self.produce_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } self.block_requests .lock() .expect("lock") @@ -631,6 +749,7 @@ impl BeaconNodeApi for MockBeaconNode { let block = ProducedBlock { fork: ForkName::Gloas, contents: Contents::Gloas { block, payload }, + producer: Default::default(), }; crate::beacon_node::validate_produced_block(request, &block)?; return Ok(block); @@ -640,6 +759,19 @@ impl BeaconNodeApi for MockBeaconNode { failure: BeaconNodeFailure::Request, detail: "no block set".into(), })?; + if self.produces_blinded + && (request.builder_boost_factor.is_some() || self.blinded_regardless) + { + let block = ProducedBlock { + fork: ForkName::Electra, + contents: Contents::Blinded { + block: BlindedBeaconBlock::from(empty_block_for(slot, proposer_index)), + }, + producer: Default::default(), + }; + crate::beacon_node::validate_produced_block(request, &block)?; + return Ok(block); + } // Electra, matching the block shape `empty_block_for` builds. A test // that needs a fork mismatch constructs one itself rather than getting // it by accident here. @@ -650,6 +782,7 @@ impl BeaconNodeApi for MockBeaconNode { kzg_proofs: Default::default(), blobs: Default::default(), }, + producer: Default::default(), }; crate::beacon_node::validate_produced_block(request, &block)?; Ok(block) @@ -664,6 +797,67 @@ impl BeaconNodeApi for MockBeaconNode { Ok(self.publish_outcome.unwrap_or(Published::Imported)) } + async fn publish_blinded_block( + &self, + _producer: Producer, + fork: ForkName, + body: &[u8], + ) -> Result { + // `failures` and `fail_everything` stand for a node that could not be + // reached, which over HTTP is a definite failure: the request never + // left. An unconfirmed publish is `with_unconfirmed_blinded_publish`. + self.guard("publish_blinded_block")?; + if let Some(status) = self.blinded_publish_refusal { + return Err(Error::BeaconNodeStatus { + status, + body: "refused".to_string(), + }); + } + self.blinded_publish_entered.notify_one(); + if let Some(gate) = &self.blinded_publish_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + self.published_blinded + .lock() + .expect("lock") + .push((fork, body.to_vec())); + match &self.blinded_publish_outcome { + None => Ok(Published::HandedToBuilder), + Some(Ok(published)) => Ok(*published), + Some(Err(detail)) => Err(Error::BlindedUnconfirmed { + status: None, + detail: detail.clone(), + }), + } + } + + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + self.registration_attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.guard("register_validators")?; + if let Some(gate) = &self.registration_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + self.registration_batches + .lock() + .expect("lock") + .push(registrations.len()); + self.registrations + .lock() + .expect("lock") + .extend(registrations.iter().cloned()); + Ok(()) + } + async fn execution_payload_envelope( &self, slot: Slot, diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 49c1c54dd..7a1706ab3 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -6,6 +6,7 @@ //! (a reorg invalidating duties, a node failing mid-slot) untestable. use async_trait::async_trait; +use ethlambda_types::beacon::builder::SignedValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::fork::ForkName; @@ -14,12 +15,12 @@ use ethlambda_types::beacon::primitives::{ }; use ethlambda_types::beacon::signing::compute_epoch_at_slot; -use crate::beacon_node::block_contents::ProducedBlock; +use crate::beacon_node::block_contents::{ProducedBlock, Producer}; use crate::beacon_node::dto::{ AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, PtcDutyDto, SingleAttestationDto, SyncCommitteeSubscriptionDto, SyncDutyDto, }; -use crate::error::Result; +use crate::error::{BeaconNodeFailure, Error, Result}; use ethlambda_types::beacon::containers::altair; use ethlambda_types::beacon::containers::electra; use ethlambda_types::beacon::containers::gloas; @@ -32,6 +33,11 @@ pub mod http; #[cfg(test)] pub mod mock; +/// Builder registrations per request. Lighthouse's batch size: large enough +/// that a few thousand validators take a handful of requests, small enough +/// that one request stays well under a node's body limit. +pub const REGISTRATION_BATCH: usize = 500; + /// Check that `data` is an answer about `slot`, as the contract on /// [`BeaconNodeApi::attestation_data`] requires. /// @@ -142,6 +148,11 @@ pub struct BlockRequest { pub randao_reveal: BlsSignature, /// Thirty-two bytes consensus never reads. pub graffiti: Bytes32, + /// The `builder_boost_factor` to send, or `None` when builder proposals are + /// off. `None` sends `0` and refuses a blinded answer; `Some` sends the + /// factor and accepts one. Ignored from gloas, where `produceBlockV4` takes + /// the builder configuration in its body. + pub builder_boost_factor: Option, } /// What became of a published block. @@ -157,6 +168,10 @@ pub enum Published { Imported, /// 202: broadcast, but the node could not import it. BroadcastNotImported, + /// 200 from `blinded_blocks`: the node handed the signed blinded block to + /// its builder, which publishes the block and reveals the payload. Nothing + /// says the node imported it. + HandedToBuilder, } /// Check that `block` is the block that was asked for. @@ -364,12 +379,13 @@ pub trait BeaconNodeApi: Send + Sync { /// returning, so that a node answering about the wrong slot or naming the /// wrong proposer is failed over from rather than signed for. /// - /// # Contract: never a blinded block + /// # Contract: a blinded block only when builder proposals are on /// - /// This client does not implement the builder flow, so it cannot publish a - /// blinded block and must not sign one. An implementation must ask for an - /// unblinded block and reject a blinded answer rather than returning - /// something the caller will sign and then be unable to send. + /// A blinded block is returned only when `request.builder_boost_factor` is + /// `Some`, since only then does the caller publish one through + /// [`Self::publish_blinded_block`]. With `None` an implementation must ask + /// for an unblinded block and reject a blinded answer rather than + /// returning something the caller will sign and then be unable to send. async fn produce_block(&self, request: &BlockRequest) -> Result; /// Publish an already-signed block, given the SSZ body and the fork it was @@ -385,6 +401,93 @@ pub trait BeaconNodeApi: Send + Sync { /// proposal, whether or not the first one succeeded. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result; + /// Publish a signed blinded block, produced under `fork` by `producer` + /// (the [`ProducedBlock::producer`] of the block it was signed from), to + /// `POST /eth/v2/beacon/blinded_blocks`. The node hands it to its builder. + /// + /// # Contract: the node that produced it, and no other + /// + /// Only the producing node's builder holds the payload, and only that node + /// claimed the slot. Failover must not offer it elsewhere: the fallback + /// sends it to `producer` alone. + /// + /// # Contract: which error means what + /// + /// The line is whether the node can have handed the block to its builder. + /// + /// - **Definite**, an `Err` other than `Error::BlindedUnconfirmed`: the + /// block was not published. A 4xx; a 503, which a syncing node answers + /// before it hands anything to a builder; and a connection that was never + /// established, refused or timed out, since the request never left. + /// - **Unconfirmed**, `Error::BlindedUnconfirmed`: every other 5xx, and any + /// failure once the connection was up (no answer in time, a reset). The + /// relay may still publish the block. + /// + /// The status cannot always tell, and the HTTP implementation documents the + /// two edges it knowingly reads the wrong way. + async fn publish_blinded_block( + &self, + producer: Producer, + fork: ForkName, + body: &[u8], + ) -> Result; + + /// Send signed builder registrations to the node in one request, which it + /// forwards to its builder. JSON, since a node may forward the body + /// unchanged to MEV-Boost. State installed on the node's builder, so + /// failover sends it to every node. + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()>; + + /// Send `registrations` in requests of at most [`REGISTRATION_BATCH`], one + /// after another, and say what became of each batch: one outcome per + /// batch, in order, `Ok` when the node accepted it. + /// + /// Batched here, below the fallback's fan-out, so that each node works + /// through its own sequence of batches beside the others: a hung node + /// holds back only its own. Batched above it, every batch would wait for + /// every node before the next one started. + /// + /// Once the node cannot be reached ([`Error::BeaconNode`]) the batches left + /// are not sent, and fail with that error: each would cost another + /// timeout, and the send waiting on them holds back the next keymanager + /// change (see `crate::registration`). A node that answered with an error + /// status is up, so it is still given the rest. + /// + /// [`Error::BeaconNode`]: crate::error::Error::BeaconNode + async fn register_validators_in_batches( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Vec> { + let mut outcomes = Vec::new(); + let mut unreachable: Option<(String, BeaconNodeFailure, String)> = None; + for batch in registrations.chunks(REGISTRATION_BATCH) { + if let Some((url, failure, detail)) = &unreachable { + outcomes.push(Err(Error::BeaconNode { + url: url.clone(), + failure: *failure, + detail: format!( + "not sent, an earlier batch could not reach the node: {detail}" + ), + })); + continue; + } + let outcome = self.register_validators(batch).await; + if let Err(Error::BeaconNode { + url, + failure, + detail, + }) = &outcome + { + unreachable = Some((url.clone(), *failure, detail.clone())); + } + outcomes.push(outcome); + } + outcomes + } + /// The envelope the node cached for the self-built payload of the block /// whose root is `block_root`, proposed at `slot`. /// diff --git a/crates/validator/src/error.rs b/crates/validator/src/error.rs index ae65aee92..b2d10aa37 100644 --- a/crates/validator/src/error.rs +++ b/crates/validator/src/error.rs @@ -62,6 +62,25 @@ pub enum Error { BeaconNodeStatus { status: u16, body: String }, #[error("beacon node is syncing")] BeaconNodeSyncing, + /// A signed blinded block was sent and the node's answer does not say + /// whether it went out: a 5xx other than 503, or no answer once connected. + /// + /// Distinct from [`Self::BeaconNodeStatus`] and [`Self::BeaconNode`], + /// which for this call mean a definite failure (a 4xx or a 503, or a + /// connection never established: the block was not published; see + /// `BeaconNodeApi::publish_blinded_block`). Here the node + /// may have handed the block to its builder before failing, and the relay + /// publishes on receipt, so the block may well be on the network. Nothing + /// can safely be retried: the slot is signed, and a second block for it + /// would be an equivocation. `status` is `None` for a transport failure. + /// + /// Classified retryable like the other beacon-node errors, but that governs + /// only whether the next duty is attempted, not this block: it is never + /// sent again. + #[error( + "the beacon node did not confirm the blinded block (it may still be published): {detail}" + )] + BlindedUnconfirmed { status: Option, detail: String }, /// The slot's attestation work ran past the end of the slot and was /// abandoned. /// @@ -72,6 +91,16 @@ pub enum Error { /// abandoned attestation itself is never retried, by design. #[error("attestation duty for slot {slot} ran past the end of its slot")] AttestationDeadline { slot: u64 }, + /// The slot's proposal ran past its deadline, the point attesters stop + /// waiting for a block, before its block was published, and was + /// abandoned. + /// + /// The proposal counterpart of [`Self::AttestationDeadline`], and + /// retryable in the same sense only. Never raised once a blinded block is + /// signed: its publish goes on past the deadline (see + /// `ProposalService::propose_by`). + #[error("proposal for slot {slot} ran past the point attesters stop waiting for its block")] + ProposalDeadline { slot: u64 }, /// Slashing protection refused to sign a block for this slot, or could not /// record it. /// @@ -161,8 +190,10 @@ impl Error { | Self::AllBeaconNodesFailed(_) | Self::BeaconNodeStatus { .. } | Self::BeaconNodeSyncing + | Self::BlindedUnconfirmed { .. } | Self::InconsistentResponse(_) | Self::AttestationDeadline { .. } + | Self::ProposalDeadline { .. } | Self::Decode(_) => true, // A refusal repeats itself, and a record that could not be written // needs an operator: retrying within the slot helps neither. diff --git a/crates/validator/src/in_flight.rs b/crates/validator/src/in_flight.rs new file mode 100644 index 000000000..ffbac8b21 --- /dev/null +++ b/crates/validator/src/in_flight.rs @@ -0,0 +1,68 @@ +//! One run at a time for work spawned off the slot loop. +//! +//! The slot loop spawns its sends to beacon nodes rather than awaiting them, so +//! a slow node cannot delay the next slot's duties. The cost is that nothing +//! stops a new run starting while the last one still waits on that node. Each +//! run holds its own copy of the work and sends it again, so the runs pile up +//! for as long as the node stays slow. A run that finds the previous one still +//! going skips instead, as Teku's registration round does. + +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Whether a run is going. +#[derive(Debug, Default)] +pub(crate) struct InFlight(AtomicBool); + +impl InFlight { + /// The guard for a new run, or `None` when one is already going. + pub(crate) fn enter(&self) -> Option> { + let running = self.0.swap(true, Ordering::AcqRel); + // The guard is built only for a run that starts: `then_some` would + // build it either way and drop it on the refused path, and dropping + // it clears the flag of the run that is still going. + if running { + None + } else { + Some(InFlightGuard(&self.0)) + } + } +} + +/// Ends the run on every exit from it, a panic included, so one failed run +/// cannot stop every later one. +pub(crate) struct InFlightGuard<'a>(&'a AtomicBool); + +impl Drop for InFlightGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_second_run_is_refused_until_the_first_ends() { + let in_flight = InFlight::default(); + let first = in_flight.enter().expect("nothing is running"); + assert!(in_flight.enter().is_none()); + assert!( + in_flight.enter().is_none(), + "a refused run leaves the running one's flag set" + ); + drop(first); + assert!(in_flight.enter().is_some()); + } + + #[test] + fn a_run_that_panics_still_ends() { + let in_flight = InFlight::default(); + let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _run = in_flight.enter().expect("nothing is running"); + panic!("the run fails"); + })); + assert!(unwound.is_err()); + assert!(in_flight.enter().is_some()); + } +} diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 24fc99dfd..67e135053 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -31,11 +31,13 @@ pub mod beacon_node; pub mod duties; pub mod error; pub mod http_api; +pub(crate) mod in_flight; pub mod keys; pub mod metrics; pub mod payload_attestation; pub mod proposal; pub mod proposer_settings; +pub mod registration; pub(crate) mod secure_fs; pub mod signing; pub mod slashing_protection; @@ -62,8 +64,9 @@ use crate::beacon_node::http::HttpBeaconNode; use crate::duties::DutiesService; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; -use crate::proposal::ProposalService; +use crate::proposal::{BlindedOutcome, ProposalService, Proposed}; use crate::proposer_settings::ProposerSettings; +use crate::registration::RegistrationService; use crate::signing::SigningContext; use crate::slot_clock::SlotClock; use crate::sync_committee::SyncCommitteeService; @@ -89,12 +92,24 @@ pub struct ValidatorConfig { /// Bind address for the keymanager API. `None` when `--enable-keymanager` /// was not passed, in which case it is never spawned. pub keymanager: Option, + /// Builder proposals, or `None` when `--builder-proposals` is off. + pub builder: Option, /// Register keys that have no slashing-protection record as never having /// signed, instead of refusing to start. See /// [`slashing_protection::SlashingProtection::open`]. pub init_slashing_protection: bool, } +/// Builder proposals: register each validator with the beacon node's builder +/// and accept a blinded block when the builder's bid wins. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BuilderProposals { + /// The `builder_boost_factor` sent with every pre-gloas block request: + /// 100 compares the bid and the local payload as they are; `u64::MAX` + /// takes the builder whenever there is a bid. + pub boost_factor: u64, +} + /// Run the validator client until the process is stopped. pub async fn run(config: ValidatorConfig) -> Result<()> { metrics::init(); @@ -112,6 +127,18 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { keymanager API." ); } + // Accepted, as every other client's flag accepts it, but said loudly: + // nobody holds the zero address's key, so what it is paid is burned, and + // a builder registration or proposer preference signs that into a + // commitment the builder honours. Only the default can be zero, since the + // keymanager API refuses it for a single validator. + if config.suggested_fee_recipient == Some(ExecutionAddress::ZERO) { + error!( + "--suggested-fee-recipient is the zero address. Every validator without its own \ + address burns its execution-layer rewards, and builder registrations and proposer \ + preferences sign that address for the builder to pay." + ); + } let settings = Arc::new(ProposerSettings::new( config.graffiti, config.suggested_fee_recipient, @@ -214,7 +241,20 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { context.clone(), settings.clone(), protection.clone(), - ); + config.builder.map(|builder| builder.boost_factor), + ) + .with_clock(clock.clone()); + let registration = config.builder.map(|builder| { + info!( + boost_factor = builder.boost_factor, + "Builder proposals on: registering validators and accepting blinded blocks" + ); + Arc::new(RegistrationService::new( + beacon_node.clone(), + context.clone(), + settings.clone(), + )) + }); if let Some(address) = config.keymanager { let token = http_api::load_or_create_token(&config.validators_dir)?; @@ -290,7 +330,19 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &context, ); match refreshed.await { - Ok(()) => last_epoch = Some(epoch), + Ok(resolved) => { + last_epoch = Some(epoch); + // Only after a refresh that got all the way through, so a + // failing one does not respawn a full send every slot, and + // spawned so a slow or hung node never delays this epoch's + // duties: a missed registration costs at most a builder + // block, never a duty. + if let Some(registration) = ®istration { + let registration = registration.clone(); + let store = store.clone(); + tokio::spawn(registration.register_all(epoch, resolved, store)); + } + } // A beacon node that is down, syncing or answering badly will // likely answer next epoch, so keep the schedule already held // and try again. Anything else will not fix itself, and looping @@ -345,6 +397,13 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &store, ) .await; + + // Off the slot loop, like the epoch's full send: a keymanager change + // to a fee recipient or gas limit reaches the builder within a slot, + // and a slow node cannot delay the next slot's duties. + if let Some(registration) = ®istration { + tokio::spawn(registration.clone().register_changed(epoch, store.clone())); + } } } @@ -640,7 +699,7 @@ async fn aggregate( /// Failures are logged and counted rather than propagated. A proposal is one /// slot's work; losing it must not stop the client attesting for the rest of /// the epoch. -async fn propose( +async fn propose( proposal: &ProposalService, clock: &SlotClock, slot: u64, @@ -648,19 +707,30 @@ async fn propose( store: &RwLock, ) { let budget = clock.until_attestation(slot, SystemTime::now()); - let attempt = tokio::time::timeout(budget, proposal.propose(slot, duty, store)); - match attempt.await { - Ok(Ok(_)) => { + let deadline = tokio::time::Instant::now() + budget; + match proposal.propose_by(slot, duty, store, deadline).await { + Ok(Proposed::Full(_) | Proposed::Blinded(BlindedOutcome::Published(_))) => { let elapsed = SystemTime::now() .duration_since(clock.start_of(slot)) .unwrap_or_default(); metrics::observe_block_publication_delay(elapsed.as_secs_f64()); } - Ok(Err(err)) => { - error!(%slot, validator = duty.validator_index, %err, "Failed to propose this slot's block"); - metrics::inc_block_proposal_failures(); + // Already logged at error and counted on its own series; the block + // may well be out, so it is neither a published-in-time delay nor a + // failure. + Ok(Proposed::Blinded(BlindedOutcome::Unconfirmed)) => {} + // The beacon node waits up to 5 s for the relay, which can outlast + // this budget. Not a missed proposal: the publish goes on, on its own + // task, which reports and counts the real outcome. + Ok(Proposed::Blinded(BlindedOutcome::StillPublishing)) => { + info!( + %slot, + validator = duty.validator_index, + budget_ms = budget.as_millis() as u64, + "Blinded block signed; its publish outlived the proposal budget and continues" + ); } - Err(_) => { + Err(Error::ProposalDeadline { .. }) => { warn!( %slot, validator = duty.validator_index, @@ -669,11 +739,18 @@ async fn propose( ); metrics::inc_block_proposal_failures(); } + Err(err) => { + error!(%slot, validator = duty.validator_index, %err, "Failed to propose this slot's block"); + metrics::inc_block_proposal_failures(); + } } } /// Resolve any newly activated validators, refresh the duty schedule, and /// re-send subnet subscriptions if it moved. +/// +/// Returns the pubkeys the beacon node resolved to an index, which is who a +/// builder registration is signed for: the node drops the rest anyway. async fn refresh_epoch( beacon_node: &Arc, duties: &mut DutiesService, @@ -682,12 +759,12 @@ async fn refresh_epoch( settings: &ProposerSettings, store: &RwLock, context: &SigningContext, -) -> Result<()> { +) -> Result> { // Nothing to resolve, nothing to schedule, nothing to register. Returning // here also keeps an empty pubkey list away from `validator_indices`, whose // endpoint reads one as "every validator on the chain". if pubkeys.is_empty() { - return Ok(()); + return Ok(Vec::new()); } if beacon_node.is_optimistic_or_syncing().await? { @@ -758,7 +835,7 @@ async fn refresh_epoch( subscriptions::subscribe(beacon_node, &duties.all(), store, context).await?; } metrics::set_beacon_node_available(true); - Ok(()) + Ok(entries.iter().map(|entry| entry.pubkey).collect()) } /// Fetch the payload timeliness committee schedule for this epoch and the next, diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index ad52aa95b..92e1466b4 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -113,6 +113,46 @@ static BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL: LazyLock = LazyLock::new .unwrap() }); +/// Builder registrations a beacon node accepted, counted per registration so a +/// full `REGISTRATION_BATCH` reads as that many. +static BUILDER_REGISTRATIONS_SENT_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_builder_registrations_sent_total", + "Builder registrations a beacon node accepted" + ) + .unwrap() +}); + +/// Registration batches no beacon node accepted. A missed registration costs +/// at most a builder block, never a duty, so this is a warning sign rather +/// than a failure. +static BUILDER_REGISTRATION_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_builder_registration_failures_total", + "Registration batches no beacon node accepted" + ) + .unwrap() +}); + +/// Blinded blocks a beacon node handed to its builder. +static BLINDED_BLOCKS_PROPOSED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_blinded_blocks_proposed_total", + "Blinded blocks a beacon node handed to its builder" + ) + .unwrap() +}); + +/// Blinded blocks the beacon node did not confirm. The relay may still have +/// published them, so this is not a count of missed proposals. +static BLINDED_BLOCKS_UNCONFIRMED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_blinded_blocks_unconfirmed_total", + "Blinded blocks the beacon node did not confirm, which the relay may still have published" + ) + .unwrap() +}); + /// Blocks this process declined to sign because slashing protection judged /// them unsafe, or could not record them. /// @@ -341,6 +381,10 @@ pub fn init() { LazyLock::force(&ATTESTATION_DEADLINE_MISSED_TOTAL); LazyLock::force(&BLOCKS_PROPOSED_TOTAL); LazyLock::force(&BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL); + LazyLock::force(&BUILDER_REGISTRATIONS_SENT_TOTAL); + LazyLock::force(&BUILDER_REGISTRATION_FAILURES_TOTAL); + LazyLock::force(&BLINDED_BLOCKS_PROPOSED_TOTAL); + LazyLock::force(&BLINDED_BLOCKS_UNCONFIRMED_TOTAL); LazyLock::force(&BLOCKS_REFUSED_TOTAL); LazyLock::force(&BLOCK_PROPOSAL_FAILURES_TOTAL); LazyLock::force(&ENVELOPES_PUBLISHED_TOTAL); @@ -407,6 +451,22 @@ pub fn inc_blocks_broadcast_not_imported() { BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL.inc(); } +pub fn inc_builder_registrations_sent(count: usize) { + BUILDER_REGISTRATIONS_SENT_TOTAL.inc_by(count as u64); +} + +pub fn inc_builder_registration_failures() { + BUILDER_REGISTRATION_FAILURES_TOTAL.inc(); +} + +pub fn inc_blinded_blocks_proposed() { + BLINDED_BLOCKS_PROPOSED_TOTAL.inc(); +} + +pub fn inc_blinded_blocks_unconfirmed() { + BLINDED_BLOCKS_UNCONFIRMED_TOTAL.inc(); +} + pub fn inc_blocks_refused() { BLOCKS_REFUSED_TOTAL.inc(); } diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index c16319f07..00dcfb21f 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -42,6 +42,34 @@ //! //! The envelope is not slashable, so slashing protection is untouched by it. //! +//! # Blinded blocks +//! +//! With builder proposals on, the node may answer with a blinded block: the +//! payload's header in place of the payload. Its root is the full block's, so +//! slashing protection, the signing root and the signature are exactly what a +//! full block gets, and the same slot-level protections apply. +//! +//! The fee recipient is not compared for a blinded block. The header names the +//! builder's address, and the relay pays the validator inside the payload, so +//! a mismatch there is the flow working rather than a node paying someone else. +//! +//! The signed block goes to `blinded_blocks` on the node that produced it and +//! on no other (the block's [`Producer`]): only that node's builder holds the +//! payload. A publish the node fails to confirm is reported as +//! [`BlindedOutcome::Unconfirmed`] rather than as a missed proposal, since the +//! relay may still publish the block. +//! +//! # The deadline +//! +//! The duty loop gives a proposal until the point attesters stop waiting for +//! the block ([`ProposalService::propose_by`]). Everything up to a full +//! block's publish ends there. A signed blinded block's publish does not: it +//! runs on its own task, since the beacon node waits up to 5 s for the relay, +//! and past the deadline the proposal returns +//! [`BlindedOutcome::StillPublishing`] while the task reports the outcome. +//! The deadline is enforced here rather than by the caller dropping the +//! future, because only here is it known whether a blinded block was signed. +//! //! # Do not retry this call within a slot //! //! For the same reason [`crate::attestation::AttestationService::attest`] must @@ -52,15 +80,17 @@ //! than dangerous, but the caller should not be relying on that. use std::sync::Arc; +use std::time::SystemTime; use ethlambda_types::beacon::constants::BUILDER_INDEX_SELF_BUILD; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress, Root, Slot}; use ethlambda_types::beacon::signing::compute_epoch_at_slot; use tokio::sync::RwLock; +use tokio::time::Instant; use tracing::{error, info, warn}; -use crate::beacon_node::block_contents::{GloasPayload, signed_envelope_ssz}; +use crate::beacon_node::block_contents::{GloasPayload, Producer, signed_envelope_ssz}; use crate::beacon_node::dto::{ProposerDutyDto, encode_hex, parse_pubkey}; use crate::beacon_node::{BeaconNodeApi, BlockRequest, Published, validate_produced_block}; use crate::error::{Error, Result}; @@ -68,6 +98,71 @@ use crate::keys::ValidatorStore; use crate::proposer_settings::ProposerSettings; use crate::signing::SigningContext; use crate::slashing_protection::{Refusal, SharedProtection}; +use crate::slot_clock::SlotClock; + +/// What became of a proposal, as the duty loop needs it. +/// +/// Every outcome here has already been logged and its block counted. What the +/// loop still owes is the publication delay of a block the node confirmed +/// while it was waiting: [`Proposed::Full`] and +/// [`BlindedOutcome::Published`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Proposed { + /// A full block, and what the node made of it. + Full(Published), + /// A blinded block, signed and sent to the node that produced it. + Blinded(BlindedOutcome), +} + +/// What became of a signed blinded block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlindedOutcome { + /// The node confirmed it: handed to its builder, or broadcast but not + /// imported (a 202). + Published(Published), + /// The node did not confirm it, so the relay may still publish it (see + /// [`Error::BlindedUnconfirmed`]). Logged at error and counted on its own + /// series: neither a published block nor a failed proposal. Not retried, + /// since the slot is signed and a second block would be an equivocation. + Unconfirmed, + /// The publish outlived the deadline and goes on, on its own task, which + /// logs and counts the outcome when it comes: a refusal as a failed + /// proposal, a confirmed block with its publication delay. + StillPublishing, +} + +/// A block signed and ready to publish: what [`ProposalService::produce_and_sign`] +/// hands the publish step. +struct SignedBlock { + pubkey: BlsPubkey, + fork: ForkName, + block_root: Root, + blinded: bool, + producer: Producer, + /// The self-built payload's envelope, taken out before the block was + /// signed into its body (gloas only). + payload: Option, + builder_index: Option, + body: Vec, +} + +/// `future`'s result, or [`Error::ProposalDeadline`] once `deadline` passes. +/// With no deadline, `future` runs to the end. +/// +/// A future that completes is never discarded, even at the deadline: the +/// timeout is checked only after the future has been polled. +async fn before( + deadline: Option, + slot: Slot, + future: impl std::future::Future>, +) -> Result { + match deadline { + Some(deadline) => tokio::time::timeout_at(deadline, future) + .await + .unwrap_or(Err(Error::ProposalDeadline { slot })), + None => future.await, + } +} pub struct ProposalService { beacon_node: Arc, @@ -82,23 +177,51 @@ pub struct ProposalService { /// What every validator has signed, shared with the attestation service /// and the keymanager. See [`crate::slashing_protection`]. protection: SharedProtection, + /// The `builder_boost_factor` sent with each block request, or `None` when + /// builder proposals are off, in which case a blinded answer is refused. + builder_boost_factor: Option, + /// Where a slot starts, so a blinded publish that outlives the deadline + /// can observe its own publication delay. `None` observes nothing. + clock: Option, } -impl ProposalService { +impl ProposalService { pub fn new( beacon_node: Arc, context: Arc, settings: Arc, protection: SharedProtection, + builder_boost_factor: Option, ) -> Self { Self { beacon_node, context, settings, protection, + builder_boost_factor, + clock: None, } } + /// Give the service the slot clock, so a blinded block's publication delay + /// is observed by the task that publishes it when that outlives + /// [`Self::propose_by`]'s deadline. + pub fn with_clock(mut self, clock: SlotClock) -> Self { + self.clock = Some(clock); + self + } + + /// Whether the produced block's fee recipient is this client's to compare. + /// + /// Not for a blinded block: its header names the builder's address, and the + /// relay pays the validator inside the payload it reveals, so a difference + /// there is the flow working, not a node paying someone else. + fn compares_fee_recipient( + produced: &crate::beacon_node::block_contents::ProducedBlock, + ) -> bool { + !produced.is_blinded() + } + /// Compare the produced block's fee recipient against what was asked for, /// and complain loudly if they differ. /// @@ -149,16 +272,89 @@ impl ProposalService { } } - /// Produce, sign and publish the block for `slot`. + /// Produce, sign and publish the block for `slot`, however long it takes. /// - /// Returns what became of it, which is not always a clean success: see - /// [`Published`]. + /// [`Self::propose_by`] with no deadline, so a blinded block's publish is + /// always waited for and never comes back as + /// [`BlindedOutcome::StillPublishing`]. pub async fn propose( &self, slot: Slot, duty: &ProposerDutyDto, store: &RwLock, - ) -> Result { + ) -> Result { + self.propose_within(slot, duty, store, None).await + } + + /// Produce, sign and publish the block for `slot`, giving up at + /// `deadline`. + /// + /// # What the deadline cuts + /// + /// Production and signing, and a full block's publish (with its envelope + /// from gloas), end at the deadline with [`Error::ProposalDeadline`]: a + /// full block that late has lost most of its value, and every further + /// second comes out of the slot's attestations. + /// + /// A signed blinded block's publish is not cut: the beacon node may wait + /// several seconds on the relay, and the block may already be out. It runs + /// on its own task, and at the deadline this returns + /// [`BlindedOutcome::StillPublishing`] while the task reports the outcome. + /// Whichever of the two ends up holding the outcome reports it, so a + /// caller that awaits this to the end, as the duty loop does, sees each + /// outcome reported once (see [`Self::publish_blinded`] for the edges). + pub async fn propose_by( + &self, + slot: Slot, + duty: &ProposerDutyDto, + store: &RwLock, + deadline: Instant, + ) -> Result { + self.propose_within(slot, duty, store, Some(deadline)).await + } + + async fn propose_within( + &self, + slot: Slot, + duty: &ProposerDutyDto, + store: &RwLock, + deadline: Option, + ) -> Result { + // Nothing awaits between signing and the end of `produce_and_sign`, + // and a completed future is never discarded at the deadline, so a + // signed block always reaches the publish step below. + let signed = before(deadline, slot, self.produce_and_sign(slot, duty, store)).await?; + let validator = duty.validator_index; + if signed.blinded { + return self + .publish_blinded( + slot, + validator, + signed.producer, + signed.fork, + signed.body, + deadline, + ) + .await; + } + before( + deadline, + slot, + self.publish_full(slot, validator, store, signed), + ) + .await + .map(Proposed::Full) + } + + /// Everything up to and including the block signature: the slashing + /// protection checks, the RANDAO reveal, the block request and the checks + /// on what came back. + async fn produce_and_sign( + &self, + slot: Slot, + duty: &ProposerDutyDto, + store: &RwLock, + ) -> Result { let pubkey = parse_pubkey(&duty.pubkey)?; let epoch = compute_epoch_at_slot(slot); @@ -192,6 +388,7 @@ impl ProposalService { proposer_index: duty.validator_index, randao_reveal, graffiti, + builder_boost_factor: self.builder_boost_factor, }; let mut produced = self.beacon_node.produce_block(&request).await?; @@ -209,6 +406,17 @@ impl ProposalService { // second block for a slot already proposed. validate_produced_block(&request, &produced)?; + // Also the implementation's contract, and checked again for the same + // reason. With builder proposals off this client never publishes a + // blinded block, so signing one would record the slot in slashing + // protection for a block that can never be sent. + if produced.is_blinded() && self.builder_boost_factor.is_none() { + return Err(Error::InconsistentResponse(format!( + "node produced a blinded block for slot {slot}; builder proposals are off, so \ + this client cannot publish one" + ))); + } + // The node's fork and this client's must agree, or the signature is // computed under a fork version the network does not accept. // @@ -229,13 +437,32 @@ impl ProposalService { ))); } - Self::check_fee_recipient(&produced, fee_recipient, slot); + if Self::compares_fee_recipient(&produced) { + Self::check_fee_recipient(&produced, fee_recipient, slot); + } else { + // Both addresses, so a log can say afterwards who built the block + // and where this client had asked to be paid. + let header = crate::beacon_node::dto::encode_hex(&produced.fee_recipient().0); + let configured = fee_recipient.map_or_else( + || "none".to_string(), + |address| crate::beacon_node::dto::encode_hex(&address.0), + ); + tracing::debug!( + %slot, + header_fee_recipient = %header, + configured_fee_recipient = %configured, + "Blinded block: its header names the builder's address; the relay pays the \ + validator in the payload" + ); + } let fork = produced.fork; + let blinded = produced.is_blinded(); info!( %slot, validator = duty.validator_index, fork = fork.as_str(), + blinded, blobs = produced.blob_count(), "Block produced; signing" ); @@ -261,16 +488,45 @@ impl ProposalService { self.context.sign_block(&store, &pubkey, block_root, slot)? }; + // Nothing below awaits: see `propose_within`. + // // Taken out before the block is signed into its body: the envelope is // published on its own, after the block, and the bare gloas block body // has no place for it. let payload = produced.take_gloas_payload(); let builder_index = produced.builder_index(); + let producer = produced.producer; + Ok(SignedBlock { + pubkey, + fork, + block_root, + blinded, + producer, + payload, + builder_index, + body: produced.into_signed_ssz(signature), + }) + } - let body = produced.into_signed_ssz(signature); - let published = self - .publish(fork, &body, slot, duty.validator_index) - .await?; + /// Publish a signed full block, then, for a self-built gloas payload, its + /// envelope. + async fn publish_full( + &self, + slot: Slot, + validator: u64, + store: &RwLock, + signed: SignedBlock, + ) -> Result { + let SignedBlock { + pubkey, + fork, + block_root, + payload, + builder_index, + body, + .. + } = signed; + let published = self.publish(fork, &body, slot, validator).await?; crate::metrics::inc_blocks_proposed(); // Only a self-built payload is this client's to reveal. @@ -279,20 +535,13 @@ impl ProposalService { // the proposal did happen; what failed is the reveal, which has its // own series because it is the failure that costs the payload. let _ = self - .publish_envelope( - slot, - duty.validator_index, - &pubkey, - block_root, - payload, - store, - ) + .publish_envelope(slot, validator, &pubkey, block_root, payload, store) .await .inspect(|()| crate::metrics::inc_envelopes_published()) .inspect_err(|err| { tracing::error!( %slot, - validator = duty.validator_index, + validator, block_root = %ethlambda_types::ShortRoot(&block_root.0), %err, "Block published but its envelope was not; the slot's payload is withheld" @@ -366,6 +615,89 @@ impl ProposalService { Ok(()) } + /// Send a signed blinded block to the node that produced it. + /// + /// # On its own task, reported by whoever holds the outcome + /// + /// The request runs on a spawned task, so neither the deadline nor a + /// dropped proposal cancels it: the beacon node may wait up to 5 s on the + /// relay, and the block may already be out. The task logs and counts the + /// node's answer ([`blinded_outcome`]) and hands the outcome back over a + /// channel. Whoever holds it then does the rest: the duty loop when this + /// returns it in time, and the task itself when this has stopped waiting + /// ([`report_unawaited`]). For a caller that awaits this to the end, the + /// channel decides which, so it is never both and never neither. + /// + /// Two edges are not covered. A task that panics after the deadline is + /// reported by nobody. A caller that drops the proposal after the task + /// has sent its outcome takes the rest of the reporting (the delay sample, + /// or a refusal's failure count) with it. The node's answer itself is + /// logged and counted by the task either way. + /// + /// # Why an unconfirmed publish is not an error + /// + /// A 5xx or a missing answer once connected does not say the block was not + /// published: the node may have handed it to the relay, which publishes + /// regardless of what the node then replied. So that outcome is logged + /// loudly and counted on its own series, and returned as + /// [`BlindedOutcome::Unconfirmed`]. It is not retried: the slot is signed, + /// and a second block for it would be an equivocation. A definite failure + /// (a 4xx, a 503, a connection never made; see + /// [`BeaconNodeApi::publish_blinded_block`]) is an error, a failed + /// proposal. + async fn publish_blinded( + &self, + slot: Slot, + validator: u64, + producer: Producer, + fork: ForkName, + body: Vec, + deadline: Option, + ) -> Result { + let beacon_node = self.beacon_node.clone(); + let slot_start = self.clock.as_ref().map(|clock| clock.start_of(slot)); + let (handoff, mut outcome) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let bytes = body.len(); + let answer = beacon_node + .publish_blinded_block(producer, fork, &body) + .await; + let outcome = blinded_outcome(slot, validator, bytes, answer); + // Fails once the receiving side has stopped waiting, which hands + // the outcome back here to report. + if let Err(outcome) = handoff.send(outcome) { + report_unawaited(slot, validator, slot_start, outcome); + } + }); + + let received = match deadline { + None => (&mut outcome).await, + Some(deadline) => match tokio::time::timeout_at(deadline, &mut outcome).await { + Ok(received) => received, + Err(_) => { + // Stop waiting. After `close`, the task's send fails and + // the task reports; an outcome sent before it is still + // here, and is taken. + outcome.close(); + match outcome.try_recv() { + Ok(sent) => Ok(sent), + Err(_) => return Ok(Proposed::Blinded(BlindedOutcome::StillPublishing)), + } + } + }, + }; + match received { + Ok(outcome) => outcome.map(Proposed::Blinded), + // The task ended without sending, so it panicked. The block may or + // may not have gone out, which is what "unconfirmed" says. + Err(_) => { + tracing::error!(%slot, validator, "The blinded publish task failed"); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(Proposed::Blinded(BlindedOutcome::Unconfirmed)) + } + } + } + /// Send the signed body and report what the node made of it. /// /// Split out so the 202 case has one place to be explained rather than @@ -397,11 +729,97 @@ impl ProposalService { ); crate::metrics::inc_blocks_broadcast_not_imported(); } + // `publish_block` never answers with this: a blinded block goes + // through `publish_blinded_block`. + Published::HandedToBuilder => {} } Ok(published) } } +/// Log and count a blinded publish's answer, as [`ProposalService::publish`] +/// does a full block's, and turn an unconfirmed send into an outcome. A +/// definite refusal stays an error, for whoever holds the outcome to report. +fn blinded_outcome( + slot: Slot, + validator: u64, + bytes: usize, + answer: Result, +) -> Result { + match answer { + Ok(published) => { + match published { + Published::HandedToBuilder => { + info!(%slot, validator, bytes, "Blinded block handed to the builder"); + } + Published::BroadcastNotImported => { + warn!( + %slot, + validator, + "Block was broadcast but the beacon node could not import it; \ + check that node's execution layer" + ); + crate::metrics::inc_blocks_broadcast_not_imported(); + } + // A full block's outcome. Reported rather than assumed away; + // the block did go out, so it is still counted. + Published::Imported => warn!( + %slot, + validator, + ?published, + "Beacon node API answered a blinded publish with an outcome that belongs to \ + another call" + ), + } + crate::metrics::inc_blocks_proposed(); + crate::metrics::inc_blinded_blocks_proposed(); + Ok(BlindedOutcome::Published(published)) + } + Err(err @ Error::BlindedUnconfirmed { .. }) => { + tracing::error!( + %slot, + validator, + %err, + "The beacon node did not confirm the blinded block; the relay may still publish \ + it. Not retried: the slot is signed, and a second block would be an equivocation" + ); + crate::metrics::inc_blinded_blocks_unconfirmed(); + Ok(BlindedOutcome::Unconfirmed) + } + Err(err) => Err(err), + } +} + +/// What the duty loop does with a blinded outcome it receives in time, done by +/// the publish task when nobody is waiting for it any more (past the deadline, +/// or a dropped proposal): the publication delay of a confirmed block, and a +/// refusal counted as a failed proposal. +fn report_unawaited( + slot: Slot, + validator: u64, + slot_start: Option, + outcome: Result, +) { + match outcome { + Ok(BlindedOutcome::Published(_)) => { + if let Some(start) = slot_start { + let elapsed = SystemTime::now().duration_since(start).unwrap_or_default(); + crate::metrics::observe_block_publication_delay(elapsed.as_secs_f64()); + } + } + Ok(BlindedOutcome::Unconfirmed | BlindedOutcome::StillPublishing) => {} + Err(err) => { + tracing::error!( + %slot, + validator, + %err, + "The beacon node refused the blinded block, after the proposal stopped waiting" + ); + crate::metrics::inc_block_proposal_failures(); + } + } +} + /// Slashing protection's answer about a block, as the duty's outcome: a judged /// refusal, or a record that could not be consulted or written (`what` names /// which), is a counted [`Error::ProposalRefused`]. @@ -483,6 +901,7 @@ mod tests { context(), Arc::new(settings), SlashingProtection::in_memory(), + None, ) } @@ -496,6 +915,7 @@ mod tests { context(), Arc::new(settings), SlashingProtection::in_memory(), + None, ) } @@ -521,7 +941,7 @@ mod tests { .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("proposes"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); let sent = node.published_blocks(); assert_eq!(sent.len(), 1); @@ -598,6 +1018,7 @@ mod tests { context(), settings.clone(), SlashingProtection::in_memory(), + None, ); settings.set_graffiti(&pubkey, Bytes32::repeat_byte(0xcd)); @@ -684,14 +1105,20 @@ mod tests { }; let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); - ProposalService::new(node.clone(), context(), settings.clone(), protection()) - .propose(slot(), &duty(&pubkey, 7, slot()), &store) - .await - .expect("first"); + ProposalService::new( + node.clone(), + context(), + settings.clone(), + protection(), + None, + ) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("first"); assert_eq!(node.published_blocks().len(), 1); let node_again = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); - let err = ProposalService::new(node_again.clone(), context(), settings, protection()) + let err = ProposalService::new(node_again.clone(), context(), settings, protection(), None) .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect_err("the slot was proposed before the restart"); @@ -717,6 +1144,7 @@ mod tests { context(), settings, Arc::new(std::sync::Mutex::new(opened)), + None, ) .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await @@ -742,7 +1170,459 @@ mod tests { .propose(slot(), &duty(&pubkey, 7, slot()), &store) .await .expect("202 is not an error"); - assert_eq!(published, Published::BroadcastNotImported); + assert_eq!(published, Proposed::Full(Published::BroadcastNotImported)); + } + + fn builder_service( + node: Arc, + fee_recipient: Option, + ) -> ProposalService { + let settings = ProposerSettings::new(Bytes32::repeat_byte(0xab), fee_recipient); + ProposalService::new( + node, + context(), + Arc::new(settings), + SlashingProtection::in_memory(), + Some(100), + ) + } + + /// A blinded block takes the `blinded_blocks` route and nothing else. Its + /// header names the builder's address, so a configured fee recipient that + /// differs is not a mismatch for it. The mismatch counter is process-global + /// and shared with the test above, so it cannot be asserted on here; what + /// is asserted is that the proposal goes through unchanged. + #[tokio::test] + async fn a_blinded_block_is_published_to_the_builder_route() { + use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; + + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + + let service = builder_service(node.clone(), Some(H160([0xfe; 20]))); + let published = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("proposes"); + assert_eq!( + published, + Proposed::Blinded(BlindedOutcome::Published(Published::HandedToBuilder)) + ); + + assert_eq!(node.block_requests()[0].builder_boost_factor, Some(100)); + assert!(node.published_blocks().is_empty()); + let sent = node.published_blinded.lock().expect("lock").clone(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].0, ForkName::Electra); + let block = SignedBlindedBeaconBlock::from_ssz_bytes(&sent[0].1).expect("decodes"); + assert_eq!(block.message.slot, slot()); + } + + /// The fee-recipient comparison is for a full block only: a blinded + /// block's header names the builder's address. + #[tokio::test] + async fn only_a_full_block_has_its_fee_recipient_compared() { + use crate::beacon_node::block_contents::{Contents, ProducedBlock, empty_block_for}; + use ethlambda_types::beacon::builder::BlindedBeaconBlock; + + let blinded = ProducedBlock { + fork: ForkName::Electra, + contents: Contents::Blinded { + block: BlindedBeaconBlock::from(empty_block_for(slot(), 7)), + }, + producer: Default::default(), + }; + assert!(!ProposalService::::compares_fee_recipient( + &blinded + )); + + let full = MockBeaconNode::new().with_block(slot(), 7); + let request = BlockRequest { + slot: slot(), + fork: ForkName::Electra, + proposer_index: 7, + randao_reveal: Default::default(), + graffiti: Default::default(), + builder_boost_factor: None, + }; + let full = full.produce_block(&request).await.expect("produces"); + assert!(ProposalService::::compares_fee_recipient( + &full + )); + } + + /// A node that does not confirm the publish may still have handed the block + /// to its relay, so it is not a missed proposal, and the slot is spent: a + /// second block would be an equivocation. + #[tokio::test] + async fn an_unconfirmed_blinded_publish_is_reported_and_the_slot_is_spent() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_unconfirmed_blinded_publish("relay timed out"), + ); + let service = builder_service(node.clone(), None); + + let published = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("an unconfirmed publish is not an error"); + assert_eq!(published, Proposed::Blinded(BlindedOutcome::Unconfirmed)); + + let err = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("slashing protection must refuse a second block for the slot"); + assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); + } + + /// Wait, up to five seconds, until `node` has recorded a blinded block. + async fn until_published_blinded(node: &MockBeaconNode) { + let published = async { + while node.published_blinded.lock().expect("lock").is_empty() { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }; + tokio::time::timeout(std::time::Duration::from_secs(5), published) + .await + .expect("the publish completes on its own task"); + } + + /// The deadline passes while the node waits on the relay. The proposal + /// returns `StillPublishing` rather than an error, and the publish, on its + /// own task, still goes out. + /// + /// The deadline is moved past by pausing the clock and advancing it once + /// the publish is at the node, which holds it until the test lets it + /// answer, rather than by picking a deadline and hoping the signing came + /// first: the slashing-protection write runs on the blocking pool and can + /// take longer than any margin on a loaded machine. + #[tokio::test] + async fn a_blinded_publish_past_the_deadline_still_goes_out() { + let (store, pubkey) = store(); + let relay_answers = Arc::new(tokio::sync::Semaphore::new(0)); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_blinded_publish_gate(relay_answers.clone()), + ); + let service = builder_service(node.clone(), None); + + let at_the_node = node.blinded_publish_entered.clone(); + let this_duty = duty(&pubkey, 7, slot()); + let hour = std::time::Duration::from_secs(3600); + let proposal = service.propose_by(slot(), &this_duty, &store, Instant::now() + hour); + tokio::pin!(proposal); + tokio::select! { + outcome = &mut proposal => { + panic!("the node is still waiting on the relay, yet the proposal ended: {outcome:?}") + } + () = at_the_node.notified() => {} + } + tokio::time::pause(); + tokio::time::advance(hour * 2).await; + let outcome = proposal + .await + .expect("past the deadline is not an error once signed"); + tokio::time::resume(); + assert_eq!(outcome, Proposed::Blinded(BlindedOutcome::StillPublishing)); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + + relay_answers.add_permits(1); + until_published_blinded(&node).await; + assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); + + // Slashing protection recorded the slot before the publish started, so + // a retry is refused without asking for a block. + let err = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("the slot is signed"); + assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); + assert_eq!(node.block_requests().len(), 1); + } + + /// The duty loop's entry point, with the deadline far off: a full block + /// and a blinded one both come back confirmed, as from `propose`. + #[tokio::test] + async fn a_proposal_within_its_deadline_returns_what_the_node_said() { + let hour = std::time::Duration::from_secs(3600); + + let (full_store, full_key) = store(); + let node = Arc::new(MockBeaconNode::new().with_block(slot(), 7)); + let full = builder_service(node.clone(), None) + .propose_by( + slot(), + &duty(&full_key, 7, slot()), + &full_store, + Instant::now() + hour, + ) + .await + .expect("proposes"); + assert_eq!(full, Proposed::Full(Published::Imported)); + assert_eq!(node.published_blocks().len(), 1); + + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + let blinded = builder_service(node.clone(), None) + .propose_by( + slot(), + &duty(&pubkey, 7, slot()), + &store, + Instant::now() + hour, + ) + .await + .expect("proposes"); + assert_eq!( + blinded, + Proposed::Blinded(BlindedOutcome::Published(Published::HandedToBuilder)) + ); + assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); + } + + /// A definite refusal that arrives within the deadline is the proposal's + /// error, for the duty loop to report, not something the task kept. + #[tokio::test] + async fn a_blinded_refusal_within_the_deadline_is_the_proposals_error() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_refused_blinded_publish(400), + ); + let hour = std::time::Duration::from_secs(3600); + let result = builder_service(node, None) + .propose_by( + slot(), + &duty(&pubkey, 7, slot()), + &store, + Instant::now() + hour, + ) + .await; + assert!( + matches!(result, Err(Error::BeaconNodeStatus { status: 400, .. })), + "got {result:?}" + ); + } + + /// A deadline that passes before anything is signed abandons the proposal: + /// an error, and nothing published. The node is still building the block + /// when the clock is moved past the deadline. + #[tokio::test] + async fn a_proposal_past_its_deadline_before_signing_is_abandoned() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_produce_gate(Arc::new(tokio::sync::Semaphore::new(0))), + ); + let service = builder_service(node.clone(), None); + + let building = node.produce_entered.clone(); + let this_duty = duty(&pubkey, 7, slot()); + let hour = std::time::Duration::from_secs(3600); + let proposal = service.propose_by(slot(), &this_duty, &store, Instant::now() + hour); + tokio::pin!(proposal); + tokio::select! { + outcome = &mut proposal => panic!("the node is still building, yet: {outcome:?}"), + () = building.notified() => {} + } + tokio::time::pause(); + tokio::time::advance(hour * 2).await; + let err = proposal.await.expect_err("abandoned at the deadline"); + tokio::time::resume(); + assert!( + matches!(err, Error::ProposalDeadline { slot: at } if at == slot()), + "got {err:?}" + ); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + assert!(node.published_blocks().is_empty()); + } + + /// A caller that drops the proposal while the node waits on the relay does + /// not cancel the publish: it runs on its own task, which reports the + /// outcome itself once nobody is waiting for it. + #[tokio::test] + async fn a_blinded_publish_survives_a_dropped_proposal() { + let (store, pubkey) = store(); + let relay_answers = Arc::new(tokio::sync::Semaphore::new(0)); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_blinded_publish_gate(relay_answers.clone()), + ); + let service = builder_service(node.clone(), None); + + let at_the_node = node.blinded_publish_entered.clone(); + let this_duty = duty(&pubkey, 7, slot()); + tokio::select! { + outcome = service.propose(slot(), &this_duty, &store) => { + panic!("the node is still waiting on the relay, yet the proposal ended: {outcome:?}") + } + () = at_the_node.notified() => {} + } + // The proposal future is dropped here. + assert!(node.published_blinded.lock().expect("lock").is_empty()); + + relay_answers.add_permits(1); + until_published_blinded(&node).await; + assert_eq!(node.published_blinded.lock().expect("lock").len(), 1); + } + + /// A 4xx is a definite refusal: the block was not published, so the + /// proposal failed. + #[tokio::test] + async fn a_definitely_refused_blinded_publish_is_an_error() { + let (store, pubkey) = store(); + let node = MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block() + .with_refused_blinded_publish(400); + + let result = builder_service(Arc::new(node), None) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await; + assert!( + matches!(result, Err(Error::BeaconNodeStatus { status: 400, .. })), + "got {result:?}" + ); + } + + #[tokio::test] + async fn without_builder_proposals_the_request_carries_no_factor() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + + let published = service(node.clone()) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("proposes"); + assert_eq!(published, Proposed::Full(Published::Imported)); + assert_eq!(node.block_requests()[0].builder_boost_factor, None); + assert!(node.published_blinded.lock().expect("lock").is_empty()); + } + + /// A node breaking its contract with a blinded answer to a request with no + /// factor is refused before signing. With builder proposals off this client + /// never publishes a blinded block, so signing would burn the slot for + /// nothing. The second attempt is refused for the same reason rather than + /// by slashing protection, which shows nothing was recorded. + #[tokio::test] + async fn a_blinded_block_is_refused_before_signing_with_builder_proposals_off() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block_regardless(), + ); + let service = service(node.clone()); + + for attempt in 0..2 { + let err = service + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect_err("a blinded block must not be signed"); + assert!( + matches!(err, Error::InconsistentResponse(_)), + "attempt {attempt}: got {err:?}" + ); + } + assert!(node.published_blinded.lock().expect("lock").is_empty()); + assert!(node.published_blocks().is_empty()); + } + + /// The signature in the published blinded block verifies over the full + /// block's root. The blinded container's root equals it, which is what lets + /// the relay put the payload back under a signature over a block the + /// validator never saw whole. + #[tokio::test] + async fn a_blinded_block_is_signed_over_the_full_blocks_root() { + use crate::beacon_node::block_contents::empty_block_for; + use blst::min_pk::{PublicKey, Signature}; + use ethlambda_types::beacon::builder::SignedBlindedBeaconBlock; + + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_block(slot(), 7) + .with_blinded_block(), + ); + builder_service(node.clone(), None) + .propose(slot(), &duty(&pubkey, 7, slot()), &store) + .await + .expect("proposes"); + + let (_, body) = node.published_blinded.lock().expect("lock").remove(0); + let published = SignedBlindedBeaconBlock::from_ssz_bytes(&body).expect("decodes"); + let full_root = empty_block_for(slot(), 7).hash_tree_root(); + assert_eq!(published.message.hash_tree_root(), full_root); + + let root = context().block_signing_root(full_root, slot()); + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(&published.signature.0).expect("valid signature"); + assert_eq!( + sig.verify( + true, + root.as_slice(), + b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_", + &[], + &pk, + true + ), + blst::BLST_ERROR::BLST_SUCCESS + ); + } + + /// A blinded answer is held to what a full one is: the slot and proposer + /// asked for, and the fork this client's schedule puts the slot in. Each + /// is refused before signing, so none spends the slot. + #[tokio::test] + async fn a_blinded_block_for_another_slot_proposer_or_fork_is_refused() { + let (store, pubkey) = store(); + // The mock always names electra, and slot 96 is phase0 on mainnet. + let cases = [ + ("slot", (slot() + 1, 7), slot()), + ("proposer", (slot(), 8), slot()), + ("fork", (96, 7), 96), + ]; + for (what, (block_slot, proposer), requested) in cases { + let node = Arc::new( + MockBeaconNode::new() + .with_block(block_slot, proposer) + .with_blinded_block(), + ); + let service = builder_service(node.clone(), None); + let err = service + .propose(requested, &duty(&pubkey, 7, requested), &store) + .await + .expect_err(what); + assert!( + matches!(err, Error::InconsistentResponse(_)), + "{what}: got {err:?}" + ); + assert!( + node.published_blinded.lock().expect("lock").is_empty(), + "{what}" + ); + } } /// A duty naming a key this client does not hold must fail before anything @@ -775,7 +1655,7 @@ mod tests { .await .expect("a wrong fee recipient must not stop the proposal"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); assert_eq!(node.published_blocks().len(), 1); } @@ -908,6 +1788,7 @@ mod gloas_tests { context(), Arc::new(settings), SlashingProtection::in_memory(), + None, ) } @@ -935,7 +1816,7 @@ mod gloas_tests { .propose(slot(), &duty(&pubkey), &store) .await .expect("proposes"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); let request = node.block_requests().remove(0); assert_eq!(request.fork, ForkName::Gloas); @@ -990,6 +1871,7 @@ mod gloas_tests { context(), settings, SlashingProtection::in_memory(), + None, ) .propose(slot(), &duty(&pubkey), &store) .await @@ -1132,7 +2014,7 @@ mod gloas_tests { .propose(slot(), &duty(&pubkey), &store) .await .expect("the block was published"); - assert_eq!(published, Published::Imported); + assert_eq!(published, Proposed::Full(Published::Imported)); assert!(node.published_envelopes().is_empty()); } } diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 01daf3f4d..2d73f4397 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -22,6 +22,7 @@ //! signing reports as a mismatch. use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{PoisonError, RwLock}; use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, H256}; @@ -59,9 +60,9 @@ pub fn graffiti_to_text(graffiti: &Bytes32) -> String { /// The gas limit a validator with no override reports. /// -/// Lighthouse's and Nimbus's default. Nothing in this client uses it yet: it -/// matters only to builder registrations, and there is no builder flow. It is -/// held so the keymanager API can serve it as the specification requires. +/// Lighthouse's and Nimbus's default. Builder registrations carry it as the +/// gas limit the builder should target, and the keymanager API serves it as the +/// specification requires. pub const DEFAULT_GAS_LIMIT: u64 = 60_000_000; /// One validator's overrides; `None` means the default applies. @@ -98,6 +99,10 @@ struct Inner { #[derive(Debug)] pub struct ProposerSettings { inner: RwLock, + /// Bumped on every override write or delete, so a reader that caches + /// something derived from the settings (the builder registrations) can + /// tell that nothing changed without reading every key. + generation: AtomicU64, } impl ProposerSettings { @@ -111,9 +116,16 @@ impl ProposerSettings { default_gas_limit: DEFAULT_GAS_LIMIT, overrides: HashMap::new(), }), + generation: AtomicU64::new(0), } } + /// A counter that moves whenever any override is written, cleared or + /// forgotten. Equal readings mean no per-key setting changed between them. + pub fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + fn read(&self, f: impl FnOnce(&Inner) -> T) -> T { f(&self.inner.read().unwrap_or_else(PoisonError::into_inner)) } @@ -127,6 +139,9 @@ impl ProposerSettings { if entry.is_empty() { inner.overrides.remove(pubkey); } + // Inside the write lock, so a reader that sees the new generation also + // sees the new value. + self.generation.fetch_add(1, Ordering::Release); } /// Where `pubkey`'s execution-layer rewards should go, or `None` when @@ -138,6 +153,29 @@ impl ProposerSettings { }) } + /// `pubkey`'s fee recipient and gas limit read together, under one lock, or + /// `None` when it has no fee recipient. + /// + /// For a signed message that carries both (a builder registration, proposer + /// preferences): two separate reads could straddle a keymanager write and + /// pair an old address with a new gas limit, a combination the operator + /// never configured, which the builder then keeps. + pub fn fee_recipient_and_gas_limit( + &self, + pubkey: &BlsPubkey, + ) -> Option<(ExecutionAddress, u64)> { + self.read(|inner| { + let overrides = inner.overrides.get(pubkey); + let fee_recipient = overrides + .and_then(|o| o.fee_recipient) + .or(inner.default_fee_recipient)?; + let gas_limit = overrides + .and_then(|o| o.gas_limit) + .unwrap_or(inner.default_gas_limit); + Some((fee_recipient, gas_limit)) + }) + } + /// The graffiti `pubkey` proposes with. pub fn graffiti(&self, pubkey: &BlsPubkey) -> Bytes32 { self.read(|inner| { @@ -183,6 +221,7 @@ impl ProposerSettings { pub fn forget(&self, pubkey: &BlsPubkey) { let mut inner = self.inner.write().unwrap_or_else(PoisonError::into_inner); inner.overrides.remove(pubkey); + self.generation.fetch_add(1, Ordering::Release); } } @@ -223,6 +262,28 @@ mod tests { assert_eq!(settings.gas_limit(&key(2)), DEFAULT_GAS_LIMIT); } + /// The pair a signed registration or preference carries agrees with the + /// two separate reads, and is `None` exactly when there is no address. + #[test] + fn the_fee_recipient_and_gas_limit_read_together_match_the_single_reads() { + let settings = ProposerSettings::new(Bytes32::ZERO, None); + assert_eq!(settings.fee_recipient_and_gas_limit(&key(1)), None); + + settings.set_gas_limit(&key(1), 30_000_000); + assert_eq!(settings.fee_recipient_and_gas_limit(&key(1)), None); + settings.set_fee_recipient(&key(1), H160([0xbb; 20])); + assert_eq!( + settings.fee_recipient_and_gas_limit(&key(1)), + Some((H160([0xbb; 20]), 30_000_000)) + ); + + let defaults = self::settings(); + assert_eq!( + defaults.fee_recipient_and_gas_limit(&key(2)), + Some((H160([0xaa; 20]), DEFAULT_GAS_LIMIT)) + ); + } + /// Clearing one field leaves the others overridden. #[test] fn clearing_an_override_restores_that_default_only() { diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs new file mode 100644 index 000000000..3f5b2cc46 --- /dev/null +++ b/crates/validator/src/registration.rs @@ -0,0 +1,1026 @@ +//! Builder registrations: what tells a beacon node's builder where to pay each +//! validator and which gas limit to target. +//! +//! A registration is a `ValidatorRegistrationV1` the validator signs, so a +//! relay can trust that the fee recipient and gas limit it builds for were +//! chosen by the key that will sign the block. The beacon node forwards them to +//! its builder, which keeps the newest per validator. +//! +//! # Why a signature is reused until the settings change +//! +//! Builders order a validator's registrations by `timestamp` and keep the +//! latest, so re-sending an unchanged registration with the same timestamp is +//! harmless, and re-signing it with a fresh one every epoch would cost a BLS +//! signature per validator for a message that says nothing new. The signed +//! registration is therefore cached per validator and re-signed, with a new +//! timestamp, only when the fee recipient or gas limit changes. The full set is +//! still re-sent every epoch: a beacon node forgets its registrations when it +//! restarts, and a relay may expire them, so repeating them is what keeps the +//! validators registered without any state beyond this cache. +//! +//! # Why it stops at gloas +//! +//! From gloas the builder market works through proposer preferences, which the +//! beacon node serves itself, so there is nothing for this client to register. +//! Both entry points return at the gloas fork epoch. +//! +//! # Why every node gets them +//! +//! A registration installs state on a node's builder, so the fallback client +//! sends to every node rather than the first that answers: a node that never +//! heard them cannot serve a bid for these validators after a failover. +//! +//! # Failures are never propagated +//! +//! A missed registration costs at most a builder block, never a duty, so a +//! failed send is logged and counted and the caller moves on. +//! +//! A per-slot send that no node accepts is sent again at every slot until one +//! does: it carries a keymanager change, and until a node takes it the builder +//! keeps paying the old fee recipient. The registrations are re-sent as they +//! were signed, so nothing is re-signed and their timestamps stay put, and +//! [`RegistrationService::signed_generation`] stays behind meanwhile, which +//! keeps the per-slot check below one atomic read. A full send that fails +//! waits for the next epoch's instead: re-sending every key each slot would +//! load a node that is already failing, and the next full send carries them +//! all anyway. A full send ends every per-slot retry, delivered or not, so a +//! node that keeps refusing costs a warning a slot for an epoch at most. +//! +//! A batch a node refuses with a 400 is not split to find the entry it +//! objects to: every entry is signed here by keys this client holds, so a +//! refusal says more about the node or the relay than about one entry, and no +//! other client splits either. +//! +//! # Each node works through the batches on its own +//! +//! The full set goes out in batches of [`REGISTRATION_BATCH`], and each node +//! is sent its batches in turn beside the other nodes rather than in step with +//! them (see [`BeaconNodeApi::register_validators_in_batches`]). A node that +//! hangs costs only its own batches, and only one timeout: what it has not +//! been sent by then is dropped until the next epoch. +//! +//! # One send at a time, and the newest registration last +//! +//! An ethlambda beacon node keeps the fee recipient of every registration it +//! is sent, newest timestamp or not, so the order sends reach a node in is the +//! order its fee recipients end up in. Every send therefore takes one lock, +//! [`RegistrationService::sending`], before it reads the cache and holds it +//! until the last batch is answered. A per-slot send can then never be +//! overtaken by an older registration from a full send that read the cache +//! before the change, and no batch of either is still in flight when the next +//! begins. The one window left is a request this client gave up on at its +//! timeout that the node still processes afterwards; the next epoch's full +//! send, which carries the newest registration, closes it. +//! +//! A per-slot send that finds the lock taken skips: it has not marked its +//! change as sent, so a later slot picks it up. While a full send runs, that +//! is the first slot after it ends, and a full send lasts as long as its +//! slowest node takes over every batch. A node has 12 s to answer each one +//! (one that takes longer is treated as unreachable and sent no more), so the +//! wait is at most 12 s per [`REGISTRATION_BATCH`] validators. An ethlambda +//! node whose relay does not answer comes close: it replies to each batch +//! with a 500 once its own 10 s wait on the relay runs out, and is still sent +//! the rest. With 5,000 validators, ten batches, a keymanager change then +//! waits about 100 s, eight slots, and two minutes at most. +//! +//! A full send waits for a running per-slot send instead, since that send is +//! short and skipping would cost an epoch. A full send that finds the +//! previous full send still running, or waiting, skips to the next epoch (see +//! [`crate::in_flight`]), so a hung node cannot queue up one waiting send per +//! epoch. +//! +//! # Signing stays off the async runtime +//! +//! The first epoch signs one BLS signature per validator. That runs on the +//! blocking pool in chunks of [`SIGN_CHUNK`], taking the store's read lock +//! afresh for each chunk, so neither the runtime's threads nor a queued +//! keymanager writer wait on thousands of signatures. The cache's own lock is +//! only ever taken under [`RegistrationService::sending`], so nothing on the +//! runtime waits on a chunk that holds it. +//! +//! # The per-slot check is one atomic read +//! +//! Only a keymanager write changes what a registration says, so +//! [`RegistrationService::register_changed`] compares +//! [`ProposerSettings::generation`] with the one it last signed against and +//! returns before touching the store when it has not moved. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use ethlambda_types::beacon::builder::{SignedValidatorRegistrationV1, ValidatorRegistrationV1}; +use ethlambda_types::beacon::fork::ForkName; +use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch}; +use tokio::sync::{Mutex, RwLock}; +use tracing::{debug, warn}; + +use crate::beacon_node::dto::encode_hex; +use crate::beacon_node::{BeaconNodeApi, REGISTRATION_BATCH}; +use crate::in_flight::InFlight; +use crate::keys::ValidatorStore; +use crate::proposer_settings::ProposerSettings; +use crate::signing::SigningContext; + +/// Keys signed per blocking task, and so per hold of the store's read lock. +const SIGN_CHUNK: usize = 64; + +/// What [`RegistrationService::current`] found for a set of validators. +struct Current { + /// Every validator's registration, cached or newly signed. + all: Vec, + /// The subset signed in this call, because it had no registration yet or + /// its fee recipient or gas limit changed. + changed: Vec, + /// Validators skipped because no fee recipient is configured for them. + no_fee_recipient: usize, + /// Chunks whose signing task failed, so whose validators are in neither + /// list. + failed_chunks: usize, +} + +impl Current { + fn extend(&mut self, other: Current) { + self.all.extend(other.all); + self.changed.extend(other.changed); + self.no_fee_recipient += other.no_fee_recipient; + self.failed_chunks += other.failed_chunks; + } +} + +/// The timestamp for a registration replacing `held`: `now_secs`, or one +/// second past `held`'s when the clock does not say later. +/// +/// A relay keeps a validator's registration with the latest timestamp and +/// drops one that is not newer. Two settings changes within one second, or a +/// wall clock stepped back between them, would otherwise sign the new settings +/// under a timestamp the relay ignores. The cached registration is then re-sent +/// every epoch while the builder keeps the old fee recipient or gas limit. +/// +/// Two changes within a second put it one second ahead of the clock, which no +/// relay refuses as being in the future. After a larger backward step it is +/// further ahead: a relay may refuse it until its own clock catches up, and the +/// epoch's full re-send then delivers it. With the clock's own reading +/// instead, the relay would drop it as older on every re-send, since the +/// cached signature keeps that timestamp until the settings change again. +fn newer_than(held: Option<&SignedValidatorRegistrationV1>, now_secs: u64) -> u64 { + held.map_or(now_secs, |held| { + now_secs.max(held.message.timestamp.saturating_add(1)) + }) +} + +/// Signs and sends builder registrations. +pub struct RegistrationService { + beacon_node: Arc, + context: Arc, + settings: Arc, + /// The validators the beacon node resolved at the last epoch refresh. + /// Taken under [`Self::sending`] only. + known: std::sync::Mutex>, + /// The latest signed registration per validator. Taken under + /// [`Self::sending`] only, so it is never contended: a `std` mutex is + /// enough, and a signing chunk holding it keeps no one else waiting. + signed: std::sync::Mutex>, + /// The [`ProposerSettings::generation`] the cache was last brought up to + /// date against. Held behind while [`Self::unsent`] is not empty, so the + /// next per-slot call retries it. + signed_generation: AtomicU64, + /// Validators whose newest registration a per-slot send could not get any + /// node to accept: sent again, as signed, at every slot until one does or + /// the next full send, which carries them anyway, ends the retry. Taken + /// under [`Self::sending`] only. + unsent: std::sync::Mutex>, + /// Held by a send from before it reads the cache until its last batch is + /// answered, so sends reach every node in the order they read the cache. + /// See the module doc. + sending: Mutex<()>, + /// The epoch's full send, while it runs or waits for [`Self::sending`]. + full_send: InFlight, +} + +impl RegistrationService { + pub fn new( + beacon_node: Arc, + context: Arc, + settings: Arc, + ) -> Self { + Self { + beacon_node, + context, + settings, + known: std::sync::Mutex::new(Vec::new()), + signed: std::sync::Mutex::new(HashMap::new()), + signed_generation: AtomicU64::new(0), + unsent: std::sync::Mutex::new(HashSet::new()), + sending: Mutex::new(()), + full_send: InFlight::default(), + } + } + + /// The registration for each of `pubkeys` as of `now_secs`, signing only + /// the ones whose `(fee_recipient, gas_limit)` differs from the cached one. + /// + /// `now_secs` is a parameter so a test can fix the timestamp. + fn current(&self, store: &ValidatorStore, pubkeys: &[BlsPubkey], now_secs: u64) -> Current { + // A poisoned lock is recovered: every write is one map operation, so + // the data behind it is never half-changed. + let mut signed = self + .signed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut current = Current { + all: Vec::with_capacity(pubkeys.len()), + changed: Vec::new(), + no_fee_recipient: 0, + failed_chunks: 0, + }; + for pubkey in pubkeys { + let Some((fee_recipient, gas_limit)) = + self.settings.fee_recipient_and_gas_limit(pubkey) + else { + current.no_fee_recipient += 1; + continue; + }; + + let held = signed.get(pubkey); + if let Some(held) = held + && held.message.fee_recipient == fee_recipient + && held.message.gas_limit == gas_limit + { + current.all.push(held.clone()); + continue; + } + + let message = ValidatorRegistrationV1 { + fee_recipient, + gas_limit, + timestamp: newer_than(held, now_secs), + pubkey: *pubkey, + }; + // A key the keymanager removed since the epoch refresh is the one + // way this fails; the next refresh no longer lists it. + let signature = match self.context.sign_validator_registration(store, &message) { + Ok(signature) => signature, + Err(err) => { + warn!(pubkey = %encode_hex(&pubkey.0), %err, "Could not sign a builder registration; skipping it"); + continue; + } + }; + let registration = SignedValidatorRegistrationV1 { message, signature }; + signed.insert(*pubkey, registration.clone()); + current.all.push(registration.clone()); + current.changed.push(registration); + } + current + } + + /// Whether `epoch` is at or after gloas, where there is nothing to register. + fn is_gloas(&self, epoch: Epoch) -> bool { + self.context.config.fork_at_epoch(epoch) >= ForkName::Gloas + } + + fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) + } + + /// [`Self::current`] over `pubkeys`, off the async runtime, [`SIGN_CHUNK`] + /// keys per blocking task with the store's read lock taken per chunk. + async fn current_blocking( + self: &Arc, + store: &Arc>, + pubkeys: &[BlsPubkey], + ) -> Current { + let now_secs = Self::now_secs(); + let mut current = Current { + all: Vec::with_capacity(pubkeys.len()), + changed: Vec::new(), + no_fee_recipient: 0, + failed_chunks: 0, + }; + for chunk in pubkeys.chunks(SIGN_CHUNK) { + let (service, store, chunk) = (self.clone(), store.clone(), chunk.to_vec()); + let part = tokio::task::spawn_blocking(move || { + let store = store.blocking_read(); + service.current(&store, &chunk, now_secs) + }) + .await; + match part { + Ok(part) => current.extend(part), + Err(err) => { + warn!(%err, "Signing builder registrations failed; skipping a chunk"); + current.failed_chunks += 1; + } + } + } + current + } + + /// Record that every setting up to `generation` is signed and sent, + /// unless a chunk of the signing failed (`failed_chunks`) or a registration + /// is waiting to be sent again ([`Self::unsent`]). Then the generation + /// stays behind, so the next slot's [`Self::register_changed`] signs what + /// that chunk held, or re-sends what no node took, rather than leaving a + /// keymanager change unsent for an epoch. + fn mark_signed(&self, generation: u64, failed_chunks: usize) { + if failed_chunks == 0 && self.unsent().is_empty() { + self.signed_generation.store(generation, Ordering::Release); + } + } + + /// [`Self::unsent`], locked. A poisoned lock is recovered: every write is + /// one set operation. + fn unsent(&self) -> std::sync::MutexGuard<'_, HashSet> { + self.unsent + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// Register every validator in `pubkeys`, the ones the beacon node resolved + /// at this epoch's refresh. Called once per epoch. + /// + /// Also forgets any validator no longer in `pubkeys`, so a key the + /// keymanager removed stops being re-signed (and warned about) every slot. + pub async fn register_all( + self: Arc, + epoch: Epoch, + pubkeys: Vec, + store: Arc>, + ) { + if self.is_gloas(epoch) { + return; + } + let Some(_running) = self.full_send.enter() else { + warn!( + epoch, + "The previous builder registration send is still running; skipping this epoch's" + ); + return; + }; + // Waits out a per-slot send rather than skipping: that one is short, + // and the full set is sent once an epoch. + let _sending = self.sending.lock().await; + // Read before signing, so a change that lands while this runs is seen + // by the next `register_changed` rather than lost. + let generation = self.settings.generation(); + let current_keys: HashSet = pubkeys.iter().copied().collect(); + self.signed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|pubkey, _| current_keys.contains(pubkey)); + *self + .known + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = pubkeys.clone(); + + let current = self.current_blocking(&store, &pubkeys).await; + if current.no_fee_recipient > 0 { + warn!( + skipped = current.no_fee_recipient, + "Validators without a fee recipient are not registered with the builder" + ); + } + self.send(¤t.all).await; + // This send carried every key, so it ends each per-slot retry: a key it + // delivered needs no more, and one it did not waits for the next + // epoch's full send with the rest of its batch. + self.unsent().clear(); + self.mark_signed(generation, current.failed_chunks); + } + + /// Send only the registrations whose fee recipient or gas limit changed + /// since they were last signed, over the validators the last + /// [`Self::register_all`] was given, plus any an earlier slot could not + /// get a node to accept ([`Self::unsent`]). Called every slot, so a + /// keymanager change reaches the builder within a slot rather than an + /// epoch. + /// + /// Returns before touching the store when no setting was written since the + /// last signing: see the module doc. + /// + /// Silent about validators with no fee recipient: that is reported once + /// per epoch by [`Self::register_all`], not repeated every slot. + pub async fn register_changed( + self: Arc, + epoch: Epoch, + store: Arc>, + ) { + if self.is_gloas(epoch) { + return; + } + if self.settings.generation() == self.signed_generation.load(Ordering::Acquire) { + return; + } + // Silently: another send is running, and the change stays unsent, so + // the next slot picks it up. + let Ok(_sending) = self.sending.try_lock() else { + return; + }; + // Again, under the lock: a full send that just ended may have signed + // this change already. + let generation = self.settings.generation(); + if generation == self.signed_generation.load(Ordering::Acquire) { + return; + } + let known = self + .known + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if known.is_empty() { + return; + } + let current = self.current_blocking(&store, &known).await; + // What changed, and what an earlier slot could not get any node to + // take. Both are read off `all`, the newest registration per key, so a + // key that changed again since its failed send goes out once, as now. + let retry = std::mem::take(&mut *self.unsent()); + let changed: HashSet = current + .changed + .iter() + .map(|registration| registration.message.pubkey) + .collect(); + let outgoing: Vec = current + .all + .into_iter() + .filter(|registration| { + let pubkey = ®istration.message.pubkey; + changed.contains(pubkey) || retry.contains(pubkey) + }) + .collect(); + if !outgoing.is_empty() { + let failed = self.send(&outgoing).await; + self.unsent().extend(failed); + } + self.mark_signed(generation, current.failed_chunks); + } + + /// Send `registrations` to the beacon node in batches, counting each batch + /// as sent when a node accepted it and as a failure when none did. + /// + /// Returns the validators in the batches no node accepted. + async fn send(&self, registrations: &[SignedValidatorRegistrationV1]) -> HashSet { + let outcomes = self + .beacon_node + .register_validators_in_batches(registrations) + .await; + let mut failed = HashSet::new(); + for (batch, outcome) in registrations.chunks(REGISTRATION_BATCH).zip(outcomes) { + let count = batch.len(); + // Only logged and counted: see "Failures are never propagated". + let refused = outcome + .inspect(|()| { + debug!(count, "Builder registrations sent"); + crate::metrics::inc_builder_registrations_sent(count); + }) + .inspect_err(|err| { + warn!(count, %err, "No beacon node accepted the builder registrations"); + crate::metrics::inc_builder_registration_failures(); + }) + .is_err(); + if refused { + failed.extend(batch.iter().map(|registration| registration.message.pubkey)); + } + } + failed + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon_node::mock::MockBeaconNode; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::primitives::{Bytes32, H160, Root}; + use tokio::sync::Semaphore; + + fn context(config: Config) -> Arc { + Arc::new(SigningContext { + config, + genesis_validators_root: Root::ZERO, + }) + } + + /// `count` distinct keys. The secrets are small integers, which are valid + /// BLS scalars, so no keystore is needed. + fn store_with_keys(count: usize) -> (ValidatorStore, Vec) { + let mut store = ValidatorStore::new(); + let pubkeys = (0..count) + .map(|index| { + let mut secret = [0u8; 32]; + secret[24..].copy_from_slice(&(index as u64 + 1).to_be_bytes()); + store.insert_secret("test", &secret).expect("inserts") + }) + .collect(); + (store, pubkeys) + } + + fn settings(fee_recipient: Option) -> Arc { + Arc::new(ProposerSettings::new(Bytes32::ZERO, fee_recipient)) + } + + fn service( + config: Config, + settings: Arc, + ) -> ( + Arc>, + Arc, + ) { + let node = Arc::new(MockBeaconNode::new()); + ( + Arc::new(RegistrationService::new( + node.clone(), + context(config), + settings, + )), + node, + ) + } + + #[test] + fn registrations_are_signed_once_and_reused() { + let (store, pubkeys) = store_with_keys(2); + let (service, _) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + + let first = service.current(&store, &pubkeys, 1_000); + assert_eq!(first.all.len(), 2); + assert_eq!(first.changed.len(), 2); + + let second = service.current(&store, &pubkeys, 2_000); + assert_eq!(second.all, first.all, "same timestamp and signature"); + assert!(second.changed.is_empty()); + assert_eq!(second.all[0].message.timestamp, 1_000); + } + + #[test] + fn a_new_fee_recipient_or_gas_limit_re_signs() { + let (store, pubkeys) = store_with_keys(3); + let settings = settings(Some(H160([0xab; 20]))); + let (service, _) = service(Config::mainnet(), settings.clone()); + let first = service.current(&store, &pubkeys, 1_000); + + settings.set_fee_recipient(&pubkeys[0], H160([0xcd; 20])); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + let second = service.current(&store, &pubkeys, 2_000); + + let changed: Vec<_> = second.changed.iter().map(|r| r.message.pubkey).collect(); + assert_eq!(changed, vec![pubkeys[0], pubkeys[1]]); + assert_eq!(second.all[0].message.fee_recipient, H160([0xcd; 20])); + assert_eq!(second.all[0].message.timestamp, 2_000); + assert_eq!(second.all[1].message.gas_limit, 30_000_000); + assert_eq!(second.all[1].message.timestamp, 2_000); + assert_ne!(second.all[0].signature, first.all[0].signature); + assert_eq!(second.all[2], first.all[2], "the third is untouched"); + } + + /// A relay drops a registration that is not newer than the one it holds, + /// so a change signed within the same second, or after the clock stepped + /// back, still gets a later timestamp than the registration it replaces. + #[test] + fn a_re_signed_registration_is_always_newer_than_the_last() { + let (store, pubkeys) = store_with_keys(1); + let settings = settings(Some(H160([0xab; 20]))); + let (service, _) = service(Config::mainnet(), settings.clone()); + let first = service.current(&store, &pubkeys, 2_000); + assert_eq!(first.all[0].message.timestamp, 2_000); + + settings.set_gas_limit(&pubkeys[0], 30_000_000); + let same_second = service.current(&store, &pubkeys, 2_000); + assert_eq!(same_second.changed[0].message.timestamp, 2_001); + + settings.set_gas_limit(&pubkeys[0], 31_000_000); + let stepped_back = service.current(&store, &pubkeys, 1_000); + assert_eq!(stepped_back.changed[0].message.timestamp, 2_002); + + // Once the clock is ahead again, its reading is used. + settings.set_gas_limit(&pubkeys[0], 32_000_000); + let caught_up = service.current(&store, &pubkeys, 5_000); + assert_eq!(caught_up.changed[0].message.timestamp, 5_000); + } + + #[test] + fn a_validator_without_a_fee_recipient_is_not_registered() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(None); + settings.set_fee_recipient(&pubkeys[1], H160([0xcd; 20])); + let (service, _) = service(Config::mainnet(), settings); + + let current = service.current(&store, &pubkeys, 1_000); + + assert_eq!(current.no_fee_recipient, 1); + assert_eq!(current.all.len(), 1); + assert_eq!(current.all[0].message.pubkey, pubkeys[1]); + } + + #[test] + fn a_removed_key_is_skipped_rather_than_failing_the_rest() { + let (mut store, pubkeys) = store_with_keys(2); + let (service, _) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + store.remove(&pubkeys[0]); + + let current = service.current(&store, &pubkeys, 1_000); + + assert_eq!(current.all.len(), 1); + assert_eq!(current.all[0].message.pubkey, pubkeys[1]); + } + + /// 501 keys is one more than a batch; signing them is a millisecond each. + #[tokio::test] + async fn register_all_sends_in_batches_of_500() { + assert_eq!(REGISTRATION_BATCH, 500); + let (store, pubkeys) = store_with_keys(REGISTRATION_BATCH + 1); + let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let store = Arc::new(RwLock::new(store)); + + service.register_all(0, pubkeys, store).await; + + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![REGISTRATION_BATCH, 1] + ); + } + + /// A service over a node whose `register_validators` calls each wait for + /// a permit from the returned gate, which starts with `permits`. + fn gated_service( + settings: Arc, + permits: usize, + ) -> ( + Arc>, + Arc, + Arc, + ) { + let gate = Arc::new(Semaphore::new(permits)); + let node = Arc::new(MockBeaconNode::new().with_registration_gate(gate.clone())); + let service = Arc::new(RegistrationService::new( + node.clone(), + context(Config::mainnet()), + settings, + )); + (service, node, gate) + } + + /// Bounds a call that must return on its own, so a regression fails the + /// test rather than hanging it. + async fn promptly(what: &str, call: F) -> F::Output { + tokio::time::timeout(std::time::Duration::from_secs(5), call) + .await + .unwrap_or_else(|_| panic!("{what}")) + } + + /// A full send that is still waiting on a slow node makes the next one + /// skip, rather than both holding the set and sending it again; once it + /// ends, the next one runs. + #[tokio::test] + async fn an_epoch_send_still_running_makes_the_next_one_skip() { + let (store, pubkeys) = store_with_keys(2); + let (service, node, gate) = gated_service(settings(Some(H160([0xab; 20]))), 0); + let store = Arc::new(RwLock::new(store)); + + let slow = tokio::spawn( + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()), + ); + node.wait_for_registration_calls(1).await; + promptly( + "skips rather than waiting on the node", + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()), + ) + .await; + gate.add_permits(1); + slow.await.expect("the first send ends"); + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2], + "only the first send reached the node" + ); + + gate.add_permits(1); + service.register_all(2, pubkeys, store).await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 2]); + } + + /// The reviewer's race: a full send holds the registrations it read + /// before a change, and a per-slot send must not deliver the change while + /// that older copy is still to reach the node, or the older one lands last + /// and the node keeps its fee recipient. The per-slot send skips instead, + /// and the next slot's delivers the change after the full send. + #[tokio::test] + async fn a_change_is_never_overtaken_by_a_full_send_that_read_before_it() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node, gate) = gated_service(settings.clone(), 0); + let store = Arc::new(RwLock::new(store)); + + // The full send has read the cache and is at the node. + let full = tokio::spawn( + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()), + ); + node.wait_for_registration_calls(1).await; + + settings.set_fee_recipient(&pubkeys[0], H160([0xcd; 20])); + promptly( + "the per-slot send skips rather than waiting on the node", + service.clone().register_changed(0, store.clone()), + ) + .await; + assert_eq!( + node.registration_attempts + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "nothing reached the node beside the full send" + ); + + gate.add_permits(1); + full.await.expect("the full send ends"); + gate.add_permits(1); + service.register_changed(0, store).await; + + let registrations = node.registrations.lock().expect("lock"); + let latest_for_first = registrations + .iter() + .rfind(|registration| registration.message.pubkey == pubkeys[0]) + .expect("registered"); + assert_eq!( + latest_for_first.message.fee_recipient, + H160([0xcd; 20]), + "the change reaches the node last" + ); + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 1]); + } + + /// The other way round, the full send waits for a running per-slot send + /// rather than skipping its epoch, and then sends the change too, since it + /// reads the cache only once the per-slot send is done. + #[tokio::test] + async fn a_full_send_waits_for_a_running_per_slot_send() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node, gate) = gated_service(settings.clone(), 1); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + settings.set_gas_limit(&pubkeys[1], 30_000_000); + let changed = tokio::spawn(service.clone().register_changed(0, store.clone())); + node.wait_for_registration_calls(2).await; + let full = tokio::spawn( + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()), + ); + // Long enough for a full send that skipped to have returned. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !full.is_finished(), + "the full send waits rather than skipping" + ); + + gate.add_permits(2); + changed.await.expect("the per-slot send ends"); + full.await.expect("the full send ends"); + + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2, 1, 2] + ); + let registrations = node.registrations.lock().expect("lock"); + assert_eq!(registrations[2].message.gas_limit, 30_000_000); + assert_eq!( + registrations[4], registrations[2], + "the full send carries the change it waited for" + ); + } + + /// Both entry points stop at gloas, and `register_changed` is checked with + /// something to send: a service that was filled before the fork and then + /// had a setting changed. + #[tokio::test] + async fn nothing_is_sent_at_or_after_gloas() { + let (store, pubkeys) = store_with_keys(2); + let config = Config { + gloas_fork_epoch: 4, + ..Config::mainnet() + }; + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(config, settings.clone()); + let store = Arc::new(RwLock::new(store)); + + // Before the fork the service registers, and holds the keys. + service + .clone() + .register_all(3, pubkeys.clone(), store.clone()) + .await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2]); + + settings.set_gas_limit(&pubkeys[0], 30_000_000); + service + .clone() + .register_all(4, pubkeys, store.clone()) + .await; + service.clone().register_changed(5, store.clone()).await; + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2], + "a changed key is still not sent at or after the fork" + ); + + // The same change is sent when the epoch is before the fork. + service.register_changed(3, store).await; + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2, 1]); + } + + #[tokio::test] + async fn register_changed_sends_only_the_changed_keys() { + let (store, pubkeys) = store_with_keys(3); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + service.clone().register_changed(0, store.clone()).await; + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![3], + "nothing changed, so nothing more is sent" + ); + + settings.set_gas_limit(&pubkeys[2], 30_000_000); + service.register_changed(0, store).await; + + let registrations = node.registrations.lock().expect("lock"); + assert_eq!(registrations.len(), 4); + assert_eq!(registrations[3].message.pubkey, pubkeys[2]); + assert_eq!(registrations[3].message.gas_limit, 30_000_000); + } + + fn refuse_registrations(node: &MockBeaconNode) { + node.failures + .lock() + .expect("lock") + .insert("register_validators", "relay down".to_string()); + } + + fn registration_calls(node: &MockBeaconNode) -> usize { + node.registration_attempts + .load(std::sync::atomic::Ordering::SeqCst) + } + + /// A change no node accepted goes out again at every slot, as it was + /// signed, until a node takes it; the slot after that sends nothing. + #[tokio::test] + async fn a_change_no_node_accepted_is_sent_again_until_one_does() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + refuse_registrations(&node); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + service.clone().register_changed(0, store.clone()).await; + service.clone().register_changed(0, store.clone()).await; + assert_eq!(registration_calls(&node), 3, "refused, then tried again"); + + node.stop_failing("register_validators"); + service.clone().register_changed(0, store.clone()).await; + let signed = service.signed.lock().expect("lock")[&pubkeys[1]].clone(); + { + let registrations = node.registrations.lock().expect("lock"); + assert_eq!( + registrations.len(), + 3, + "the full send's two, then the change" + ); + assert_eq!(registrations[2].message.gas_limit, 30_000_000); + assert_eq!( + registrations[2], signed, + "re-sent as signed, not signed again" + ); + } + + service.clone().register_changed(0, store.clone()).await; + assert_eq!(registration_calls(&node), 4, "delivered, so nothing more"); + } + + /// A full send that delivers a key waiting for its retry ends the retry, + /// and the slots after it go back to sending nothing. + #[tokio::test] + async fn a_full_send_that_delivers_a_waiting_change_ends_its_retry() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + refuse_registrations(&node); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + service.clone().register_changed(0, store.clone()).await; + + node.stop_failing("register_validators"); + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()) + .await; + service.clone().register_changed(1, store.clone()).await; + assert_eq!( + *node.registration_batches.lock().expect("lock"), + vec![2, 2], + "the two full sends, and no retry after the second" + ); + } + + /// A full send ends the retry even when it is refused too: the change then + /// waits for the next epoch's full send with the rest of the set, rather + /// than costing a refused send at every slot for as long as the node + /// refuses. + #[tokio::test] + async fn a_refused_full_send_ends_the_retry_too() { + let (store, pubkeys) = store_with_keys(2); + let settings = settings(Some(H160([0xab; 20]))); + let (service, node) = service(Config::mainnet(), settings.clone()); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + + refuse_registrations(&node); + settings.set_gas_limit(&pubkeys[1], 30_000_000); + service.clone().register_changed(0, store.clone()).await; + service + .clone() + .register_all(1, pubkeys.clone(), store.clone()) + .await; + assert_eq!(registration_calls(&node), 3, "full, per-slot, full"); + + service.clone().register_changed(1, store.clone()).await; + assert_eq!(registration_calls(&node), 3, "no retry after the full send"); + } + + /// The per-slot call is one atomic read when no setting moved: with the + /// store write-locked, so any read of it would wait forever, it still + /// returns at once, and sends nothing. + #[tokio::test] + async fn a_slot_with_no_change_makes_no_store_read_and_no_send() { + let (store, pubkeys) = store_with_keys(2); + let (service, node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let store = Arc::new(RwLock::new(store)); + service + .clone() + .register_all(0, pubkeys, store.clone()) + .await; + let signed_before = service.signed.lock().expect("lock").clone(); + + let _writer = store.write().await; + tokio::time::timeout( + std::time::Duration::from_millis(200), + service.clone().register_changed(0, store.clone()), + ) + .await + .expect("returns without waiting on the store"); + + assert_eq!(*node.registration_batches.lock().expect("lock"), vec![2]); + assert_eq!(*service.signed.lock().expect("lock"), signed_before); + } + + /// A key that left the resolved set is dropped from the cache, so it is not + /// re-signed (and warned about) at every change. + #[tokio::test] + async fn a_key_no_longer_resolved_is_forgotten() { + let (store, pubkeys) = store_with_keys(2); + let (service, _node) = service(Config::mainnet(), settings(Some(H160([0xab; 20])))); + let store = Arc::new(RwLock::new(store)); + + service + .clone() + .register_all(0, pubkeys.clone(), store.clone()) + .await; + service + .clone() + .register_all(0, vec![pubkeys[1]], store) + .await; + + let signed = service.signed.lock().expect("lock"); + assert_eq!(signed.keys().copied().collect::>(), vec![pubkeys[1]]); + assert_eq!(*service.known.lock().expect("lock"), vec![pubkeys[1]]); + } +} diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index 9317ab341..d1c9767c1 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -5,6 +5,7 @@ //! beacon node sent: the wire representation is a transport detail, and the //! chain only ever agrees about the merkle root. +use ethlambda_types::beacon::builder::ValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants::{ DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_BUILDER, @@ -25,7 +26,7 @@ use ethlambda_types::beacon::primitives::{ // `AttestationData`, `Checkpoint`, `Fork` and `SigningData` all live in // `containers::shared`, verified against `containers/shared.rs:158,120,97,220`. use ethlambda_types::beacon::signing::{ - compute_domain, compute_epoch_at_slot, compute_signing_root, + compute_builder_domain, compute_domain, compute_epoch_at_slot, compute_signing_root, }; use crate::error::{Error, Result}; @@ -297,6 +298,24 @@ impl SigningContext { self.sign_root(store, pubkey, self.envelope_signing_root(envelope, slot)) } + /// A builder registration's signature. + /// + /// Signed under `compute_builder_domain(genesis_fork_version)`: the genesis + /// fork version and a zero validators root, per builder-specs, so a + /// registration verifies whatever fork the chain is in. Not + /// `DOMAIN_BEACON_BUILDER`, which is gloas's envelope domain. Not + /// slashable: a registration commits to nothing about the chain, and a + /// builder keeps only the newest one per validator. + pub fn sign_validator_registration( + &self, + store: &ValidatorStore, + registration: &ValidatorRegistrationV1, + ) -> Result { + let domain = compute_builder_domain(self.config.genesis_fork_version); + let root = compute_signing_root(registration.hash_tree_root(), domain); + self.sign_root(store, ®istration.pubkey, root) + } + /// Sign a payload timeliness committee vote on behalf of `pubkey`. /// /// Not slashable by the specification, and deduplicated per validator and @@ -507,6 +526,71 @@ mod tests { (store, pubkey) } + fn registration(pubkey: BlsPubkey) -> ValidatorRegistrationV1 { + ValidatorRegistrationV1 { + fee_recipient: ethlambda_types::beacon::primitives::H160([0xab; 20]), + gas_limit: 36_000_000, + timestamp: 1_700_000_000, + pubkey, + } + } + + /// A context whose genesis validators root is not zero, so the builder + /// domain (which zeroes it) and a context-derived one cannot coincide. + fn context_with_validators_root() -> SigningContext { + SigningContext { + config: Config::mainnet(), + genesis_validators_root: Root::repeat_byte(0x42), + } + } + + #[test] + fn a_registration_is_signed_under_the_builder_domain() { + use ethlambda_types::beacon::constants::DOMAIN_APPLICATION_BUILDER; + + let (store, pubkey) = store_with_key(); + let context = context_with_validators_root(); + let registration = registration(pubkey); + + let signature = context + .sign_validator_registration(&store, ®istration) + .expect("signs"); + + let root = compute_signing_root( + registration.hash_tree_root(), + compute_builder_domain(Config::mainnet().genesis_fork_version), + ); + assert!(verify(&pubkey, &signature, root)); + + // The same domain type under this chain's validators root is a + // different domain: a context-derived one would not verify. + let with_root = compute_signing_root( + registration.hash_tree_root(), + context.domain(DOMAIN_APPLICATION_BUILDER, 0), + ); + assert!(!verify(&pubkey, &signature, with_root)); + } + + /// The registration domain is not gloas's envelope domain, whose type + /// constant is similar in name only: a signature under one must not verify + /// under the other. + #[test] + fn a_registration_does_not_verify_under_the_envelope_domain() { + let (store, pubkey) = store_with_key(); + let context = context_with_validators_root(); + let registration = registration(pubkey); + + let signature = context + .sign_validator_registration(&store, ®istration) + .expect("signs"); + + let envelope_root = compute_signing_root( + registration.hash_tree_root(), + context.domain(DOMAIN_BEACON_BUILDER, 0), + ); + assert!(!verify(&pubkey, &signature, envelope_root)); + } + #[test] fn a_randao_reveal_verifies_under_the_randao_signing_root() { let (store, pubkey) = store_with_key(); diff --git a/docs/cli.md b/docs/cli.md index f230f745c..b01b9243c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -297,9 +297,12 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--http-address` | `127.0.0.1` | Bind address for the metrics and keymanager servers | | `--metrics-port` | `5064` | Prometheus metrics port | | `--keymanager-port` | `5062` | Keymanager API port. Only bound with `--enable-keymanager` | -| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours | +| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours. The zero address is accepted, as every other client accepts it, but reported at error at startup: it burns the rewards, and builder registrations and proposer preferences sign it for the builder to pay | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | +| `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | +| `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` prefers the local payload unless it cannot be built. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | +| `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid: this client sends `builder_boost_factor = u64::MAX`. The beacon node still keeps its local payload when its execution client sets `shouldOverrideBuilder`. Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | | `--init-slashing-protection` | off | Register every loaded key with no slashing-protection record as never having signed, instead of refusing to start. Dangerous: only for keys that have signed nothing anywhere for at least two epochs, and even then it rules out a double vote only, not a surround of an older vote; a key with a history should bring it instead. Never lowers an existing record; remove it once the keys are registered, and startup warns while it is set for nothing | ### What it does today @@ -314,6 +317,17 @@ validator's fee recipient with the beacon node. The subscription also tells the node which committees this client will aggregate for, which it has to know in advance so it can collect the votes. +With `--builder-proposals`, before gloas, it also signs a builder registration +(fee recipient and gas limit) for each validator the node resolved and sends +them to every node: the full set after each epoch's refresh, in batches, and +every slot only the registrations whose fee recipient or gas limit changed +through the keymanager API. A block request then carries a +`builder_boost_factor`, and when the node answers with a blinded block it is +signed like a full one and published to `blinded_blocks` on the node that +produced it, with no failover. A publish that node does not confirm is logged +at error and counted as unconfirmed, not as a missed proposal, since the relay +may still publish the block. + Each slot it wakes at the boundary. If one of its validators proposes that slot, it signs the RANDAO reveal, asks the beacon node for a block, checks that the block is for the slot and proposer it asked about, signs it and publishes @@ -334,7 +348,12 @@ slot. The specification defines no block-production deadline; the nearest thing it defines is the point attesters stop waiting for a block, which is the same instant. A proposal still running then has already lost most of the block's value, and every second past it comes out of this client's own attestations, so -it is abandoned and the slot's attesters still vote. +it is abandoned and the slot's attesters still vote. A blinded block that was +already signed is the exception: its publish runs on its own task, since the +beacon node may wait several seconds on the relay, so an overrun after signing +is logged at info and not counted as a failed proposal, and the task counts the +real outcome (a refusal as a failed proposal, success with its publication +delay). Electra is the earliest fork it will propose under. A deneb block body has one field fewer than an electra one, so it would need a container pair of its own, @@ -427,9 +446,9 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch | +| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no registration, so the builder keeps paying the address it was last given | | `graffiti` | `--graffiti` | The next proposal | -| `gas_limit` | `DEFAULT_GAS_LIMIT` | Stored and served only: it matters to builder registrations, and there is no builder flow | +| `gas_limit` | `DEFAULT_GAS_LIMIT` | With `--builder-proposals`, the next builder registration, which the builder builds toward: usually at the next slot, later while the epoch's full registration send is still running, and again every slot until a node accepts it. Otherwise stored and served only | **These are held in memory only.** A restart puts every validator back on the defaults, so an operator driving these endpoints must set them again after one. diff --git a/docs/metrics.md b/docs/metrics.md index 70de4ac02..646f2f913 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -748,6 +748,10 @@ and absent is not the same as zero. | `ethlambda_validator_aggregates_published_total` | Counter | Aggregates accepted by a beacon node. Bursty rather than steady: a validator is selected a few times a day, so hours at zero are normal for a small deployment | | `ethlambda_validator_aggregation_failures_total` | Counter | Aggregation duties that ended in no published aggregate, including ones abandoned for overrunning the slot | | `ethlambda_validator_fee_recipient_mismatches_total` | Counter | Blocks paying execution rewards to an address this client did not request. Should read zero forever; a non-zero value means every proposal is paying somewhere else | +| `ethlambda_validator_builder_registrations_sent_total` | Counter | Builder registrations a beacon node accepted (`--builder-proposals`) | +| `ethlambda_validator_builder_registration_failures_total` | Counter | Registration batches no beacon node accepted. A missed registration costs at most a builder block, never a duty | +| `ethlambda_validator_blinded_blocks_proposed_total` | Counter | Blinded blocks a beacon node handed to its builder. A subset of `blocks_proposed_total` | +| `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx other than 503, or no answer once connected), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in | | `ethlambda_validator_beacon_node_available` | Gauge | 1 when a beacon node answered the last duty refresh or attestation | | `ethlambda_validator_signing_duration_seconds` | Histogram | Time to sign one slot's batch, store read lock included | @@ -764,13 +768,17 @@ second points at the beacon nodes; a rise in the third points at the clock or the duty schedule and is worth investigating even though the attestation was correctly suppressed. -The proposal series split the same way, plus one that is neither a success nor +The proposal series split the same way, plus two that are neither a success nor a failure. `blocks_proposed_total` counts blocks a node accepted; `block_proposal_failures_total` counts duties that produced none; `blocks_refused_total` counts blocks deliberately not signed. Between them, `blocks_broadcast_not_imported_total` counts blocks that did reach the network but that the node answering could not import, which is a beacon-node fault rather than a validator one and is why it is not folded into either. +`blinded_blocks_unconfirmed_total` is the other: a blinded block the node did +not confirm, which the relay may still have published. It is not in +`blocks_proposed_total` nor in the publication delay histogram, and it is not a +failure either. Gloas adds two pairs. A gloas proposal ends in a block and then an envelope, and the envelope's failure does not fail the proposal: `blocks_proposed_total` diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index a42f2b17d..5c49fdc4f 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -760,3 +760,44 @@ comes first. The validator client signs once, at the deadline (`SYNC_MESSAGE_DUE_BPS`, or its gloas variant), over the head root it then reads, and refuses to when that root is optimistic (`specs/bellatrix/optimistic-sync.md`). + +## The validator client's builder proposals + +With `--builder-proposals` the validator client registers its validators with +the beacon node's builder and may sign a blinded block (builder-specs and +`validator.md`'s builder flow), for fulu and earlier slots. A blinded block +has the root of the full block it stands for, so slashing protection records +its slot like any other block's, durably and before it is signed: one slot, one +signature, whichever way the node answered. Three departures: + +- **A blinded block is published only on the beacon node that produced it, with + no failover.** This is this client's choice, not the specification's: only + that node's builder holds the payload and only that node claimed the slot, so + offering the signed block to another node would hand it to a builder that + never saw the bid. If the producing node fails, the publish fails, and + slashing protection has already recorded the slot, so nothing retries. A 5xx + other than 503, or no answer once connected, is reported as unconfirmed + (logged at error, counted on `blinded_blocks_unconfirmed_total`) and not as a + missed proposal, since the relay may still publish the block. A 4xx, a 503 or + a connection never made (refused, or past the client's 3 s connect timeout) + is a missed proposal: the node cannot have handed the block on. The block + carries the node that produced it (`Producer`, filled in by the fallback), + and the proposal path sends the signed block back to it; Teku pins a blinded + block to its producer too, in a cache keyed by slot and root, while + Lighthouse, Lodestar and Nimbus fail over or broadcast instead. + Two status codes are read the wrong way: some of ethlambda's own 500s are + definite (no builder configured, the block failing the node's checks) and + count as unconfirmed, and a proxy's 503 sent after it forwarded the request + counts as definite. Neither leads to a retry or a second signature. +- **A blinded block stays signed but undelivered for longer.** The node waits + up to 5 s on the relay and the client up to 8 s on the node, against well + under a second for a full block. The publish runs on its own task so the + proposal budget cannot cut it off: past the budget the proposal returns + "still publishing" and the task reports the outcome itself. The process + handles no signals, though: a SIGINT in that window ends the task with the + request. +- **Registrations stop at gloas.** The client sends no `register_validator` from + the gloas fork epoch on: the builder market works through proposer + preferences there, and `produceBlockV4` keeps its self-build configuration. + Validators with no fee recipient are not registered either (the field is + required), and only validators the beacon node resolved are signed for.