From db6138890799877f9182551743640baf7388fedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:24:35 -0300 Subject: [PATCH 1/6] fix(sync): judge the sync gate against peers' Status heads A node reported itself synced while far behind (#559): the tracker took the freshest block the network knows from the node's own imported chain, which trails the head during a catch-up, so its network-stall override fired and the node attested and proposed on a stale head. The beacon follower had the same bug and reported synced through whole catch-ups. Follow Lighthouse and Prysm instead: a node is syncing only when its head trails the wall clock and the head its peers report in Status is ahead of its own. P2P keeps each connected peer's latest Status head (re-requested every few slots, dropped on disconnect, a head past the clock ignored) and publishes the second-highest of three or more on the Store. Thresholds are per chain: leanSpec's 4/2 on lean, Lighthouse's two epochs of clock lag and one epoch of peer lead on beacon. --- crates/blockchain/src/lib.rs | 90 +++++--- crates/blockchain/src/sync_status.rs | 286 ++++++++++++++++++------ crates/net/p2p/src/lib.rs | 73 ++++++ crates/net/p2p/src/peer_heads.rs | 216 ++++++++++++++++++ crates/net/p2p/src/req_resp/handlers.rs | 27 +++ crates/storage/src/store.rs | 42 ++++ docs/architecture.md | 16 +- docs/beacon_wire.md | 10 +- docs/rpc.md | 2 +- docs/spec_deviations.md | 39 ++++ 10 files changed, 689 insertions(+), 112 deletions(-) create mode 100644 crates/net/p2p/src/peer_heads.rs diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 251048b7a..f35555eab 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -34,7 +34,7 @@ use crate::aggregation::{ PRIOR_WORKER_JOIN_TIMEOUT, aggregation_deadline, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; -use crate::sync_status::SyncStatusTracker; +use crate::sync_status::{SyncStatusTracker, SyncThresholds}; use spawned_concurrency::actor; use spawned_concurrency::error::ActorError; use spawned_concurrency::protocol; @@ -304,7 +304,7 @@ impl BlockChain { Self::start_actor( store, - SyncStatusTracker::new(gate_duties), + SyncStatusTracker::new(gate_duties, SyncThresholds::LEAN), sync_status_controller, events, ChainDuties::Lean(Box::new(lean)), @@ -322,7 +322,7 @@ impl BlockChain { /// [`ChainDuties::Beacon`]). Feeding it gossip and calling this from /// `run_node` are later slices; this constructor only builds the actor. /// - /// Uses [`SyncStatusTracker::new`]`(false)`: the tracker still drives the + /// Uses [`SyncStatusTracker::new`]`(false, ..)`: the tracker still drives the /// `lean_node_sync_status` metric (the name predates beacon support), but /// there are no duties on this arm for it to gate. /// @@ -356,7 +356,7 @@ impl BlockChain { Self::start_actor( store, - SyncStatusTracker::new(false), + SyncStatusTracker::new(false, SyncThresholds::BEACON), sync_status_controller, events, ChainDuties::Beacon, @@ -2018,7 +2018,7 @@ impl BlockChainServer { safe_slots_to_import_optimistically, last_tick_instant: None, epoch_precompute_in_flight: None, - sync_status: SyncStatusTracker::new(false), + sync_status: SyncStatusTracker::new(false, SyncThresholds::BEACON), sync_status_controller: SyncStatusController::default(), events: ChainEvents::new(EventBus::default(), &store), duties: ChainDuties::Beacon, @@ -3584,18 +3584,16 @@ impl BlockChainServer { /// /// Reads the head through [`Self::head_slot`], which is where the /// per-chain part of that lives (lean's `Store::head_slot` and beacon's - /// `Store::beacon_head` decode different tables). The tracker itself is - /// chain-agnostic, but the freshest block seen is not: on a beacon - /// follower, once the clock is past a fork the node does not follow (see - /// [`Self::clock_is_past_followed_forks`]) the network is treated as fresh. + /// `Store::beacon_head` decode different tables), and the network's head + /// from [`Store::network_head`], the head the node's peers report in + /// `Status`. The tracker itself is chain-agnostic, with per-chain + /// thresholds; see [`SyncThresholds`]. On a beacon follower, once the clock + /// is past a fork the node does not follow (see + /// [`Self::clock_is_past_followed_forks`]) the network is treated as being + /// at the clock, since no peer on that fork can tell it so. fn update_sync_status(&mut self, current_slot: u64) { let head_slot = self.head_slot(); - let live_max_slot = self - .store - .max_live_chain_slot() - .expect("max live chain slot exists") - .unwrap_or(head_slot); - let max_seen_slot = match self.store.chain() { + let network_head = match self.store.chain() { Chain::Beacon if Self::clock_is_past_followed_forks( &self.store.config(), @@ -3603,13 +3601,13 @@ impl BlockChainServer { head_slot, ) => { - current_slot + Some(current_slot) } - Chain::Beacon | Chain::Lean => live_max_slot, + Chain::Beacon | Chain::Lean => self.store.network_head(), }; let status = self .sync_status - .update(current_slot, head_slot, max_seen_slot); + .update(current_slot, head_slot, network_head); metrics::set_node_sync_status(status); self.sync_status_controller.set(status); } @@ -3620,16 +3618,17 @@ impl BlockChainServer { /// Every fork is followed today, so this is false; it stays as the guard /// for the next fork the node cannot follow. /// - /// Blocks past the fork do not reach the live chain from the network, so the freshest block - /// the store knows stays at the head and the sync tracker would read the - /// network as stalled `NETWORK_STALL_THRESHOLD` slots after the fork and - /// report `Synced` for good on a node that follows nothing. Nothing else - /// tells the node the network is ahead: gossip blocks past the fork are - /// ignored in p2p. The fork digest rolls over at runtime, so a node that - /// runs across a fork it can follow switches topics and Status digests on - /// its own; for a fork it cannot follow that switch brings nothing in. The - /// clock is the one signal that is always there, and treating the network - /// as fresh lets head lag decide. + /// Past such a fork the peers that moved on answer `Status` from a digest + /// this node cannot sync from, so none of their heads is recorded (see the + /// Status handlers in p2p), and the peers left behind report heads as + /// stale as ours. The sync tracker would read that silence as a stalled + /// network and report `Synced` for good on a node that follows nothing. + /// Nothing else tells the node the network is ahead: gossip blocks past the + /// fork are ignored in p2p. The fork digest rolls over at runtime, so a node + /// that runs across a fork it can follow switches topics and Status + /// digests on its own; for a fork it cannot follow that switch brings + /// nothing in. The clock is the one signal that is always there, and + /// treating the network as being at the clock lets head lag decide. fn clock_is_past_followed_forks(config: &Config, current_slot: u64, head_slot: u64) -> bool { let fork_at = |slot: u64| config.fork_at_epoch(compute_epoch_at_slot(slot)); !fork_at(current_slot).is_followed() && fork_at(head_slot).is_followed() @@ -4490,7 +4489,7 @@ mod tests { safe_slots_to_import_optimistically: constants::SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, last_tick_instant: None, epoch_precompute_in_flight: None, - sync_status: SyncStatusTracker::new(false), + sync_status: SyncStatusTracker::new(false, SyncThresholds::BEACON), sync_status_controller: SyncStatusController::default(), duties: ChainDuties::Beacon, } @@ -5277,15 +5276,18 @@ mod tests { beacon_server(beacon_store_with_config(GENESIS_TIME, 0, config)) } + /// Three epochs in: past the beacon tracker's two-epoch clock tolerance + /// with the head still at the anchor (slot 0). + const THREE_EPOCHS: u64 = 3 * preset::SLOTS_PER_EPOCH; + #[test] fn the_clock_reaching_a_followed_fork_does_not_force_syncing() { - // Gloas is followed, so blocks past the fork do reach the live chain - // and the tracker's own stall detection applies as before: with no - // block at all it reads a stalled network as synced, the same as when - // no fork is scheduled. + // Gloas is followed, so the fork rule stays out of it and the peers + // decide: every peer as stale as we are is a stalled network, read as + // synced, the same as when no fork is scheduled. let mut server = syncing_server(Some(1)); - let past_the_fork = preset::SLOTS_PER_EPOCH + 8; - server.update_sync_status(past_the_fork); + server.store.set_network_head(Some(0)); + server.update_sync_status(THREE_EPOCHS); assert_eq!( server.sync_status_controller.get(), crate::metrics::SyncStatus::Synced @@ -5297,13 +5299,29 @@ mod tests { // The same silence with no gloas scheduled is a stalled network, which // the tracker deliberately reports as synced. let mut server = syncing_server(None); - server.update_sync_status(preset::SLOTS_PER_EPOCH + 8); + server.store.set_network_head(Some(0)); + server.update_sync_status(THREE_EPOCHS); assert_eq!( server.sync_status_controller.get(), crate::metrics::SyncStatus::Synced ); } + #[test] + fn a_follower_behind_its_peers_reads_as_syncing() { + // The mainnet follower `docs/beacon_wire.md` describes: catching up, + // two epochs and more behind the clock, with peers at the clock. It + // used to read synced, since the freshest block it knew of was its own + // head. + let mut server = syncing_server(None); + server.store.set_network_head(Some(THREE_EPOCHS)); + server.update_sync_status(THREE_EPOCHS); + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Syncing + ); + } + /// Gloas at epoch 1, so slot `SLOTS_PER_EPOCH` is the first gloas slot. fn gloas_at_epoch_one() -> Config { Config::mainnet().with_fork_epoch(ForkName::Gloas, 1) diff --git a/crates/blockchain/src/sync_status.rs b/crates/blockchain/src/sync_status.rs index 27dff999b..52a04d5fd 100644 --- a/crates/blockchain/src/sync_status.rs +++ b/crates/blockchain/src/sync_status.rs @@ -1,7 +1,8 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU8, Ordering}; -use tracing::debug; +use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; +use tracing::info; use crate::metrics::SyncStatus; @@ -44,17 +45,70 @@ impl Default for SyncStatusController { } } -/// Local head lag beyond which the node is considered to be syncing. +/// How far behind a node may fall before it reports itself syncing, per chain. /// -/// See: leanSpec PR #708. -const SYNC_LAG_THRESHOLD: u64 = 4; -/// Freshest-known block lag beyond which the network is considered stalled. +/// The node is syncing only when **both** of these hold: /// -/// During a network-wide stall the node remains synced so validators can help -/// the chain recover. -const NETWORK_STALL_THRESHOLD: u64 = 8; -/// Recovery band that prevents the sync status from flapping near the threshold. -const SYNC_HYSTERESIS_BAND: u64 = 2; +/// - its head trails the wall clock by more than [`Self::clock_lag`], which is +/// local evidence no peer can forge, and +/// - the head its peers report (see `Store::network_head`) is more than +/// [`Self::peer_lead`] slots ahead of its own. +/// +/// This is Lighthouse's and Prysm's shape. Lighthouse serves duties unless a +/// range sync is running *and* the head is over `sync_tolerance_epochs` (2) +/// behind the clock, and only a peer more than 32 slots ahead starts a range +/// sync. Prysm resyncs only when the head is two epochs behind the clock *and* +/// peers report a head two epochs ahead. Peers decide between "the network is +/// ahead of me" and "the network is stalled"; the clock keeps a peer's claim +/// from idling a node whose own head is current. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SyncThresholds { + /// Slots the head may trail the wall clock before the node can be syncing. + pub(crate) clock_lag: u64, + /// Slots the peers' head must lead ours before the node can be syncing. + pub(crate) peer_lead: u64, + /// Recovery band: once syncing, both lags must drop this many slots below + /// their thresholds before the node reports synced again, so the status + /// does not flap at the boundary. + pub(crate) hysteresis: u64, + /// Slots after the first tick during which, with no peer having reported a + /// head yet, the node assumes the network is at the wall clock. + /// + /// The first tick runs before any peer has answered a `Status`, so without + /// this a node resuming a stale database would read "no peer is ahead" as a + /// stalled network and do its duties on that stale head until the first + /// handshake lands. Prysm holds a starting node in initial sync until it + /// has peers; this bounds that hold, so a node that never finds a peer (a + /// single-node devnet, or one cut off from the start) still falls back to + /// treating the silence as a stall. + pub(crate) startup_grace: u64, +} + +impl SyncThresholds { + /// Lean consensus: leanSpec PR #708's lag threshold and hysteresis band, + /// applied to both conditions, with the stall override replaced by the peer + /// lead (see `docs/spec_deviations.md`). Lean has no epochs, so neither + /// client's epoch-sized tolerance carries over. + pub(crate) const LEAN: Self = Self { + clock_lag: 4, + peer_lead: 4, + hysteresis: 2, + startup_grace: 8, + }; + + /// The Beacon Chain: Lighthouse's numbers. Two epochs of clock lag + /// (`DEFAULT_SYNC_TOLERANCE_EPOCHS`, Prysm's `shouldReSync` too) and a peer + /// lead of one epoch (`SLOT_IMPORT_TOLERANCE`, the lead that makes a + /// Lighthouse peer `Advanced`). Neither client has a hysteresis band. The + /// startup hold is an epoch, since finding beacon peers through discovery + /// can take a while; Prysm holds until it has peers, however long. + pub(crate) const BEACON: Self = Self { + clock_lag: 2 * SLOTS_PER_EPOCH, + peer_lead: SLOTS_PER_EPOCH, + hysteresis: 0, + startup_grace: SLOTS_PER_EPOCH, + }; +} pub(crate) struct SyncStatusTracker { syncing: bool, @@ -65,51 +119,68 @@ pub(crate) struct SyncStatusTracker { /// returns `true`: the gate is observe-only. Seeded from the CLI /// `--disable-duty-sync-gate` flag (gating stays on by default). gate_duties: bool, -} - -impl Default for SyncStatusTracker { - fn default() -> Self { - Self { - syncing: false, - gate_duties: true, - } - } + thresholds: SyncThresholds, + /// The slot at which the startup hold ends; set by the first update. + startup_ends_at: Option, + /// Whether any peer has reported a head yet. Once one has, losing every + /// peer reads as a stall, not as startup. + heard_from_peers: bool, } impl SyncStatusTracker { /// Build a tracker, choosing whether the syncing state gates duties. - pub(crate) fn new(gate_duties: bool) -> Self { + pub(crate) fn new(gate_duties: bool, thresholds: SyncThresholds) -> Self { Self { + syncing: false, gate_duties, - ..Self::default() + thresholds, + startup_ends_at: None, + heard_from_peers: false, } } + /// Re-judge the status at `current_slot` (the wall clock's slot). + /// + /// `network_head` is the head slot the node's peers report, or `None` when + /// no connected peer has reported one. With no report the node cannot be + /// behind anyone, so it reads as synced (a stalled or empty network, where + /// its validators should keep working), except during the startup hold. pub(crate) fn update( &mut self, current_slot: u64, head_slot: u64, - max_seen_slot: u64, + network_head: Option, ) -> SyncStatus { - let head_lag = current_slot.saturating_sub(head_slot); - let network_lag = current_slot.saturating_sub(max_seen_slot); + let SyncThresholds { + clock_lag: clock_threshold, + peer_lead: lead_threshold, + hysteresis, + startup_grace, + } = self.thresholds; + let startup_ends_at = *self + .startup_ends_at + .get_or_insert(current_slot.saturating_add(startup_grace)); + self.heard_from_peers |= network_head.is_some(); + let starting_up = !self.heard_from_peers && current_slot < startup_ends_at; + + // During the startup hold, assume the network is at the clock. + let network_head = network_head.or(starting_up.then_some(current_slot)); + let clock_lag = current_slot.saturating_sub(head_slot); + let peer_lead = network_head.map_or(0, |slot| slot.saturating_sub(head_slot)); + let band = if self.syncing { hysteresis } else { 0 }; let was_syncing = self.syncing; - if network_lag > NETWORK_STALL_THRESHOLD { - self.syncing = false; - } else if self.syncing { - self.syncing = head_lag > SYNC_LAG_THRESHOLD.saturating_sub(SYNC_HYSTERESIS_BAND); - } else { - self.syncing = head_lag > SYNC_LAG_THRESHOLD; - } + self.syncing = clock_lag > clock_threshold.saturating_sub(band) + && peer_lead > lead_threshold.saturating_sub(band); if self.syncing != was_syncing { - debug!( + info!( current_slot, head_slot, - max_seen_slot, - head_lag, - network_lag, + network_head, + clock_lag, + peer_lead, + starting_up, syncing = self.syncing, "Sync status changed" ); @@ -138,61 +209,146 @@ impl SyncStatusTracker { mod tests { use super::*; + const LEAN: SyncThresholds = SyncThresholds::LEAN; + + /// A lean tracker past its startup hold, with peers heard from, so each + /// test exercises the steady-state rule unless it says otherwise. + fn steady_lean() -> SyncStatusTracker { + let mut tracker = SyncStatusTracker::new(true, LEAN); + tracker.update(0, 0, Some(0)); + tracker + } + #[test] - fn sync_status_allows_lag_through_threshold() { - let mut tracker = SyncStatusTracker::default(); + fn behind_both_the_clock_and_the_peers_is_syncing() { + let mut tracker = steady_lean(); + // Issue #559: the head is 550 slots back and peers are at the clock. + assert_eq!(tracker.update(600, 50, Some(600)), SyncStatus::Syncing); + assert!(!tracker.duties_allowed()); + } - for lag in 0..=SYNC_LAG_THRESHOLD { - assert_eq!(tracker.update(10 + lag, 10, 10 + lag), SyncStatus::Synced); + #[test] + fn lag_up_to_the_threshold_is_synced() { + let mut tracker = steady_lean(); + for lag in 0..=LEAN.clock_lag { + let current = 10 + lag; + assert_eq!( + tracker.update(current, 10, Some(current)), + SyncStatus::Synced + ); } - - let first_syncing_slot = 10 + SYNC_LAG_THRESHOLD + 1; + let current = 10 + LEAN.clock_lag + 1; assert_eq!( - tracker.update(first_syncing_slot, 10, first_syncing_slot), + tracker.update(current, 10, Some(current)), SyncStatus::Syncing ); } #[test] - fn sync_status_detects_local_lag_when_fresh_blocks_are_known() { - let mut tracker = SyncStatusTracker::default(); - let current_slot = 10 + SYNC_LAG_THRESHOLD + 1; + fn a_stalled_network_keeps_duties_on_however_long_it_lasts() { + // Nobody is ahead of us: the network is not producing, so validators + // keep working to restart it, at any clock lag. leanSpec's + // `test_network_wide_stall_keeps_duties_live`. + let mut tracker = steady_lean(); + for current in [10, 50, 1_000] { + assert_eq!(tracker.update(current, 0, Some(0)), SyncStatus::Synced); + } + } - assert_eq!( - tracker.update(current_slot, 10, current_slot), - SyncStatus::Syncing - ); + #[test] + fn missed_proposals_do_not_close_the_gate() { + // leanSpec #1194: several empty slots in a row push the clock lag past + // the threshold on every honest node, but no peer is ahead of any of + // them, so none stops working. + let mut tracker = steady_lean(); + assert_eq!(tracker.update(16, 10, Some(10)), SyncStatus::Synced); + assert_eq!(tracker.update(16, 10, Some(11)), SyncStatus::Synced); + } + + #[test] + fn a_peer_ahead_cannot_idle_a_node_at_the_clock() { + // The clock condition is what bounds a lying peer: our own head is + // current, so no claimed lead turns duties off. + let mut tracker = steady_lean(); + assert_eq!(tracker.update(100, 98, Some(500)), SyncStatus::Synced); } #[test] - fn sync_status_treats_stale_known_blocks_as_network_stall() { - let mut tracker = SyncStatusTracker::default(); + fn losing_every_peer_reads_as_a_stall() { + let mut tracker = steady_lean(); + assert_eq!(tracker.update(100, 50, Some(100)), SyncStatus::Syncing); + // Cut off: no peer reports anything any more. + assert_eq!(tracker.update(120, 50, None), SyncStatus::Synced); + // Back in touch, still behind. + assert_eq!(tracker.update(130, 50, Some(130)), SyncStatus::Syncing); + } - assert_eq!(tracker.update(100, 0, 0), SyncStatus::Synced); + #[test] + fn startup_holds_a_stale_node_until_a_peer_reports() { + let mut tracker = SyncStatusTracker::new(true, LEAN); + // Resumed 300 slots behind, nobody heard from yet. + assert_eq!(tracker.update(300, 0, None), SyncStatus::Syncing); + // The first handshake says the network is with us after all. + assert_eq!(tracker.update(301, 0, Some(0)), SyncStatus::Synced); } #[test] - fn sync_status_hysteresis_prevents_flapping() { - let mut tracker = SyncStatusTracker::default(); + fn the_startup_hold_ends_without_peers() { + // A node that never finds a peer stops holding after the grace period + // and treats the silence as a stall, so a single node still works. + let mut tracker = SyncStatusTracker::new(true, LEAN); + assert_eq!(tracker.update(300, 0, None), SyncStatus::Syncing); + let after_grace = 300 + LEAN.startup_grace; + assert_eq!(tracker.update(after_grace, 0, None), SyncStatus::Synced); + } - assert_eq!(tracker.update(15, 10, 15), SyncStatus::Syncing); - assert_eq!(tracker.update(15, 11, 15), SyncStatus::Syncing); - assert_eq!(tracker.update(15, 10, 15), SyncStatus::Syncing); - assert_eq!(tracker.update(15, 13, 15), SyncStatus::Synced); + #[test] + fn a_current_node_is_synced_from_the_first_tick() { + let mut tracker = SyncStatusTracker::new(true, LEAN); + assert_eq!(tracker.update(300, 299, None), SyncStatus::Synced); } #[test] - fn network_stall_reopens_sync_status() { - let mut tracker = SyncStatusTracker::default(); + fn hysteresis_prevents_flapping() { + let mut tracker = steady_lean(); + assert_eq!(tracker.update(15, 10, Some(15)), SyncStatus::Syncing); + // Inside the band on both counts: still syncing. + assert_eq!(tracker.update(15, 12, Some(15)), SyncStatus::Syncing); + // Below the band on the clock: synced. + assert_eq!(tracker.update(15, 13, Some(15)), SyncStatus::Synced); + } - assert_eq!(tracker.update(20, 0, 20), SyncStatus::Syncing); - assert_eq!(tracker.update(30, 0, 20), SyncStatus::Synced); + #[test] + fn the_beacon_thresholds_are_lighthouses() { + let mut tracker = SyncStatusTracker::new(true, SyncThresholds::BEACON); + tracker.update(0, 0, Some(0)); + let epoch = SLOTS_PER_EPOCH; + // A checkpoint-synced follower 100 slots back with peers at the clock. + assert_eq!(tracker.update(1_000, 900, Some(1_000)), SyncStatus::Syncing); + // Within two epochs of the clock, Lighthouse serves duties. + assert_eq!( + tracker.update(1_000, 1_000 - 2 * epoch, Some(1_000)), + SyncStatus::Synced + ); + // Peers within an epoch of us are not `Advanced`. + assert_eq!( + tracker.update(1_000, 900, Some(900 + epoch)), + SyncStatus::Synced + ); } #[test] - fn future_head_saturates_lag_at_zero() { - let mut tracker = SyncStatusTracker::default(); + fn a_disabled_gate_still_tracks_but_never_blocks() { + let mut tracker = SyncStatusTracker::new(false, LEAN); + tracker.update(0, 0, Some(0)); + assert_eq!(tracker.update(600, 50, Some(600)), SyncStatus::Syncing); + assert!(tracker.is_syncing()); + assert!(tracker.duties_allowed()); + } - assert_eq!(tracker.update(15, 20, 20), SyncStatus::Synced); + #[test] + fn a_future_head_saturates_at_zero() { + let mut tracker = steady_lean(); + assert_eq!(tracker.update(15, 20, Some(20)), SyncStatus::Synced); } } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index cc9ca2bd1..7febf4b54 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -115,6 +115,7 @@ pub mod discovery; mod gossipsub; pub mod lean; pub mod metrics; +mod peer_heads; mod req_resp; pub(crate) mod swarm_adapter; @@ -1212,6 +1213,7 @@ impl P2P { .beacon() .map(|beacon| beacon::scoring::PeerScoring::new(&beacon.config, beacon.fork_digest)); let local_metadata = built.wire.beacon().map(|beacon| beacon.metadata.clone()); + let first_status_refresh = status_refresh_interval(&store); let server = P2PServer { swarm_handle, @@ -1226,6 +1228,7 @@ impl P2P { outbound_requests: HashMap::new(), range_sync_state: None, beacon_fetched_through, + peer_heads: Default::default(), bootnode_addrs: built.bootnode_addrs, node_names, discovery: discovery.map(|handle| DiscoveryState::new(handle, built.local_peer_id)), @@ -1283,6 +1286,13 @@ impl P2P { handle.context(), p2p_protocol::LeaveExpiredAggregatorSubnets, ); + // The handshake covers a new peer, so the first refresh waits a full + // interval. + send_after( + first_status_refresh, + handle.context(), + p2p_protocol::RefreshPeerStatus, + ); // The dial loop's first tick. Nothing else schedules one, so without // discovery the loop never runs. if discovery_enabled { @@ -1384,6 +1394,9 @@ pub struct P2PServer { /// whether or not it has been imported yet. Gossip never advances it, so /// range sync reads it beside the store's head; see `beacon_sync_target`. pub(crate) beacon_fetched_through: u64, + /// Every connected peer's latest `Status` head, aggregated onto + /// `Store::network_head` for the sync tracker. See [`peer_heads`]. + pub(crate) peer_heads: peer_heads::PeerHeads, bootnode_addrs: HashMap>, node_names: HashMap, @@ -1561,6 +1574,9 @@ pub(crate) trait P2PProtocol: Send + Sync { fn advance_fork_schedule(&self) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly fn refresh_peer_scoring(&self) -> Result<(), ActorError>; + /// Re-ask every connected peer for its `Status`; see [`peer_heads`]. + #[allow(dead_code)] // invoked via send_after, not called directly + fn refresh_peer_status(&self) -> Result<(), ActorError>; } #[actor(protocol = P2PProtocol)] @@ -1734,6 +1750,27 @@ impl P2PServer { beacon::scoring::refresh(self); } + /// Re-ask every connected peer for its `Status`, so the head each one + /// reported at the handshake does not go stale while it stays connected. + /// Both chains; see [`peer_heads`] for the pace. + #[send_handler] + async fn handle_refresh_peer_status( + &mut self, + _msg: p2p_protocol::RefreshPeerStatus, + ctx: &Context, + ) { + send_after( + status_refresh_interval(&self.store), + ctx.clone(), + p2p_protocol::RefreshPeerStatus, + ); + let peers: Vec = self.connected_peers.keys().copied().collect(); + for peer in peers { + self.peer_heads.begin_refresh(peer); + request_status(self, peer).await; + } + } + /// The deadline of a range batch held back for custody. Scheduled once, /// when the batch starts waiting; see [`RANGE_BATCH_CUSTODY_WAIT`]. #[send_handler] @@ -1940,6 +1977,40 @@ async fn fetch_missing_columns(server: &mut P2PServer, block_root: H256, columns // --- Manual Handler for swarm events --- +/// How often to re-ask every connected peer for its `Status`; see +/// [`peer_heads`] for the pace on each chain. +fn status_refresh_interval(store: &Store) -> Duration { + let slots = match store.chain() { + Chain::Lean => peer_heads::STATUS_REFRESH_SLOTS_LEAN, + Chain::Beacon => peer_heads::STATUS_REFRESH_SLOTS_BEACON, + }; + Duration::from_millis(store.config().slot_duration_ms.saturating_mul(slots)) +} + +/// Send `peer` our `Status`, asking for its own: the periodic refresh. The +/// handshake in `ConnectionEstablished` sends the same request, with its own +/// logging around it. +async fn request_status(server: &P2PServer, peer: PeerId) { + // Built before the send, so no borrow of `server.wire` is alive across it. + let beacon_status = server.wire.beacon().map(|wire| { + beacon::handler::build_status(&server.store, wire, beacon::handler::StatusVersion::V1) + }); + match beacon_status { + Some(status) => beacon::handler::send_status(server, peer, status).await, + None => { + let our_status = build_status(&server.store); + server + .swarm_handle + .send_request( + peer, + Request::LeanStatus(our_status), + ReqRespProtocol::LeanStatus, + ) + .await; + } + } +} + impl Handler for P2PServer { async fn handle(&mut self, msg: WrappedSwarmEvent, ctx: &Context) { handle_swarm_event(self, msg.0, ctx).await; @@ -2072,6 +2143,7 @@ async fn handle_swarm_event( .remove(&peer_id) .unwrap_or(closed_direction); forget_discovered_peer(server, &peer_id); + peer_heads::forget(server, &peer_id); server.refresh_peer_metrics(); let peer_count = server.connected_peers.len(); metrics::notify_peer_disconnected( @@ -3042,6 +3114,7 @@ pub(crate) mod test_support { outbound_requests: HashMap::new(), range_sync_state: None, beacon_fetched_through: 0, + peer_heads: Default::default(), bootnode_addrs: HashMap::new(), node_names: HashMap::new(), discovery: Some(crate::discovery::dial::DiscoveryState::new( diff --git a/crates/net/p2p/src/peer_heads.rs b/crates/net/p2p/src/peer_heads.rs new file mode 100644 index 000000000..4700d6588 --- /dev/null +++ b/crates/net/p2p/src/peer_heads.rs @@ -0,0 +1,216 @@ +//! The head slot this node's peers report in `Status`, published on the +//! [`Store`](ethlambda_storage::Store) as the sync tracker's evidence that the +//! network is ahead of this node. +//! +//! Lighthouse and Prysm both decide "I am behind" against what their peers +//! report rather than against blocks they have imported: a node whose import +//! has stalled, or that is still catching up, has imported nothing recent, so +//! its own blocks cannot tell it the network moved on. A peer's head is a +//! claim, not a verified block, so it is bounded three ways: a head past the +//! wall clock is dropped (both clients refuse such a `Status` at the +//! handshake), several peers must agree before the value moves, and the sync +//! tracker also requires the node's own head to trail the clock, so no claim +//! can stop a node that is current. +//! +//! Peers send `Status` once per connection, so a claim would go stale while +//! the peer stays connected. [`STATUS_REFRESH_SLOTS_LEAN`] and +//! [`STATUS_REFRESH_SLOTS_BEACON`] pace a re-request, the way Lighthouse +//! re-statuses every 300 s and Prysm every half epoch. + +use std::collections::{HashMap, HashSet}; + +use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; +use ethlambda_types::time::unix_now_ms; +use libp2p::PeerId; +use tracing::debug; + +use crate::P2PServer; + +/// From this many reporting peers on, the network head is the second-highest +/// report rather than the highest, so one peer alone cannot move it. Below +/// it the highest report counts: with one or two peers, a stale or still +/// syncing peer must not veto a fresher one, and the tracker's clock condition +/// already keeps a lone liar from idling a node that is current. +pub(crate) const PEER_HEAD_QUORUM: usize = 3; + +/// Slots a `Status` head may sit past the wall clock and still count: Lighthouse's +/// `FUTURE_SLOT_TOLERANCE` and Prysm's `maxFutureStatusHeadSlot`. +pub(crate) const FUTURE_HEAD_TOLERANCE: u64 = 1; + +/// How often a lean node re-asks every connected peer for its `Status`: every +/// lead the lean sync tracker tolerates (`SyncThresholds::LEAN.peer_lead`), so +/// a claim is never staler than the gap it is meant to detect. +pub(crate) const STATUS_REFRESH_SLOTS_LEAN: u64 = 4; + +/// How often a beacon node re-asks every connected peer for its `Status`: +/// Prysm's half epoch (`maintainPeerStatuses`). +pub(crate) const STATUS_REFRESH_SLOTS_BEACON: u64 = SLOTS_PER_EPOCH / 2; + +/// How far ahead a peer answering a *refresh* must be before its answer starts +/// a range sync, per chain. A handshake answer starts one at any lead, as it +/// always has; a refresh lands every few slots on a node that is keeping up +/// with gossip, where a peer one slot ahead is ordinary and a one-block range +/// request would only race gossip. Lighthouse makes a peer `Advanced`, the one +/// kind its range sync follows, only beyond `SLOT_IMPORT_TOLERANCE` (32). +pub(crate) const REFRESH_RANGE_SYNC_LEAD_LEAN: u64 = 4; +/// The beacon counterpart of [`REFRESH_RANGE_SYNC_LEAD_LEAN`]. +pub(crate) const REFRESH_RANGE_SYNC_LEAD_BEACON: u64 = SLOTS_PER_EPOCH; + +/// Every connected peer's latest `Status` head. +#[derive(Debug, Default)] +pub(crate) struct PeerHeads { + /// The head slot each peer last reported, while it stays connected. + heads: HashMap, + /// Peers whose outstanding `Status` request is a refresh rather than the + /// handshake; see [`REFRESH_RANGE_SYNC_LEAD_LEAN`]. + refreshing: HashSet, +} + +impl PeerHeads { + /// Record `peer`'s head, unless it sits past `wall_slot` by more than + /// [`FUTURE_HEAD_TOLERANCE`]; a peer whose head is dropped loses any + /// earlier report too, since a peer that claims the future is not one to + /// believe about the present. Returns whether the head was recorded. + pub(crate) fn record(&mut self, peer: PeerId, head_slot: u64, wall_slot: u64) -> bool { + if head_slot > wall_slot.saturating_add(FUTURE_HEAD_TOLERANCE) { + self.heads.remove(&peer); + return false; + } + self.heads.insert(peer, head_slot); + true + } + + /// Forget a disconnected peer. + pub(crate) fn forget(&mut self, peer: &PeerId) { + self.heads.remove(peer); + self.refreshing.remove(peer); + } + + /// The network's head as the peers report it: the second-highest report + /// once [`PEER_HEAD_QUORUM`] peers have reported, the highest before that, + /// `None` with no report at all. + pub(crate) fn network_head(&self) -> Option { + let mut heads: Vec = self.heads.values().copied().collect(); + heads.sort_unstable_by(|a, b| b.cmp(a)); + let rank = if heads.len() >= PEER_HEAD_QUORUM { + 1 + } else { + 0 + }; + heads.get(rank).copied() + } + + /// Note that the `Status` request about to go to `peer` is a refresh. + pub(crate) fn begin_refresh(&mut self, peer: PeerId) { + self.refreshing.insert(peer); + } + + /// Whether `peer`'s `Status` answer is to a refresh, clearing the mark. + pub(crate) fn take_refresh(&mut self, peer: &PeerId) -> bool { + self.refreshing.remove(peer) + } +} + +/// The slot the wall clock is in, on the store's time grid. +fn wall_clock_slot(server: &P2PServer) -> u64 { + let grid = server.store.config().time_grid(); + unix_now_ms().saturating_sub(grid.genesis_time_ms()) / grid.milliseconds_per_slot +} + +/// Record the head `peer` reported in a `Status` (request or answer) and +/// republish the network head. +pub(crate) fn observe(server: &mut P2PServer, peer: PeerId, head_slot: u64) { + let wall_slot = wall_clock_slot(server); + if !server.peer_heads.record(peer, head_slot, wall_slot) { + debug!( + %peer, + peer_head_slot = head_slot, + wall_slot, + "Ignoring a Status head past the wall clock" + ); + } + publish(server); +} + +/// Forget a disconnected peer's head and republish the network head. +pub(crate) fn forget(server: &mut P2PServer, peer: &PeerId) { + server.peer_heads.forget(peer); + publish(server); +} + +fn publish(server: &P2PServer) { + server + .store + .set_network_head(server.peer_heads.network_head()); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_report_is_no_network_head() { + assert_eq!(PeerHeads::default().network_head(), None); + } + + #[test] + fn below_quorum_the_highest_report_counts() { + // Two peers, one of them stale: the fresher one is not vetoed. + let mut heads = PeerHeads::default(); + heads.record(PeerId::random(), 100, 100); + heads.record(PeerId::random(), 40, 100); + assert_eq!(heads.network_head(), Some(100)); + } + + #[test] + fn at_quorum_one_peer_cannot_move_the_head() { + let mut heads = PeerHeads::default(); + heads.record(PeerId::random(), 100, 100); + heads.record(PeerId::random(), 50, 100); + heads.record(PeerId::random(), 50, 100); + assert_eq!(heads.network_head(), Some(50)); + // A second peer at the clock is enough. + heads.record(PeerId::random(), 100, 100); + assert_eq!(heads.network_head(), Some(100)); + } + + #[test] + fn a_head_past_the_clock_is_dropped_with_the_peers_earlier_report() { + let mut heads = PeerHeads::default(); + let liar = PeerId::random(); + assert!(heads.record(liar, 90, 100)); + assert!(heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE, 100)); + assert!(!heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE + 1, 100)); + assert_eq!(heads.network_head(), None); + } + + #[test] + fn a_later_report_replaces_an_earlier_one() { + // Latest, not highest: a peer that reorged to a lower head is believed. + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + heads.record(peer, 100, 100); + heads.record(peer, 95, 100); + assert_eq!(heads.network_head(), Some(95)); + } + + #[test] + fn a_disconnected_peer_is_forgotten() { + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + heads.record(peer, 100, 100); + heads.begin_refresh(peer); + heads.forget(&peer); + assert_eq!(heads.network_head(), None); + assert!(!heads.take_refresh(&peer)); + } + + #[test] + fn a_refresh_mark_is_taken_once() { + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + heads.begin_refresh(peer); + assert!(heads.take_refresh(&peer)); + assert!(!heads.take_refresh(&peer)); + } +} diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 5801a84f2..57556dd67 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -53,6 +53,7 @@ use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; use crate::lean::messages::{BlocksByRootRequest, RequestedBlockRoots, Status}; use crate::lean::protocols::MAX_REQUEST_BLOCKS; +use crate::peer_heads; use crate::req_resp::messages::BlocksByRangeRequest; use crate::{ BACKOFF_MULTIPLIER, CustodyWait, INITIAL_BACKOFF_MS, MAX_FETCH_RETRIES, MAX_SYNC_RANGE, @@ -598,17 +599,25 @@ async fn handle_lean_status_request( peer: PeerId, ) { trace!(finalized_slot=%request.finalized.slot, head_slot=%request.head.slot, "Received status request from peer {peer}"); + peer_heads::observe(server, peer, request.head.slot); let our_status = build_status(&server.store); respond(server, channel, ResponsePayload::LeanStatus(our_status)); } async fn handle_lean_status_response(server: &mut P2PServer, status: Status, peer: PeerId) { trace!(finalized_slot=%status.finalized.slot, head_slot=%status.head.slot, "Received status response from peer {peer}"); + let refresh = server.peer_heads.take_refresh(&peer); + peer_heads::observe(server, peer, status.head.slot); let our_head_slot = server.store.head_slot(); if status.head.slot <= our_head_slot { return; } + // A refresh answer only updates the peer's head unless it shows this node + // well behind; see `REFRESH_RANGE_SYNC_LEAD_LEAN`. + if refresh && status.head.slot - our_head_slot <= peer_heads::REFRESH_RANGE_SYNC_LEAD_LEAN { + return; + } let gap = status.head.slot - our_head_slot; debug!( %peer, @@ -1730,7 +1739,14 @@ async fn handle_status_request( peer_finalized_epoch = peer_status.finalized_epoch(), "Beacon status received" ); + // Recorded only from a digest this node syncs from, as an answer is + // below: a peer still on a superseded digest is not on the chain being + // followed. + if transition::is_syncable_digest(wire, peer_status.fork_digest(), unix_now_ms()) { + peer_heads::observe(server, peer, peer_status.head_slot()); + } } + let wire = server.wire.beacon().expect("checked above"); let our_status = beacon_handler::build_status(&server.store, wire, StatusVersion::of(&peer_status)); respond(server, channel, ResponsePayload::Status(our_status)); @@ -1836,6 +1852,7 @@ async fn handle_status_response( status: BeaconStatus, ctx: &Context, ) { + let refresh = server.peer_heads.take_refresh(&peer); let Some(wire) = server.wire.beacon() else { return; }; @@ -1866,8 +1883,10 @@ async fn handle_status_response( %peer, peer_head_slot, peer_finalized_epoch = status.finalized_epoch(), + refresh, "Beacon handshake complete" ); + peer_heads::observe(server, peer, peer_head_slot); // A whole-block decode per handshake, the same one `build_status` already // pays to send ours. @@ -1877,6 +1896,13 @@ async fn handle_status_response( else { return; }; + // A refresh answer only updates the peer's head unless it shows this node + // an epoch behind; see `REFRESH_RANGE_SYNC_LEAD_BEACON`. + if refresh + && target_range.end - target_range.start <= peer_heads::REFRESH_RANGE_SYNC_LEAD_BEACON + { + return; + } debug!( %peer, @@ -2780,6 +2806,7 @@ pub(crate) mod tests { outbound_requests: HashMap::new(), range_sync_state: None, beacon_fetched_through: 0, + peer_heads: Default::default(), bootnode_addrs: HashMap::new(), node_names: HashMap::new(), discovery: Some(crate::discovery::dial::DiscoveryState::new( diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 14ac9116f..9cdb313c0 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::num::NonZeroUsize; use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; use std::time::Instant; @@ -253,6 +254,9 @@ impl Chain { /// snapshot read or a diff-chain reconstruction. const STATE_CACHE_CAPACITY: usize = 32; +/// [`Store::network_head`]'s "no peer has reported" marker. No slot reaches it. +const NO_NETWORK_HEAD: u64 = u64::MAX; + /// Keep block proofs for at least this many slots below the tip, even once /// finalized. Proofs older than this window are pruned only when the window /// lies entirely within finalized history; see [`Store::prune_old_block_proofs`]. @@ -1196,6 +1200,11 @@ pub struct Store { /// node's own API), which is why it lives on the `Store` all three share. /// See [`ObservedLiveness`]. Always empty on lean. observed_liveness: Arc, + /// The head slot this node's peers report, as P2P last aggregated it from + /// their `Status` messages, or [`NO_NETWORK_HEAD`] while no connected peer + /// has reported one. Written by P2P, read by the chain actor's sync + /// tracker; see [`Store::network_head`]. + network_head: Arc, /// The total active balance per epoch, shared the way /// [`Self::committee_cache`] is: by the chain actor's state transition /// and anything else holding a `Store` clone. See @@ -1846,6 +1855,7 @@ impl Store { pending_states, committee_cache: Arc::new(CommitteeCache::default()), observed_liveness: Arc::new(ObservedLiveness::default()), + network_head: Arc::new(AtomicU64::new(NO_NETWORK_HEAD)), active_balance_cache: Arc::new(ActiveBalanceCache::default()), beacon: Default::default(), state_writer, @@ -3294,6 +3304,27 @@ impl Store { &self.observed_liveness } + /// The head slot this node's peers report, or `None` while no connected + /// peer has reported one. + /// + /// The sync tracker's evidence that the network is ahead of this node. It + /// is a peer *claim*, not a verified block: P2P drops a head past the wall + /// clock and asks several peers to agree before the value moves, and the + /// tracker additionally requires the node's own head to trail the clock, + /// so no claim alone can stop a node that is current. Runtime-only and + /// shared by every clone of this `Store`. + pub fn network_head(&self) -> Option { + let slot = self.network_head.load(Ordering::Relaxed); + (slot != NO_NETWORK_HEAD).then_some(slot) + } + + /// Publish the peers' head; see [`Self::network_head`]. P2P is the only + /// writer. + pub fn set_network_head(&self, slot: Option) { + let slot = slot.unwrap_or(NO_NETWORK_HEAD); + self.network_head.store(slot, Ordering::Relaxed); + } + /// The total-active-balance cache shared by every clone of this store. /// /// An owned `Arc` for the same reason as [`Self::committee_cache`]. @@ -7192,6 +7223,17 @@ mod tests { assert!(store.cached_state(key).is_some()); } + #[test] + fn the_network_head_is_shared_across_store_clones() { + let store = Store::test_store(); + let clone = store.clone(); + assert_eq!(store.network_head(), None); + clone.set_network_head(Some(42)); + assert_eq!(store.network_head(), Some(42)); + clone.set_network_head(None); + assert_eq!(store.network_head(), None); + } + #[test] fn observed_liveness_is_shared_across_store_clones() { let store = beacon_test_store(Arc::new(InMemoryBackend::new())); diff --git a/docs/architecture.md b/docs/architecture.md index 9a4df60e0..14d0c27f8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -166,12 +166,16 @@ flag unset on a placement that does not cover every subnet. ### Sync gate -`sync_status.rs` tracks how far the local head lags the slot clock. Past the threshold the -node reports itself syncing and stops attesting and proposing, since a head derived from a -partial view is not worth voting for. A hysteresis band stops the state from flapping at the -boundary, and a network-wide stall (nobody else is ahead either) leaves the node synced so -its validators can help the chain recover. The same status feeds the `lean_node_sync_status` -metric and, through a shared controller, the `/lean/v0/node/syncing` endpoint. +`sync_status.rs` decides whether the node is behind. It is syncing, and stops attesting +and proposing, when its head trails the slot clock **and** the head its peers report is +ahead of its own, since a head derived from a partial view is not worth voting for. The +peers' head is each connected peer's latest `Status`, re-requested every few slots, which +p2p aggregates onto `Store::network_head` (`crates/net/p2p/src/peer_heads.rs`). A +hysteresis band stops the state from flapping at the boundary, and a network-wide stall +(nobody else is ahead either) leaves the node synced so its validators can help the chain +recover. This departs from leanSpec's rule; see `docs/spec_deviations.md`. The same status +feeds the `lean_node_sync_status` metric and, through a shared controller, the +`/lean/v0/node/syncing` endpoint. `--disable-duty-sync-gate` reduces the gate to observe-only. ### Missing parents diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index c2138785b..14b44ca72 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -295,10 +295,12 @@ Three differences from lighthouse: `MESH_DELIVERY_MAX_HEAD_LAG` slots, and back on only after an epoch within that lag, so the counters have refilled. While P3 is off its threshold and weight are zero but its cap is not, so the counters keep counting to their - real ceiling. `SyncStatus` cannot serve as this gate: on the mainnet - follower it reported `synced` through a 10-minute catch-up up to 98 slots - behind, since its network-stall rule reads a lagging freshest-known block - as a stalled network. + real ceiling. `SyncStatus` is not this gate: it reports syncing only past + two epochs of lag, and only once peers report a head ahead, where P3 must + be off from `MESH_DELIVERY_MAX_HEAD_LAG` slots. (It also used to read a + lagging freshest-known block as a stalled network, and reported `synced` + through a 10-minute catch-up up to 98 slots behind on the mainnet + follower.) - **Leaving a topic waits out the peers' backoff.** Gossipsub's default backoff after unsubscribing (10 s) is shorter than its prune backoff (60 s), and a rust-libp2p peer that sees a mesh peer unsubscribe backs off diff --git a/docs/rpc.md b/docs/rpc.md index bdcc260ec..74b8cb83f 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -190,7 +190,7 @@ The fork-choice tree from the finalized root, with LMD-GHOST weights computed ov { "is_syncing": false, "head_slot": 1024, "sync_distance": 1, "finalized_slot": 986 } ``` -`is_syncing` is the node's own stateful sync decision: head-vs-wall-clock lag with hysteresis and a network-stall override, updated each tick. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. +`is_syncing` is the node's own stateful sync decision, updated each tick: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. `sync_distance` is the raw number of slots between the node's current head and the current wall-clock slot, computed per request. Because `is_syncing` carries hysteresis and stall handling and is not recomputed from `sync_distance`, the two can point different ways near the threshold or during a network-wide stall. diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 49ee15c03..6bce1a744 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -28,6 +28,45 @@ interval, and stops early once it runs out of time. - **leanSpec:** `aggregate()` is called inline and synchronously from `tick_interval`, at interval 2 only. It walks every attestation data with fresh evidence, with no job cap, no time budget, no worker, and no cancellation. - **Equivalence:** on cancellation the worker emits only the groups that finished, so a slot may pack fewer aggregates than the synchronous path would; any such subset still yields a valid block, affecting how many votes are included rather than signature validity. The job cap has the same character: it bounds prover work per slot, not what a block may carry. +## The duty sync gate compares the head with peers' heads, not with imported blocks + +A node reports itself syncing, and stops proposing and attesting, only when its +head trails the wall clock **and** its peers report a head ahead of it. + +- **leanSpec:** `ValidatorService._is_synced_for_duties` (PR #708) takes the + freshest block it knows from imported blocks only. When that block is more + than 8 slots behind the wall clock it reads the network as stalled and allows + duties; otherwise the node is syncing once its head is more than 4 slots + behind the clock, with a hysteresis band of 2. + `test_no_extra_blocks_treats_isolation_as_network_stall` pins the case of a + node 100 slots behind with nothing newer imported, which does its duties. +- **ethlambda:** the node is syncing when its head is more than 4 slots behind + the wall clock and the head its peers report is more than 4 slots ahead of + its own, with the same band of 2 on both (`SyncThresholds::LEAN`, + `crates/blockchain/src/sync_status.rs`). The peers' head comes from `Status` + (`crates/net/p2p/src/peer_heads.rs`): each connected peer's latest report, + re-requested every 4 slots, a head past the wall clock by more than a slot + dropped, and the second-highest report counted once 3 or more peers have + reported (the highest below that). With no report at all the node cannot be + behind anyone and reads as synced, except during an 8-slot hold at startup + for a node whose head already trails the clock. +- **Why:** imported blocks cannot tell "the network is stalled" from "this node + is behind". A node catching up has imported nothing recent either, so the + specification's rule reports it synced for the whole catch-up and it proposes + and attests on a stale head (lambdaclass/ethlambda#559). Lighthouse and Prysm + decide against what their peers report, and require the head to trail the + clock too, so no peer's claim can idle a node that is current. Comparing with + peers also keeps a run of missed proposals (leanSpec #1194) from closing the + gate on every honest node, since none of them is behind another. +- **Equivalence:** the two agree whenever no peer is further ahead of the node + than its newest imported block. They differ for a node behind its peers with + nothing newer imported, which the specification lets do its duties and this + node does not. + +The beacon node runs the same rule with Lighthouse's numbers (two epochs of +clock lag, one epoch of peer lead, a refresh every half epoch). consensus-specs +has no sync gate, and the Beacon API leaves `is_syncing` to the client. + ## Attestation scoring on block building Attestations are scored and selected when packing a block, rather than taken in From 5e0a9d19ac24632b7a6e5162e37df5b80b5d4653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:33:04 -0300 Subject: [PATCH 2/6] fix(sync): judge the gate on every block arrival, not only on the tick The beacon tick is a message in the same mailbox range sync forwards a catch-up into, and each import holds the actor for an engine round trip, so on a catch-up the tick waited behind every block of it. The gate was judged only there, so it kept answering `Synced` for the whole catch-up: on a kurtosis devnet a node reconnecting 100 slots behind under a slow EL read synced for 53 s and its validator client proposed on a stale head. Judge it in the head recompute every beacon arrival, tick and envelope release ends in, ahead of the forkchoiceUpdated call, and after each lean arrival, whose tick waits the same way. --- crates/blockchain/src/lib.rs | 44 ++++++++++++++++++++++++++++++++++++ docs/architecture.md | 5 +++- docs/rpc.md | 2 +- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f35555eab..9eacc8963 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -2088,8 +2088,15 @@ impl BlockChainServer { // the store clock, which only `on_tick` and an early arrival // advance. metrics::update_current_slot(self.wall_clock_slot()); + // Judges the sync gate too. let head = self.recompute_beacon_head().await; cascade.absorb_head(head); + } else { + // Lean duties read the gate in the tick that judges it, but the + // tick waits behind a catch-up's blocks the same way beacon's + // does, and the Lean API, the status gauge and the next arrival's + // reaggregation check read it in between. + self.update_sync_status(self.wall_clock_slot()); } cascade.observe(blocks); @@ -2306,8 +2313,16 @@ impl BlockChainServer { /// `forkchoiceUpdated` verdict's) happens inside it. After the /// `forkchoiceUpdated` round trip, so `execution_optimistic` reflects the /// verdict it carried. + /// + /// And where the sync gate is judged between ticks, for the same reason + /// the head is: on a catch-up the tick waits behind every block range sync + /// forwarded, so a gate judged only there kept answering `Synced` to the + /// validator client for the whole catch-up, however far behind. Judged + /// before the `forkchoiceUpdated` round trip, so a slow execution client + /// does not delay it by one more engine call. async fn recompute_beacon_head(&mut self) -> HeadTimings { let mut timings = self.update_head_from_fork_choice(); + self.update_sync_status(self.wall_clock_slot()); if let Some((start, end)) = self.notify_forkchoice_updated().await { timings.fcu_start = Some(start); timings.fcu_end = Some(end); @@ -5322,6 +5337,35 @@ mod tests { ); } + #[tokio::test] + async fn a_block_arrival_judges_the_gate_without_waiting_for_the_tick() { + // A node reconnecting to peers far ahead of it: range sync forwards + // the whole catch-up at once, and the tick waits in the mailbox behind + // every block of it. The gate has to move on the arrivals themselves, + // or it answers `Synced` for the whole catch-up. A block whose parent + // is unknown is enough: whatever becomes of a block, its arrival ends + // in the head recompute. + let mut server = syncing_server(None); + let orphan = || bare_block(1, H256::repeat_byte(0xee)); + + // Peers as stale as the node: a stall, synced. + server.store.set_network_head(Some(0)); + server.on_block(orphan(), ImportTimings::default()).await; + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Synced + ); + + // A peer reports a head at the clock; the next arrival closes the gate. + let wall_clock_slot = server.wall_clock_slot(); + server.store.set_network_head(Some(wall_clock_slot)); + server.on_block(orphan(), ImportTimings::default()).await; + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Syncing + ); + } + /// Gloas at epoch 1, so slot `SLOTS_PER_EPOCH` is the first gloas slot. fn gloas_at_epoch_one() -> Config { Config::mainnet().with_fork_epoch(ForkName::Gloas, 1) diff --git a/docs/architecture.md b/docs/architecture.md index 14d0c27f8..9178ea47b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -173,7 +173,10 @@ peers' head is each connected peer's latest `Status`, re-requested every few slo p2p aggregates onto `Store::network_head` (`crates/net/p2p/src/peer_heads.rs`). A hysteresis band stops the state from flapping at the boundary, and a network-wide stall (nobody else is ahead either) leaves the node synced so its validators can help the chain -recover. This departs from leanSpec's rule; see `docs/spec_deviations.md`. The same status +recover. This departs from leanSpec's rule; see `docs/spec_deviations.md`. The gate is +judged on every tick and again after every block arrival: on a catch-up, range sync puts +the whole span into the actor's mailbox at once and the tick waits behind all of it, so a +gate judged only on the tick kept reading synced until the catch-up was over. The same status feeds the `lean_node_sync_status` metric and, through a shared controller, the `/lean/v0/node/syncing` endpoint. `--disable-duty-sync-gate` reduces the gate to observe-only. diff --git a/docs/rpc.md b/docs/rpc.md index 74b8cb83f..4fa487709 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -190,7 +190,7 @@ The fork-choice tree from the finalized root, with LMD-GHOST weights computed ov { "is_syncing": false, "head_slot": 1024, "sync_distance": 1, "finalized_slot": 986 } ``` -`is_syncing` is the node's own stateful sync decision, updated each tick: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. +`is_syncing` is the node's own stateful sync decision, updated each tick and after each block arrival: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. `sync_distance` is the raw number of slots between the node's current head and the current wall-clock slot, computed per request. Because `is_syncing` carries hysteresis and stall handling and is not recomputed from `sync_distance`, the two can point different ways near the threshold or during a network-wide stall. From d907b600723bb36b067f7b08228fc5faedfc668f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:27:50 -0300 Subject: [PATCH 3/6] docs(sync): the half-epoch Status refresh is Prysm's cadence, not Lighthouse's --- docs/spec_deviations.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 6bce1a744..51976e44f 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -64,7 +64,8 @@ head trails the wall clock **and** its peers report a head ahead of it. node does not. The beacon node runs the same rule with Lighthouse's numbers (two epochs of -clock lag, one epoch of peer lead, a refresh every half epoch). consensus-specs +clock lag, one epoch of peer lead) and Prysm's `Status` refresh, every half +epoch (Lighthouse refreshes every 300 s). consensus-specs has no sync gate, and the Beacon API leaves `is_syncing` to the client. ## Attestation scoring on block building From 6f08c9820dc5c3f6c3dddc9689a6174f330c4a82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:02:57 -0300 Subject: [PATCH 4/6] perf(p2p): spread Status refreshes over the interval instead of one burst The refresh timer re-asked every connected peer for its Status in the same instant: 200 peers on a beacon node with the default target. The bytes are small (about 0.5 KB per round trip), but 200 streams opened together, and 200 answers handled together, are a needless burst. Each peer is now due one interval after its own last Status exchange in either direction, and the timer ticks every slot and asks at most ceil(peers / interval) of the due peers, longest-waiting first, the way Lighthouse and Prysm track a status time per peer. 200 beacon peers that connected together are re-asked at most 13 per slot and stay spread afterwards. --- crates/net/p2p/src/lib.rs | 45 +++++++++----- crates/net/p2p/src/peer_heads.rs | 100 +++++++++++++++++++++++++++++-- docs/architecture.md | 3 +- 3 files changed, 128 insertions(+), 20 deletions(-) diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 7febf4b54..5a21333d1 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -1213,7 +1213,7 @@ impl P2P { .beacon() .map(|beacon| beacon::scoring::PeerScoring::new(&beacon.config, beacon.fork_digest)); let local_metadata = built.wire.beacon().map(|beacon| beacon.metadata.clone()); - let first_status_refresh = status_refresh_interval(&store); + let first_status_refresh = status_refresh_tick(&store); let server = P2PServer { swarm_handle, @@ -1286,8 +1286,8 @@ impl P2P { handle.context(), p2p_protocol::LeaveExpiredAggregatorSubnets, ); - // The handshake covers a new peer, so the first refresh waits a full - // interval. + // The refresh timer's first tick. A peer is first due an interval + // after its handshake, which `peer_heads` tracks per peer. send_after( first_status_refresh, handle.context(), @@ -1574,7 +1574,8 @@ pub(crate) trait P2PProtocol: Send + Sync { fn advance_fork_schedule(&self) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly fn refresh_peer_scoring(&self) -> Result<(), ActorError>; - /// Re-ask every connected peer for its `Status`; see [`peer_heads`]. + /// Re-ask the connected peers that are due for their `Status`; see + /// [`peer_heads`]. #[allow(dead_code)] // invoked via send_after, not called directly fn refresh_peer_status(&self) -> Result<(), ActorError>; } @@ -1750,9 +1751,12 @@ impl P2PServer { beacon::scoring::refresh(self); } - /// Re-ask every connected peer for its `Status`, so the head each one - /// reported at the handshake does not go stale while it stays connected. - /// Both chains; see [`peer_heads`] for the pace. + /// Re-ask the connected peers whose `Status` is an interval old, so the + /// head each one reported at the handshake does not go stale while it + /// stays connected. Ticks every slot and asks only that slot's share of + /// the peers, so the requests are spread over the interval rather than + /// sent to every peer at once. Both chains; see [`peer_heads`] for the + /// pace. #[send_handler] async fn handle_refresh_peer_status( &mut self, @@ -1760,11 +1764,17 @@ impl P2PServer { ctx: &Context, ) { send_after( - status_refresh_interval(&self.store), + status_refresh_tick(&self.store), ctx.clone(), p2p_protocol::RefreshPeerStatus, ); - let peers: Vec = self.connected_peers.keys().copied().collect(); + let wall_slot = peer_heads::wall_clock_slot(self); + let interval = status_refresh_slots(&self.store); + let peers = self.peer_heads.due_for_refresh( + self.connected_peers.keys().copied(), + wall_slot, + interval, + ); for peer in peers { self.peer_heads.begin_refresh(peer); request_status(self, peer).await; @@ -1977,14 +1987,19 @@ async fn fetch_missing_columns(server: &mut P2PServer, block_root: H256, columns // --- Manual Handler for swarm events --- -/// How often to re-ask every connected peer for its `Status`; see -/// [`peer_heads`] for the pace on each chain. -fn status_refresh_interval(store: &Store) -> Duration { - let slots = match store.chain() { +/// Slots between two `Status` requests to the same peer; see [`peer_heads`] +/// for the pace on each chain. +fn status_refresh_slots(store: &Store) -> u64 { + match store.chain() { Chain::Lean => peer_heads::STATUS_REFRESH_SLOTS_LEAN, Chain::Beacon => peer_heads::STATUS_REFRESH_SLOTS_BEACON, - }; - Duration::from_millis(store.config().slot_duration_ms.saturating_mul(slots)) + } +} + +/// The refresh timer's period: one slot, each tick asking that slot's share +/// of the peers that are due. +fn status_refresh_tick(store: &Store) -> Duration { + Duration::from_millis(store.config().slot_duration_ms) } /// Send `peer` our `Status`, asking for its own: the periodic refresh. The diff --git a/crates/net/p2p/src/peer_heads.rs b/crates/net/p2p/src/peer_heads.rs index 4700d6588..e20535184 100644 --- a/crates/net/p2p/src/peer_heads.rs +++ b/crates/net/p2p/src/peer_heads.rs @@ -15,7 +15,11 @@ //! Peers send `Status` once per connection, so a claim would go stale while //! the peer stays connected. [`STATUS_REFRESH_SLOTS_LEAN`] and //! [`STATUS_REFRESH_SLOTS_BEACON`] pace a re-request, the way Lighthouse -//! re-statuses every 300 s and Prysm every half epoch. +//! re-statuses every 300 s and Prysm every half epoch. The pace is per peer, +//! counted from that peer's own last exchange, and no slot asks more than its +//! share of the peers (see [`PeerHeads::due_for_refresh`]), so a node with +//! hundreds of peers spreads the requests over the interval instead of sending +//! them all at once. use std::collections::{HashMap, HashSet}; @@ -37,12 +41,12 @@ pub(crate) const PEER_HEAD_QUORUM: usize = 3; /// `FUTURE_SLOT_TOLERANCE` and Prysm's `maxFutureStatusHeadSlot`. pub(crate) const FUTURE_HEAD_TOLERANCE: u64 = 1; -/// How often a lean node re-asks every connected peer for its `Status`: every +/// How often a lean node re-asks each connected peer for its `Status`: every /// lead the lean sync tracker tolerates (`SyncThresholds::LEAN.peer_lead`), so /// a claim is never staler than the gap it is meant to detect. pub(crate) const STATUS_REFRESH_SLOTS_LEAN: u64 = 4; -/// How often a beacon node re-asks every connected peer for its `Status`: +/// How often a beacon node re-asks each connected peer for its `Status`: /// Prysm's half epoch (`maintainPeerStatuses`). pub(crate) const STATUS_REFRESH_SLOTS_BEACON: u64 = SLOTS_PER_EPOCH / 2; @@ -64,6 +68,9 @@ pub(crate) struct PeerHeads { /// Peers whose outstanding `Status` request is a refresh rather than the /// handshake; see [`REFRESH_RANGE_SYNC_LEAD_LEAN`]. refreshing: HashSet, + /// The wall-clock slot of each connected peer's last `Status` exchange, in + /// either direction: the last time it told us its head, or we asked. + last_exchange: HashMap, } impl PeerHeads { @@ -71,7 +78,9 @@ impl PeerHeads { /// [`FUTURE_HEAD_TOLERANCE`]; a peer whose head is dropped loses any /// earlier report too, since a peer that claims the future is not one to /// believe about the present. Returns whether the head was recorded. + /// Either way the exchange postpones `peer`'s next refresh. pub(crate) fn record(&mut self, peer: PeerId, head_slot: u64, wall_slot: u64) -> bool { + self.last_exchange.insert(peer, wall_slot); if head_slot > wall_slot.saturating_add(FUTURE_HEAD_TOLERANCE) { self.heads.remove(&peer); return false; @@ -84,6 +93,7 @@ impl PeerHeads { pub(crate) fn forget(&mut self, peer: &PeerId) { self.heads.remove(peer); self.refreshing.remove(peer); + self.last_exchange.remove(peer); } /// The network's head as the peers report it: the second-highest report @@ -100,6 +110,43 @@ impl PeerHeads { heads.get(rank).copied() } + /// The connected peers to re-ask for their `Status` at `wall_slot`, at + /// most one slot's share of them. + /// + /// A peer falls due `interval` slots after its last exchange. Each call + /// sends at most `connected / interval` (rounded up) of the due peers, + /// longest-waiting first, so 200 peers that connected together are + /// re-asked across a whole interval rather than in one burst, and from + /// then on stay spread out. A peer seen here for the first time is still + /// in its handshake, so it counts as having just exchanged. The peers + /// returned count as asked at `wall_slot`. + pub(crate) fn due_for_refresh( + &mut self, + connected: impl IntoIterator, + wall_slot: u64, + interval: u64, + ) -> Vec { + let interval = interval.max(1); + let mut connected_count = 0usize; + let mut due = Vec::new(); + for peer in connected { + connected_count += 1; + let last = *self.last_exchange.entry(peer).or_insert(wall_slot); + if last.saturating_add(interval) <= wall_slot { + due.push((last, peer)); + } + } + let per_slot = connected_count.div_ceil(usize::try_from(interval).unwrap_or(usize::MAX)); + due.sort_unstable_by_key(|(last, _)| *last); + due.truncate(per_slot); + due.into_iter() + .map(|(_, peer)| { + self.last_exchange.insert(peer, wall_slot); + peer + }) + .collect() + } + /// Note that the `Status` request about to go to `peer` is a refresh. pub(crate) fn begin_refresh(&mut self, peer: PeerId) { self.refreshing.insert(peer); @@ -112,7 +159,7 @@ impl PeerHeads { } /// The slot the wall clock is in, on the store's time grid. -fn wall_clock_slot(server: &P2PServer) -> u64 { +pub(crate) fn wall_clock_slot(server: &P2PServer) -> u64 { let grid = server.store.config().time_grid(); unix_now_ms().saturating_sub(grid.genesis_time_ms()) / grid.milliseconds_per_slot } @@ -203,6 +250,51 @@ mod tests { heads.forget(&peer); assert_eq!(heads.network_head(), None); assert!(!heads.take_refresh(&peer)); + // A reconnect is a fresh handshake, not a refresh that is overdue. + assert!(heads.due_for_refresh([peer], 1_000, 4).is_empty()); + } + + #[test] + fn a_peer_is_re_asked_one_interval_after_its_last_exchange() { + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + // First seen at slot 10, mid-handshake. + assert!(heads.due_for_refresh([peer], 10, 4).is_empty()); + assert!(heads.due_for_refresh([peer], 13, 4).is_empty()); + assert_eq!(heads.due_for_refresh([peer], 14, 4), vec![peer]); + // Asked at 14, so not again until 18. + assert!(heads.due_for_refresh([peer], 17, 4).is_empty()); + assert_eq!(heads.due_for_refresh([peer], 18, 4), vec![peer]); + } + + #[test] + fn a_status_from_the_peer_postpones_its_refresh() { + // A peer that sent us its `Status` at slot 12 is as fresh as one we + // asked then. + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + heads.due_for_refresh([peer], 10, 4); + heads.record(peer, 12, 12); + assert!(heads.due_for_refresh([peer], 14, 4).is_empty()); + assert_eq!(heads.due_for_refresh([peer], 16, 4), vec![peer]); + } + + #[test] + fn peers_that_connected_together_are_re_asked_across_an_interval() { + // A beacon node's default 200 peers, all connected at slot 0 and due + // together at slot 16: no slot asks more than 13 of them, and every + // one is asked within the interval after they fell due. + let peers: Vec = (0..200).map(|_| PeerId::random()).collect(); + let mut heads = PeerHeads::default(); + heads.due_for_refresh(peers.iter().copied(), 0, STATUS_REFRESH_SLOTS_BEACON); + let mut asked = HashSet::new(); + for slot in 16..32 { + let batch = + heads.due_for_refresh(peers.iter().copied(), slot, STATUS_REFRESH_SLOTS_BEACON); + assert!(batch.len() <= 13, "slot {slot} asked {} peers", batch.len()); + asked.extend(batch); + } + assert_eq!(asked.len(), peers.len()); } #[test] diff --git a/docs/architecture.md b/docs/architecture.md index 9178ea47b..e063a90b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -169,7 +169,8 @@ flag unset on a placement that does not cover every subnet. `sync_status.rs` decides whether the node is behind. It is syncing, and stops attesting and proposing, when its head trails the slot clock **and** the head its peers report is ahead of its own, since a head derived from a partial view is not worth voting for. The -peers' head is each connected peer's latest `Status`, re-requested every few slots, which +peers' head is each connected peer's latest `Status`, re-requested every few slots per peer +(spread over the interval, so a node with hundreds of peers never asks them all at once), which p2p aggregates onto `Store::network_head` (`crates/net/p2p/src/peer_heads.rs`). A hysteresis band stops the state from flapping at the boundary, and a network-wide stall (nobody else is ahead either) leaves the node synced so its validators can help the chain From d2135a52419fbc240956f0c1b1b65a8f9d160003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:36:01 -0300 Subject: [PATCH 5/6] fix(sync): bound peer claims by progress and count the highest report Review fixes for the peer-head sync gate. - Count the highest connected peer's Status head instead of the second-highest of three, so one honest peer is enough. A lying peer is bounded by a new no-progress backstop instead: a node that reads Syncing on peer reports while its head stays flat for `stall_window` slots (16 lean, an epoch beacon) treats the network as stalled until its head advances. The startup hold and the unfollowed-fork guard are exempt. - A head this node already imported (its head, or a block it holds a state for) is not counted as a lead and starts no range sync. - The startup hold runs its whole window; a report can add lead during it but never end it early. - Move the thresholds and the Status refresh cadence into `ethlambda_network_api::sync::SyncThresholds`, shared by the tracker and p2p, with a const assertion that a report as stale as the refresh allows still shows the lead of a node past the clock threshold. Lean: peer lead 4 -> 1, refresh 4 -> 2 slots. Beacon: a 4-slot hysteresis band. - Beacon judges the gate inside `update_head_from_fork_choice`, so the INVALID-verdict paths are covered, the head is decoded once, and the tick no longer judges it a second time. - Refreshes go out in each peer's last Status version, the Status is built once per tick, and a refused `status/2` is never retried. - Lean range sync uses the fetched-through watermark too (renamed `range_fetched_through`), and refresh answers on both chains go through `status_sync_target`. - The beacon digest-mismatch warning and counter fire once per move onto a foreign digest, not on every Status. - One shared `wall_clock_slot` replaces `builder_market::wall_slot`. --- CLAUDE.md | 4 +- crates/blockchain/src/lib.rs | 119 ++++-- crates/blockchain/src/sync_status.rs | 441 +++++++++++++++----- crates/net/api/src/lib.rs | 2 + crates/net/api/src/sync.rs | 131 ++++++ crates/net/p2p/src/beacon/builder_market.rs | 17 +- crates/net/p2p/src/beacon/handler.rs | 39 +- crates/net/p2p/src/beacon/messages.rs | 7 + crates/net/p2p/src/beacon/verdict.rs | 9 +- crates/net/p2p/src/lib.rs | 127 +++--- crates/net/p2p/src/metrics.rs | 5 +- crates/net/p2p/src/peer_heads.rs | 308 +++++++++----- crates/net/p2p/src/req_resp/handlers.rs | 366 ++++++++++++---- crates/storage/src/store.rs | 21 +- docs/architecture.md | 27 +- docs/beacon_wire.md | 18 +- docs/rpc.md | 2 +- docs/spec_deviations.md | 44 +- 18 files changed, 1253 insertions(+), 434 deletions(-) create mode 100644 crates/net/api/src/sync.rs diff --git a/CLAUDE.md b/CLAUDE.md index d08ee404f..e51d05289 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ crates/ ├─ metrics/ # Prometheus re-exports, TimingGuard, gather utilities └─ test-fixtures/ # Spec-fixture loading (prod dep of rpc's Hive test driver) net/ - ├─ api/ # Actor protocol traits wiring BlockChain ↔ P2P + ├─ api/ # Actor protocol traits wiring BlockChain ↔ P2P, plus the shared sync-gate thresholds (`sync.rs`) ├─ p2p/ # libp2p: gossipsub + req-resp (Status, BlocksByRoot, BlocksByRange, gloas envelopes) │ ├─ src/gossipsub/ # Topic encoding, message handling │ ├─ src/req_resp/ # Request/response codec and handlers @@ -379,7 +379,7 @@ actual_slot = finalized_slot + 1 + relative_index Both serve from the checkpoint-anchored store, and `build_status` advertises it (head, finalized checkpoint, and the anchor's slot as `earliest_available_slot`), so peers have a reason to ask. Asking runs too: a peer's Status starts a range session from the - higher of `P2PServer::beacon_fetched_through` (the highest slot a range answer handed to + higher of `P2PServer::range_fetched_through` (the highest slot a range answer handed to the chain actor, since the store's head trails a delivered batch by the whole actor mailbox) and the store's head (which gossip imports move and the watermark never sees). Fetched blocks reach the actor as `BlockSource::Sync`. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 9eacc8963..014d64547 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -564,7 +564,7 @@ pub struct BlockChainServer { /// Shared, read-only mirror of `sync_status` for readers outside the actor /// (the RPC `/lean/v0/node/syncing` endpoint). Written from - /// `update_sync_status` with the same `SyncStatus` fed to the metric. + /// `judge_sync_status` with the same `SyncStatus` fed to the metric. sync_status_controller: SyncStatusController, /// Chain-event publication: the bus and what decides what goes out on @@ -996,7 +996,13 @@ impl BlockChainServer { // Update current slot metric metrics::update_current_slot(slot); - self.update_sync_status(slot); + // Beacon judges the gate in the head recompute this tick ends in (see + // `update_head_from_fork_choice`), off the head it has just read; + // judging here too would only be overwritten moments later, at the + // cost of another whole-block decode. + if self.store.chain() == Chain::Lean { + self.update_sync_status(slot); + } // Snapshot the aggregator flag once per tick so all read sites within // the tick see a consistent value even if the admin API toggles it @@ -2317,12 +2323,13 @@ impl BlockChainServer { /// And where the sync gate is judged between ticks, for the same reason /// the head is: on a catch-up the tick waits behind every block range sync /// forwarded, so a gate judged only there kept answering `Synced` to the - /// validator client for the whole catch-up, however far behind. Judged - /// before the `forkchoiceUpdated` round trip, so a slow execution client - /// does not delay it by one more engine call. + /// validator client for the whole catch-up, however far behind. The + /// judgment is in [`Self::update_head_from_fork_choice`], so it lands + /// before the `forkchoiceUpdated` round trip (a slow execution client does + /// not delay it by one more engine call) and again whenever an `INVALID` + /// verdict moves the head inside that call. async fn recompute_beacon_head(&mut self) -> HeadTimings { let mut timings = self.update_head_from_fork_choice(); - self.update_sync_status(self.wall_clock_slot()); if let Some((start, end)) = self.notify_forkchoice_updated().await { timings.fcu_start = Some(start); timings.fcu_end = Some(end); @@ -2503,6 +2510,14 @@ impl BlockChainServer { /// A failure here means fork choice could not find a head (for instance /// every known block is unjustifiable), which is a condition to log and /// wait out, not a reason to crash a follower. + /// + /// Also the beacon sync gate's one judge, since the gate reads the head: + /// every tick, arrival and envelope release ends here through + /// [`Self::recompute_beacon_head`], and so does an `INVALID` + /// `forkchoiceUpdated` verdict that moves the head back, which would + /// otherwise leave the gate judging a head that no longer is. Judged off + /// the head read here rather than reading it again: on beacon that read is + /// a whole-block decode. pub(crate) fn update_head_from_fork_choice(&mut self) -> HeadTimings { let _timing = metrics::time_beacon_head_compute(); let mut timings = HeadTimings { @@ -2518,13 +2533,18 @@ impl BlockChainServer { // what this does: the head simply does not move. warn!(%err, "Failed to compute beacon head"); timings.head_end = Some(Instant::now()); + // The head did not move, but the clock did. + self.update_sync_status(self.wall_clock_slot()); return timings; } timings.head_end = Some(Instant::now()); - if let Some((head_slot, _)) = self.store.beacon_head() { + let head = self.store.beacon_head(); + if let Some((head_slot, _)) = head { metrics::update_head_slot(head_slot); } - self.pin_head_shufflings(); + self.pin_head_shufflings(head); + let head_slot = head.map_or(0, |(slot, _)| slot); + self.judge_sync_status(self.wall_clock_slot(), head_slot); timings } @@ -2539,8 +2559,8 @@ impl BlockChainServer { /// longer cached (a reorg back to an old block) keeps the previous head's /// pinning until a later head is resident, which at worst lets one of its /// shufflings be evicted and rebuilt, never serves a wrong committee. - fn pin_head_shufflings(&mut self) { - let Some((_, head_root)) = self.store.beacon_head() else { + fn pin_head_shufflings(&mut self, head: Option<(u64, H256)>) { + let Some((_, head_root)) = head else { return; }; let committees = self.store.committee_cache(); @@ -3594,35 +3614,36 @@ impl BlockChainServer { } } + /// [`Self::judge_sync_status`] off the head read through + /// [`Self::head_slot`], which is where the per-chain part of that lives + /// (lean's `Store::head_slot` and beacon's `Store::beacon_head` decode + /// different tables). For a caller that has not just read the head. + fn update_sync_status(&mut self, current_slot: u64) { + let head_slot = self.head_slot(); + self.judge_sync_status(current_slot, head_slot); + } + /// Refresh the sync-status tracker and its two outputs (the /// `lean_node_sync_status` metric and [`SyncStatusController`]). /// - /// Reads the head through [`Self::head_slot`], which is where the - /// per-chain part of that lives (lean's `Store::head_slot` and beacon's - /// `Store::beacon_head` decode different tables), and the network's head - /// from [`Store::network_head`], the head the node's peers report in - /// `Status`. The tracker itself is chain-agnostic, with per-chain - /// thresholds; see [`SyncThresholds`]. On a beacon follower, once the clock - /// is past a fork the node does not follow (see - /// [`Self::clock_is_past_followed_forks`]) the network is treated as being - /// at the clock, since no peer on that fork can tell it so. - fn update_sync_status(&mut self, current_slot: u64) { - let head_slot = self.head_slot(); - let network_head = match self.store.chain() { - Chain::Beacon - if Self::clock_is_past_followed_forks( - &self.store.config(), - current_slot, - head_slot, - ) => - { - Some(current_slot) - } - Chain::Beacon | Chain::Lean => self.store.network_head(), + /// The network's head comes from [`Store::network_head`], the highest head + /// the node's peers report in `Status` that it does not already hold. The + /// tracker itself is chain-agnostic, with per-chain thresholds; see + /// [`SyncThresholds`]. On a beacon follower, once the clock is past a fork + /// the node does not follow (see [`Self::clock_is_past_followed_forks`]) + /// the network is treated as being at the clock, since no peer on that + /// fork can tell it so. + fn judge_sync_status(&mut self, current_slot: u64, head_slot: u64) { + let past_followed_forks = self.store.chain() == Chain::Beacon + && Self::clock_is_past_followed_forks(&self.store.config(), current_slot, head_slot); + let status = if past_followed_forks { + self.sync_status + .update_with_network_at_clock(current_slot, head_slot) + } else { + let network_head = self.store.network_head(); + self.sync_status + .update(current_slot, head_slot, network_head) }; - let status = self - .sync_status - .update(current_slot, head_slot, network_head); metrics::set_node_sync_status(status); self.sync_status_controller.set(status); } @@ -5295,12 +5316,19 @@ mod tests { /// with the head still at the anchor (slot 0). const THREE_EPOCHS: u64 = 3 * preset::SLOTS_PER_EPOCH; + /// Start the tracker's clock at slot 0, so its startup hold (an epoch) is + /// over by [`THREE_EPOCHS`] and the peers' reports decide. + fn past_startup(server: &mut BlockChainServer) { + server.update_sync_status(0); + } + #[test] fn the_clock_reaching_a_followed_fork_does_not_force_syncing() { // Gloas is followed, so the fork rule stays out of it and the peers // decide: every peer as stale as we are is a stalled network, read as // synced, the same as when no fork is scheduled. let mut server = syncing_server(Some(1)); + past_startup(&mut server); server.store.set_network_head(Some(0)); server.update_sync_status(THREE_EPOCHS); assert_eq!( @@ -5314,6 +5342,7 @@ mod tests { // The same silence with no gloas scheduled is a stalled network, which // the tracker deliberately reports as synced. let mut server = syncing_server(None); + past_startup(&mut server); server.store.set_network_head(Some(0)); server.update_sync_status(THREE_EPOCHS); assert_eq!( @@ -5329,6 +5358,7 @@ mod tests { // used to read synced, since the freshest block it knew of was its own // head. let mut server = syncing_server(None); + past_startup(&mut server); server.store.set_network_head(Some(THREE_EPOCHS)); server.update_sync_status(THREE_EPOCHS); assert_eq!( @@ -5337,6 +5367,24 @@ mod tests { ); } + #[test] + fn a_head_recompute_judges_the_gate_on_its_own() { + // An `INVALID` `forkchoiceUpdated` verdict moves the head through + // `update_head_from_fork_choice` alone, inside the engine call and + // after `recompute_beacon_head` has judged the gate. The gate has to + // follow that head too, so the recompute itself is what judges it. + let mut server = syncing_server(None); + past_startup(&mut server); + server + .store + .set_network_head(Some(server.wall_clock_slot())); + server.update_head_from_fork_choice(); + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Syncing + ); + } + #[tokio::test] async fn a_block_arrival_judges_the_gate_without_waiting_for_the_tick() { // A node reconnecting to peers far ahead of it: range sync forwards @@ -5346,6 +5394,7 @@ mod tests { // is unknown is enough: whatever becomes of a block, its arrival ends // in the head recompute. let mut server = syncing_server(None); + past_startup(&mut server); let orphan = || bare_block(1, H256::repeat_byte(0xee)); // Peers as stale as the node: a stall, synced. diff --git a/crates/blockchain/src/sync_status.rs b/crates/blockchain/src/sync_status.rs index 52a04d5fd..e09453a3c 100644 --- a/crates/blockchain/src/sync_status.rs +++ b/crates/blockchain/src/sync_status.rs @@ -1,16 +1,17 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU8, Ordering}; -use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; -use tracing::info; +pub(crate) use ethlambda_network_api::sync::SyncThresholds; +use tracing::{info, warn}; use crate::metrics::SyncStatus; /// Shared, runtime-readable node sync status. /// -/// The blockchain actor is the sole writer: once per tick it calls +/// The blockchain actor is the sole writer: whenever it judges the gate (every +/// tick, every head recompute, and every lean block arrival) it calls /// [`SyncStatusController::set`] with the same [`SyncStatus`] it feeds the -/// `lean_node_sync_status` metric (see `BlockChainServer::update_sync_status`). +/// `lean_node_sync_status` metric (see `BlockChainServer::judge_sync_status`). /// Read paths outside the actor — notably the RPC `/lean/v0/node/syncing` /// endpoint — clone the controller and read the current status without touching /// actor state. Mirrors [`ethlambda_types::aggregator::AggregatorController`]. @@ -45,71 +46,16 @@ impl Default for SyncStatusController { } } -/// How far behind a node may fall before it reports itself syncing, per chain. -/// -/// The node is syncing only when **both** of these hold: -/// -/// - its head trails the wall clock by more than [`Self::clock_lag`], which is -/// local evidence no peer can forge, and -/// - the head its peers report (see `Store::network_head`) is more than -/// [`Self::peer_lead`] slots ahead of its own. -/// -/// This is Lighthouse's and Prysm's shape. Lighthouse serves duties unless a -/// range sync is running *and* the head is over `sync_tolerance_epochs` (2) -/// behind the clock, and only a peer more than 32 slots ahead starts a range -/// sync. Prysm resyncs only when the head is two epochs behind the clock *and* -/// peers report a head two epochs ahead. Peers decide between "the network is -/// ahead of me" and "the network is stalled"; the clock keeps a peer's claim -/// from idling a node whose own head is current. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct SyncThresholds { - /// Slots the head may trail the wall clock before the node can be syncing. - pub(crate) clock_lag: u64, - /// Slots the peers' head must lead ours before the node can be syncing. - pub(crate) peer_lead: u64, - /// Recovery band: once syncing, both lags must drop this many slots below - /// their thresholds before the node reports synced again, so the status - /// does not flap at the boundary. - pub(crate) hysteresis: u64, - /// Slots after the first tick during which, with no peer having reported a - /// head yet, the node assumes the network is at the wall clock. - /// - /// The first tick runs before any peer has answered a `Status`, so without - /// this a node resuming a stale database would read "no peer is ahead" as a - /// stalled network and do its duties on that stale head until the first - /// handshake lands. Prysm holds a starting node in initial sync until it - /// has peers; this bounds that hold, so a node that never finds a peer (a - /// single-node devnet, or one cut off from the start) still falls back to - /// treating the silence as a stall. - pub(crate) startup_grace: u64, -} - -impl SyncThresholds { - /// Lean consensus: leanSpec PR #708's lag threshold and hysteresis band, - /// applied to both conditions, with the stall override replaced by the peer - /// lead (see `docs/spec_deviations.md`). Lean has no epochs, so neither - /// client's epoch-sized tolerance carries over. - pub(crate) const LEAN: Self = Self { - clock_lag: 4, - peer_lead: 4, - hysteresis: 2, - startup_grace: 8, - }; - - /// The Beacon Chain: Lighthouse's numbers. Two epochs of clock lag - /// (`DEFAULT_SYNC_TOLERANCE_EPOCHS`, Prysm's `shouldReSync` too) and a peer - /// lead of one epoch (`SLOT_IMPORT_TOLERANCE`, the lead that makes a - /// Lighthouse peer `Advanced`). Neither client has a hysteresis band. The - /// startup hold is an epoch, since finding beacon peers through discovery - /// can take a while; Prysm holds until it has peers, however long. - pub(crate) const BEACON: Self = Self { - clock_lag: 2 * SLOTS_PER_EPOCH, - peer_lead: SLOTS_PER_EPOCH, - hysteresis: 0, - startup_grace: SLOTS_PER_EPOCH, - }; +/// The head slot the tracker last saw rise, and the wall-clock slot it rose at. +#[derive(Clone, Copy, Debug)] +struct Progress { + head_slot: u64, + at: u64, } +/// The sync gate: judges, from the clock, the node's own head and the head its +/// peers report, whether the node is too far behind to do validator duties. +/// See [`SyncThresholds`] for the rule and the numbers. pub(crate) struct SyncStatusTracker { syncing: bool, /// Whether the syncing state suppresses validator duties. @@ -122,9 +68,17 @@ pub(crate) struct SyncStatusTracker { thresholds: SyncThresholds, /// The slot at which the startup hold ends; set by the first update. startup_ends_at: Option, - /// Whether any peer has reported a head yet. Once one has, losing every - /// peer reads as a stall, not as startup. - heard_from_peers: bool, + /// When the head last advanced; set by the first update. + progress: Option, + /// The wall-clock slot from which the node has been behind both the clock + /// and its peers, while it still is. + behind_since: Option, + /// The head slot at which a peer-reported lead was given up on for making + /// no progress (see [`SyncThresholds::stall_window`]). Holds the node + /// synced until the head moves past it: the peers that claimed the lead + /// repeat it on every refresh, so a fresh report is no reason to believe + /// them again. + stalled_at_head: Option, } impl SyncStatusTracker { @@ -135,43 +89,131 @@ impl SyncStatusTracker { gate_duties, thresholds, startup_ends_at: None, - heard_from_peers: false, + progress: None, + behind_since: None, + stalled_at_head: None, } } /// Re-judge the status at `current_slot` (the wall clock's slot). /// - /// `network_head` is the head slot the node's peers report, or `None` when - /// no connected peer has reported one. With no report the node cannot be - /// behind anyone, so it reads as synced (a stalled or empty network, where - /// its validators should keep working), except during the startup hold. + /// `network_head` is the highest head the node's peers report that it + /// does not already hold, or `None` when no connected peer reports one. + /// With no report the node cannot be behind anyone, so it reads as synced + /// (a stalled or empty network, where its validators should keep + /// working), except during the startup hold, which assumes the network is + /// at least at the clock. pub(crate) fn update( &mut self, current_slot: u64, head_slot: u64, network_head: Option, + ) -> SyncStatus { + self.judge(current_slot, head_slot, network_head, false) + } + + /// [`Self::update`] for a node that knows the network is at least at the + /// clock without any peer saying so: a beacon follower whose clock is past + /// a fork it does not follow (`clock_is_past_followed_forks`). Exempt from + /// the stall backstop like the startup hold is, since the head of a node + /// that cannot follow the fork never advances, and that is the point. + pub(crate) fn update_with_network_at_clock( + &mut self, + current_slot: u64, + head_slot: u64, + ) -> SyncStatus { + self.judge(current_slot, head_slot, None, true) + } + + fn judge( + &mut self, + current_slot: u64, + head_slot: u64, + network_head: Option, + network_at_clock: bool, ) -> SyncStatus { let SyncThresholds { clock_lag: clock_threshold, peer_lead: lead_threshold, hysteresis, startup_grace, + stall_window, + .. } = self.thresholds; let startup_ends_at = *self .startup_ends_at .get_or_insert(current_slot.saturating_add(startup_grace)); - self.heard_from_peers |= network_head.is_some(); - let starting_up = !self.heard_from_peers && current_slot < startup_ends_at; + let starting_up = current_slot < startup_ends_at; + + let progress = self.progress.get_or_insert(Progress { + head_slot, + at: current_slot, + }); + if head_slot > progress.head_slot { + *progress = Progress { + head_slot, + at: current_slot, + }; + } else if head_slot < progress.head_slot { + // The head moved back (an `INVALID` execution verdict cutting a + // branch out): progress is measured from where it now is, or the + // re-imports below the old mark would not count. Moving back is + // not itself progress, so `at` stays. + progress.head_slot = head_slot; + } + let progress = *progress; + match self.stalled_at_head { + Some(stalled) if head_slot > stalled => self.stalled_at_head = None, + // Same for a head that moved back below where the lead was given + // up on: the next advance from there releases it. + Some(stalled) if head_slot < stalled => self.stalled_at_head = Some(head_slot), + _ => {} + } - // During the startup hold, assume the network is at the clock. - let network_head = network_head.or(starting_up.then_some(current_slot)); + // During the startup hold, assume the network is at least at the + // clock: a report can add lead, never take it away, since the first + // peer to answer may be as stale as we are. + let assumed = starting_up || network_at_clock; + let network_head = if assumed { + Some(network_head.map_or(current_slot, |slot| slot.max(current_slot))) + } else { + network_head + }; let clock_lag = current_slot.saturating_sub(head_slot); let peer_lead = network_head.map_or(0, |slot| slot.saturating_sub(head_slot)); let band = if self.syncing { hysteresis } else { 0 }; - let was_syncing = self.syncing; - - self.syncing = clock_lag > clock_threshold.saturating_sub(band) + let behind = clock_lag > clock_threshold.saturating_sub(band) && peer_lead > lead_threshold.saturating_sub(band); + // Counted only while behind on the peers' word: time spent behind on + // an assumption (the startup hold) must not use up the window, or a + // node whose first peer answers late in the hold would give up on it + // before its first range batch could land. + let behind_since = if behind && !assumed { + *self.behind_since.get_or_insert(current_slot) + } else { + self.behind_since = None; + current_slot + }; + + // The backstop: behind on the peers' word alone (the startup hold has + // its own bound), with the head flat for a whole window since the + // node fell behind on that word or last advanced. Whoever claims the + // lead is not delivering it, which is what a stalled network looks + // like too. + let flat_for = current_slot.saturating_sub(behind_since.max(progress.at)); + if behind && !assumed && self.stalled_at_head.is_none() && flat_for > stall_window { + warn!( + current_slot, + head_slot, + network_head, + flat_for, + "Peers report a head this node is not reaching; treating the network as stalled" + ); + self.stalled_at_head = Some(head_slot); + } + + let was_syncing = self.syncing; + self.syncing = behind && (assumed || self.stalled_at_head.is_none()); if self.syncing != was_syncing { info!( @@ -181,6 +223,7 @@ impl SyncStatusTracker { clock_lag, peer_lead, starting_up, + stalled = self.stalled_at_head.is_some(), syncing = self.syncing, "Sync status changed" ); @@ -207,18 +250,27 @@ impl SyncStatusTracker { #[cfg(test)] mod tests { + use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + use super::*; const LEAN: SyncThresholds = SyncThresholds::LEAN; + const BEACON: SyncThresholds = SyncThresholds::BEACON; - /// A lean tracker past its startup hold, with peers heard from, so each - /// test exercises the steady-state rule unless it says otherwise. - fn steady_lean() -> SyncStatusTracker { - let mut tracker = SyncStatusTracker::new(true, LEAN); + /// A tracker past its startup hold, with the head at genesis and every + /// peer with it, so each test exercises the steady-state rule unless it + /// says otherwise. + fn steady(thresholds: SyncThresholds) -> SyncStatusTracker { + let mut tracker = SyncStatusTracker::new(true, thresholds); tracker.update(0, 0, Some(0)); + tracker.update(thresholds.startup_grace, 0, Some(0)); tracker } + fn steady_lean() -> SyncStatusTracker { + steady(LEAN) + } + #[test] fn behind_both_the_clock_and_the_peers_is_syncing() { let mut tracker = steady_lean(); @@ -244,6 +296,21 @@ mod tests { ); } + #[test] + fn a_refresh_old_report_still_shows_a_node_past_the_clock_threshold() { + // Finding: a lean report is up to `status_refresh_slots` old. A node + // just past the clock threshold, whose peers were a slot behind the + // clock when they last reported, must still read as behind them. + let mut tracker = steady_lean(); + let current = 100; + let head = current - LEAN.clock_lag - 1; + let stale_report = current - 1 - LEAN.status_refresh_slots; + assert_eq!( + tracker.update(current, head, Some(stale_report)), + SyncStatus::Syncing + ); + } + #[test] fn a_stalled_network_keeps_duties_on_however_long_it_lasts() { // Nobody is ahead of us: the network is not producing, so validators @@ -259,7 +326,7 @@ mod tests { fn missed_proposals_do_not_close_the_gate() { // leanSpec #1194: several empty slots in a row push the clock lag past // the threshold on every honest node, but no peer is ahead of any of - // them, so none stops working. + // them by more than propagation, so none stops working. let mut tracker = steady_lean(); assert_eq!(tracker.update(16, 10, Some(10)), SyncStatus::Synced); assert_eq!(tracker.update(16, 10, Some(11)), SyncStatus::Synced); @@ -284,12 +351,158 @@ mod tests { } #[test] - fn startup_holds_a_stale_node_until_a_peer_reports() { + fn a_lead_that_never_turns_into_progress_is_a_stall() { + // A peer claims a head it does not serve (an empty answer, invalid + // blocks, no answer) while the network is stalled with us behind the + // clock. It costs one stall window, not the rest of the stall. + let mut tracker = steady_lean(); + assert_eq!(tracker.update(100, 50, Some(100)), SyncStatus::Syncing); + let last_syncing = 100 + LEAN.stall_window; + assert_eq!( + tracker.update(last_syncing, 50, Some(last_syncing)), + SyncStatus::Syncing + ); + assert_eq!( + tracker.update(last_syncing + 1, 50, Some(last_syncing + 1)), + SyncStatus::Synced + ); + assert!(tracker.duties_allowed()); + // The claim keeps coming on every refresh; it is not believed again + // while the head stays where it was given up on. + assert_eq!(tracker.update(500, 50, Some(500)), SyncStatus::Synced); + } + + #[test] + fn steady_progress_keeps_a_long_catch_up_syncing() { + // A slow catch-up advancing a slot every few, far behind peers at the + // clock, never trips the backstop. + let mut tracker = steady_lean(); + for current in 100..400 { + let head = 50 + (current - 100) / 4; + assert_eq!( + tracker.update(current, head, Some(current)), + SyncStatus::Syncing, + "slot {current}" + ); + } + } + + #[test] + fn a_head_advance_after_a_trip_re_arms_the_gate() { + let mut tracker = steady_lean(); + tracker.update(100, 50, Some(100)); + let tripped = 101 + LEAN.stall_window; + assert_eq!( + tracker.update(tripped, 50, Some(tripped)), + SyncStatus::Synced + ); + // A block lands but leaves the node behind: the peers were right + // after all, and the window starts over from the advance. + assert_eq!( + tracker.update(tripped + 1, 51, Some(tripped + 1)), + SyncStatus::Syncing + ); + let end = tripped + 1 + LEAN.stall_window; + assert_eq!(tracker.update(end, 51, Some(end)), SyncStatus::Syncing); + assert_eq!( + tracker.update(end + 1, 51, Some(end + 1)), + SyncStatus::Synced + ); + } + + #[test] + fn the_backstop_does_not_shorten_the_startup_hold() { + // A stall window shorter than the hold: the head stays flat through + // the whole hold, and the node holds for all of it. + let thresholds = SyncThresholds { + stall_window: 2, + ..LEAN + }; + let mut tracker = SyncStatusTracker::new(true, thresholds); + for current in 300..300 + LEAN.startup_grace { + assert_eq!( + tracker.update(current, 0, None), + SyncStatus::Syncing, + "slot {current}" + ); + } + } + + #[test] + fn a_report_late_in_the_startup_hold_still_gets_a_whole_stall_window() { + // A beacon node resumed 100 slots behind finds its first peer only in + // the hold's last slot. The window the head has to start moving in + // runs from the end of the hold, not from startup: the hold and the + // window are both an epoch, so counting from startup would give the + // first range batch no time at all. + let mut tracker = SyncStatusTracker::new(true, BEACON); + let start = 1_000; + let head = start - 100; + let hold_ends = start + BEACON.startup_grace; + for current in start..hold_ends - 1 { + assert_eq!(tracker.update(current, head, None), SyncStatus::Syncing); + } + let last_held = hold_ends - 1; + assert_eq!( + tracker.update(last_held, head, Some(last_held)), + SyncStatus::Syncing + ); + let window_end = hold_ends + BEACON.stall_window; + for current in hold_ends..=window_end { + assert_eq!( + tracker.update(current, head, Some(current)), + SyncStatus::Syncing, + "slot {current}" + ); + } + } + + #[test] + fn a_head_that_moves_back_and_climbs_again_is_progress() { + // An `INVALID` execution verdict moves the head back. Re-importing + // from there is progress even while still below the old head. + let mut tracker = steady_lean(); + assert_eq!(tracker.update(100, 60, Some(100)), SyncStatus::Syncing); + assert_eq!(tracker.update(105, 50, Some(105)), SyncStatus::Syncing); + assert_eq!(tracker.update(110, 51, Some(110)), SyncStatus::Syncing); + let past_the_old_window = 101 + LEAN.stall_window; + assert_eq!( + tracker.update(past_the_old_window, 51, Some(past_the_old_window)), + SyncStatus::Syncing + ); + } + + #[test] + fn a_network_known_to_be_at_the_clock_is_not_a_stall() { + // Past a fork the node cannot follow, the head never advances again; + // that must not read as a stall the way an unserved claim does. + let mut tracker = steady_lean(); + for current in [100, 100 + LEAN.stall_window + 1, 1_000] { + assert_eq!( + tracker.update_with_network_at_clock(current, 50), + SyncStatus::Syncing, + "slot {current}" + ); + } + } + + #[test] + fn startup_holds_a_stale_node_without_peers() { let mut tracker = SyncStatusTracker::new(true, LEAN); // Resumed 300 slots behind, nobody heard from yet. assert_eq!(tracker.update(300, 0, None), SyncStatus::Syncing); - // The first handshake says the network is with us after all. - assert_eq!(tracker.update(301, 0, Some(0)), SyncStatus::Synced); + } + + #[test] + fn a_stale_first_report_does_not_end_the_startup_hold() { + // Finding: the first peer to answer may be as stale as we are, or + // still syncing itself. Its report can only add lead during the hold. + let mut tracker = SyncStatusTracker::new(true, LEAN); + assert_eq!(tracker.update(300, 0, None), SyncStatus::Syncing); + assert_eq!(tracker.update(301, 0, Some(0)), SyncStatus::Syncing); + // Once the hold is over, the same report reads as a stall. + let after_grace = 300 + LEAN.startup_grace; + assert_eq!(tracker.update(after_grace, 0, Some(0)), SyncStatus::Synced); } #[test] @@ -320,19 +533,57 @@ mod tests { #[test] fn the_beacon_thresholds_are_lighthouses() { - let mut tracker = SyncStatusTracker::new(true, SyncThresholds::BEACON); - tracker.update(0, 0, Some(0)); let epoch = SLOTS_PER_EPOCH; // A checkpoint-synced follower 100 slots back with peers at the clock. - assert_eq!(tracker.update(1_000, 900, Some(1_000)), SyncStatus::Syncing); + assert_eq!( + steady(BEACON).update(1_000, 900, Some(1_000)), + SyncStatus::Syncing + ); // Within two epochs of the clock, Lighthouse serves duties. assert_eq!( - tracker.update(1_000, 1_000 - 2 * epoch, Some(1_000)), + steady(BEACON).update(1_000, 1_000 - 2 * epoch, Some(1_000)), SyncStatus::Synced ); // Peers within an epoch of us are not `Advanced`. assert_eq!( - tracker.update(1_000, 900, Some(900 + epoch)), + steady(BEACON).update(1_000, 900, Some(900 + epoch)), + SyncStatus::Synced + ); + } + + #[test] + fn a_beacon_follower_at_the_clock_threshold_does_not_flap() { + // Finding: importing about a block a slot right at the two-epoch + // boundary, the lag alternates between just over and just at the + // threshold, once per tick and once per arrival. + let mut tracker = steady(BEACON); + let boundary = BEACON.clock_lag; + let mut head = 1_000 - boundary - 1; + assert_eq!( + tracker.update(1_000, head, Some(1_000)), + SyncStatus::Syncing + ); + for current in 1_000..1_100 { + // An arrival brings the lag back down to the threshold... + head += 1; + assert_eq!( + tracker.update(current, head, Some(current)), + SyncStatus::Syncing, + "arrival at slot {current}" + ); + // ...and the next tick pushes it over again. + assert_eq!( + tracker.update(current + 1, head, Some(current + 1)), + SyncStatus::Syncing, + "tick at slot {}", + current + 1 + ); + } + // Gaining on the clock by the band releases it. + let current = 1_100; + let caught_up = current - (boundary - BEACON.hysteresis); + assert_eq!( + tracker.update(current, caught_up, Some(current)), SyncStatus::Synced ); } diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index a0aa347d2..281ba5b15 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -21,6 +21,8 @@ use spawned_concurrency::error::ActorError; use spawned_concurrency::message::Message; use spawned_concurrency::protocol; +pub mod sync; + // --- Protocol: BlockChain -> P2P --- #[protocol] diff --git a/crates/net/api/src/sync.rs b/crates/net/api/src/sync.rs new file mode 100644 index 000000000..f9faa58ce --- /dev/null +++ b/crates/net/api/src/sync.rs @@ -0,0 +1,131 @@ +//! The sync gate's thresholds, shared by the two actors that act on them. +//! +//! The chain actor's sync tracker judges "behind" with these numbers, and the +//! p2p layer paces its `Status` refresh and gates the range syncs a refresh +//! answer may start by the same ones. Kept in one place so the two cannot +//! drift: a refresh cadence that no longer fits the lead it is meant to detect +//! is a silent sensitivity loss, not a compile error. + +use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + +/// How far behind a node may fall before it reports itself syncing, per chain. +/// +/// The node is syncing only when **both** of these hold: +/// +/// - its head trails the wall clock by more than [`Self::clock_lag`], which is +/// local evidence no peer can forge, and +/// - the head its peers report is more than [`Self::peer_lead`] slots ahead of +/// its own. +/// +/// This is Lighthouse's and Prysm's shape. Lighthouse serves duties unless a +/// range sync is running *and* the head is over `sync_tolerance_epochs` (2) +/// behind the clock, and only a peer more than 32 slots ahead starts a range +/// sync. Prysm resyncs only when the head is two epochs behind the clock *and* +/// peers report a head two epochs ahead. Peers decide between "the network is +/// ahead of me" and "the network is stalled"; the clock keeps a peer's claim +/// from idling a node whose own head is current. +/// +/// A peer's head is a claim, and a claim that never turns into progress is +/// bounded by [`Self::stall_window`]: syncing means getting somewhere, and a +/// lead nobody delivers reads the same as a stalled network. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SyncThresholds { + /// Slots the head may trail the wall clock before the node can be syncing. + pub clock_lag: u64, + /// Slots the peers' head must lead ours before the node can be syncing. + /// + /// Also how far ahead a peer answering a `Status` *refresh* must be + /// before its answer starts a range sync. A handshake answer starts one + /// at any lead, but a refresh lands every few slots on a node that is + /// keeping up with gossip, where a peer a slot ahead is ordinary and a + /// one-block range request would only race gossip. + pub peer_lead: u64, + /// Recovery band: once syncing, both lags must drop this many slots below + /// their thresholds before the node reports synced again, so the status + /// does not flap at the boundary. + pub hysteresis: u64, + /// Slots after the first judgment during which the node assumes the + /// network is at least at the wall clock, whatever its peers report. + /// + /// The first judgment runs before any peer has answered a `Status`, and + /// the first peer to answer may itself be behind, so without this a node + /// resuming a stale database would read "no peer is ahead" as a stalled + /// network and do its duties on that stale head. Prysm holds a starting + /// node in initial sync until it has peers; this bounds that hold, so a + /// node that never finds a peer, or restarts into a real stall, still + /// falls back to treating the silence as a stall. + pub startup_grace: u64, + /// Slots the node may read as syncing on a peer-reported lead without its + /// head advancing, before it treats the network as stalled after all. + /// + /// A peer that claims a head it does not serve (an empty answer, invalid + /// blocks, no answer) would otherwise keep a node that is behind the + /// clock syncing for as long as it repeats the claim. Sized above what can + /// legitimately hold a catching-up head still: the range batch custody + /// wait (30 s), an execution client's retry ladder (about 26 s), lean + /// proof verification of a whole batch. + pub stall_window: u64, + /// How often the p2p layer re-asks each connected peer for its `Status`. + /// + /// A peer's report is up to this many slots stale when the tracker reads + /// it, which shrinks the lead it shows by as much. [`Self::is_consistent`] + /// keeps that staleness inside the clock threshold, so a node the clock + /// says is behind still reads as behind its peers. + pub status_refresh_slots: u64, +} + +impl SyncThresholds { + /// Lean consensus: leanSpec PR #708's lag threshold and hysteresis band, + /// with the stall override replaced by the peer lead (see + /// `docs/spec_deviations.md`). Lean has no epochs, so neither client's + /// epoch-sized tolerance carries over. + /// + /// The peer lead is one slot rather than the clock's four because a + /// report is up to [`Self::status_refresh_slots`] old: a node five slots + /// behind a producing network sees peers that were current two slots ago + /// at least two slots ahead, which a lead threshold of one detects. A lead + /// of one is still not enough on its own: peers a single slot apart are + /// ordinary propagation, and a run of missed proposals leaves every peer + /// within that. + pub const LEAN: Self = Self { + clock_lag: 4, + peer_lead: 1, + hysteresis: 2, + startup_grace: 8, + stall_window: 16, + status_refresh_slots: 2, + }; + + /// The Beacon Chain: Lighthouse's numbers. Two epochs of clock lag + /// (`DEFAULT_SYNC_TOLERANCE_EPOCHS`, Prysm's `shouldReSync` too) and a peer + /// lead of one epoch (`SLOT_IMPORT_TOLERANCE`, the lead that makes a + /// Lighthouse peer `Advanced`). Neither client has a hysteresis band, but + /// neither re-judges on every block arrival either; this node does, and a + /// follower importing about one block a slot right at the boundary would + /// otherwise flip twice a slot. The startup hold and the stall window are + /// an epoch: finding beacon peers through discovery can take a while, and + /// Prysm holds until it has peers, however long. The `Status` refresh is + /// Prysm's half epoch (`maintainPeerStatuses`). + pub const BEACON: Self = Self { + clock_lag: 2 * SLOTS_PER_EPOCH, + peer_lead: SLOTS_PER_EPOCH, + hysteresis: 4, + startup_grace: SLOTS_PER_EPOCH, + stall_window: SLOTS_PER_EPOCH, + status_refresh_slots: SLOTS_PER_EPOCH / 2, + }; + + /// Whether a report as stale as the refresh allows still shows the lead + /// of a node the clock says is behind: a node just past + /// [`Self::clock_lag`] behind peers that are a slot behind the clock and + /// last reported [`Self::status_refresh_slots`] ago sees a lead of + /// `clock_lag - status_refresh_slots`, which has to clear + /// [`Self::peer_lead`]. + pub const fn is_consistent(&self) -> bool { + self.peer_lead + self.status_refresh_slots < self.clock_lag + && self.hysteresis <= self.clock_lag + } +} + +const _: () = assert!(SyncThresholds::LEAN.is_consistent()); +const _: () = assert!(SyncThresholds::BEACON.is_consistent()); diff --git a/crates/net/p2p/src/beacon/builder_market.rs b/crates/net/p2p/src/beacon/builder_market.rs index f44ce4bbd..5c9f87fde 100644 --- a/crates/net/p2p/src/beacon/builder_market.rs +++ b/crates/net/p2p/src/beacon/builder_market.rs @@ -180,13 +180,6 @@ pub(crate) fn publish_proposer_preferences( ); } -/// The wall-clock slot, from the store's config. -pub(crate) fn wall_slot(server: &P2PServer) -> Slot { - let config = server.store.config(); - let genesis_ms = config.genesis_time_ms(); - unix_now_ms().saturating_sub(genesis_ms) / config.slot_duration_ms.max(1) -} - #[cfg(test)] mod tests { use ethlambda_state_transition::beacon::gossip::{IgnoreReason, Outcome, RejectReason}; @@ -273,7 +266,7 @@ mod tests { #[tokio::test] async fn a_far_slot_bid_is_not_current_or_next() { let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; - let bid = bid_at(wall_slot(&server) + 1_000); + let bid = bid_at(crate::wall_clock_slot(&server.store) + 1_000); assert!(matches!( triage_execution_payload_bid(&server, &bid.to_ssz()), Dispatch::Report(Outcome::Ignore(IgnoreReason::NotCurrentOrNextSlot)) @@ -283,7 +276,7 @@ mod tests { #[tokio::test] async fn a_bid_with_a_nonzero_payment_is_rejected() { let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; - let mut bid = bid_at(wall_slot(&server) + 1); + let mut bid = bid_at(crate::wall_clock_slot(&server.store) + 1); bid.message.execution_payment = 1; assert!(matches!( triage_execution_payload_bid(&server, &bid.to_ssz()), @@ -297,7 +290,7 @@ mod tests { #[tokio::test] async fn a_valid_shaped_bid_is_validated_unless_its_builder_key_was_seen() { let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; - let bid = bid_at(wall_slot(&server) + 1); + let bid = bid_at(crate::wall_clock_slot(&server.store) + 1); let payload = bid.to_ssz(); match triage_execution_payload_bid(&server, &payload) { Dispatch::Validate(Validated::ExecutionPayloadBid { @@ -320,7 +313,7 @@ mod tests { #[tokio::test] async fn preferences_are_judged_on_the_clock() { let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; - let wall = wall_slot(&server); + let wall = crate::wall_clock_slot(&server.store); let past = preferences_at(wall.saturating_sub(5)); assert!(matches!( triage_proposer_preferences(&server, &past.to_ssz()), @@ -336,7 +329,7 @@ mod tests { #[tokio::test] async fn preferences_for_a_cached_key_are_already_seen() { let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; - let wall = wall_slot(&server); + let wall = crate::wall_clock_slot(&server.store); let preferences = preferences_at(wall + 2); let payload = preferences.to_ssz(); assert!(matches!( diff --git a/crates/net/p2p/src/beacon/handler.rs b/crates/net/p2p/src/beacon/handler.rs index 2e566f2dc..ad7ba87ef 100644 --- a/crates/net/p2p/src/beacon/handler.rs +++ b/crates/net/p2p/src/beacon/handler.rs @@ -51,6 +51,13 @@ use crate::req_resp::Request; /// stream is eight bytes short and the codec refuses it, which killed the /// connection outright. Answering a request means answering in its own version. pub fn build_status(store: &Store, wire: &BeaconWire, version: StatusVersion) -> BeaconStatus { + status_in_version(&build_status_v2(store, wire), version) +} + +/// [`build_status`] in v2, the version that carries every field, for a caller +/// that sends one `Status` in several versions (the periodic refresh) and so +/// should read the head, a whole-block decode, once. +pub fn build_status_v2(store: &Store, wire: &BeaconWire) -> StatusV2 { let (finalized_root, finalized_epoch, head_root, head_slot, earliest_available_slot) = match store.beacon_head() { Some((head_slot, head_root)) => { @@ -72,23 +79,27 @@ pub fn build_status(store: &Store, wire: &BeaconWire, version: StatusVersion) -> None => (Root::ZERO, 0, Root::ZERO, 0, 0), }; - let v1 = StatusV1 { + StatusV2 { fork_digest: wire.fork_digest, finalized_root, finalized_epoch, head_root, head_slot, - }; + earliest_available_slot, + } +} + +/// `status` in `version`: v1 is v2 without `earliest_available_slot`. +pub fn status_in_version(status: &StatusV2, version: StatusVersion) -> BeaconStatus { match version { - StatusVersion::V1 => BeaconStatus::V1(v1), - StatusVersion::V2 => BeaconStatus::V2(StatusV2 { - fork_digest: v1.fork_digest, - finalized_root: v1.finalized_root, - finalized_epoch: v1.finalized_epoch, - head_root: v1.head_root, - head_slot: v1.head_slot, - earliest_available_slot, + StatusVersion::V1 => BeaconStatus::V1(StatusV1 { + fork_digest: status.fork_digest, + finalized_root: status.finalized_root, + finalized_epoch: status.finalized_epoch, + head_root: status.head_root, + head_slot: status.head_slot, }), + StatusVersion::V2 => BeaconStatus::V2(status.clone()), } } @@ -225,11 +236,13 @@ pub async fn request_metadata(server: &P2PServer, peer_id: PeerId) { .await; } -/// Re-open a refused handshake on the other `Status` version. +/// Re-open a refused `status/1` request on `status/2`. /// /// A peer that supports neither version was never going to talk to us, and one -/// retry cannot loop: the retry is sent in the version the first attempt was -/// not. +/// retry cannot loop: the caller retries only a refused `status/1`, and this +/// sends `status/2`. A refresh asks in the version the peer last answered in, +/// so a peer that has dropped v1 is asked in v2 from then on rather than +/// refused and retried every time. pub async fn retry_status_on_other_version(server: &P2PServer, peer_id: PeerId) { let Some(wire) = server.wire.beacon() else { return; diff --git a/crates/net/p2p/src/beacon/messages.rs b/crates/net/p2p/src/beacon/messages.rs index 2a89713ae..9ade4b943 100644 --- a/crates/net/p2p/src/beacon/messages.rs +++ b/crates/net/p2p/src/beacon/messages.rs @@ -70,6 +70,13 @@ impl BeaconStatus { } } + pub fn head_root(&self) -> Root { + match self { + BeaconStatus::V1(status) => status.head_root, + BeaconStatus::V2(status) => status.head_root, + } + } + pub fn finalized_epoch(&self) -> Epoch { match self { BeaconStatus::V1(status) => status.finalized_epoch, diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index ab5651c26..15cf22ceb 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -40,7 +40,7 @@ use spawned_concurrency::message::Message; use spawned_concurrency::tasks::{Context, Handler}; use tracing::{debug, error, warn}; -use crate::beacon::{builder_market, column_checks}; +use crate::beacon::column_checks; use crate::gossipsub::operation_kind; use crate::{P2PServer, metrics}; @@ -213,9 +213,10 @@ impl Validated { Self::ExecutionPayloadBid { bid, .. } => { server.builder_market.record_bid((**bid).clone()) } - Self::ProposerPreferences(preferences) => server - .builder_market - .record_preferences((**preferences).clone(), builder_market::wall_slot(server)), + Self::ProposerPreferences(preferences) => server.builder_market.record_preferences( + (**preferences).clone(), + crate::wall_clock_slot(&server.store), + ), Self::PayloadAttestation(message) => server .seen_payload_attestations .record(message.data.slot, message.validator_index), diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 5a21333d1..e269d76c8 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -68,6 +68,7 @@ use ethlambda_state_transition::beacon::{ use ethlambda_storage::{Chain, Store}; use ethlambda_types::beacon::preset::{MAX_VALIDATORS_PER_COMMITTEE, SLOTS_PER_EPOCH}; use ethlambda_types::primitives::H256; +use ethlambda_types::time::unix_now_ms; use ethrex_p2p::types::NodeRecord; use ethrex_rlp::decode::RLPDecode; use futures::StreamExt; @@ -101,7 +102,7 @@ use crate::{ publish_attestation, publish_beacon_aggregate, publish_beacon_attestation, publish_beacon_block, publish_beacon_operation, publish_block, }, - lean::protocols::MAX_REQUEST_BLOCKS, + lean::{messages::Status, protocols::MAX_REQUEST_BLOCKS}, req_resp::{ Codec, MAX_COMPRESSED_PAYLOAD_SIZE, ReqResp, ReqRespEvent, Request, build_status, fetch_block_from_peer, fetch_data_columns_from_peer, fetch_envelope_from_peer, @@ -1198,7 +1199,8 @@ impl P2P { // Seeded from the anchor the store was bootstrapped at, so the first // range request starts where the chain does rather than at slot 0. - let beacon_fetched_through = match store.chain() { + // Lean reads its head beside it, which is the same floor. + let range_fetched_through = match store.chain() { Chain::Beacon => store.beacon_head().map_or(0, |(slot, _)| slot), Chain::Lean => 0, }; @@ -1227,7 +1229,7 @@ impl P2P { pending_column_requests: HashMap::new(), outbound_requests: HashMap::new(), range_sync_state: None, - beacon_fetched_through, + range_fetched_through, peer_heads: Default::default(), bootnode_addrs: built.bootnode_addrs, node_names, @@ -1390,11 +1392,12 @@ pub struct P2PServer { pub(crate) outbound_requests: HashMap, pub(crate) range_sync_state: Option, - /// Highest beacon slot a range answer has handed to the chain actor, - /// whether or not it has been imported yet. Gossip never advances it, so - /// range sync reads it beside the store's head; see `beacon_sync_target`. - pub(crate) beacon_fetched_through: u64, - /// Every connected peer's latest `Status` head, aggregated onto + /// Highest slot a range answer has handed to the chain actor, whether or + /// not it has been imported yet, on either chain. Gossip never advances + /// it, so range sync reads it beside the store's head; see + /// `range_sync_target`. + pub(crate) range_fetched_through: u64, + /// Every connected peer's latest `Status` head, published on /// `Store::network_head` for the sync tracker. See [`peer_heads`]. pub(crate) peer_heads: peer_heads::PeerHeads, bootnode_addrs: HashMap>, @@ -1768,17 +1771,7 @@ impl P2PServer { ctx.clone(), p2p_protocol::RefreshPeerStatus, ); - let wall_slot = peer_heads::wall_clock_slot(self); - let interval = status_refresh_slots(&self.store); - let peers = self.peer_heads.due_for_refresh( - self.connected_peers.keys().copied(), - wall_slot, - interval, - ); - for peer in peers { - self.peer_heads.begin_refresh(peer); - request_status(self, peer).await; - } + self.refresh_due_peer_statuses().await; } /// The deadline of a range batch held back for custody. Scheduled once, @@ -1987,13 +1980,16 @@ async fn fetch_missing_columns(server: &mut P2PServer, block_root: H256, columns // --- Manual Handler for swarm events --- -/// Slots between two `Status` requests to the same peer; see [`peer_heads`] -/// for the pace on each chain. -fn status_refresh_slots(store: &Store) -> u64 { - match store.chain() { - Chain::Lean => peer_heads::STATUS_REFRESH_SLOTS_LEAN, - Chain::Beacon => peer_heads::STATUS_REFRESH_SLOTS_BEACON, - } +/// The slot the wall clock is in, on the store's time grid. Zero before +/// genesis. +/// +/// The one p2p-wide answer: the builder market's gossip verdicts and the +/// peer-head tracker all read "now" through it. [`beacon::BeaconWire::wall_slot`] +/// is the same formula off the wire's copy of the config, for code that holds +/// a wire rather than a store. +pub(crate) fn wall_clock_slot(store: &Store) -> u64 { + let grid = store.config().time_grid(); + unix_now_ms().saturating_sub(grid.genesis_time_ms()) / grid.milliseconds_per_slot.max(1) } /// The refresh timer's period: one slot, each tick asking that slot's share @@ -2002,26 +1998,63 @@ fn status_refresh_tick(store: &Store) -> Duration { Duration::from_millis(store.config().slot_duration_ms) } -/// Send `peer` our `Status`, asking for its own: the periodic refresh. The -/// handshake in `ConnectionEstablished` sends the same request, with its own -/// logging around it. -async fn request_status(server: &P2PServer, peer: PeerId) { - // Built before the send, so no borrow of `server.wire` is alive across it. - let beacon_status = server.wire.beacon().map(|wire| { - beacon::handler::build_status(&server.store, wire, beacon::handler::StatusVersion::V1) - }); - match beacon_status { - Some(status) => beacon::handler::send_status(server, peer, status).await, - None => { - let our_status = build_status(&server.store); - server - .swarm_handle - .send_request( - peer, - Request::LeanStatus(our_status), - ReqRespProtocol::LeanStatus, - ) - .await; +/// The `Status` the periodic refresh sends, built once per refresh tick +/// however many peers it asks: on beacon, building one decodes the head +/// block. +enum RefreshStatus { + Lean(Status), + /// Built in v2, the superset; a peer that speaks only v1 gets it with + /// `earliest_available_slot` dropped. + Beacon(beacon::messages::StatusV2), +} + +impl RefreshStatus { + fn build(server: &P2PServer) -> Self { + match server.wire.beacon() { + Some(wire) => Self::Beacon(beacon::handler::build_status_v2(&server.store, wire)), + None => Self::Lean(build_status(&server.store)), + } + } +} + +impl P2PServer { + /// Re-ask this slot's share of the connected peers that are due for their + /// `Status`; see [`peer_heads`]. The half of + /// [`Self::handle_refresh_peer_status`] that needs no actor context. + /// + /// Asks each beacon peer in the version its last exchange ran on, so a + /// peer that has dropped `status/1` is not refused (and then retried) on + /// every refresh. + pub(crate) async fn refresh_due_peer_statuses(&mut self) { + let wall_slot = wall_clock_slot(&self.store); + let interval = peer_heads::thresholds(&self.store).status_refresh_slots; + let peers = self.peer_heads.due_for_refresh( + self.connected_peers.keys().copied(), + wall_slot, + interval, + ); + if peers.is_empty() { + return; + } + let status = RefreshStatus::build(self); + for peer in peers { + self.peer_heads.begin_refresh(peer); + match &status { + RefreshStatus::Beacon(v2) => { + let version = self.peer_heads.status_version(&peer); + let status = beacon::handler::status_in_version(v2, version); + beacon::handler::send_status(self, peer, status).await; + } + RefreshStatus::Lean(status) => { + self.swarm_handle + .send_request( + peer, + Request::LeanStatus(status.clone()), + ReqRespProtocol::LeanStatus, + ) + .await; + } + } } } } @@ -3128,7 +3161,7 @@ pub(crate) mod test_support { pending_column_requests: HashMap::new(), outbound_requests: HashMap::new(), range_sync_state: None, - beacon_fetched_through: 0, + range_fetched_through: 0, peer_heads: Default::default(), bootnode_addrs: HashMap::new(), node_names: HashMap::new(), diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index 643300967..ed1093414 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -354,7 +354,7 @@ static LEAN_BEACON_GOSSIP_MESSAGES_TOTAL: LazyLock = LazyLock::ne static LEAN_BEACON_STATUS_DIGEST_MISMATCH_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( "lean_beacon_status_digest_mismatch_total", - "Beacon Status requests whose fork digest did not match ours" + "Peers whose beacon Status moved onto a fork digest we do not hold, counted once per move" ) .unwrap() }); @@ -474,6 +474,9 @@ pub fn inc_beacon_gossip_verdict_expired(kind: &str) { .inc(); } +/// Count a peer whose `Status` has moved onto a fork digest this node does not +/// hold: once when it moves there, not on every `Status` it sends from there, +/// so the periodic refresh does not inflate it with connection age. pub fn inc_beacon_status_digest_mismatch() { LEAN_BEACON_STATUS_DIGEST_MISMATCH_TOTAL.inc(); } diff --git a/crates/net/p2p/src/peer_heads.rs b/crates/net/p2p/src/peer_heads.rs index e20535184..389c21b18 100644 --- a/crates/net/p2p/src/peer_heads.rs +++ b/crates/net/p2p/src/peer_heads.rs @@ -1,92 +1,115 @@ //! The head slot this node's peers report in `Status`, published on the -//! [`Store`](ethlambda_storage::Store) as the sync tracker's evidence that the -//! network is ahead of this node. +//! [`Store`] as the sync tracker's evidence that the network is ahead of this +//! node. //! //! Lighthouse and Prysm both decide "I am behind" against what their peers //! report rather than against blocks they have imported: a node whose import //! has stalled, or that is still catching up, has imported nothing recent, so -//! its own blocks cannot tell it the network moved on. A peer's head is a -//! claim, not a verified block, so it is bounded three ways: a head past the -//! wall clock is dropped (both clients refuse such a `Status` at the -//! handshake), several peers must agree before the value moves, and the sync -//! tracker also requires the node's own head to trail the clock, so no claim -//! can stop a node that is current. +//! its own blocks cannot tell it the network moved on. The published value is +//! the highest report, as in Lighthouse, where a single `Advanced` peer starts +//! a range sync. A peer's head is a claim, not a verified block, so it is +//! bounded four ways: a head past the wall clock is dropped (both clients +//! refuse such a `Status` at the handshake); a head this node has already +//! imported is not a lead, since it says nothing about blocks the node lacks +//! (a peer on a lighter fork whose tip we hold is not ahead of us); the sync +//! tracker requires the node's own head to trail the clock, so no claim can +//! stop a node that is current; and the tracker gives up on a lead its head +//! does not advance towards (`SyncThresholds::stall_window`), so a claim +//! nobody delivers costs one window, not a stall's whole length. //! //! Peers send `Status` once per connection, so a claim would go stale while -//! the peer stays connected. [`STATUS_REFRESH_SLOTS_LEAN`] and -//! [`STATUS_REFRESH_SLOTS_BEACON`] pace a re-request, the way Lighthouse -//! re-statuses every 300 s and Prysm every half epoch. The pace is per peer, -//! counted from that peer's own last exchange, and no slot asks more than its -//! share of the peers (see [`PeerHeads::due_for_refresh`]), so a node with -//! hundreds of peers spreads the requests over the interval instead of sending -//! them all at once. +//! the peer stays connected. `SyncThresholds::status_refresh_slots` paces a +//! re-request, the way Lighthouse re-statuses every 300 s and Prysm every half +//! epoch. The pace is per peer, counted from that peer's own last exchange, +//! and no slot asks more than its share of the peers (see +//! [`PeerHeads::due_for_refresh`]), so a node with hundreds of peers spreads +//! the requests over the interval instead of sending them all at once. use std::collections::{HashMap, HashSet}; -use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; -use ethlambda_types::time::unix_now_ms; +use ethlambda_network_api::sync::SyncThresholds; +use ethlambda_storage::{Chain, Store}; +use ethlambda_types::primitives::H256; use libp2p::PeerId; use tracing::debug; use crate::P2PServer; - -/// From this many reporting peers on, the network head is the second-highest -/// report rather than the highest, so one peer alone cannot move it. Below -/// it the highest report counts: with one or two peers, a stale or still -/// syncing peer must not veto a fresher one, and the tracker's clock condition -/// already keeps a lone liar from idling a node that is current. -pub(crate) const PEER_HEAD_QUORUM: usize = 3; +use crate::beacon::handler::StatusVersion; /// Slots a `Status` head may sit past the wall clock and still count: Lighthouse's /// `FUTURE_SLOT_TOLERANCE` and Prysm's `maxFutureStatusHeadSlot`. pub(crate) const FUTURE_HEAD_TOLERANCE: u64 = 1; -/// How often a lean node re-asks each connected peer for its `Status`: every -/// lead the lean sync tracker tolerates (`SyncThresholds::LEAN.peer_lead`), so -/// a claim is never staler than the gap it is meant to detect. -pub(crate) const STATUS_REFRESH_SLOTS_LEAN: u64 = 4; - -/// How often a beacon node re-asks each connected peer for its `Status`: -/// Prysm's half epoch (`maintainPeerStatuses`). -pub(crate) const STATUS_REFRESH_SLOTS_BEACON: u64 = SLOTS_PER_EPOCH / 2; - -/// How far ahead a peer answering a *refresh* must be before its answer starts -/// a range sync, per chain. A handshake answer starts one at any lead, as it -/// always has; a refresh lands every few slots on a node that is keeping up -/// with gossip, where a peer one slot ahead is ordinary and a one-block range -/// request would only race gossip. Lighthouse makes a peer `Advanced`, the one -/// kind its range sync follows, only beyond `SLOT_IMPORT_TOLERANCE` (32). -pub(crate) const REFRESH_RANGE_SYNC_LEAD_LEAN: u64 = 4; -/// The beacon counterpart of [`REFRESH_RANGE_SYNC_LEAD_LEAN`]. -pub(crate) const REFRESH_RANGE_SYNC_LEAD_BEACON: u64 = SLOTS_PER_EPOCH; - -/// Every connected peer's latest `Status` head. +/// This chain's sync thresholds. The chain actor's tracker judges the gate by +/// the same ones, which is why they live in `ethlambda-network-api`: the +/// refresh cadence ([`SyncThresholds::status_refresh_slots`]) and the lead a +/// refresh answer needs before it starts a range sync +/// ([`SyncThresholds::peer_lead`]) cannot drift from the gate they serve. +pub(crate) fn thresholds(store: &Store) -> SyncThresholds { + match store.chain() { + Chain::Lean => SyncThresholds::LEAN, + Chain::Beacon => SyncThresholds::BEACON, + } +} + +/// What became of a reported head. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Report { + /// Counted towards the network head. + Counted, + /// A block this node has already imported: the peer is not ahead of it, + /// whatever its slot, so it does not count. + Held, + /// Past the wall clock: dropped, along with the peer's earlier report. + Future, +} + +/// Every connected peer's latest `Status` head, and the per-peer state of the +/// `Status` exchange that keeps it fresh. #[derive(Debug, Default)] pub(crate) struct PeerHeads { - /// The head slot each peer last reported, while it stays connected. + /// The head slot each peer last reported, while it stays connected and + /// the head is not one this node holds. heads: HashMap, /// Peers whose outstanding `Status` request is a refresh rather than the - /// handshake; see [`REFRESH_RANGE_SYNC_LEAD_LEAN`]. + /// handshake; a refresh answer needs a larger lead to start a range sync. refreshing: HashSet, /// The wall-clock slot of each connected peer's last `Status` exchange, in /// either direction: the last time it told us its head, or we asked. last_exchange: HashMap, + /// Peers whose last beacon `Status` came from a fork digest this node does + /// not hold, so a refresh that finds one still there is not a new event. + foreign_digest: HashSet, + /// The `Status` version each beacon peer last exchanged in, so a refresh + /// asks in the version the peer is known to speak. + status_versions: HashMap, } impl PeerHeads { /// Record `peer`'s head, unless it sits past `wall_slot` by more than - /// [`FUTURE_HEAD_TOLERANCE`]; a peer whose head is dropped loses any - /// earlier report too, since a peer that claims the future is not one to - /// believe about the present. Returns whether the head was recorded. - /// Either way the exchange postpones `peer`'s next refresh. - pub(crate) fn record(&mut self, peer: PeerId, head_slot: u64, wall_slot: u64) -> bool { + /// [`FUTURE_HEAD_TOLERANCE`] or names a block this node already `held`. + /// A peer whose head is dropped loses any earlier report too: a peer that + /// claims the future is not one to believe about the present, and one + /// whose head we hold is no longer ahead of us. Either way the exchange + /// postpones `peer`'s next refresh. + pub(crate) fn record( + &mut self, + peer: PeerId, + head_slot: u64, + wall_slot: u64, + held: bool, + ) -> Report { self.last_exchange.insert(peer, wall_slot); if head_slot > wall_slot.saturating_add(FUTURE_HEAD_TOLERANCE) { self.heads.remove(&peer); - return false; + return Report::Future; + } + if held { + self.heads.remove(&peer); + return Report::Held; } self.heads.insert(peer, head_slot); - true + Report::Counted } /// Forget a disconnected peer. @@ -94,20 +117,14 @@ impl PeerHeads { self.heads.remove(peer); self.refreshing.remove(peer); self.last_exchange.remove(peer); + self.foreign_digest.remove(peer); + self.status_versions.remove(peer); } - /// The network's head as the peers report it: the second-highest report - /// once [`PEER_HEAD_QUORUM`] peers have reported, the highest before that, - /// `None` with no report at all. + /// The network's head as the peers report it: the highest counted report, + /// `None` with none at all. pub(crate) fn network_head(&self) -> Option { - let mut heads: Vec = self.heads.values().copied().collect(); - heads.sort_unstable_by(|a, b| b.cmp(a)); - let rank = if heads.len() >= PEER_HEAD_QUORUM { - 1 - } else { - 0 - }; - heads.get(rank).copied() + self.heads.values().copied().max() } /// The connected peers to re-ask for their `Status` at `wall_slot`, at @@ -156,19 +173,56 @@ impl PeerHeads { pub(crate) fn take_refresh(&mut self, peer: &PeerId) -> bool { self.refreshing.remove(peer) } + + /// Note whether `peer`'s latest beacon `Status` came from a fork digest + /// this node holds. Returns whether the peer has only now moved to one it + /// does not: the event worth a warning and a count, once, rather than on + /// every refresh of a connection that has been on a foreign digest all + /// along. + pub(crate) fn note_digest(&mut self, peer: PeerId, held: bool) -> bool { + if held { + self.foreign_digest.remove(&peer); + false + } else { + self.foreign_digest.insert(peer) + } + } + + /// Note the `Status` version `peer` last exchanged in. + pub(crate) fn note_status_version(&mut self, peer: PeerId, version: StatusVersion) { + self.status_versions.insert(peer, version); + } + + /// The `Status` version to ask `peer` in: the one it last exchanged in, + /// or `status/1`, the handshake's own opening, before it has. + pub(crate) fn status_version(&self, peer: &PeerId) -> StatusVersion { + self.status_versions + .get(peer) + .copied() + .unwrap_or(StatusVersion::V1) + } } -/// The slot the wall clock is in, on the store's time grid. -pub(crate) fn wall_clock_slot(server: &P2PServer) -> u64 { - let grid = server.store.config().time_grid(); - unix_now_ms().saturating_sub(grid.genesis_time_ms()) / grid.milliseconds_per_slot +/// Whether this node has already imported `root`: its own head, or any block +/// it holds a post-state for. A pending or held block has no post-state, so it +/// does not count, the same as a block the node has never seen. +fn holds_block(store: &Store, root: &H256) -> bool { + store.head().is_ok_and(|head| head == *root) || store.has_state(root).unwrap_or(false) } /// Record the head `peer` reported in a `Status` (request or answer) and -/// republish the network head. -pub(crate) fn observe(server: &mut P2PServer, peer: PeerId, head_slot: u64) { - let wall_slot = wall_clock_slot(server); - if !server.peer_heads.record(peer, head_slot, wall_slot) { +/// republish the network head. Returns what became of it, so an answer whose +/// head this node already holds can skip looking for a range to sync. +pub(crate) fn observe( + server: &mut P2PServer, + peer: PeerId, + head_slot: u64, + head_root: H256, +) -> Report { + let wall_slot = crate::wall_clock_slot(&server.store); + let held = holds_block(&server.store, &head_root); + let report = server.peer_heads.record(peer, head_slot, wall_slot, held); + if report == Report::Future { debug!( %peer, peer_head_slot = head_slot, @@ -177,6 +231,7 @@ pub(crate) fn observe(server: &mut P2PServer, peer: PeerId, head_slot: u64) { ); } publish(server); + report } /// Forget a disconnected peer's head and republish the network head. @@ -195,39 +250,64 @@ fn publish(server: &P2PServer) { mod tests { use super::*; + const BEACON_REFRESH: u64 = SyncThresholds::BEACON.status_refresh_slots; + #[test] fn no_report_is_no_network_head() { assert_eq!(PeerHeads::default().network_head(), None); } #[test] - fn below_quorum_the_highest_report_counts() { - // Two peers, one of them stale: the fresher one is not vetoed. + fn the_highest_report_counts() { + // A stale or still-syncing peer does not veto a fresher one. let mut heads = PeerHeads::default(); - heads.record(PeerId::random(), 100, 100); - heads.record(PeerId::random(), 40, 100); + heads.record(PeerId::random(), 100, 100, false); + heads.record(PeerId::random(), 40, 100, false); assert_eq!(heads.network_head(), Some(100)); } #[test] - fn at_quorum_one_peer_cannot_move_the_head() { + fn one_current_peer_among_stale_ones_is_enough() { + // Finding: nodes restarted together, behind, and peered with each + // other and one healthy peer. Their own reports must not outvote the + // one peer that is current; the sync tracker's stall window is what + // bounds a single peer that lies. let mut heads = PeerHeads::default(); - heads.record(PeerId::random(), 100, 100); - heads.record(PeerId::random(), 50, 100); - heads.record(PeerId::random(), 50, 100); - assert_eq!(heads.network_head(), Some(50)); - // A second peer at the clock is enough. - heads.record(PeerId::random(), 100, 100); - assert_eq!(heads.network_head(), Some(100)); + heads.record(PeerId::random(), 600, 600, false); + heads.record(PeerId::random(), 50, 600, false); + heads.record(PeerId::random(), 50, 600, false); + assert_eq!(heads.network_head(), Some(600)); + } + + #[test] + fn a_head_this_node_holds_is_not_a_lead() { + // Finding: peers on a lighter fork whose tip we imported report a + // higher slot than our head, but we are not missing anything from + // them. + let mut heads = PeerHeads::default(); + let on_lighter_fork = PeerId::random(); + assert_eq!(heads.record(on_lighter_fork, 110, 110, true), Report::Held); + assert_eq!(heads.network_head(), None); + // A peer that had reported an unknown head is dropped once its head is + // one we hold. + heads.record(on_lighter_fork, 109, 110, false); + heads.record(on_lighter_fork, 110, 110, true); + assert_eq!(heads.network_head(), None); } #[test] fn a_head_past_the_clock_is_dropped_with_the_peers_earlier_report() { let mut heads = PeerHeads::default(); let liar = PeerId::random(); - assert!(heads.record(liar, 90, 100)); - assert!(heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE, 100)); - assert!(!heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE + 1, 100)); + assert_eq!(heads.record(liar, 90, 100, false), Report::Counted); + assert_eq!( + heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE, 100, false), + Report::Counted + ); + assert_eq!( + heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE + 1, 100, false), + Report::Future + ); assert_eq!(heads.network_head(), None); } @@ -236,8 +316,8 @@ mod tests { // Latest, not highest: a peer that reorged to a lower head is believed. let mut heads = PeerHeads::default(); let peer = PeerId::random(); - heads.record(peer, 100, 100); - heads.record(peer, 95, 100); + heads.record(peer, 100, 100, false); + heads.record(peer, 95, 100, false); assert_eq!(heads.network_head(), Some(95)); } @@ -245,15 +325,42 @@ mod tests { fn a_disconnected_peer_is_forgotten() { let mut heads = PeerHeads::default(); let peer = PeerId::random(); - heads.record(peer, 100, 100); + heads.record(peer, 100, 100, false); heads.begin_refresh(peer); + heads.note_digest(peer, false); + heads.note_status_version(peer, StatusVersion::V2); heads.forget(&peer); assert_eq!(heads.network_head(), None); assert!(!heads.take_refresh(&peer)); + assert_eq!(heads.status_version(&peer), StatusVersion::V1); + // A reconnect on a foreign digest is news again. + assert!(heads.note_digest(peer, false)); // A reconnect is a fresh handshake, not a refresh that is overdue. assert!(heads.due_for_refresh([peer], 1_000, 4).is_empty()); } + #[test] + fn a_foreign_digest_is_news_once_per_move() { + // Finding: the refresh re-asks peers on a digest we do not hold every + // half epoch; only the move onto that digest is worth a warning. + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + assert!(heads.note_digest(peer, false)); + assert!(!heads.note_digest(peer, false)); + // Back on a held digest, then off again: a new move. + assert!(!heads.note_digest(peer, true)); + assert!(heads.note_digest(peer, false)); + } + + #[test] + fn a_refresh_asks_in_the_version_the_peer_speaks() { + let mut heads = PeerHeads::default(); + let peer = PeerId::random(); + assert_eq!(heads.status_version(&peer), StatusVersion::V1); + heads.note_status_version(peer, StatusVersion::V2); + assert_eq!(heads.status_version(&peer), StatusVersion::V2); + } + #[test] fn a_peer_is_re_asked_one_interval_after_its_last_exchange() { let mut heads = PeerHeads::default(); @@ -274,7 +381,7 @@ mod tests { let mut heads = PeerHeads::default(); let peer = PeerId::random(); heads.due_for_refresh([peer], 10, 4); - heads.record(peer, 12, 12); + heads.record(peer, 12, 12, false); assert!(heads.due_for_refresh([peer], 14, 4).is_empty()); assert_eq!(heads.due_for_refresh([peer], 16, 4), vec![peer]); } @@ -282,16 +389,21 @@ mod tests { #[test] fn peers_that_connected_together_are_re_asked_across_an_interval() { // A beacon node's default 200 peers, all connected at slot 0 and due - // together at slot 16: no slot asks more than 13 of them, and every - // one is asked within the interval after they fell due. + // together one interval later: no slot asks more than its share of + // them, and every one is asked within the interval after they fell + // due. let peers: Vec = (0..200).map(|_| PeerId::random()).collect(); + let share = peers.len().div_ceil(BEACON_REFRESH as usize); let mut heads = PeerHeads::default(); - heads.due_for_refresh(peers.iter().copied(), 0, STATUS_REFRESH_SLOTS_BEACON); + heads.due_for_refresh(peers.iter().copied(), 0, BEACON_REFRESH); let mut asked = HashSet::new(); - for slot in 16..32 { - let batch = - heads.due_for_refresh(peers.iter().copied(), slot, STATUS_REFRESH_SLOTS_BEACON); - assert!(batch.len() <= 13, "slot {slot} asked {} peers", batch.len()); + for slot in BEACON_REFRESH..2 * BEACON_REFRESH { + let batch = heads.due_for_refresh(peers.iter().copied(), slot, BEACON_REFRESH); + assert!( + batch.len() <= share, + "slot {slot} asked {} peers", + batch.len() + ); asked.extend(batch); } assert_eq!(asked.len(), peers.len()); diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 57556dd67..9bfe7d645 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -53,7 +53,7 @@ use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; use crate::lean::messages::{BlocksByRootRequest, RequestedBlockRoots, Status}; use crate::lean::protocols::MAX_REQUEST_BLOCKS; -use crate::peer_heads; +use crate::peer_heads::{self, Report}; use crate::req_resp::messages::BlocksByRangeRequest; use crate::{ BACKOFF_MULTIPLIER, CustodyWait, INITIAL_BACKOFF_MS, MAX_FETCH_RETRIES, MAX_SYNC_RANGE, @@ -515,12 +515,17 @@ pub async fn handle_req_resp_message( // untracked handshake or an id this process never recorded. // Only the former is worth acting on: a peer that has dropped // `status/1` refuses the stream outright, and without a - // handshake it never enters the sync peer set at all. + // handshake it never enters the sync peer set at all. Only a + // refused `status/1`: the retry is `status/2`, so retrying a + // refused `status/2` (a refresh asks in whichever version the + // peer last answered) would only be refused again, forever. None => { - if matches!( - error, - request_response::OutboundFailure::UnsupportedProtocols - ) { + if request_id.protocol == ReqRespProtocol::BeaconStatusV1 + && matches!( + error, + request_response::OutboundFailure::UnsupportedProtocols + ) + { crate::beacon::handler::retry_status_on_other_version(server, peer).await; } } @@ -599,50 +604,59 @@ async fn handle_lean_status_request( peer: PeerId, ) { trace!(finalized_slot=%request.finalized.slot, head_slot=%request.head.slot, "Received status request from peer {peer}"); - peer_heads::observe(server, peer, request.head.slot); + peer_heads::observe(server, peer, request.head.slot, request.head.root); let our_status = build_status(&server.store); respond(server, channel, ResponsePayload::LeanStatus(our_status)); } +/// Record the peer's answer to our `Status`, and start or extend range sync +/// when [`status_sync_target`] says it leaves this node behind. +/// +/// Measured from the higher of the store's head and +/// [`P2PServer::range_fetched_through`], as beacon's is: a range session that +/// handed its blocks to the chain actor and completed leaves them queued +/// behind the actor's mailbox, and an answer measured from the head alone +/// would fetch them all again. async fn handle_lean_status_response(server: &mut P2PServer, status: Status, peer: PeerId) { trace!(finalized_slot=%status.finalized.slot, head_slot=%status.head.slot, "Received status response from peer {peer}"); let refresh = server.peer_heads.take_refresh(&peer); - peer_heads::observe(server, peer, status.head.slot); - - let our_head_slot = server.store.head_slot(); - if status.head.slot <= our_head_slot { + // A head this node already imported leaves nothing to fetch from this peer. + if peer_heads::observe(server, peer, status.head.slot, status.head.root) == Report::Held { return; } - // A refresh answer only updates the peer's head unless it shows this node - // well behind; see `REFRESH_RANGE_SYNC_LEAD_LEAN`. - if refresh && status.head.slot - our_head_slot <= peer_heads::REFRESH_RANGE_SYNC_LEAD_LEAN { + + let our_head_slot = server.store.head_slot(); + let refresh_lead = refresh.then(|| peer_heads::thresholds(&server.store).peer_lead); + let Some(target_range) = status_sync_target( + server.range_fetched_through, + our_head_slot, + status.head.slot, + refresh_lead, + ) else { return; - } - let gap = status.head.slot - our_head_slot; + }; debug!( %peer, peer_head_slot = status.head.slot, local_head_slot = our_head_slot, - slot_gap = gap, - "Peer status head is ahead of local head" + fetched_through = server.range_fetched_through, + start_slot = target_range.start, + end_exclusive = target_range.end, + "Peer status head is ahead of what has been fetched" ); - let start_slot = our_head_slot.saturating_add(1); - let end_exclusive = start_slot.saturating_add(gap.min(MAX_SYNC_RANGE)); - + let start_slot = target_range.start; + let end_exclusive = target_range.end; match &mut server.range_sync_state { Some(state) => state.merge_peer(peer, status.head.slot, end_exclusive), None => { - server.range_sync_state = Some(RangeSyncState::new( - start_slot..end_exclusive, - peer, - status.head.slot, - )); + server.range_sync_state = + Some(RangeSyncState::new(target_range, peer, status.head.slot)); } } request_next_range_batch(server).await; - trace!(%peer, start_slot, gap, "Long-range sync: using BlocksByRange"); + trace!(%peer, start_slot, end_exclusive, "Long-range sync: using BlocksByRange"); } async fn handle_lean_blocks_by_root_request( @@ -843,6 +857,9 @@ async fn handle_lean_blocks_by_range_response( continue; } + // Highest slot handed to the actor, not the highest imported; see + // `range_sync_target`. + server.range_fetched_through = server.range_fetched_through.max(slot); let block_root = block.message.hash_tree_root(); if let Err(err) = blockchain.new_block( SignedBeaconBlock::Lean(block), @@ -1720,30 +1737,48 @@ async fn handle_status_request( // Re-taken as a shared borrow of just `server.wire`, now that the // `refuse` above (which needs `&mut server`) is behind us. let wire = server.wire.beacon().expect("checked above"); + let peer_digest = peer_status.fork_digest(); + let our_digest = wire.fork_digest; + let held = wire.holds_digest(peer_digest); + // Recorded only from a digest this node syncs from, as an answer is + // below: a peer still on a superseded digest is not on the chain being + // followed. + let syncable = held && transition::is_syncable_digest(wire, peer_digest, unix_now_ms()); - if !wire.holds_digest(peer_status.fork_digest()) { + server + .peer_heads + .note_status_version(peer, StatusVersion::of(&peer_status)); + if !held { // Not grounds for closing the stream: the peer told us who it is and we // answer honestly. Counting it is how a digest that has moved under us - // becomes visible. - warn!( - %peer, - peer_digest = %hex::encode(peer_status.fork_digest()), - our_digest = %hex::encode(wire.fork_digest), - "Peer is on another fork digest" - ); - metrics::inc_beacon_status_digest_mismatch(); + // becomes visible, so it is counted when the peer moves onto it, not + // again on every `Status` the peer sends from there. + if server.peer_heads.note_digest(peer, false) { + warn!( + %peer, + peer_digest = %hex::encode(peer_digest), + our_digest = %hex::encode(our_digest), + "Peer is on another fork digest" + ); + metrics::inc_beacon_status_digest_mismatch(); + } else { + debug!(%peer, peer_digest = %hex::encode(peer_digest), "Peer still on another fork digest"); + } } else { + server.peer_heads.note_digest(peer, true); trace!( %peer, peer_head_slot = peer_status.head_slot(), peer_finalized_epoch = peer_status.finalized_epoch(), "Beacon status received" ); - // Recorded only from a digest this node syncs from, as an answer is - // below: a peer still on a superseded digest is not on the chain being - // followed. - if transition::is_syncable_digest(wire, peer_status.fork_digest(), unix_now_ms()) { - peer_heads::observe(server, peer, peer_status.head_slot()); + if syncable { + peer_heads::observe( + server, + peer, + peer_status.head_slot(), + peer_status.head_root(), + ); } } let wire = server.wire.beacon().expect("checked above"); @@ -1805,7 +1840,7 @@ fn handle_goodbye(peer: PeerId, goodbye: Goodbye) { /// How far this node has the chain is the higher of two floors, because each /// one misses blocks the other sees: /// -/// - `fetched_through` is `server.beacon_fetched_through`, the highest slot a +/// - `fetched_through` is `server.range_fetched_through`, the highest slot a /// range answer has handed to the chain actor. Delivery is a message and /// import is work, so the store's head lags a delivered batch by the whole /// actor mailbox. Driven off the store's head alone, the live follower kept @@ -1819,9 +1854,9 @@ fn handle_goodbye(peer: PeerId, goodbye: Goodbye) { /// which held a gossip block 4.2 s in the chain actor's queue. The head is /// a safe floor, since a head block's whole ancestry is already imported. /// -/// Bounded by [`MAX_SYNC_RANGE`], the same ceiling -/// `handle_lean_status_response` bounds its own request span by. -fn beacon_sync_target( +/// Bounded by [`MAX_SYNC_RANGE`], the same ceiling both chains' range syncs +/// are bounded by. +fn range_sync_target( fetched_through: u64, head_slot: u64, peer_head_slot: u64, @@ -1836,14 +1871,36 @@ fn beacon_sync_target( Some(start_slot..end_exclusive) } -/// Record the peer's answer to our handshake, and start or extend the -/// anchor-to-head range sync when [`beacon_sync_target`] says it leaves this +/// The range a peer's `Status` answer should start or extend a range sync +/// over, on either chain: [`range_sync_target`], unless the answer is to a +/// *refresh* (`refresh_lead` is then `SyncThresholds::peer_lead`) and leads +/// what this node has fetched by no more than that. +/// +/// A handshake answer starts a range sync at any lead, as it always has. A +/// refresh lands every few slots on a node that is keeping up with gossip, +/// where a peer a slot ahead is ordinary and a one-block range request would +/// only race gossip; past the lead the sync gate reads as behind, the refresh +/// is what notices, and nothing else would start the sync. +fn status_sync_target( + fetched_through: u64, + head_slot: u64, + peer_head_slot: u64, + refresh_lead: Option, +) -> Option> { + let target = range_sync_target(fetched_through, head_slot, peer_head_slot)?; + match refresh_lead { + Some(lead) if target.end - target.start <= lead => None, + _ => Some(target), + } +} + +/// Record the peer's answer to our `Status`, and start or extend the +/// anchor-to-head range sync when [`status_sync_target`] says it leaves this /// node behind. /// /// The beacon counterpart of `handle_lean_status_response`: same merge-or- /// create on `range_sync_state` and the same kick of the first batch, -/// differing only in what "behind" is measured against (see -/// [`beacon_sync_target`]) and in which function sends the batch +/// differing only in which function sends the batch /// ([`request_next_beacon_range_batch`], so the beacon protocol id and /// `MAX_REQUEST_BLOCKS_DENEB` apply). async fn handle_status_response( @@ -1856,24 +1913,39 @@ async fn handle_status_response( let Some(wire) = server.wire.beacon() else { return; }; - if !wire.holds_digest(status.fork_digest()) { - warn!( - %peer, - peer_digest = %hex::encode(status.fork_digest()), - our_digest = %hex::encode(wire.fork_digest), - "Handshake answered from another fork digest" - ); - metrics::inc_beacon_status_digest_mismatch(); + let peer_digest = status.fork_digest(); + let our_digest = wire.fork_digest; + let held = wire.holds_digest(peer_digest); + let syncable = held && transition::is_syncable_digest(wire, peer_digest, unix_now_ms()); + server + .peer_heads + .note_status_version(peer, StatusVersion::of(&status)); + if !held { + // Counted when the peer moves onto the digest, not again on every + // refresh of a connection that stays there; see the request side. + if server.peer_heads.note_digest(peer, false) { + warn!( + %peer, + peer_digest = %hex::encode(peer_digest), + our_digest = %hex::encode(our_digest), + refresh, + "Handshake answered from another fork digest" + ); + metrics::inc_beacon_status_digest_mismatch(); + } else { + debug!(%peer, peer_digest = %hex::encode(peer_digest), "Peer still on another fork digest"); + } return; } + server.peer_heads.note_digest(peer, true); // Answered at any held digest, but only synced from on the current one (or // the next, within clock skew): after a boundary, a peer still on the old // digest has not upgraded, and its chain is not the one being followed. - if !transition::is_syncable_digest(wire, status.fork_digest(), unix_now_ms()) { + if !syncable { debug!( %peer, - peer_digest = %hex::encode(status.fork_digest()), - our_digest = %hex::encode(wire.fork_digest), + peer_digest = %hex::encode(peer_digest), + our_digest = %hex::encode(our_digest), "Not syncing from a peer on a superseded fork digest" ); return; @@ -1886,28 +1958,29 @@ async fn handle_status_response( refresh, "Beacon handshake complete" ); - peer_heads::observe(server, peer, peer_head_slot); + // A head this node already imported leaves nothing to fetch from this + // peer, and saves the head decode below: on a node keeping up, that is + // nearly every refresh answer. + if peer_heads::observe(server, peer, peer_head_slot, status.head_root()) == Report::Held { + return; + } - // A whole-block decode per handshake, the same one `build_status` already - // pays to send ours. + // A whole-block decode, the same one `build_status` pays to send ours. let head_slot = server.store.beacon_head().map_or(0, |(slot, _)| slot); - let Some(target_range) = - beacon_sync_target(server.beacon_fetched_through, head_slot, peer_head_slot) - else { + let refresh_lead = refresh.then(|| peer_heads::thresholds(&server.store).peer_lead); + let Some(target_range) = status_sync_target( + server.range_fetched_through, + head_slot, + peer_head_slot, + refresh_lead, + ) else { return; }; - // A refresh answer only updates the peer's head unless it shows this node - // an epoch behind; see `REFRESH_RANGE_SYNC_LEAD_BEACON`. - if refresh - && target_range.end - target_range.start <= peer_heads::REFRESH_RANGE_SYNC_LEAD_BEACON - { - return; - } debug!( %peer, peer_head_slot, - fetched_through = server.beacon_fetched_through, + fetched_through = server.range_fetched_through, head_slot, start_slot = target_range.start, end_exclusive = target_range.end, @@ -2672,10 +2745,10 @@ async fn handle_beacon_blocks_by_range_response( .await; // Highest slot *handed to* the actor, not the highest imported: see - // [`beacon_sync_target`] for why range sync reads this beside the store's + // [`range_sync_target`] for why range sync reads this beside the store's // head rather than the head alone. if let Some(highest) = highest_forwarded_slot { - server.beacon_fetched_through = server.beacon_fetched_through.max(highest); + server.range_fetched_through = server.range_fetched_through.max(highest); } if let Some(state) = &mut server.range_sync_state { state.complete_batch(end_slot); @@ -2719,6 +2792,7 @@ async fn retire_unanswered( pub(crate) mod tests { use super::*; use crate::ConnectionDirection; + use ethlambda_network_api::sync::SyncThresholds; use ethlambda_storage::{ForkCheckpoints, backend::InMemoryBackend}; use ethlambda_types::beacon::containers::gloas; use ethlambda_types::constants::DEFAULT_MILLISECONDS_PER_SLOT; @@ -2805,7 +2879,7 @@ pub(crate) mod tests { pending_column_requests: HashMap::new(), outbound_requests: HashMap::new(), range_sync_state: None, - beacon_fetched_through: 0, + range_fetched_through: 0, peer_heads: Default::default(), bootnode_addrs: HashMap::new(), node_names: HashMap::new(), @@ -3187,39 +3261,155 @@ pub(crate) mod tests { } #[test] - fn beacon_sync_target_keys_off_fetched_through_while_the_head_lags() { + fn range_sync_target_keys_off_fetched_through_while_the_head_lags() { // A batch already handed to the chain actor but not yet imported // leaves the store's own head behind `fetched_through`; the sync // target must still be computed from `fetched_through`. See - // `beacon_sync_target`'s own doc comment for why the store's head + // `range_sync_target`'s own doc comment for why the store's head // alone is the wrong signal to drive this off: it is what pulled // 11,213 blocks off the wire to import 100 on the live follower. - assert_eq!(beacon_sync_target(100, 40, 150), Some(101..151)); + assert_eq!(range_sync_target(100, 40, 150), Some(101..151)); // A peer at or behind what has already been fetched has nothing to // offer, regardless of what the store's own (possibly much lower) // head happens to be. - assert_eq!(beacon_sync_target(150, 40, 150), None); - assert_eq!(beacon_sync_target(150, 40, 100), None); + assert_eq!(range_sync_target(150, 40, 150), None); + assert_eq!(range_sync_target(150, 40, 100), None); } #[test] - fn beacon_sync_target_starts_after_a_head_gossip_moved_past_fetched_through() { + fn range_sync_target_starts_after_a_head_gossip_moved_past_fetched_through() { // A caught-up follower imports from gossip, which leaves // `fetched_through` at its last range batch. A peer one block ahead // is owed that one block, not every block since that batch. - assert_eq!(beacon_sync_target(100, 149, 150), Some(150..151)); + assert_eq!(range_sync_target(100, 149, 150), Some(150..151)); // A peer at or behind the head has nothing to offer, however far // behind `fetched_through` is. - assert_eq!(beacon_sync_target(100, 150, 150), None); - assert_eq!(beacon_sync_target(100, 160, 150), None); + assert_eq!(range_sync_target(100, 150, 150), None); + assert_eq!(range_sync_target(100, 160, 150), None); } #[test] - fn beacon_sync_target_is_bounded_by_max_sync_range() { - let target = beacon_sync_target(0, 0, u64::MAX).expect("peer is far ahead"); + fn range_sync_target_is_bounded_by_max_sync_range() { + let target = range_sync_target(0, 0, u64::MAX).expect("peer is far ahead"); assert_eq!(target, 1..(1 + MAX_SYNC_RANGE)); } + #[test] + fn a_handshake_answer_starts_a_range_sync_at_any_lead() { + assert_eq!(status_sync_target(0, 10, 11, None), Some(11..12)); + } + + #[test] + fn a_refresh_answer_starts_one_only_past_the_peer_lead() { + let lead = SyncThresholds::BEACON.peer_lead; + assert_eq!(status_sync_target(0, 100, 100 + lead, Some(lead)), None); + assert_eq!( + status_sync_target(0, 100, 101 + lead, Some(lead)), + Some(101..(102 + lead)) + ); + } + + #[test] + fn a_refresh_answer_measures_its_lead_past_what_was_fetched() { + // Finding: a lean catch-up handed slots up to 2000 to the chain actor, + // which has imported only up to 1200. A peer at 2050 is 50 past what + // was fetched, so the range starts at 2001, not back at 1201. + let lead = SyncThresholds::LEAN.peer_lead; + assert_eq!( + status_sync_target(2000, 1200, 2050, Some(lead)), + Some(2001..2051) + ); + assert_eq!( + status_sync_target(2000, 1200, 2000 + lead, Some(lead)), + None + ); + } + + fn lean_status_at(root: H256, slot: u64) -> Status { + Status { + finalized: Checkpoint::default(), + head: Checkpoint { root, slot }, + } + } + + #[tokio::test] + async fn a_lean_refresh_answer_starts_range_sync_only_past_the_lead() { + let mut server = unconnected_server().await; + let peer = PeerId::random(); + let lead = SyncThresholds::LEAN.peer_lead; + let unknown = H256::repeat_byte(0x11); + + server.peer_heads.begin_refresh(peer); + handle_lean_status_response(&mut server, lean_status_at(unknown, lead), peer).await; + assert!(server.range_sync_state.is_none()); + // The answer still counts towards the network head the gate reads. + assert_eq!(server.store.network_head(), Some(lead)); + + server.peer_heads.begin_refresh(peer); + handle_lean_status_response(&mut server, lean_status_at(unknown, lead + 1), peer).await; + let state = server.range_sync_state.as_ref().expect("past the lead"); + assert_eq!(state.current_range, 1..(lead + 2)); + } + + #[tokio::test] + async fn a_lean_status_answer_skips_blocks_already_handed_to_the_chain() { + // Finding: lean had no fetched-through watermark, so every answer + // during a catch-up re-requested blocks still queued for import. + let mut server = unconnected_server().await; + let peer = PeerId::random(); + let unknown = H256::repeat_byte(0x11); + server.range_fetched_through = 2000; + + handle_lean_status_response(&mut server, lean_status_at(unknown, 2000), peer).await; + assert!(server.range_sync_state.is_none()); + + handle_lean_status_response(&mut server, lean_status_at(unknown, 2050), peer).await; + let state = server + .range_sync_state + .as_ref() + .expect("past what was fetched"); + assert_eq!(state.current_range.start, 2001); + } + + #[tokio::test] + async fn a_lean_answer_naming_a_block_this_node_holds_is_no_lead() { + // Finding: a peer whose head we already imported (here our own head, + // under any slot it likes) is not ahead of us, so it neither moves the + // network head nor starts a range sync. + let mut server = unconnected_server().await; + let peer = PeerId::random(); + let our_head = server.store.head().expect("head exists"); + + handle_lean_status_response(&mut server, lean_status_at(our_head, 50), peer).await; + assert!(server.range_sync_state.is_none()); + assert_eq!(server.store.network_head(), None); + } + + #[tokio::test] + async fn the_refresh_asks_one_slots_share_of_the_due_peers_and_marks_them() { + let mut server = unconnected_server().await; + let interval = SyncThresholds::LEAN.status_refresh_slots; + let wall = crate::wall_clock_slot(&server.store); + let peers: Vec = (0..4).map(|_| PeerId::random()).collect(); + for &peer in &peers { + server + .connected_peers + .insert(peer, ConnectionDirection::Inbound); + // Last heard from an interval ago, so due now. + server.peer_heads.record(peer, 0, wall - interval, false); + } + + server.refresh_due_peer_statuses().await; + + // Marked as refreshes, so their answers need the lead to start a sync, + // and no more of them than this slot's share. + let asked = peers + .iter() + .filter(|peer| server.peer_heads.take_refresh(peer)) + .count(); + assert_eq!(asked, peers.len().div_ceil(interval as usize)); + } + #[test] fn a_range_session_is_exhausted_by_an_empty_range_or_an_empty_peer_set() { let peer = PeerId::random(); diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 9cdb313c0..a2ac6f4d7 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -1200,10 +1200,10 @@ pub struct Store { /// node's own API), which is why it lives on the `Store` all three share. /// See [`ObservedLiveness`]. Always empty on lean. observed_liveness: Arc, - /// The head slot this node's peers report, as P2P last aggregated it from - /// their `Status` messages, or [`NO_NETWORK_HEAD`] while no connected peer - /// has reported one. Written by P2P, read by the chain actor's sync - /// tracker; see [`Store::network_head`]. + /// The highest head slot this node's peers report in `Status` that it has + /// not already imported, as P2P last published it, or [`NO_NETWORK_HEAD`] + /// while no connected peer reports one. Written by P2P, read by the chain + /// actor's sync tracker; see [`Store::network_head`]. network_head: Arc, /// The total active balance per epoch, shared the way /// [`Self::committee_cache`] is: by the chain actor's state transition @@ -3304,15 +3304,16 @@ impl Store { &self.observed_liveness } - /// The head slot this node's peers report, or `None` while no connected - /// peer has reported one. + /// The highest head slot this node's peers report that it has not already + /// imported, or `None` while no connected peer reports one. /// /// The sync tracker's evidence that the network is ahead of this node. It /// is a peer *claim*, not a verified block: P2P drops a head past the wall - /// clock and asks several peers to agree before the value moves, and the - /// tracker additionally requires the node's own head to trail the clock, - /// so no claim alone can stop a node that is current. Runtime-only and - /// shared by every clone of this `Store`. + /// clock and does not count a head this node already holds, and the + /// tracker requires the node's own head to trail the clock, so no claim + /// can stop a node that is current, and gives up on a lead its head does + /// not advance towards, so no claim can hold a stalled node for long. + /// Runtime-only and shared by every clone of this `Store`. pub fn network_head(&self) -> Option { let slot = self.network_head.load(Ordering::Relaxed); (slot != NO_NETWORK_HEAD).then_some(slot) diff --git a/docs/architecture.md b/docs/architecture.md index e063a90b6..afb90cc05 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -169,17 +169,22 @@ flag unset on a placement that does not cover every subnet. `sync_status.rs` decides whether the node is behind. It is syncing, and stops attesting and proposing, when its head trails the slot clock **and** the head its peers report is ahead of its own, since a head derived from a partial view is not worth voting for. The -peers' head is each connected peer's latest `Status`, re-requested every few slots per peer -(spread over the interval, so a node with hundreds of peers never asks them all at once), which -p2p aggregates onto `Store::network_head` (`crates/net/p2p/src/peer_heads.rs`). A -hysteresis band stops the state from flapping at the boundary, and a network-wide stall -(nobody else is ahead either) leaves the node synced so its validators can help the chain -recover. This departs from leanSpec's rule; see `docs/spec_deviations.md`. The gate is -judged on every tick and again after every block arrival: on a catch-up, range sync puts -the whole span into the actor's mailbox at once and the tick waits behind all of it, so a -gate judged only on the tick kept reading synced until the catch-up was over. The same status -feeds the `lean_node_sync_status` metric and, through a shared controller, the -`/lean/v0/node/syncing` endpoint. +peers' head is the highest of each connected peer's latest `Status`, re-requested every few +slots per peer (spread over the interval, so a node with hundreds of peers never asks them +all at once), not counting a head the node has already imported, which p2p publishes on +`Store::network_head` (`crates/net/p2p/src/peer_heads.rs`). The thresholds, and the refresh +cadence they depend on, are `SyncThresholds` in `crates/net/api/src/sync.rs`, shared by both +actors. A hysteresis band stops the state from flapping at the boundary, and a network-wide +stall (nobody else is ahead either) leaves the node synced so its validators can help the +chain recover. A lead the head does not advance towards for a whole stall window reads as a +stall too, so a peer claiming a head it never serves cannot idle the node for longer than +that. This departs from leanSpec's rule; see `docs/spec_deviations.md`. The gate is judged +on every tick and again whenever the head can move between ticks (every lean block arrival, +every beacon head recompute, `INVALID` `forkchoiceUpdated` verdicts included): on a +catch-up, range sync puts the whole span into the actor's mailbox at once and the tick +waits behind all of it, so a gate judged only on the tick kept reading synced until the +catch-up was over. The same status feeds the `lean_node_sync_status` metric and, through a +shared controller, the `/lean/v0/node/syncing` endpoint. `--disable-duty-sync-gate` reduces the gate to observe-only. ### Missing parents diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 14b44ca72..ed7642200 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -739,7 +739,7 @@ Both paths now import. A fetched block reaches the chain actor as `BlockSource::Sync`, keeping every per-block range and root check on the way in, and the by-root path is what resolves a gossiped block's missing parent. -Range sync is paced by `P2PServer::beacon_fetched_through`, the highest slot +Range sync is paced by `P2PServer::range_fetched_through`, the highest slot handed to the actor, not by the store's head. Delivery is a message and import is work, so the store trails a delivered batch by the whole actor mailbox. Driven off the head, every resync tick re-requested the part still draining, @@ -752,8 +752,18 @@ follower that had caught up and then imported from gossip kept it at its last range batch, and every peer that connected later was asked again for every block since: 748 blocks in 66 bursts on one follower. A `Status` answer is therefore measured against the higher of the watermark and the store's head -(`beacon_sync_target`). The head is a safe floor, since a head block's whole -ancestry is already imported. +(`range_sync_target`). The head is a safe floor, since a head block's whole +ancestry is already imported. Lean range sync reads the same watermark, for +the same reason. + +Each peer is also re-asked for its `Status` every half epoch, so the head it +reported at the handshake does not go stale for the sync gate (see +`crates/net/p2p/src/peer_heads.rs`). An answer to that refresh starts a range +sync only when it leads the same floor by more than an epoch +(`status_sync_target`): a node keeping up with gossip sees peers a slot ahead +all the time, and a one-block range request would only race gossip. An answer +naming a head this node has already imported starts nothing, and skips the head +decode the comparison would cost. `Status` is store-derived: head from `Store::beacon_head`, the finalized checkpoint from `Store::beacon_finalized_checkpoint`, and @@ -808,7 +818,7 @@ as a query filter, so a `quic`-only record is invisible to it. | `lean_beacon_gossip_messages_total{topic,result}` | Gossip received, by topic and by `decoded` / `decode_failed` / `decompress_failed` / `unsupported_fork` | | `lean_beacon_gossip_validation_total{kind,outcome,reason}` | Gossip verdicts, now including `beacon_aggregate_and_proof` and `beacon_attestation`; see [metrics.md](./metrics.md#beacon-gossip-validation) | | `lean_beacon_gossip_verdict_expired_total{kind}` | Verdicts that came too late to propagate anything; should stay at zero | -| `lean_beacon_status_digest_mismatch_total` | Handshakes seen from another fork digest | +| `lean_beacon_status_digest_mismatch_total` | Peers whose `Status` moved onto another fork digest, once per move (not per refresh) | | `lean_beacon_fork_digest{digest}` | The digest computed at startup, as a label | | `lean_beacon_aggregate_decode_seconds` | Time spent decoding one aggregate off the wire | | `lean_beacon_aggregate_mailbox_wait_seconds` | How long an aggregate sat in the chain actor's mailbox | diff --git a/docs/rpc.md b/docs/rpc.md index 4fa487709..101237adc 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -190,7 +190,7 @@ The fork-choice tree from the finalized root, with LMD-GHOST weights computed ov { "is_syncing": false, "head_slot": 1024, "sync_distance": 1, "finalized_slot": 986 } ``` -`is_syncing` is the node's own stateful sync decision, updated each tick and after each block arrival: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. +`is_syncing` is the node's own stateful sync decision, updated each tick and after each block arrival: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false, except during the first 8 slots after startup, when a node whose head trails the clock reads as syncing whatever its first peers report. It is also false once the head has stayed flat for 16 slots while peers claim a lead, until the head next advances. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. `sync_distance` is the raw number of slots between the node's current head and the current wall-clock slot, computed per request. Because `is_syncing` carries hysteresis and stall handling and is not recomputed from `sync_distance`, the two can point different ways near the threshold or during a network-wide stall. diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 51976e44f..fbd32add9 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -41,15 +41,27 @@ head trails the wall clock **and** its peers report a head ahead of it. `test_no_extra_blocks_treats_isolation_as_network_stall` pins the case of a node 100 slots behind with nothing newer imported, which does its duties. - **ethlambda:** the node is syncing when its head is more than 4 slots behind - the wall clock and the head its peers report is more than 4 slots ahead of - its own, with the same band of 2 on both (`SyncThresholds::LEAN`, + the wall clock and the head its peers report is more than 1 slot ahead of its + own, with the same band of 2 on both (`SyncThresholds::LEAN`, + `crates/net/api/src/sync.rs`, judged in `crates/blockchain/src/sync_status.rs`). The peers' head comes from `Status` - (`crates/net/p2p/src/peer_heads.rs`): each connected peer's latest report, - re-requested every 4 slots, a head past the wall clock by more than a slot - dropped, and the second-highest report counted once 3 or more peers have - reported (the highest below that). With no report at all the node cannot be - behind anyone and reads as synced, except during an 8-slot hold at startup - for a node whose head already trails the clock. + (`crates/net/p2p/src/peer_heads.rs`): the highest of each connected peer's + latest report, re-requested every 2 slots, with a head past the wall clock by + more than a slot dropped and a head the node has already imported not counted + (the peer is not ahead of it, whatever its slot). The peer lead is smaller + than the clock lag because a report is up to 2 slots old: a node 5 slots + behind a producing network sees its peers' 2-slot-old heads 2 slots ahead. + With no report at all the node cannot be behind anyone and reads as synced, + except during an 8-slot hold at startup, during which the network is assumed + to be at least at the clock, whatever the first peers report. +- **Stall window:** a lead that does not turn into progress is given up on. A + node that reads as syncing on its peers' word with its head flat for 16 slots + (`SyncThresholds::stall_window`) treats the network as stalled and does its + duties, until its head next advances. A peer that claims a head it does not + serve (an empty answer, invalid blocks, no answer at all) therefore costs at + most that window, not the rest of a network stall. The window is sized above + what can legitimately hold a catching-up head still, and counts only from the + end of the startup hold. - **Why:** imported blocks cannot tell "the network is stalled" from "this node is behind". A node catching up has imported nothing recent either, so the specification's rule reports it synced for the whole catch-up and it proposes @@ -57,16 +69,22 @@ head trails the wall clock **and** its peers report a head ahead of it. decide against what their peers report, and require the head to trail the clock too, so no peer's claim can idle a node that is current. Comparing with peers also keeps a run of missed proposals (leanSpec #1194) from closing the - gate on every honest node, since none of them is behind another. + gate on every honest node, since none of them is more than propagation behind + another. - **Equivalence:** the two agree whenever no peer is further ahead of the node than its newest imported block. They differ for a node behind its peers with nothing newer imported, which the specification lets do its duties and this - node does not. + node does not, until its head has been flat for the stall window. They also + differ for a node restarted into a network stall, which the specification + lets do its duties at once and this node holds for the 8-slot startup window. The beacon node runs the same rule with Lighthouse's numbers (two epochs of -clock lag, one epoch of peer lead) and Prysm's `Status` refresh, every half -epoch (Lighthouse refreshes every 300 s). consensus-specs -has no sync gate, and the Beacon API leaves `is_syncing` to the client. +clock lag, one epoch of peer lead), a band of 4 slots that neither Lighthouse +nor Prysm has (this node re-judges on every block arrival, and would otherwise +flip twice a slot at the boundary), an epoch of startup hold and of stall +window, and Prysm's `Status` refresh, every half epoch (Lighthouse refreshes +every 300 s). consensus-specs has no sync gate, and the Beacon API leaves +`is_syncing` to the client. ## Attestation scoring on block building From ad9a587a857f9e1942359fb2ec33693c052395ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:07:36 -0300 Subject: [PATCH 6/6] feat(sync): end the startup hold once 5 peers have reported The startup hold assumed the network head was at the clock for its whole window (8 slots lean, an epoch beacon), so a node that started level with its peers kept its duties off for that long even after every peer had told it where the chain was. It now ends early once 5 connected peers have reported a Status head, and stays over once it has: the window remains the fallback for a node with fewer peers. A peer counts once its latest Status is not past the clock, including a head this node already holds, since a peer level with us is the slow-start case. A disconnect drops it from the count. After the hold the highest report is the network head, so one current peer among the five still shows the lead. p2p publishes the network head and the reporting-peer count together as a `PeerView` (`Store::set_peer_view` / `Store::peer_view`, replacing `set_network_head`). The head is written before the count and the count read before the head, so the tracker never sees the fifth peer counted without its head. --- crates/blockchain/src/lib.rs | 69 +++++++++++++++++++++----- crates/blockchain/src/sync_status.rs | 72 ++++++++++++++++++++++++++-- crates/net/api/src/sync.rs | 18 ++++++- crates/net/p2p/src/peer_heads.rs | 53 ++++++++++++++++++-- crates/storage/src/lib.rs | 2 +- crates/storage/src/store.rs | 70 ++++++++++++++++++++++----- docs/architecture.md | 4 +- docs/rpc.md | 2 +- docs/spec_deviations.md | 15 ++++-- 9 files changed, 265 insertions(+), 40 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 014d64547..7cd4a7b61 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3626,8 +3626,10 @@ impl BlockChainServer { /// Refresh the sync-status tracker and its two outputs (the /// `lean_node_sync_status` metric and [`SyncStatusController`]). /// - /// The network's head comes from [`Store::network_head`], the highest head - /// the node's peers report in `Status` that it does not already hold. The + /// The network's head comes from [`Store::peer_view`], the highest head + /// the node's peers report in `Status` that it does not already hold, + /// together with how many peers report one, which can end the tracker's + /// startup hold early. The /// tracker itself is chain-agnostic, with per-chain thresholds; see /// [`SyncThresholds`]. On a beacon follower, once the clock is past a fork /// the node does not follow (see [`Self::clock_is_past_followed_forks`]) @@ -3636,13 +3638,16 @@ impl BlockChainServer { fn judge_sync_status(&mut self, current_slot: u64, head_slot: u64) { let past_followed_forks = self.store.chain() == Chain::Beacon && Self::clock_is_past_followed_forks(&self.store.config(), current_slot, head_slot); + // One read for both halves, so the count that may end the startup + // hold comes with a head at least as new as the one published with it. + let peers = self.store.peer_view(); + self.sync_status.note_reporting_peers(peers.reporting_peers); let status = if past_followed_forks { self.sync_status .update_with_network_at_clock(current_slot, head_slot) } else { - let network_head = self.store.network_head(); self.sync_status - .update(current_slot, head_slot, network_head) + .update(current_slot, head_slot, peers.network_head) }; metrics::set_node_sync_status(status); self.sync_status_controller.set(status); @@ -4213,6 +4218,7 @@ mod tests { use super::*; use ethlambda_state_transition::beacon::fork_choice::seconds_to_milliseconds; + use ethlambda_storage::PeerView; use ethlambda_storage::backend::InMemoryBackend; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::{ @@ -5322,6 +5328,29 @@ mod tests { server.update_sync_status(0); } + #[test] + fn five_reporting_peers_published_by_p2p_end_the_startup_hold() { + // The tracker's first judgment is three epochs past the head, so the + // hold reads it as syncing. Five peers reporting heads it holds (no + // network head) end the hold well inside its epoch, and the peers' + // word, a stalled network, stands. + let mut server = syncing_server(None); + server.update_sync_status(THREE_EPOCHS); + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Syncing + ); + server.store.set_peer_view(PeerView { + network_head: None, + reporting_peers: SyncThresholds::BEACON.startup_peers, + }); + server.update_sync_status(THREE_EPOCHS + 1); + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Synced + ); + } + #[test] fn the_clock_reaching_a_followed_fork_does_not_force_syncing() { // Gloas is followed, so the fork rule stays out of it and the peers @@ -5329,7 +5358,10 @@ mod tests { // synced, the same as when no fork is scheduled. let mut server = syncing_server(Some(1)); past_startup(&mut server); - server.store.set_network_head(Some(0)); + server.store.set_peer_view(PeerView { + network_head: Some(0), + reporting_peers: 0, + }); server.update_sync_status(THREE_EPOCHS); assert_eq!( server.sync_status_controller.get(), @@ -5343,7 +5375,10 @@ mod tests { // the tracker deliberately reports as synced. let mut server = syncing_server(None); past_startup(&mut server); - server.store.set_network_head(Some(0)); + server.store.set_peer_view(PeerView { + network_head: Some(0), + reporting_peers: 0, + }); server.update_sync_status(THREE_EPOCHS); assert_eq!( server.sync_status_controller.get(), @@ -5359,7 +5394,10 @@ mod tests { // head. let mut server = syncing_server(None); past_startup(&mut server); - server.store.set_network_head(Some(THREE_EPOCHS)); + server.store.set_peer_view(PeerView { + network_head: Some(THREE_EPOCHS), + reporting_peers: 0, + }); server.update_sync_status(THREE_EPOCHS); assert_eq!( server.sync_status_controller.get(), @@ -5375,9 +5413,10 @@ mod tests { // follow that head too, so the recompute itself is what judges it. let mut server = syncing_server(None); past_startup(&mut server); - server - .store - .set_network_head(Some(server.wall_clock_slot())); + server.store.set_peer_view(PeerView { + network_head: Some(server.wall_clock_slot()), + reporting_peers: 0, + }); server.update_head_from_fork_choice(); assert_eq!( server.sync_status_controller.get(), @@ -5398,7 +5437,10 @@ mod tests { let orphan = || bare_block(1, H256::repeat_byte(0xee)); // Peers as stale as the node: a stall, synced. - server.store.set_network_head(Some(0)); + server.store.set_peer_view(PeerView { + network_head: Some(0), + reporting_peers: 0, + }); server.on_block(orphan(), ImportTimings::default()).await; assert_eq!( server.sync_status_controller.get(), @@ -5407,7 +5449,10 @@ mod tests { // A peer reports a head at the clock; the next arrival closes the gate. let wall_clock_slot = server.wall_clock_slot(); - server.store.set_network_head(Some(wall_clock_slot)); + server.store.set_peer_view(PeerView { + network_head: Some(wall_clock_slot), + reporting_peers: 0, + }); server.on_block(orphan(), ImportTimings::default()).await; assert_eq!( server.sync_status_controller.get(), diff --git a/crates/blockchain/src/sync_status.rs b/crates/blockchain/src/sync_status.rs index e09453a3c..1bc42946a 100644 --- a/crates/blockchain/src/sync_status.rs +++ b/crates/blockchain/src/sync_status.rs @@ -68,6 +68,13 @@ pub(crate) struct SyncStatusTracker { thresholds: SyncThresholds, /// The slot at which the startup hold ends; set by the first update. startup_ends_at: Option, + /// How many connected peers last reported a head; see + /// [`Self::note_reporting_peers`]. + reporting_peers: usize, + /// Whether [`SyncThresholds::startup_peers`] peers have reported, which + /// ends the startup hold for good: peers dropping off afterwards make the + /// view thinner, not the assumption true again. + heard_from_peers: bool, /// When the head last advanced; set by the first update. progress: Option, /// The wall-clock slot from which the node has been behind both the clock @@ -89,12 +96,22 @@ impl SyncStatusTracker { gate_duties, thresholds, startup_ends_at: None, + reporting_peers: 0, + heard_from_peers: false, progress: None, behind_since: None, stalled_at_head: None, } } + /// Note how many connected peers have reported a `Status` head, counting + /// those this node already holds (`PeerView::reporting_peers`). Read by + /// the next judgment, which ends the startup hold once it reaches + /// [`SyncThresholds::startup_peers`]. + pub(crate) fn note_reporting_peers(&mut self, reporting_peers: usize) { + self.reporting_peers = reporting_peers; + } + /// Re-judge the status at `current_slot` (the wall clock's slot). /// /// `network_head` is the highest head the node's peers report that it @@ -102,7 +119,8 @@ impl SyncStatusTracker { /// With no report the node cannot be behind anyone, so it reads as synced /// (a stalled or empty network, where its validators should keep /// working), except during the startup hold, which assumes the network is - /// at least at the clock. + /// at least at the clock until [`SyncThresholds::startup_peers`] peers + /// have reported or [`SyncThresholds::startup_grace`] slots have passed. pub(crate) fn update( &mut self, current_slot: u64, @@ -137,13 +155,26 @@ impl SyncStatusTracker { peer_lead: lead_threshold, hysteresis, startup_grace, + startup_peers, stall_window, .. } = self.thresholds; let startup_ends_at = *self .startup_ends_at .get_or_insert(current_slot.saturating_add(startup_grace)); - let starting_up = current_slot < startup_ends_at; + if !self.heard_from_peers && self.reporting_peers >= startup_peers { + self.heard_from_peers = true; + if current_slot < startup_ends_at { + info!( + current_slot, + head_slot, + network_head, + reporting_peers = self.reporting_peers, + "Enough peers have reported; ending the startup hold early" + ); + } + } + let starting_up = !self.heard_from_peers && current_slot < startup_ends_at; let progress = self.progress.get_or_insert(Progress { head_slot, @@ -172,7 +203,8 @@ impl SyncStatusTracker { // During the startup hold, assume the network is at least at the // clock: a report can add lead, never take it away, since the first - // peer to answer may be as stale as we are. + // peer to answer may be as stale as we are. Enough of them reporting + // ends the hold (see above); one does not. let assumed = starting_up || network_at_clock; let network_head = if assumed { Some(network_head.map_or(current_slot, |slot| slot.max(current_slot))) @@ -505,6 +537,40 @@ mod tests { assert_eq!(tracker.update(after_grace, 0, Some(0)), SyncStatus::Synced); } + #[test] + fn enough_reporting_peers_end_the_startup_hold() { + // A slow network start, not a stale node: ten slots behind the clock + // inside the hold, with every peer level with us (their heads are + // ones we hold, so there is no network head). One short of the count + // keeps the hold; reaching it lets the peers' word stand. + let mut tracker = SyncStatusTracker::new(true, LEAN); + assert_eq!(tracker.update(300, 290, None), SyncStatus::Syncing); + tracker.note_reporting_peers(LEAN.startup_peers - 1); + assert_eq!(tracker.update(301, 290, None), SyncStatus::Syncing); + tracker.note_reporting_peers(LEAN.startup_peers); + assert_eq!(tracker.update(302, 290, None), SyncStatus::Synced); + } + + #[test] + fn the_startup_hold_stays_over_once_enough_peers_reported() { + let mut tracker = SyncStatusTracker::new(true, LEAN); + tracker.note_reporting_peers(LEAN.startup_peers); + assert_eq!(tracker.update(300, 290, None), SyncStatus::Synced); + // Peers dropping off thin the view; they do not bring the hold back. + tracker.note_reporting_peers(1); + assert_eq!(tracker.update(301, 290, None), SyncStatus::Synced); + } + + #[test] + fn one_current_peer_among_the_reporters_still_shows_the_lead() { + // The hold is over, but a stale node is still caught: the highest + // report is the network head, so one current peer among the five is + // enough. + let mut tracker = SyncStatusTracker::new(true, BEACON); + tracker.note_reporting_peers(BEACON.startup_peers); + assert_eq!(tracker.update(1000, 100, Some(999)), SyncStatus::Syncing); + } + #[test] fn the_startup_hold_ends_without_peers() { // A node that never finds a peer stops holding after the grace period diff --git a/crates/net/api/src/sync.rs b/crates/net/api/src/sync.rs index f9faa58ce..a13503041 100644 --- a/crates/net/api/src/sync.rs +++ b/crates/net/api/src/sync.rs @@ -45,7 +45,8 @@ pub struct SyncThresholds { /// does not flap at the boundary. pub hysteresis: u64, /// Slots after the first judgment during which the node assumes the - /// network is at least at the wall clock, whatever its peers report. + /// network is at least at the wall clock, unless [`Self::startup_peers`] + /// peers have reported before then. /// /// The first judgment runs before any peer has answered a `Status`, and /// the first peer to answer may itself be behind, so without this a node @@ -55,6 +56,16 @@ pub struct SyncThresholds { /// node that never finds a peer, or restarts into a real stall, still /// falls back to treating the silence as a stall. pub startup_grace: u64, + /// Connected peers whose `Status` ends the startup hold early. + /// + /// Once this many have reported a head (one past the wall clock does + /// not count, one this node already holds does), the node judges by + /// their reports instead of the hold's assumption. One report is not + /// enough, since the first peer to answer may be as stale as this node, + /// but the highest of several is: a single current peer among them shows + /// the lead. A node with fewer peers than this keeps the whole + /// [`Self::startup_grace`] window. The same count on both chains. + pub startup_peers: usize, /// Slots the node may read as syncing on a peer-reported lead without its /// head advancing, before it treats the network as stalled after all. /// @@ -92,6 +103,7 @@ impl SyncThresholds { peer_lead: 1, hysteresis: 2, startup_grace: 8, + startup_peers: 5, stall_window: 16, status_refresh_slots: 2, }; @@ -104,13 +116,15 @@ impl SyncThresholds { /// follower importing about one block a slot right at the boundary would /// otherwise flip twice a slot. The startup hold and the stall window are /// an epoch: finding beacon peers through discovery can take a while, and - /// Prysm holds until it has peers, however long. The `Status` refresh is + /// Prysm holds until it has peers, however long; five peers reporting end + /// the hold sooner. The `Status` refresh is /// Prysm's half epoch (`maintainPeerStatuses`). pub const BEACON: Self = Self { clock_lag: 2 * SLOTS_PER_EPOCH, peer_lead: SLOTS_PER_EPOCH, hysteresis: 4, startup_grace: SLOTS_PER_EPOCH, + startup_peers: 5, stall_window: SLOTS_PER_EPOCH, status_refresh_slots: SLOTS_PER_EPOCH / 2, }; diff --git a/crates/net/p2p/src/peer_heads.rs b/crates/net/p2p/src/peer_heads.rs index 389c21b18..7bdba2321 100644 --- a/crates/net/p2p/src/peer_heads.rs +++ b/crates/net/p2p/src/peer_heads.rs @@ -28,7 +28,7 @@ use std::collections::{HashMap, HashSet}; use ethlambda_network_api::sync::SyncThresholds; -use ethlambda_storage::{Chain, Store}; +use ethlambda_storage::{Chain, PeerView, Store}; use ethlambda_types::primitives::H256; use libp2p::PeerId; use tracing::debug; @@ -71,6 +71,12 @@ pub(crate) struct PeerHeads { /// The head slot each peer last reported, while it stays connected and /// the head is not one this node holds. heads: HashMap, + /// Connected peers whose latest report was not past the wall clock, + /// including those whose head this node holds and so left out of + /// `heads`: the sync tracker ends its startup hold on how many peers have + /// spoken, and a peer that is level with us has spoken as much as one + /// that is ahead. + reporting: HashSet, /// Peers whose outstanding `Status` request is a refresh rather than the /// handshake; a refresh answer needs a larger lead to start a range sync. refreshing: HashSet, @@ -102,8 +108,10 @@ impl PeerHeads { self.last_exchange.insert(peer, wall_slot); if head_slot > wall_slot.saturating_add(FUTURE_HEAD_TOLERANCE) { self.heads.remove(&peer); + self.reporting.remove(&peer); return Report::Future; } + self.reporting.insert(peer); if held { self.heads.remove(&peer); return Report::Held; @@ -115,6 +123,7 @@ impl PeerHeads { /// Forget a disconnected peer. pub(crate) fn forget(&mut self, peer: &PeerId) { self.heads.remove(peer); + self.reporting.remove(peer); self.refreshing.remove(peer); self.last_exchange.remove(peer); self.foreign_digest.remove(peer); @@ -127,6 +136,12 @@ impl PeerHeads { self.heads.values().copied().max() } + /// How many connected peers have reported a head not past the wall + /// clock, whether or not this node holds it. + pub(crate) fn reporting_peers(&self) -> usize { + self.reporting.len() + } + /// The connected peers to re-ask for their `Status` at `wall_slot`, at /// most one slot's share of them. /// @@ -241,9 +256,10 @@ pub(crate) fn forget(server: &mut P2PServer, peer: &PeerId) { } fn publish(server: &P2PServer) { - server - .store - .set_network_head(server.peer_heads.network_head()); + server.store.set_peer_view(PeerView { + network_head: server.peer_heads.network_head(), + reporting_peers: server.peer_heads.reporting_peers(), + }); } #[cfg(test)] @@ -295,6 +311,35 @@ mod tests { assert_eq!(heads.network_head(), None); } + #[test] + fn peers_level_with_us_count_as_reporting() { + // At genesis every peer reports a head this node already holds. None + // of them is a lead, but all of them have spoken, which is what ends + // the sync tracker's startup hold. + let mut heads = PeerHeads::default(); + let peers: Vec<_> = (0..5).map(|_| PeerId::random()).collect(); + for peer in &peers { + assert_eq!(heads.record(*peer, 3, 6, true), Report::Held); + } + assert_eq!(heads.network_head(), None); + assert_eq!(heads.reporting_peers(), 5); + // A later report from one of them is still one peer. + heads.record(peers[0], 4, 7, false); + assert_eq!(heads.reporting_peers(), 5); + heads.forget(&peers[1]); + assert_eq!(heads.reporting_peers(), 4); + } + + #[test] + fn a_peer_reporting_the_future_is_not_reporting() { + let mut heads = PeerHeads::default(); + let liar = PeerId::random(); + heads.record(liar, 100, 100, false); + assert_eq!(heads.reporting_peers(), 1); + heads.record(liar, 100 + FUTURE_HEAD_TOLERANCE + 1, 100, false); + assert_eq!(heads.reporting_peers(), 0); + } + #[test] fn a_head_past_the_clock_is_dropped_with_the_peers_earlier_report() { let mut heads = PeerHeads::default(); diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 85ad30c5a..4491b944d 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -27,5 +27,5 @@ pub use liveness::ObservedLiveness; pub use state_writer::CacheKey; pub use store::{ Chain, DB_VERSION, ForkCheckpoints, GetForkchoiceStoreError, HeadVoteWindow, - MAX_RESUMABLE_DB_STATE_AGE, NEW_PAYLOAD_CAP, Store, + MAX_RESUMABLE_DB_STATE_AGE, NEW_PAYLOAD_CAP, PeerView, Store, }; diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index a2ac6f4d7..c77614f3e 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::num::NonZeroUsize; use std::ops::Range; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; use std::time::Instant; @@ -257,6 +257,19 @@ const STATE_CACHE_CAPACITY: usize = 32; /// [`Store::network_head`]'s "no peer has reported" marker. No slot reaches it. const NO_NETWORK_HEAD: u64 = u64::MAX; +/// What this node's connected peers report in `Status`, as P2P last published +/// it; see [`Store::peer_view`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PeerView { + /// The highest head slot the peers report that this node has not already + /// imported, or `None` while no connected peer reports one. + pub network_head: Option, + /// Connected peers whose latest `Status` head was not past the wall + /// clock, counted whether or not this node already holds that head: how + /// many peers the view rests on, which `network_head` alone cannot say. + pub reporting_peers: usize, +} + /// Keep block proofs for at least this many slots below the tip, even once /// finalized. Proofs older than this window are pruned only when the window /// lies entirely within finalized history; see [`Store::prune_old_block_proofs`]. @@ -1205,6 +1218,9 @@ pub struct Store { /// while no connected peer reports one. Written by P2P, read by the chain /// actor's sync tracker; see [`Store::network_head`]. network_head: Arc, + /// [`PeerView::reporting_peers`], published with `network_head`; see + /// [`Store::set_peer_view`] for the order the two are written and read in. + reporting_peers: Arc, /// The total active balance per epoch, shared the way /// [`Self::committee_cache`] is: by the chain actor's state transition /// and anything else holding a `Store` clone. See @@ -1856,6 +1872,7 @@ impl Store { committee_cache: Arc::new(CommitteeCache::default()), observed_liveness: Arc::new(ObservedLiveness::default()), network_head: Arc::new(AtomicU64::new(NO_NETWORK_HEAD)), + reporting_peers: Arc::new(AtomicUsize::new(0)), active_balance_cache: Arc::new(ActiveBalanceCache::default()), beacon: Default::default(), state_writer, @@ -3315,15 +3332,35 @@ impl Store { /// not advance towards, so no claim can hold a stalled node for long. /// Runtime-only and shared by every clone of this `Store`. pub fn network_head(&self) -> Option { - let slot = self.network_head.load(Ordering::Relaxed); - (slot != NO_NETWORK_HEAD).then_some(slot) + self.peer_view().network_head + } + + /// The peers' head together with how many peers report one; see + /// [`PeerView`]. The sync tracker reads both from one call, since it ends + /// its startup hold on the count and then judges by the head. + pub fn peer_view(&self) -> PeerView { + // Count first, head second: the reverse of `set_peer_view`, so a count + // read here comes with a head at least as new as the one published + // with it, never one from before it. + let reporting_peers = self.reporting_peers.load(Ordering::SeqCst); + let slot = self.network_head.load(Ordering::SeqCst); + PeerView { + network_head: (slot != NO_NETWORK_HEAD).then_some(slot), + reporting_peers, + } } - /// Publish the peers' head; see [`Self::network_head`]. P2P is the only + /// Publish what the peers report; see [`Self::peer_view`]. P2P is the only /// writer. - pub fn set_network_head(&self, slot: Option) { - let slot = slot.unwrap_or(NO_NETWORK_HEAD); - self.network_head.store(slot, Ordering::Relaxed); + /// + /// The head goes first and the count after it. A tracker that read the + /// new count with the old head could end its startup hold on the fifth + /// report while missing that report's lead, the one it was waiting for. + pub fn set_peer_view(&self, view: PeerView) { + let slot = view.network_head.unwrap_or(NO_NETWORK_HEAD); + self.network_head.store(slot, Ordering::SeqCst); + self.reporting_peers + .store(view.reporting_peers, Ordering::SeqCst); } /// The total-active-balance cache shared by every clone of this store. @@ -7225,13 +7262,24 @@ mod tests { } #[test] - fn the_network_head_is_shared_across_store_clones() { + fn the_peer_view_is_shared_across_store_clones() { let store = Store::test_store(); let clone = store.clone(); - assert_eq!(store.network_head(), None); - clone.set_network_head(Some(42)); + assert_eq!(store.peer_view(), PeerView::default()); + let view = PeerView { + network_head: Some(42), + reporting_peers: 3, + }; + clone.set_peer_view(view); + assert_eq!(store.peer_view(), view); assert_eq!(store.network_head(), Some(42)); - clone.set_network_head(None); + // Peers whose heads this node holds still count, with no head left. + let held = PeerView { + network_head: None, + reporting_peers: 3, + }; + clone.set_peer_view(held); + assert_eq!(store.peer_view(), held); assert_eq!(store.network_head(), None); } diff --git a/docs/architecture.md b/docs/architecture.md index afb90cc05..4275fb981 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -172,7 +172,9 @@ ahead of its own, since a head derived from a partial view is not worth voting f peers' head is the highest of each connected peer's latest `Status`, re-requested every few slots per peer (spread over the interval, so a node with hundreds of peers never asks them all at once), not counting a head the node has already imported, which p2p publishes on -`Store::network_head` (`crates/net/p2p/src/peer_heads.rs`). The thresholds, and the refresh +`Store::peer_view` with how many peers have reported (`crates/net/p2p/src/peer_heads.rs`). +Until 5 peers have, for a bounded window after startup, the network is assumed to be at the +clock, since the first peer to answer may be as stale as the node. The thresholds, and the refresh cadence they depend on, are `SyncThresholds` in `crates/net/api/src/sync.rs`, shared by both actors. A hysteresis band stops the state from flapping at the boundary, and a network-wide stall (nobody else is ahead either) leaves the node synced so its validators can help the diff --git a/docs/rpc.md b/docs/rpc.md index 101237adc..fb08ffb5e 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -190,7 +190,7 @@ The fork-choice tree from the finalized root, with LMD-GHOST weights computed ov { "is_syncing": false, "head_slot": 1024, "sync_distance": 1, "finalized_slot": 986 } ``` -`is_syncing` is the node's own stateful sync decision, updated each tick and after each block arrival: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false, except during the first 8 slots after startup, when a node whose head trails the clock reads as syncing whatever its first peers report. It is also false once the head has stayed flat for 16 slots while peers claim a lead, until the head next advances. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. +`is_syncing` is the node's own stateful sync decision, updated each tick and after each block arrival: true while the head trails the wall clock and the head the node's peers report in `Status` is ahead of it, with hysteresis. With no peer ahead (a stalled network, or no peers) it is false, except during the first 8 slots after startup, or until 5 peers have reported their head if that comes sooner, when a node whose head trails the clock reads as syncing whatever its first peers report. It is also false once the head has stayed flat for 16 slots while peers claim a lead, until the head next advances. It is the same signal that gates validator duties and drives the `lean_node_sync_status` metric, so the endpoint, the gate, and the metric always agree. `sync_distance` is the raw number of slots between the node's current head and the current wall-clock slot, computed per request. Because `is_syncing` carries hysteresis and stall handling and is not recomputed from `sync_distance`, the two can point different ways near the threshold or during a network-wide stall. diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index fbd32add9..a067f824b 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -52,8 +52,12 @@ head trails the wall clock **and** its peers report a head ahead of it. than the clock lag because a report is up to 2 slots old: a node 5 slots behind a producing network sees its peers' 2-slot-old heads 2 slots ahead. With no report at all the node cannot be behind anyone and reads as synced, - except during an 8-slot hold at startup, during which the network is assumed - to be at least at the clock, whatever the first peers report. + except during a hold at startup, during which the network is assumed to be + at least at the clock. The hold lasts 8 slots, or until 5 connected peers + have reported a head (one the node already holds counts; one past the clock + does not), whichever comes first: the first peer to answer may be as stale + as the node, but the highest of five shows the lead if any one of them is + current. - **Stall window:** a lead that does not turn into progress is given up on. A node that reads as syncing on its peers' word with its head flat for 16 slots (`SyncThresholds::stall_window`) treats the network as stalled and does its @@ -76,13 +80,14 @@ head trails the wall clock **and** its peers report a head ahead of it. nothing newer imported, which the specification lets do its duties and this node does not, until its head has been flat for the stall window. They also differ for a node restarted into a network stall, which the specification - lets do its duties at once and this node holds for the 8-slot startup window. + lets do its duties at once and this node holds until 5 peers have reported + or the 8-slot startup window has passed. The beacon node runs the same rule with Lighthouse's numbers (two epochs of clock lag, one epoch of peer lead), a band of 4 slots that neither Lighthouse nor Prysm has (this node re-judges on every block arrival, and would otherwise -flip twice a slot at the boundary), an epoch of startup hold and of stall -window, and Prysm's `Status` refresh, every half epoch (Lighthouse refreshes +flip twice a slot at the boundary), an epoch of startup hold (ended sooner by +the same 5 reporting peers) and of stall window, and Prysm's `Status` refresh, every half epoch (Lighthouse refreshes every 300 s). consensus-specs has no sync gate, and the Beacon API leaves `is_syncing` to the client.