From ea7c9d7c85bd5efbd11097722a1ed40c91eff468 Mon Sep 17 00:00:00 2001 From: Pablo Deymonnaz Date: Thu, 5 Nov 2026 16:58:46 -0300 Subject: [PATCH 1/2] Persist each validator's fee recipient, graffiti and gas limit on its entry in validator_definitions.yml, in Lighthouse's layout (suggested_fee_recipient, graffiti, gas_limit), so a restart runs with what the keymanager API was given instead of putting every validator back on the command line's defaults. The keymanager's settings routes now write the file under the same definitions lock the keystore routes take, and before the in-memory change: a failed write is a 500 that changes nothing. Startup seeds the settings from the enabled entries, refusing a malformed value the way a bad keystore is refused. A re-imported key keeps the overrides its entry held, and a deleted key's entry takes them with it. The fields may also be written by hand, and a Lighthouse file reads as is. --- crates/validator/src/http_api/keystores.rs | 54 +++- crates/validator/src/http_api/proposer.rs | 301 ++++++++++++++++----- crates/validator/src/keys/definitions.rs | 239 +++++++++++++++- crates/validator/src/keys/store.rs | 6 + crates/validator/src/lib.rs | 12 + crates/validator/src/proposer_settings.rs | 138 ++++++++-- docs/cli.md | 11 +- 7 files changed, 669 insertions(+), 92 deletions(-) diff --git a/crates/validator/src/http_api/keystores.rs b/crates/validator/src/http_api/keystores.rs index 2232679b3..132e0910f 100644 --- a/crates/validator/src/http_api/keystores.rs +++ b/crates/validator/src/http_api/keystores.rs @@ -314,6 +314,9 @@ fn persist_import( voting_public_key: base.clone(), voting_keystore_path: keystore_path, voting_keystore_password_path: password_path, + suggested_fee_recipient: None, + graffiti: None, + gas_limit: None, }; // How to put `definitions` back if the save below fails. // @@ -341,10 +344,15 @@ fn persist_import( .iter() .position(|existing| existing.voting_public_key == base) { - Some(index) => Undo::Replaced { - index, - previous: std::mem::replace(&mut definitions.0[index], definition), - }, + Some(index) => { + // A re-imported key keeps the fee recipient, graffiti and gas + // limit its entry held: only the keystore and password move. + let definition = definition.with_overrides_of(&definitions.0[index]); + Undo::Replaced { + index, + previous: std::mem::replace(&mut definitions.0[index], definition), + } + } None => { definitions.0.push(definition); Undo::Appended @@ -718,6 +726,9 @@ mod tests { voting_public_key: encode_hex(&pubkey.0), voting_keystore_path: PathBuf::from("old.json"), voting_keystore_password_path: PathBuf::from("old.txt"), + suggested_fee_recipient: None, + graffiti: Some("kept".to_string()), + gas_limit: None, }; let mut definitions = ValidatorDefinitions(vec![existing.clone()]); @@ -865,6 +876,41 @@ mod tests { assert!(store.read().await.is_empty()); } + /// Re-importing a key (a password rotation, or a retry) rebuilds its + /// entry from the new keystore; the fee recipient, graffiti and gas limit + /// the entry held stay on it. + #[tokio::test] + async fn a_reimported_key_keeps_its_persisted_overrides() { + let dir = tempfile::tempdir().expect("temp dir"); + let store = Arc::new(RwLock::new(ValidatorStore::new())); + let context = context_with_dirs(store.clone(), dir.path()); + let app = router(context.clone(), TOKEN.to_string()); + let import = serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }); + + let (status, body) = send( + app.clone(), + authed("POST", "/eth/v1/keystores", import.clone()), + ) + .await; + assert_eq!(status, StatusCode::OK, "got {body}"); + let pubkey = store.read().await.pubkeys()[0]; + let set = serde_json::json!({ "graffiti": "kept" }); + let path = format!("/eth/v1/validator/{}/graffiti", encode_hex(&pubkey.0)); + let (status, _) = send(app.clone(), authed("POST", &path, set)).await; + assert_eq!(status, StatusCode::ACCEPTED); + + let (status, body) = send(app, authed("POST", "/eth/v1/keystores", import)).await; + assert_eq!(status, StatusCode::OK, "got {body}"); + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + + let definitions = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(definitions.0.len(), 1, "one entry, not a duplicate"); + assert_eq!(definitions.0[0].graffiti.as_deref(), Some("kept")); + } + /// The regression test for the gap that let the keymanager mutate only /// the in-memory store: a fresh `ValidatorStore::load` from the same /// directory, independent of the one the API mutated, is the only way to diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index beec84d3c..4ef899fe9 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -1,10 +1,10 @@ //! `GET`, `POST` and `DELETE /eth/v1/validator/{pubkey}/feerecipient`, //! `/graffiti` and `/gas_limit`. //! -//! Each overrides one validator's setting in [`ProposerSettings`], which holds -//! them in memory only: a restart puts every validator back on the command -//! line's defaults. See [`crate::proposer_settings`] for why, and for when a -//! change takes effect. +//! Each overrides one validator's setting in [`ProposerSettings`], and first +//! writes it onto that validator's entry in `validator_definitions.yml`, so a +//! restart runs with it. See [`crate::proposer_settings`] for when a change +//! takes effect. //! //! `GET` answers the validator's own value, or the process-wide default when it //! has none, as the specification asks. `DELETE` drops the validator's own @@ -15,24 +15,26 @@ //! the change happen under one read of the validator store, so a keystore //! delete, which takes that store's write lock and then forgets the key's //! settings, cannot land between them and leave an override behind for a key -//! that is gone. +//! that is gone. The file is written under the same `definitions_lock` the +//! keystore routes take, and before memory changes: a failed write is a 500 +//! that changes nothing, so what the operator sees is what the next restart +//! loads. use axum::Json; use axum::extract::rejection::JsonRejection; use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; -use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress, H160}; +use ethlambda_types::beacon::primitives::BlsPubkey; use serde::{Deserialize, Serialize}; use tracing::info; use crate::beacon_node::dto::{encode_hex, parse_pubkey}; use crate::http_api::KeymanagerContext; -use crate::keys::ValidatorStore; -use crate::proposer_settings::{GRAFFITI_BYTES, graffiti_from_text, graffiti_to_text}; - -/// An execution address is exactly this wide. -const ADDRESS_BYTES: usize = 20; +use crate::keys::{ValidatorDefinitions, ValidatorStore}; +use crate::proposer_settings::{ + GRAFFITI_BYTES, SettingChange, fee_recipient_from_text, graffiti_from_text, graffiti_to_text, +}; /// The keymanager API's error body, `{"message": ...}`. #[derive(Debug)] @@ -117,22 +119,53 @@ fn require_held(store: &ValidatorStore, pubkey: &BlsPubkey) -> Result<(), ApiErr } } -/// `0x` and forty hex digits, the specification's `EthAddress` pattern. -fn parse_address(text: &str) -> Result { - let Some(digits) = text.strip_prefix("0x") else { - return Err(ApiError::BadRequest( - "ethaddress must be 0x-prefixed hex".to_string(), - )); - }; - let bytes = hex::decode(digits) - .map_err(|err| ApiError::BadRequest(format!("ethaddress is not hex: {err}")))?; - let bytes: [u8; ADDRESS_BYTES] = bytes.try_into().map_err(|got: Vec| { - ApiError::BadRequest(format!( - "ethaddress is {} bytes, expected {ADDRESS_BYTES}", - got.len() - )) - })?; - Ok(H160(bytes)) +/// Persist `change` on `pubkey`'s definitions entry, then apply it in memory. +/// +/// Locks in the order the keystore routes take them: `definitions_lock` +/// first, which serialises this open-mutate-save cycle against a concurrent +/// import or delete, then `store`'s read lock for the held check, kept until +/// the change is applied so a delete cannot land in between. +/// +/// A held key with no entry in the file can only mean the file was edited +/// under a running client. That is a 500 naming the file rather than a +/// memory-only change the operator would believe persisted. +async fn persist_and_apply( + context: &KeymanagerContext, + pubkey: &BlsPubkey, + change: SettingChange, +) -> Result<(), ApiError> { + let _definitions_guard = context.definitions_lock.lock().await; + let store = context.store.read().await; + require_held(&store, pubkey)?; + + let path = context + .validators_dir + .join(crate::keys::definitions::DEFINITIONS_FILE); + let mut definitions = ValidatorDefinitions::open(&context.validators_dir) + .map_err(|err| ApiError::Internal(format!("could not read {}: {err}", path.display())))?; + let entry = definitions + .0 + .iter_mut() + .find(|definition| { + parse_pubkey(&definition.voting_public_key) + .map(|parsed| parsed == *pubkey) + .unwrap_or(false) + }) + .ok_or_else(|| { + ApiError::Internal(format!( + "validator {} is loaded but has no entry in {}; the file was changed under a \ + running client, so the change was not made", + encode_hex(&pubkey.0), + path.display() + )) + })?; + entry.apply(change); + definitions + .save(&context.validators_dir) + .map_err(|err| ApiError::Internal(format!("could not write {}: {err}", path.display())))?; + + context.settings.apply(pubkey, change); + Ok(()) } /// A decimal `u64` with no sign and no leading zero, the specification's @@ -182,19 +215,15 @@ pub async fn set_fee_recipient( ) -> Result { let pubkey = parse_path_key(&pubkey)?; let Json(request) = body?; - let address = parse_address(&request.ethaddress)?; - // The specification forbids it outright: it would read as an address - // chosen on purpose, and pays nobody. - if address == H160([0; ADDRESS_BYTES]) { - return Err(ApiError::BadRequest( - "the zero address cannot be set as a fee recipient".to_string(), - )); - } + let address = fee_recipient_from_text(&request.ethaddress) + .map_err(|why| ApiError::BadRequest(format!("ethaddress {why}")))?; - let store = context.store.read().await; - require_held(&store, &pubkey)?; - context.settings.set_fee_recipient(&pubkey, address); - drop(store); + persist_and_apply( + &context, + &pubkey, + SettingChange::FeeRecipient(Some(address)), + ) + .await?; info!( pubkey = %encode_hex(&pubkey.0), @@ -209,10 +238,7 @@ pub async fn delete_fee_recipient( Path(pubkey): Path, ) -> Result { let pubkey = parse_path_key(&pubkey)?; - let store = context.store.read().await; - require_held(&store, &pubkey)?; - context.settings.clear_fee_recipient(&pubkey); - drop(store); + persist_and_apply(&context, &pubkey, SettingChange::FeeRecipient(None)).await?; info!(pubkey = %encode_hex(&pubkey.0), "Fee recipient reset to the default"); Ok(StatusCode::NO_CONTENT) @@ -247,10 +273,7 @@ pub async fn set_graffiti( ))); }; - let store = context.store.read().await; - require_held(&store, &pubkey)?; - context.settings.set_graffiti(&pubkey, graffiti); - drop(store); + persist_and_apply(&context, &pubkey, SettingChange::Graffiti(Some(graffiti))).await?; info!(pubkey = %encode_hex(&pubkey.0), graffiti = ?request.graffiti, "Graffiti set"); Ok(StatusCode::ACCEPTED) @@ -261,10 +284,7 @@ pub async fn delete_graffiti( Path(pubkey): Path, ) -> Result { let pubkey = parse_path_key(&pubkey)?; - let store = context.store.read().await; - require_held(&store, &pubkey)?; - context.settings.clear_graffiti(&pubkey); - drop(store); + persist_and_apply(&context, &pubkey, SettingChange::Graffiti(None)).await?; info!(pubkey = %encode_hex(&pubkey.0), "Graffiti reset to the default"); Ok(StatusCode::NO_CONTENT) @@ -294,10 +314,7 @@ pub async fn set_gas_limit( let Json(request) = body?; let gas_limit = parse_uint64(&request.gas_limit)?; - let store = context.store.read().await; - require_held(&store, &pubkey)?; - context.settings.set_gas_limit(&pubkey, gas_limit); - drop(store); + persist_and_apply(&context, &pubkey, SettingChange::GasLimit(Some(gas_limit))).await?; info!(pubkey = %encode_hex(&pubkey.0), gas_limit, "Gas limit set"); Ok(StatusCode::ACCEPTED) @@ -308,10 +325,7 @@ pub async fn delete_gas_limit( Path(pubkey): Path, ) -> Result { let pubkey = parse_path_key(&pubkey)?; - let store = context.store.read().await; - require_held(&store, &pubkey)?; - context.settings.clear_gas_limit(&pubkey); - drop(store); + persist_and_apply(&context, &pubkey, SettingChange::GasLimit(None)).await?; info!(pubkey = %encode_hex(&pubkey.0), "Gas limit reset to the default"); Ok(StatusCode::NO_CONTENT) @@ -323,14 +337,15 @@ mod tests { use axum::body::Body; use axum::http::Request; - use ethlambda_types::beacon::primitives::Bytes32; + use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, H160}; use http_body_util::BodyExt as _; use tokio::sync::RwLock; use tower::ServiceExt as _; use super::*; use crate::http_api::router; - use crate::proposer_settings::{DEFAULT_GAS_LIMIT, ProposerSettings}; + use crate::keys::ValidatorDefinition; + use crate::proposer_settings::{ADDRESS_BYTES, DEFAULT_GAS_LIMIT, ProposerSettings}; const TOKEN: &str = "test-token"; @@ -338,9 +353,21 @@ mod tests { app: axum::Router, settings: Arc, pubkey: BlsPubkey, - /// Kept alive for the test that deletes a keystore, which writes the - /// definitions file. - _dir: tempfile::TempDir, + /// The validators directory, holding the definitions file the routes + /// write through. + dir: tempfile::TempDir, + } + + impl Harness { + /// The key's entry as the file holds it now. + fn entry(&self) -> ValidatorDefinition { + ValidatorDefinitions::open(self.dir.path()) + .expect("opens") + .0 + .into_iter() + .find(|entry| entry.voting_public_key == encode_hex(&self.pubkey.0)) + .expect("the key's entry") + } } fn secret() -> [u8; 32] { @@ -355,8 +382,27 @@ mod tests { } /// A router over a store holding one key, with a default graffiti of - /// `default` and, when `fee_recipient` is `Some`, a default address. + /// `default` and, when `fee_recipient` is `Some`, a default address. The + /// definitions file names the key, as it would for any loaded key. fn harness(fee_recipient: Option) -> Harness { + let harness = harness_without_definitions(fee_recipient); + ValidatorDefinitions(vec![ValidatorDefinition { + enabled: true, + voting_public_key: encode_hex(&harness.pubkey.0), + voting_keystore_path: harness.dir.path().join("voting-keystore.json"), + voting_keystore_password_path: harness.dir.path().join("password"), + suggested_fee_recipient: None, + graffiti: None, + gas_limit: None, + }]) + .save(harness.dir.path()) + .expect("writes the definitions file"); + harness + } + + /// [`harness`] with no definitions file at all: a key in memory the file + /// does not describe. + fn harness_without_definitions(fee_recipient: Option) -> Harness { let mut store = ValidatorStore::new(); let pubkey = store.insert_secret("test", &secret()).expect("inserts"); let settings = Arc::new(ProposerSettings::new(default_graffiti(), fee_recipient)); @@ -372,7 +418,7 @@ mod tests { app: router(context, TOKEN.to_string()), settings, pubkey, - _dir: dir, + dir, } } @@ -611,6 +657,129 @@ mod tests { assert!(body["message"].is_string(), "got {body}"); } + /// Every change lands on the key's entry in the definitions file before + /// it is answered, and a delete takes the field out again. + #[tokio::test] + async fn a_change_is_written_to_the_definitions_file_and_a_delete_removes_it() { + let harness = harness(Some(default_address())); + let own = format!("0x{}", "bb".repeat(ADDRESS_BYTES)); + + let set = serde_json::json!({ "ethaddress": own }); + let path = uri(&harness.pubkey, "feerecipient"); + let (status, _) = send(&harness.app, request("POST", &path, Some(set))).await; + assert_eq!(status, StatusCode::ACCEPTED); + assert_eq!( + harness.entry().suggested_fee_recipient.as_deref(), + Some(own.as_str()) + ); + + let set = serde_json::json!({ "graffiti": "hello" }); + let path = uri(&harness.pubkey, "graffiti"); + send(&harness.app, request("POST", &path, Some(set))).await; + assert_eq!(harness.entry().graffiti.as_deref(), Some("hello")); + + let set = serde_json::json!({ "gas_limit": "30000000" }); + let path = uri(&harness.pubkey, "gas_limit"); + send(&harness.app, request("POST", &path, Some(set))).await; + assert_eq!(harness.entry().gas_limit, Some(30_000_000)); + + for setting in ["feerecipient", "graffiti", "gas_limit"] { + let request = request("DELETE", &uri(&harness.pubkey, setting), None); + let (status, _) = send(&harness.app, request).await; + assert_eq!(status, StatusCode::NO_CONTENT, "DELETE {setting}"); + } + let entry = harness.entry(); + assert_eq!(entry.suggested_fee_recipient, None); + assert_eq!(entry.graffiti, None); + assert_eq!(entry.gas_limit, None); + } + + /// What a restart does with the file: the overrides a previous run + /// persisted are the ones the next run proposes with. + #[tokio::test] + async fn persisted_overrides_survive_a_restart() { + let harness = harness(Some(default_address())); + let own = format!("0x{}", "bb".repeat(ADDRESS_BYTES)); + let set = serde_json::json!({ "ethaddress": own }); + let path = uri(&harness.pubkey, "feerecipient"); + send(&harness.app, request("POST", &path, Some(set))).await; + let set = serde_json::json!({ "graffiti": "persisted" }); + let path = uri(&harness.pubkey, "graffiti"); + send(&harness.app, request("POST", &path, Some(set))).await; + + // A fresh `ProposerSettings`, as a restart builds, seeded as startup + // seeds it. + let restarted = ProposerSettings::new(default_graffiti(), Some(default_address())); + let seeded = ValidatorDefinitions::open(harness.dir.path()) + .expect("opens") + .seed_overrides(&restarted) + .expect("seeds"); + assert_eq!(seeded, 1); + assert_eq!( + restarted.fee_recipient(&harness.pubkey), + Some(H160([0xbb; ADDRESS_BYTES])) + ); + assert_eq!( + restarted.graffiti(&harness.pubkey), + graffiti_from_text("persisted").expect("fits") + ); + assert_eq!(restarted.gas_limit(&harness.pubkey), DEFAULT_GAS_LIMIT); + } + + /// A loaded key the file does not describe means the file was changed + /// under the client. The change is refused rather than kept in memory + /// alone, where the operator would believe it persisted. + #[tokio::test] + async fn a_key_missing_from_the_definitions_file_is_a_500_and_memory_is_unchanged() { + let harness = harness_without_definitions(None); + let set = serde_json::json!({ "graffiti": "hello" }); + let path = uri(&harness.pubkey, "graffiti"); + let (status, body) = send(&harness.app, request("POST", &path, Some(set))).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert!( + body["message"] + .as_str() + .is_some_and(|message| message.contains("validator_definitions.yml")), + "got {body}" + ); + assert_eq!( + harness.settings.graffiti(&harness.pubkey), + default_graffiti(), + "a change that was not persisted must not take effect" + ); + let (status, _) = send(&harness.app, request("DELETE", &path, None)).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + } + + /// A file that cannot be written is a 500 that changes nothing. + #[cfg(unix)] + #[tokio::test] + async fn a_file_that_cannot_be_written_is_a_500_and_memory_is_unchanged() { + use std::os::unix::fs::PermissionsExt as _; + + let harness = harness(None); + // Saving writes a sibling temporary file, which a read-only directory + // refuses. Root is exempt from permission bits, so the test is moot + // there rather than wrong. + if unsafe { libc::geteuid() } == 0 { + return; + } + std::fs::set_permissions(harness.dir.path(), std::fs::Permissions::from_mode(0o500)) + .expect("chmod"); + + let set = serde_json::json!({ "graffiti": "hello" }); + let path = uri(&harness.pubkey, "graffiti"); + let (status, _) = send(&harness.app, request("POST", &path, Some(set))).await; + std::fs::set_permissions(harness.dir.path(), std::fs::Permissions::from_mode(0o700)) + .expect("chmod back, so the directory can be removed"); + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + harness.settings.graffiti(&harness.pubkey), + default_graffiti() + ); + assert_eq!(harness.entry().graffiti, None, "the file is unchanged too"); + } + /// A deleted key's overrides go with it, so the same key imported again /// starts on the defaults. #[tokio::test] diff --git a/crates/validator/src/keys/definitions.rs b/crates/validator/src/keys/definitions.rs index ab48a8db0..d19006fff 100644 --- a/crates/validator/src/keys/definitions.rs +++ b/crates/validator/src/keys/definitions.rs @@ -2,13 +2,19 @@ //! //! One YAML-free, JSON-Lines-free plain list, deliberately separate from the //! keystores themselves. It is what the keymanager API mutates, so a key -//! imported at runtime is still there after a restart. +//! imported at runtime is still there after a restart, and so are the fee +//! recipient, graffiti and gas limit it was given. use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use crate::beacon_node::dto::parse_pubkey; use crate::error::{Error, Result}; +use crate::proposer_settings::{ + GRAFFITI_BYTES, Overrides, ProposerSettings, SettingChange, fee_recipient_from_text, + fee_recipient_to_text, graffiti_from_text, graffiti_to_text, +}; use crate::secure_fs; /// The file's name inside the validators directory. @@ -25,6 +31,80 @@ pub struct ValidatorDefinition { pub voting_keystore_path: PathBuf, /// Path to the file holding that keystore's password. pub voting_keystore_password_path: PathBuf, + /// This validator's own fee recipient, `0x`-prefixed hex, in place of + /// `--suggested-fee-recipient`. The three fields below are Lighthouse's, + /// name for name and form for form, so its file reads here and this one + /// reads there. Each is absent from the file, not `null`, when unset. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub suggested_fee_recipient: Option, + /// This validator's own graffiti, as text, in place of `--graffiti`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub graffiti: Option, + /// This validator's own gas limit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gas_limit: Option, +} + +impl ValidatorDefinition { + /// The entry's three overrides, parsed. A value the file holds in the + /// wrong form is an error naming the entry and the field: a validator + /// proposing to an address the operator did not mean is worse than a + /// refused start. + pub fn overrides(&self) -> Result { + let reject = |field: &str, reason: String| Error::Keystore { + path: format!("validator_definitions.yml entry {}", self.voting_public_key), + reason: format!("{field} {reason}"), + }; + let fee_recipient = self + .suggested_fee_recipient + .as_deref() + .map(|text| { + fee_recipient_from_text(text).map_err(|why| reject("suggested_fee_recipient", why)) + }) + .transpose()?; + let graffiti = self + .graffiti + .as_deref() + .map(|text| { + graffiti_from_text(text).ok_or_else(|| { + reject( + "graffiti", + format!( + "is {} bytes encoded as UTF-8; the field holds {GRAFFITI_BYTES}", + text.len() + ), + ) + }) + }) + .transpose()?; + Ok(Overrides { + fee_recipient, + graffiti, + gas_limit: self.gas_limit, + }) + } + + /// Write one change into the entry, in the form the file carries. + pub fn apply(&mut self, change: SettingChange) { + match change { + SettingChange::FeeRecipient(address) => { + self.suggested_fee_recipient = address.as_ref().map(fee_recipient_to_text); + } + SettingChange::Graffiti(graffiti) => { + self.graffiti = graffiti.as_ref().map(graffiti_to_text); + } + SettingChange::GasLimit(gas_limit) => self.gas_limit = gas_limit, + } + } + + /// Carry another entry's overrides onto this one, for a re-imported key + /// whose entry is rebuilt from the new keystore. + pub fn with_overrides_of(mut self, other: &ValidatorDefinition) -> Self { + self.suggested_fee_recipient = other.suggested_fee_recipient.clone(); + self.graffiti = other.graffiti.clone(); + self.gas_limit = other.gas_limit; + self + } } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -127,6 +207,9 @@ impl ValidatorDefinitions { voting_keystore_password_path: secrets_dir.join(&pubkey), voting_public_key: pubkey, voting_keystore_path: keystore_path, + suggested_fee_recipient: None, + graffiti: None, + gas_limit: None, }); } // Directory order is not stable across filesystems; sorting makes the @@ -142,6 +225,31 @@ impl ValidatorDefinitions { pub fn enabled(&self) -> impl Iterator { self.0.iter().filter(|definition| definition.enabled) } + + /// Install every enabled entry's overrides into `settings`, returning how + /// many entries carried one. What startup does once the keys are loaded. + /// A disabled entry keeps its fields in the file for when it is enabled + /// again, and is skipped here as its key is skipped by `load`. + pub fn seed_overrides(&self, settings: &ProposerSettings) -> Result { + let mut seeded = 0; + for definition in self.enabled() { + let overrides = definition.overrides()?; + if overrides.is_empty() { + continue; + } + let pubkey = + parse_pubkey(&definition.voting_public_key).map_err(|err| Error::Keystore { + path: format!( + "validator_definitions.yml entry {}", + definition.voting_public_key + ), + reason: format!("the voting_public_key is unreadable: {err}"), + })?; + settings.seed(&pubkey, overrides); + seeded += 1; + } + Ok(seeded) + } } #[cfg(test)] @@ -212,9 +320,138 @@ mod tests { voting_public_key: pubkey.to_string(), voting_keystore_path: PathBuf::from("keystore.json"), voting_keystore_password_path: PathBuf::from("password.txt"), + suggested_fee_recipient: None, + graffiti: None, + gas_limit: None, } } + /// An entry with no overrides is written exactly as it always was: the + /// three fields are absent, not `null`, so a file this client writes + /// stays readable by a reader that predates them. + #[test] + fn an_entry_without_overrides_is_written_without_the_fields() { + let yaml = serde_yaml_ng::to_string(&ValidatorDefinitions(vec![definition("0xaa", true)])) + .expect("serializes"); + assert!(!yaml.contains("suggested_fee_recipient"), "{yaml}"); + assert!(!yaml.contains("graffiti"), "{yaml}"); + assert!(!yaml.contains("gas_limit"), "{yaml}"); + } + + #[test] + fn overrides_round_trip_through_the_file() { + use ethlambda_types::beacon::primitives::H160; + + let dir = tempfile::tempdir().expect("temp dir"); + let mut entry = definition("0xaa", true); + entry.apply(SettingChange::FeeRecipient(Some(H160([0xbb; 20])))); + entry.apply(SettingChange::Graffiti(Some( + graffiti_from_text("hello").expect("fits"), + ))); + entry.apply(SettingChange::GasLimit(Some(30_000_000))); + ValidatorDefinitions(vec![entry.clone(), definition("0xbb", true)]) + .save(dir.path()) + .expect("saves"); + + let read = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(read.0[0], entry); + assert_eq!( + read.0[0].overrides().expect("parses"), + Overrides { + fee_recipient: Some(H160([0xbb; 20])), + graffiti: Some(graffiti_from_text("hello").expect("fits")), + gas_limit: Some(30_000_000), + } + ); + assert_eq!(read.0[1].overrides().expect("parses"), Overrides::default()); + } + + /// Lighthouse's own file, fields it writes that this client does not + /// included, reads here with the three overrides picked up. + #[test] + fn a_lighthouse_written_entry_parses_with_its_overrides() { + use ethlambda_types::beacon::primitives::H160; + + let yaml = format!( + "- enabled: true + voting_public_key: 0x{} + description: \"\" + \ + graffiti: from lighthouse + suggested_fee_recipient: 0x{} + gas_limit: 36000000 + \ + builder_proposals: false + type: local_keystore + \ + voting_keystore_path: /keys/voting-keystore.json + \ + voting_keystore_password_path: /secrets/password +", + "ab".repeat(48), + "cd".repeat(20) + ); + let definitions: ValidatorDefinitions = serde_yaml_ng::from_str(&yaml).expect("parses"); + let overrides = definitions.0[0].overrides().expect("valid"); + assert_eq!(overrides.fee_recipient, Some(H160([0xcd; 20]))); + assert_eq!( + overrides.graffiti, + Some(graffiti_from_text("from lighthouse").expect("fits")) + ); + assert_eq!(overrides.gas_limit, Some(36_000_000)); + } + + #[test] + fn a_malformed_override_is_an_error_naming_the_field() { + let mut entry = definition("0xaa", true); + entry.suggested_fee_recipient = Some("0x1234".to_string()); + let err = entry.overrides().expect_err("two bytes is not an address"); + assert!( + err.to_string().contains("suggested_fee_recipient") && err.to_string().contains("0xaa"), + "got {err}" + ); + + let mut entry = definition("0xaa", true); + entry.graffiti = Some("a".repeat(GRAFFITI_BYTES + 1)); + let err = entry.overrides().expect_err("33 bytes does not fit"); + assert!(err.to_string().contains("graffiti"), "got {err}"); + } + + /// Startup seeding: enabled entries' overrides land in the settings, a + /// disabled entry's do not. + #[test] + fn seeding_installs_enabled_entries_overrides_only() { + use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, H160}; + + let enabled_key = format!("0x{}", "11".repeat(48)); + let disabled_key = format!("0x{}", "22".repeat(48)); + let mut enabled = definition(&enabled_key, true); + enabled.apply(SettingChange::FeeRecipient(Some(H160([0xbb; 20])))); + let mut disabled = definition(&disabled_key, false); + disabled.apply(SettingChange::GasLimit(Some(1))); + let definitions = ValidatorDefinitions(vec![enabled, disabled, definition("0xcc", true)]); + + let settings = ProposerSettings::new(Bytes32::ZERO, None); + let seeded = definitions.seed_overrides(&settings).expect("seeds"); + assert_eq!(seeded, 1); + assert_eq!( + settings.fee_recipient(&BlsPubkey([0x11; 48])), + Some(H160([0xbb; 20])) + ); + assert_eq!( + settings.overrides(&BlsPubkey([0x22; 48])), + Overrides::default() + ); + } + + #[test] + fn clearing_an_override_removes_the_field() { + let mut entry = definition("0xaa", true); + entry.apply(SettingChange::GasLimit(Some(1))); + entry.apply(SettingChange::GasLimit(None)); + assert_eq!(entry, definition("0xaa", true)); + } + #[test] fn a_missing_file_is_an_empty_set_not_an_error() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/crates/validator/src/keys/store.rs b/crates/validator/src/keys/store.rs index 216855253..01ad461e3 100644 --- a/crates/validator/src/keys/store.rs +++ b/crates/validator/src/keys/store.rs @@ -280,6 +280,9 @@ mod tests { voting_public_key: "0x9612d7a727c9d0a22e185a1c768478dfe919cada9266988cb32359c11f2b7b27f4ae4040902382ae2910c15e2b420d07".to_string(), voting_keystore_path: PathBuf::from("keystore.json"), voting_keystore_password_path: PathBuf::from("password.txt"), + suggested_fee_recipient: None, + graffiti: None, + gas_limit: None, }]); definitions.save(dir.path()).expect("saves definitions"); @@ -306,6 +309,9 @@ mod tests { voting_public_key: voting_public_key.to_string(), voting_keystore_path: PathBuf::from("keystore.json"), voting_keystore_password_path: PathBuf::from("password.txt"), + suggested_fee_recipient: None, + graffiti: None, + gas_limit: None, }]) .save(dir) .expect("saves definitions"); diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index ebec73b1f..3e0742950 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -150,6 +150,18 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { if store.is_empty() { warn!("No validators loaded; the client will idle"); } + // The per-validator fee recipients, graffiti and gas limits the keymanager + // persisted, or the operator wrote by hand. A malformed one is fatal, as a + // bad keystore is: a validator paying an address the operator did not + // mean is worse than a refused start. + let seeded = + keys::ValidatorDefinitions::open(&config.validators_dir)?.seed_overrides(&settings)?; + if seeded > 0 { + info!( + count = seeded, + "Loaded per-validator proposer settings from the definitions file" + ); + } metrics::set_validators_loaded(store.len() as u64); // The keymanager mutates this at runtime while the duty loop reads it, so // it is shared rather than owned outright from here on. diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 01daf3f4d..71b5aa60c 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -4,13 +4,15 @@ //! The keymanager API's `feerecipient`, `graffiti` and `gas_limit` endpoints //! override them per key, and `DELETE` puts a key back on the defaults. //! -//! # In memory only +//! # The definitions file is the record; this is the cache //! -//! Overrides are not written anywhere, so a restart puts every validator back -//! on the defaults. Unlike an imported key, which must survive a restart or a -//! validator silently stops signing, an override that is lost is at worst a -//! block paying the default address or carrying the default text, and an -//! operator driving these endpoints can re-apply them after a restart. +//! Each override is persisted on its validator's entry in +//! `validator_definitions.yml` (`suggested_fee_recipient`, `graffiti`, +//! `gas_limit`, Lighthouse's layout, so a file written by either client reads +//! in the other). The keymanager writes the file before it changes anything +//! here, and startup seeds this from the file, so what an operator set is what +//! the next restart runs with. An operator may also put the fields in the file +//! by hand before starting the client. //! //! # When a change takes effect //! @@ -24,7 +26,7 @@ use std::collections::HashMap; use std::sync::{PoisonError, RwLock}; -use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, H256}; +use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, H160, H256}; /// A block's graffiti field is exactly this wide. pub const GRAFFITI_BYTES: usize = 32; @@ -64,18 +66,63 @@ pub fn graffiti_to_text(graffiti: &Bytes32) -> String { /// held so the keymanager API can serve it as the specification requires. pub const DEFAULT_GAS_LIMIT: u64 = 60_000_000; +/// `text` as a fee recipient: `0x` and forty hex digits, the keymanager +/// specification's `EthAddress` pattern. The zero address is refused: the +/// specification forbids it outright, since it would read as an address +/// chosen on purpose and pays nobody. +pub fn fee_recipient_from_text(text: &str) -> Result { + let Some(digits) = text.strip_prefix("0x") else { + return Err("must be 0x-prefixed hex".to_string()); + }; + let bytes = hex::decode(digits).map_err(|err| format!("is not hex: {err}"))?; + let bytes: [u8; ADDRESS_BYTES] = bytes + .try_into() + .map_err(|got: Vec| format!("is {} bytes, expected {ADDRESS_BYTES}", got.len()))?; + if bytes == [0; ADDRESS_BYTES] { + return Err("is the zero address, which cannot be a fee recipient".to_string()); + } + Ok(H160(bytes)) +} + +/// A fee recipient as the `0x`-prefixed hex the file and the API carry. +pub fn fee_recipient_to_text(address: &ExecutionAddress) -> String { + format!("0x{}", hex::encode(address.0)) +} + +/// An execution address is exactly this wide. +pub const ADDRESS_BYTES: usize = 20; + /// One validator's overrides; `None` means the default applies. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -struct Overrides { - fee_recipient: Option, - graffiti: Option, - gas_limit: Option, +pub struct Overrides { + pub fee_recipient: Option, + pub graffiti: Option, + pub gas_limit: Option, } impl Overrides { - fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { *self == Self::default() } + + /// `change` applied on top of these. + pub fn apply(&mut self, change: SettingChange) { + match change { + SettingChange::FeeRecipient(address) => self.fee_recipient = address, + SettingChange::Graffiti(graffiti) => self.graffiti = graffiti, + SettingChange::GasLimit(gas_limit) => self.gas_limit = gas_limit, + } + } +} + +/// One change to one validator's overrides: a new value, or `None` to put +/// that setting back on the default. One type for the three, so the +/// keymanager's persist-then-apply step is written once rather than six times. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SettingChange { + FeeRecipient(Option), + Graffiti(Option), + GasLimit(Option), } #[derive(Debug)] @@ -154,28 +201,45 @@ impl ProposerSettings { }) } + /// Every override `pubkey` holds, defaults left as `None`. + pub fn overrides(&self, pubkey: &BlsPubkey) -> Overrides { + self.read(|inner| inner.overrides.get(pubkey).copied().unwrap_or_default()) + } + + /// Apply one change to `pubkey`'s overrides. The keymanager calls this + /// only after the same change is in the definitions file. + pub fn apply(&self, pubkey: &BlsPubkey, change: SettingChange) { + self.update(pubkey, |entry| entry.apply(change)); + } + + /// Install `pubkey`'s overrides wholesale, replacing any it had. What + /// startup does with each entry read from the definitions file. + pub fn seed(&self, pubkey: &BlsPubkey, overrides: Overrides) { + self.update(pubkey, |entry| *entry = overrides); + } + pub fn set_fee_recipient(&self, pubkey: &BlsPubkey, address: ExecutionAddress) { - self.update(pubkey, |entry| entry.fee_recipient = Some(address)); + self.apply(pubkey, SettingChange::FeeRecipient(Some(address))); } pub fn clear_fee_recipient(&self, pubkey: &BlsPubkey) { - self.update(pubkey, |entry| entry.fee_recipient = None); + self.apply(pubkey, SettingChange::FeeRecipient(None)); } pub fn set_graffiti(&self, pubkey: &BlsPubkey, graffiti: Bytes32) { - self.update(pubkey, |entry| entry.graffiti = Some(graffiti)); + self.apply(pubkey, SettingChange::Graffiti(Some(graffiti))); } pub fn clear_graffiti(&self, pubkey: &BlsPubkey) { - self.update(pubkey, |entry| entry.graffiti = None); + self.apply(pubkey, SettingChange::Graffiti(None)); } pub fn set_gas_limit(&self, pubkey: &BlsPubkey, gas_limit: u64) { - self.update(pubkey, |entry| entry.gas_limit = Some(gas_limit)); + self.apply(pubkey, SettingChange::GasLimit(Some(gas_limit))); } pub fn clear_gas_limit(&self, pubkey: &BlsPubkey) { - self.update(pubkey, |entry| entry.gas_limit = None); + self.apply(pubkey, SettingChange::GasLimit(None)); } /// Drop every override `pubkey` holds, for a key this client no longer @@ -276,6 +340,44 @@ mod tests { assert!(graffiti_from_text(&"ñ".repeat(17)).is_none()); } + #[test] + fn seeding_replaces_a_keys_overrides_wholesale() { + let settings = settings(); + settings.set_gas_limit(&key(1), 30_000_000); + settings.seed( + &key(1), + Overrides { + fee_recipient: Some(H160([0xbb; 20])), + graffiti: None, + gas_limit: None, + }, + ); + assert_eq!(settings.fee_recipient(&key(1)), Some(H160([0xbb; 20]))); + assert_eq!( + settings.gas_limit(&key(1)), + DEFAULT_GAS_LIMIT, + "an override the seed does not carry is gone" + ); + settings.seed(&key(1), Overrides::default()); + assert!(settings.read(|inner| inner.overrides.is_empty())); + } + + #[test] + fn a_fee_recipient_round_trips_through_text_and_the_zero_address_is_refused() { + let text = format!("0x{}", "ab".repeat(ADDRESS_BYTES)); + let address = fee_recipient_from_text(&text).expect("valid"); + assert_eq!(address, H160([0xab; ADDRESS_BYTES])); + assert_eq!(fee_recipient_to_text(&address), text); + for bad in [ + "ab".repeat(ADDRESS_BYTES), + format!("0x{}", "ab".repeat(ADDRESS_BYTES - 1)), + format!("0x{}", "zz".repeat(ADDRESS_BYTES)), + format!("0x{}", "00".repeat(ADDRESS_BYTES)), + ] { + assert!(fee_recipient_from_text(&bad).is_err(), "{bad}"); + } + } + #[test] fn a_key_cleared_of_every_override_leaves_no_entry() { let settings = settings(); diff --git a/docs/cli.md b/docs/cli.md index 2dcbf6703..fb893889a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -415,9 +415,14 @@ give one validator its own value in place of the command line's default: | `graffiti` | `--graffiti` | The next proposal | | `gas_limit` | `DEFAULT_GAS_LIMIT` | Stored and served only: it matters to builder registrations, and there is no builder flow | -**These are held in memory only.** A restart puts every validator back on the -defaults, so an operator driving these endpoints must set them again after one. -Deleting a keystore drops its values too. +**These persist.** Each is written onto the validator's entry in +`validator_definitions.yml` (`suggested_fee_recipient`, `graffiti`, `gas_limit`, +Lighthouse's layout, so a file written by either client reads in the other) +before the request is answered, and loaded again at startup, so a restart runs +with what the operator set. A write that fails is a `500` that changes nothing. +The fields may also be put in the file by hand before the client starts; a +malformed one refuses startup. Deleting a keystore drops its entry and its +values with it. A few answers worth knowing: `GET feerecipient` is a `500` when the validator has no address and there is no `--suggested-fee-recipient` default, as From 79f0cb60166cf3a767be901fe57300144053ac8e Mon Sep 17 00:00:00 2001 From: Pablo Deymonnaz Date: Fri, 13 Nov 2026 15:27:50 -0300 Subject: [PATCH 2/2] Address the security review of the proposer-settings PR: make an import load and validate the overrides its entry carries, flush the definitions file before relying on it, and keep the validator store's lock off the file write. - Import: an entry the import enables (one that was disabled, or one a failed delete left behind) kept its fee recipient, graffiti and gas limit in the file while the process ran on the defaults until a restart switched them. The overrides are now parsed before anything is written, so a malformed one fails the import rather than the next start, and seeded into the settings once the save succeeds. The entry is found by its parsed key, as delete and the settings routes find it, not by how the file spells it. - Durability: secure_fs flushes every file it writes, and ValidatorDefinitions::save flushes the directory after the rename. A rename that outlived its data left an empty definitions file, which parses as no validators rather than as an error. - Locking: persist_and_apply reads the store's lock for the held check only. definitions_lock already keeps a delete out, and a read held across the file work let an import's queued write stall the duty loop's signing behind it. --- crates/validator/src/http_api/keystores.rs | 194 ++++++++++++++++++++- crates/validator/src/http_api/proposer.rs | 33 ++-- crates/validator/src/keys/definitions.rs | 27 ++- crates/validator/src/secure_fs.rs | 53 +++++- docs/cli.md | 5 +- 5 files changed, 282 insertions(+), 30 deletions(-) diff --git a/crates/validator/src/http_api/keystores.rs b/crates/validator/src/http_api/keystores.rs index 132e0910f..c2e77dd19 100644 --- a/crates/validator/src/http_api/keystores.rs +++ b/crates/validator/src/http_api/keystores.rs @@ -33,6 +33,7 @@ use crate::beacon_node::dto::{encode_hex, parse_pubkey}; use crate::http_api::KeymanagerContext; use crate::keys::keystore::Keystore; use crate::keys::{ValidatorDefinition, ValidatorDefinitions, ValidatorStore}; +use crate::proposer_settings::Overrides; use crate::secure_fs; /// The EIP-3076 interchange version this client reports when it has no history. @@ -294,6 +295,31 @@ fn persist_import( let keystore_path = absolute(context.validators_dir.join(format!("{base}.json")))?; let password_path = absolute(context.secrets_dir.join(&base))?; + // Re-importing an already-known key updates its entry in place instead + // of appending a duplicate: `voting_public_key` is this file's natural + // key, and a duplicate would make `ValidatorStore::load` process the + // same keystore twice on the next restart. + // + // Compared as keys, the way the delete and the settings routes find an + // entry, not as text: a hand-written entry may spell its key in upper + // case or without the `0x`, and one this lookup missed would be appended + // beside rather than replaced, leaving its overrides on an entry the + // settings below no longer describe. + let position = definitions.0.iter().position(|existing| { + parse_pubkey(&existing.voting_public_key).is_ok_and(|parsed| parsed == *pubkey) + }); + + // The overrides the rebuilt entry will carry (see `with_overrides_of` + // below), parsed before anything is written. Startup parses enabled + // entries only, so the entry being replaced may hold a value nothing has + // checked yet, and this import is what enables it: carried over unparsed, + // a malformed one would be reported `imported` and then refuse the next + // start, for every validator rather than this one. + let overrides = match position { + Some(index) => definitions.0[index].overrides()?, + None => Overrides::default(), + }; + // `write_private_no_symlink` still overwrites an existing *regular* // file, so re-importing the same key (rotating its password, or simply // retrying) behaves exactly as a plain `std::fs::write` would; it only @@ -335,15 +361,7 @@ fn persist_import( }, } - // Re-importing an already-known key updates its entry in place instead - // of appending a duplicate: `voting_public_key` is this file's natural - // key, and a duplicate would make `ValidatorStore::load` process the - // same keystore twice on the next restart. - let undo = match definitions - .0 - .iter() - .position(|existing| existing.voting_public_key == base) - { + let undo = match position { Some(index) => { // A re-imported key keeps the fee recipient, graffiti and gas // limit its entry held: only the keystore and password move. @@ -371,6 +389,16 @@ fn persist_import( } return Err(err); } + + // The file now holds this key's entry, enabled, with these overrides, so + // the settings must hold them too. For a key already loaded this changes + // nothing. For an entry this import enables (one that was disabled, or + // one a failed delete left behind after forgetting the key's settings) + // nothing else would load them before the next restart: the file would + // name one fee recipient while the process paid the default, and the + // restart would switch it with no request having asked. A fresh import + // has none, which clears anything left over for the key. + context.settings.seed(pubkey, overrides); Ok(()) } @@ -911,6 +939,154 @@ mod tests { assert_eq!(definitions.0[0].graffiti.as_deref(), Some("kept")); } + /// The public key of [`keystore_json`]'s keystore. + fn keystore_pubkey() -> BlsPubkey { + parse_pubkey( + "9612d7a727c9d0a22e185a1c768478dfe919cada9266988cb32359c11f2b7b27f4ae4040902382ae2910c15e2b420d07", + ) + .expect("valid pubkey") + } + + /// A disabled entry for [`keystore_json`]'s key carrying overrides of its + /// own, as an operator would have left one in the file. + fn disabled_entry(suggested_fee_recipient: &str) -> ValidatorDefinition { + ValidatorDefinition { + enabled: false, + voting_public_key: encode_hex(&keystore_pubkey().0), + voting_keystore_path: PathBuf::from("old.json"), + voting_keystore_password_path: PathBuf::from("old.txt"), + suggested_fee_recipient: Some(suggested_fee_recipient.to_string()), + graffiti: Some("kept".to_string()), + gas_limit: Some(30_000_000), + } + } + + fn import_request() -> Request { + let import = serde_json::json!({ + "keystores": [keystore_json()], + "passwords": [PASSWORD], + }); + authed("POST", "/eth/v1/keystores", import) + } + + /// An import is what enables a disabled entry, so it is also what has to + /// load the overrides the entry carries: startup seeds enabled entries + /// only. Without that the file names one fee recipient while the process + /// pays the default and `GET feerecipient` reports the default, until a + /// restart switches the payout with no request having asked for it. + #[tokio::test] + async fn an_import_that_enables_an_entry_loads_the_overrides_it_carries() { + let dir = tempfile::tempdir().expect("temp dir"); + let store = Arc::new(RwLock::new(ValidatorStore::new())); + let context = context_with_dirs(store.clone(), dir.path()); + let app = router(context.clone(), TOKEN.to_string()); + let pubkey = keystore_pubkey(); + let own = format!("0x{}", "bb".repeat(20)); + ValidatorDefinitions(vec![disabled_entry(&own)]) + .save(dir.path()) + .expect("writes the definitions file"); + + let (status, body) = send(app.clone(), import_request()).await; + assert_eq!(status, StatusCode::OK, "got {body}"); + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + + let definitions = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(definitions.0.len(), 1, "one entry, not a duplicate"); + assert!(definitions.0[0].enabled); + let persisted = definitions.0[0].overrides().expect("valid"); + assert_eq!( + persisted.fee_recipient, + Some(ethlambda_types::beacon::primitives::H160([0xbb; 20])), + "the entry keeps what it carried" + ); + assert_eq!(persisted.gas_limit, Some(30_000_000)); + assert_eq!( + context.settings.overrides(&pubkey), + persisted, + "the process must run with what the file says, not the defaults" + ); + + // And it is what the API reports. This context has no default fee + // recipient, so an import that left the settings alone answers 500. + let path = format!("/eth/v1/validator/{}/feerecipient", encode_hex(&pubkey.0)); + let request = Request::builder() + .uri(&path) + .header("authorization", format!("Bearer {TOKEN}")) + .body(Body::empty()) + .expect("request"); + let (status, body) = send(app, request).await; + assert_eq!(status, StatusCode::OK, "got {body}"); + assert_eq!(body["data"]["ethaddress"], own, "got {body}"); + } + + /// Startup parses enabled entries only, so a disabled entry can hold a + /// value nothing has checked. An import that enabled it unparsed would + /// report `imported` and then refuse the next start, for every validator. + /// It is refused here instead, before anything is written. + #[tokio::test] + async fn a_malformed_override_on_the_entry_fails_the_import_and_changes_nothing() { + let dir = tempfile::tempdir().expect("temp dir"); + let store = Arc::new(RwLock::new(ValidatorStore::new())); + let context = context_with_dirs(store.clone(), dir.path()); + let app = router(context.clone(), TOKEN.to_string()); + let pubkey = keystore_pubkey(); + let existing = disabled_entry("0x1234"); + ValidatorDefinitions(vec![existing.clone()]) + .save(dir.path()) + .expect("writes the definitions file"); + + let (status, body) = send(app, import_request()).await; + assert_eq!(status, StatusCode::OK, "got {body}"); + assert_eq!(body["data"][0]["status"], "error", "got {body}"); + assert!( + body["data"][0]["message"] + .as_str() + .is_some_and(|message| message.contains("suggested_fee_recipient")), + "the error should name the field: {body}" + ); + + assert_eq!( + ValidatorDefinitions::open(dir.path()).expect("opens").0, + vec![existing], + "the entry stays as it was, disabled" + ); + assert!(store.read().await.is_empty(), "the key is not signing"); + assert_eq!(context.settings.overrides(&pubkey), Overrides::default()); + let keystore = dir.path().join(format!("{}.json", encode_hex(&pubkey.0))); + assert!(!keystore.exists(), "refused before the keystore is written"); + } + + /// The entry is found by its key, not by how the file spells it: a + /// hand-written `voting_public_key` in upper case with no `0x` loads like + /// any other, so a re-import must replace it, overrides carried, rather + /// than append a second entry beside it. + #[tokio::test] + async fn a_reimport_replaces_an_entry_that_spells_its_key_differently() { + let dir = tempfile::tempdir().expect("temp dir"); + let store = Arc::new(RwLock::new(ValidatorStore::new())); + let context = context_with_dirs(store.clone(), dir.path()); + let app = router(context.clone(), TOKEN.to_string()); + let pubkey = keystore_pubkey(); + let mut existing = disabled_entry(&format!("0x{}", "bb".repeat(20))); + existing.voting_public_key = hex::encode_upper(pubkey.0); + ValidatorDefinitions(vec![existing]) + .save(dir.path()) + .expect("writes the definitions file"); + + let (status, body) = send(app, import_request()).await; + assert_eq!(status, StatusCode::OK, "got {body}"); + assert_eq!(body["data"][0]["status"], "imported", "got {body}"); + + let definitions = ValidatorDefinitions::open(dir.path()).expect("opens"); + assert_eq!(definitions.0.len(), 1, "replaced, not appended beside"); + assert_eq!(definitions.0[0].voting_public_key, encode_hex(&pubkey.0)); + assert_eq!(definitions.0[0].graffiti.as_deref(), Some("kept")); + assert_eq!( + context.settings.overrides(&pubkey), + definitions.0[0].overrides().expect("valid") + ); + } + /// The regression test for the gap that let the keymanager mutate only /// the in-memory store: a fresh `ValidatorStore::load` from the same /// directory, independent of the one the API mutated, is the only way to diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index 4ef899fe9..ce9e86f20 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -12,13 +12,14 @@ //! one to drop. //! //! Every route answers 404 for a key this client does not hold. The check and -//! the change happen under one read of the validator store, so a keystore -//! delete, which takes that store's write lock and then forgets the key's -//! settings, cannot land between them and leave an override behind for a key -//! that is gone. The file is written under the same `definitions_lock` the -//! keystore routes take, and before memory changes: a failed write is a 500 -//! that changes nothing, so what the operator sees is what the next restart -//! loads. +//! the change happen under one hold of `definitions_lock`, which a keystore +//! delete also holds from before it removes the key until after it has +//! forgotten the key's settings, so a delete cannot land between them and +//! leave an override behind for a key that is gone. The validator store's own +//! lock is read for the check alone and released before the file is touched, +//! since the duty loop signs under it. The file is written before memory +//! changes: a failed write is a 500 that changes nothing, so what the operator +//! sees is what the next restart loads. use axum::Json; use axum::extract::rejection::JsonRejection; @@ -123,8 +124,17 @@ fn require_held(store: &ValidatorStore, pubkey: &BlsPubkey) -> Result<(), ApiErr /// /// Locks in the order the keystore routes take them: `definitions_lock` /// first, which serialises this open-mutate-save cycle against a concurrent -/// import or delete, then `store`'s read lock for the held check, kept until -/// the change is applied so a delete cannot land in between. +/// import or delete, then `store`'s read lock for the held check. +/// +/// The read lock is released as soon as the check has answered, not kept +/// across the file work below. It does not need to be: only a delete removes +/// a key, and a delete holds `definitions_lock` for its whole duration, so +/// none can land before the change is applied. And it must not be: an import +/// takes `store`'s write lock after it has released `definitions_lock`, that +/// write would queue behind a read still held here, and +/// `tokio::sync::RwLock` is write-preferring, so every signature the duty +/// loop owes would wait behind it for as long as this file took to read, +/// write and flush. /// /// A held key with no entry in the file can only mean the file was edited /// under a running client. That is a 500 naming the file rather than a @@ -135,8 +145,9 @@ async fn persist_and_apply( change: SettingChange, ) -> Result<(), ApiError> { let _definitions_guard = context.definitions_lock.lock().await; - let store = context.store.read().await; - require_held(&store, pubkey)?; + // The guard is a temporary of this statement, so it is gone before the + // file is opened. + require_held(&*context.store.read().await, pubkey)?; let path = context .validators_dir diff --git a/crates/validator/src/keys/definitions.rs b/crates/validator/src/keys/definitions.rs index d19006fff..5f207078f 100644 --- a/crates/validator/src/keys/definitions.rs +++ b/crates/validator/src/keys/definitions.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use tracing::warn; use crate::beacon_node::dto::parse_pubkey; use crate::error::{Error, Result}; @@ -137,6 +138,15 @@ impl ValidatorDefinitions { /// would leave a truncated file that stops the client booting, losing every /// validator rather than the one being changed. `rename` within one /// directory is atomic, so a reader sees the old file or the new one. + /// + /// That holds across a power loss too, not only a crash of this process: + /// `write_private` flushes the temporary file before the rename, so the + /// name never points at data that has not reached the device. Without it + /// the rename can survive and the data not, and an empty definitions file + /// is not an error: it parses as no validators at all, so the client would + /// start, idle, and say so in one warning. The directory is flushed after + /// the rename so that a change the keymanager has answered for is not + /// undone by one either. pub fn save(&self, validators_dir: &Path) -> Result<()> { let path = validators_dir.join(DEFINITIONS_FILE); let contents = serde_yaml_ng::to_string(self).map_err(|err| Error::Keystore { @@ -155,7 +165,22 @@ impl ValidatorDefinitions { std::fs::rename(&temporary, &path).map_err(|source| Error::Io { path: path.display().to_string(), source, - }) + })?; + + // Logged, not returned. The rename above is the commit: the new file + // is already what every reader sees, and every caller takes an error + // from here to mean the file is unchanged (an import undoes its entry, + // a settings route leaves memory alone). Failing now would make that + // false, to report a rename that is in place but not yet flushed. + let _ = secure_fs::sync_dir(validators_dir).inspect_err(|err| { + warn!( + path = %validators_dir.display(), + %err, + "Wrote the validator definitions file but could not flush its directory; the \ + change is in place and may not survive a power loss" + ) + }); + Ok(()) } /// The entries that should be loaded and signed with. diff --git a/crates/validator/src/secure_fs.rs b/crates/validator/src/secure_fs.rs index 574056e9f..aee9a9e35 100644 --- a/crates/validator/src/secure_fs.rs +++ b/crates/validator/src/secure_fs.rs @@ -8,9 +8,38 @@ //! convention here is `0600`; everything in this crate that writes such a //! file must go through one of these two functions instead, to actually //! follow that convention rather than only claim to. +//! +//! Both also flush what they wrote to the device before returning. Each of +//! these files is one a later step relies on having reached disk: the +//! definitions file's `.tmp` sibling is renamed over the live file, and a +//! rename can survive a power loss that the data behind it did not, leaving +//! an empty definitions file, which parses as "no validators" rather than as +//! an error; an imported keystore and its password are named by a definitions +//! entry that is made durable right after them. use std::path::Path; +/// Write `contents` to `file` and flush them to the device. +fn write_durably(mut file: std::fs::File, contents: impl AsRef<[u8]>) -> std::io::Result<()> { + use std::io::Write as _; + file.write_all(contents.as_ref())?; + file.sync_all() +} + +/// Flush a directory, which is what makes a `rename` into it durable: until +/// then a power loss can bring the old name's target back. +#[cfg(unix)] +pub(crate) fn sync_dir(dir: &Path) -> std::io::Result<()> { + std::fs::File::open(dir)?.sync_all() +} + +/// A directory cannot be opened as a file here; the rename is left to the +/// platform. +#[cfg(not(unix))] +pub(crate) fn sync_dir(_dir: &Path) -> std::io::Result<()> { + Ok(()) +} + /// Write `contents`, creating the file if it does not exist and truncating it /// if it does, at mode `0600` on unix. /// @@ -22,20 +51,19 @@ use std::path::Path; /// [`write_private_no_symlink`] for the ones that are. #[cfg(unix)] pub(crate) fn write_private(path: &Path, contents: impl AsRef<[u8]>) -> std::io::Result<()> { - use std::io::Write as _; use std::os::unix::fs::OpenOptionsExt as _; - let mut file = std::fs::OpenOptions::new() + let file = std::fs::OpenOptions::new() .write(true) .create(true) .truncate(true) .mode(0o600) .open(path)?; - file.write_all(contents.as_ref()) + write_durably(file, contents) } #[cfg(not(unix))] pub(crate) fn write_private(path: &Path, contents: impl AsRef<[u8]>) -> std::io::Result<()> { - std::fs::write(path, contents) + write_durably(std::fs::File::create(path)?, contents) } /// Write `contents` at mode `0600`, refusing to follow an existing symlink at @@ -55,16 +83,15 @@ pub(crate) fn write_private_no_symlink( path: &Path, contents: impl AsRef<[u8]>, ) -> std::io::Result<()> { - use std::io::Write as _; use std::os::unix::fs::OpenOptionsExt as _; - let mut file = std::fs::OpenOptions::new() + let file = std::fs::OpenOptions::new() .write(true) .create(true) .truncate(true) .mode(0o600) .custom_flags(libc::O_NOFOLLOW) .open(path)?; - file.write_all(contents.as_ref()) + write_durably(file, contents) } #[cfg(not(unix))] @@ -72,7 +99,7 @@ pub(crate) fn write_private_no_symlink( path: &Path, contents: impl AsRef<[u8]>, ) -> std::io::Result<()> { - std::fs::write(path, contents) + write_durably(std::fs::File::create(path)?, contents) } #[cfg(all(test, unix))] @@ -94,6 +121,16 @@ mod tests { assert_eq!(std::fs::read_to_string(&path).expect("reads"), "shh"); } + /// `ValidatorDefinitions::save` calls this after every rename. It only + /// logs a failure, so a platform that refused to flush a directory would + /// show up as a warning on every keymanager write and nowhere else. + #[test] + fn sync_dir_flushes_a_directory() { + let dir = tempfile::tempdir().expect("temp dir"); + write_private(&dir.path().join("secret"), "shh").expect("writes"); + sync_dir(dir.path()).expect("a directory can be flushed"); + } + #[test] fn write_private_no_symlink_creates_a_mode_0600_file() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/docs/cli.md b/docs/cli.md index fb893889a..2f324f80f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -422,7 +422,10 @@ before the request is answered, and loaded again at startup, so a restart runs with what the operator set. A write that fails is a `500` that changes nothing. The fields may also be put in the file by hand before the client starts; a malformed one refuses startup. Deleting a keystore drops its entry and its -values with it. +values with it. Importing a key whose entry is already in the file, a +re-import or one that enables a disabled entry, keeps the entry's values and +runs with them at once; if one of them is malformed the import is refused and +nothing is written. A few answers worth knowing: `GET feerecipient` is a `500` when the validator has no address and there is no `--suggested-fee-recipient` default, as