diff --git a/Cargo.lock b/Cargo.lock index 0a2fcb8c92..025ba026d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5093,8 +5093,13 @@ dependencies = [ name = "openshell-supervisor" version = "0.0.0" dependencies = [ + "bytes", "clap", "futures", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", "miette", "nix 0.29.0", "openshell-core", diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index a97deab7d6..75672d0263 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -5,21 +5,23 @@ //! `OpenShell` server. //! //! Every request carries a sandbox bearer credential in the `Authorization` -//! header. The token is resolved at startup from one of three sources: +//! header. Each sandbox instance owns its [`GatewayCredentials`], usually +//! built from the trusted launch bundle with +//! [`GatewayCredentials::from_auth_bundle`]. Diagnostic tools can instead +//! resolve a token with [`GatewayCredentials::from_environment`] from one of: //! //! 1. `OPENSHELL_SANDBOX_TOKEN` — raw JWT in the env (test harness path). -//! 2. `OPENSHELL_SANDBOX_TOKEN_FILE` — file containing the JWT (Docker / -//! Podman / VM drivers write this to a bundle file at sandbox-create -//! time). +//! 2. `OPENSHELL_SANDBOX_TOKEN_FILE` — file containing the JWT. //! 3. `OPENSHELL_K8S_SA_TOKEN_FILE` — projected `ServiceAccount` JWT; the -//! supervisor exchanges it for a gateway JWT via `IssueSandboxToken` -//! once at startup. +//! caller exchanges it for a gateway JWT via `IssueSandboxToken`. //! -//! The resolved bearer credential is held in process memory thereafter and -//! injected on every outbound call by [`AuthInterceptor`]. +//! Callers reach the gateway through a [`GatewayEndpoint`], which pairs the +//! URL with one sandbox instance's credentials. No credential is installed +//! process-wide, so instances hosted by one supervisor process cannot use or +//! rotate each other's identity. use std::collections::HashMap; -use std::sync::{Arc, OnceLock, RwLock}; +use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::endpoint_status::{EndpointResult, EndpointStatusSnapshot}; @@ -63,8 +65,8 @@ pub const MAX_SUPERVISOR_STREAM_MESSAGE_BYTES: usize = 9 * 1024 * 1024; /// Shared, refreshable Bearer header. All [`AuthInterceptor`] clones read /// the same slot, so the renewal task can replace the token in place without -/// rebuilding the channel. -type TokenSlot = Arc>; +/// rebuilding the channel. `None` means the owning sandbox instance revoked it. +type TokenSlot = Arc>>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TokenSource { @@ -73,38 +75,224 @@ enum TokenSource { K8sServiceAccount, } -/// Process-wide token slot. Initialized by the first [`connect_channel`] -/// call and shared with every subsequent client and the renewal loop. -static TOKEN_SLOT: OnceLock = OnceLock::new(); +/// Gateway credentials owned by one sandbox instance. +/// +/// Clients built from a [`GatewayEndpoint`] read only these slots, and the +/// renewal task started by the first connection updates only them. Dropping +/// the last clone or calling [`GatewayCredentials::revoke`] stops renewal. +#[derive(Clone)] +pub struct GatewayCredentials { + inner: Arc, +} -/// Refresh strategy used by the process-wide token slot. -static TOKEN_REFRESH_MODE: OnceLock = OnceLock::new(); +struct CredentialState { + bearer: TokenSlot, + source: TokenSource, + #[cfg(feature = "jwt")] + sandbox_bearer: Option, + renewal: std::sync::Mutex, +} -/// Serializes the first token acquisition. Several supervisor subsystems -/// connect during startup; without this guard they can all observe an empty -/// [`TOKEN_SLOT`] and perform duplicate K8s bootstrap exchanges. -static TOKEN_INIT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); +enum Renewal { + Idle, + Started(tokio::task::AbortHandle), + Revoked, +} -/// One-shot guard so the renewal loop spawns at most once per process. -static REFRESH_SPAWNED: OnceLock<()> = OnceLock::new(); +impl Drop for CredentialState { + fn drop(&mut self) { + let renewal = self + .renewal + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Renewal::Started(task) = renewal { + task.abort(); + } + } +} -#[cfg(feature = "jwt")] -static SANDBOX_BEARER_SLOT: OnceLock = OnceLock::new(); +impl std::fmt::Debug for GatewayCredentials { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GatewayCredentials") + .field("source", &self.inner.source) + .finish_non_exhaustive() + } +} -#[derive(Clone, Debug)] -enum RefreshMode { - GatewayJwt(TokenSource), +impl GatewayCredentials { + fn new(bearer: AsciiMetadataValue, source: TokenSource) -> Self { + Self { + inner: Arc::new(CredentialState { + bearer: Arc::new(RwLock::new(Some(bearer))), + source, + #[cfg(feature = "jwt")] + sandbox_bearer: None, + renewal: std::sync::Mutex::new(Renewal::Idle), + }), + } + } + + /// Credentials issued for one exact sandbox launch. Renewal also rotates + /// the bundle's Sandbox Protocol credential, returned by + /// [`GatewayCredentials::sandbox_bearer`]. + #[cfg(feature = "jwt")] + pub fn from_auth_bundle(bundle: &crate::jwt::SupervisorAuthBundle) -> Result { + let bearer = validate_gateway_bearer(bundle.gateway_token.expose_secret())?; + let sandbox_bearer = bundle + .sandbox_bearer_slot() + .into_diagnostic() + .wrap_err("invalid Sandbox Protocol credential")?; + Ok(Self { + inner: Arc::new(CredentialState { + bearer: Arc::new(RwLock::new(Some(bearer))), + source: TokenSource::File, + sandbox_bearer: Some(sandbox_bearer), + renewal: std::sync::Mutex::new(Renewal::Idle), + }), + }) + } + + /// Use a caller-supplied gateway JWT. Renewal cannot rebootstrap it. + pub fn from_gateway_token(token: &str) -> Result { + Ok(Self::new(validate_gateway_bearer(token)?, TokenSource::Env)) + } + + /// Resolve a token for diagnostic tools from the environment, exchanging a + /// projected `ServiceAccount` token when no gateway JWT is present. + pub async fn from_environment(endpoint: &str) -> Result { + let plain_channel = build_plain_channel(endpoint).await?; + let acquired = acquire_sandbox_token(endpoint, &plain_channel).await?; + Ok(Self::new( + validate_gateway_bearer(&acquired.token)?, + acquired.source, + )) + } + + /// The Sandbox Protocol credential renewed with these gateway credentials. + #[cfg(feature = "jwt")] + #[must_use] + pub fn sandbox_bearer(&self) -> Option { + self.inner.sandbox_bearer.clone() + } + + /// Clear both credentials and stop renewal. Existing clients fail their + /// next request locally instead of presenting a revoked identity. + pub fn revoke(&self) { + let mut renewal = self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Renewal::Started(task) = &*renewal { + task.abort(); + } + *renewal = Renewal::Revoked; + *self + .inner + .bearer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + #[cfg(feature = "jwt")] + if let Some(sandbox_bearer) = &self.inner.sandbox_bearer { + sandbox_bearer.clear(); + } + } + + /// Start the renewal task once for these credentials, on the caller's + /// runtime, so a hosted sandbox instance's renewal stops with that instance. + fn start_renewal(&self, channel: &AuthedChannel, plain_channel: &Channel, endpoint: &str) { + let mut renewal = self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !matches!(*renewal, Renewal::Idle) { + return; + } + let task = tokio::spawn(refresh_token_loop(RenewalTarget { + channel: channel.clone(), + bearer: self.inner.bearer.clone(), + #[cfg(feature = "jwt")] + sandbox_bearer: self.inner.sandbox_bearer.clone(), + source: self.inner.source, + endpoint: endpoint.to_string(), + plain_channel: plain_channel.clone(), + })); + *renewal = Renewal::Started(task.abort_handle()); + } + + fn is_revoked(&self) -> bool { + matches!( + *self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + Renewal::Revoked + ) + } + + fn install_gateway_bearer(&self, bearer: AsciiMetadataValue) { + let mut slot = self + .inner + .bearer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // Revoked instance credentials must not be revived by an in-flight + // response. + if slot.is_some() { + *slot = Some(bearer); + } + } +} + +/// The gateway URL a sandbox instance uses, paired with that instance's +/// credentials. +#[derive(Clone)] +pub struct GatewayEndpoint { + url: Arc, + credentials: GatewayCredentials, +} + +impl GatewayEndpoint { + pub fn new(url: impl Into>, credentials: GatewayCredentials) -> Self { + Self { + url: url.into(), + credentials, + } + } + + #[must_use] + pub fn url(&self) -> &str { + &self.url + } + + #[must_use] + pub fn credentials(&self) -> &GatewayCredentials { + &self.credentials + } + + /// Reach another gateway address with the same sandbox instance credentials. + #[must_use] + pub fn with_url(&self, url: impl Into>) -> Self { + Self::new(url, self.credentials.clone()) + } +} + +impl std::fmt::Debug for GatewayEndpoint { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GatewayEndpoint") + .field("url", &self.url) + .finish_non_exhaustive() + } } #[derive(Debug)] struct AcquiredToken { token: String, - refresh_mode: RefreshMode, -} - -fn install_token_slot(token: &str) -> Result { - let bearer = validate_gateway_bearer(token)?; - Ok(install_validated_token_slot(bearer)) + source: TokenSource, } fn validate_gateway_bearer(token: &str) -> Result { @@ -113,18 +301,6 @@ fn validate_gateway_bearer(token: &str) -> Result { .wrap_err("sandbox JWT contained characters not valid for a header value") } -fn install_validated_token_slot(bearer: AsciiMetadataValue) -> TokenSlot { - if let Some(existing) = TOKEN_SLOT.get() { - *existing - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = bearer; - return existing.clone(); - } - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); - let _ = TOKEN_SLOT.set(slot.clone()); - TOKEN_SLOT.get().cloned().unwrap_or(slot) -} - #[cfg(feature = "jwt")] struct ValidatedSandboxRefresh { token: crate::jwt::SecretJwt, @@ -159,22 +335,6 @@ fn validate_sandbox_refresh( }) } -/// Install the gateway-session credential supplied in trusted supervisor -/// launch state before any gateway client is constructed. -#[cfg(feature = "jwt")] -pub fn install_supervisor_auth_bundle( - bundle: &crate::jwt::SupervisorAuthBundle, -) -> Result { - install_token_slot(bundle.gateway_token.expose_secret())?; - let _ = TOKEN_REFRESH_MODE.set(RefreshMode::GatewayJwt(TokenSource::File)); - let slot = bundle - .sandbox_bearer_slot() - .into_diagnostic() - .wrap_err("invalid Sandbox Protocol credential")?; - let _ = SANDBOX_BEARER_SLOT.set(slot.clone()); - Ok(SANDBOX_BEARER_SLOT.get().cloned().unwrap_or(slot)) -} - /// gRPC interceptor that injects `authorization: Bearer ` on every /// outbound request. The token lives in a shared [`TokenSlot`] so the renewal /// task can replace it without rebuilding clients. @@ -197,8 +357,11 @@ impl tonic::service::Interceptor for AuthInterceptor { let bearer = self .bearer .read() - .expect("auth interceptor token slot poisoned") - .clone(); + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + .ok_or_else(|| { + Status::unauthenticated("sandbox instance gateway credentials were revoked") + })?; req.metadata_mut().insert("authorization", bearer); #[cfg(feature = "trace-context")] let req = @@ -299,56 +462,26 @@ impl Drop for ClientSpanStatus { } } -/// Build a Bearer-authenticated channel to the gateway. -/// -/// First call per process resolves the sandbox JWT via the three-step -/// lookup (env → file → K8s SA bootstrap exchange) and installs it into -/// the process-wide [`TOKEN_SLOT`]. Subsequent calls reuse the cached -/// slot — the renewal loop keeps the value fresh, so re-running the -/// bootstrap is both unnecessary and (on the K8s SA path) expensive -/// (one apiserver round-trip per call). The renewal loop itself is -/// spawned once per process via [`REFRESH_SPAWNED`]. -async fn connect_channel(endpoint: &str) -> Result { - let channel = build_plain_channel(endpoint).await?; - let (slot, refresh_mode) = token_slot(endpoint, &channel).await?; - let plain_channel = channel.clone(); - let intercepted = InterceptedService::new(channel, AuthInterceptor::new(slot.clone())); - if REFRESH_SPAWNED.set(()).is_ok() { - let RefreshMode::GatewayJwt(source) = refresh_mode; - let refresh_channel = intercepted.clone(); - let endpoint = endpoint.to_string(); - tokio::spawn(async move { - refresh_token_loop(refresh_channel, slot, source, endpoint, plain_channel).await; - }); +/// Build a Bearer-authenticated channel to the gateway with one sandbox +/// instance's credentials. The first connection starts that instance's +/// renewal task. +async fn connect_channel(gateway: &GatewayEndpoint) -> Result { + if gateway.credentials.is_revoked() { + return Err(miette::miette!( + "sandbox instance gateway credentials were revoked" + )); } + let channel = build_plain_channel(gateway.url()).await?; + let intercepted = InterceptedService::new( + channel.clone(), + AuthInterceptor::new(gateway.credentials.inner.bearer.clone()), + ); + gateway + .credentials + .start_renewal(&intercepted, &channel, gateway.url()); Ok(intercepted) } -async fn token_slot(endpoint: &str, plain_channel: &Channel) -> Result<(TokenSlot, RefreshMode)> { - if let Some(existing) = TOKEN_SLOT.get() { - let refresh_mode = TOKEN_REFRESH_MODE - .get() - .cloned() - .unwrap_or(RefreshMode::GatewayJwt(TokenSource::Env)); - return Ok((existing.clone(), refresh_mode)); - } - - let _guard = TOKEN_INIT_LOCK.lock().await; - - if let Some(existing) = TOKEN_SLOT.get() { - let refresh_mode = TOKEN_REFRESH_MODE - .get() - .cloned() - .unwrap_or(RefreshMode::GatewayJwt(TokenSource::Env)); - return Ok((existing.clone(), refresh_mode)); - } - - let acquired = acquire_sandbox_token(endpoint, plain_channel).await?; - let slot = install_token_slot(&acquired.token)?; - let _ = TOKEN_REFRESH_MODE.set(acquired.refresh_mode.clone()); - Ok((slot, acquired.refresh_mode)) -} - /// Resolve the sandbox JWT used to authenticate every outbound RPC. /// /// `endpoint` is logged on errors but never used for transport here; the @@ -362,7 +495,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul debug!(source = "env", "loaded sandbox token"); return Ok(AcquiredToken { token: t, - refresh_mode: RefreshMode::GatewayJwt(TokenSource::Env), + source: TokenSource::Env, }); } @@ -375,7 +508,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul debug!(source = "file", path = %path, "loaded sandbox token"); return Ok(AcquiredToken { token: contents.trim().to_string(), - refresh_mode: RefreshMode::GatewayJwt(TokenSource::File), + source: TokenSource::File, }); } @@ -384,7 +517,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul { return Ok(AcquiredToken { token: acquire_k8s_sandbox_token(endpoint, plain_channel, &sa_path).await?, - refresh_mode: RefreshMode::GatewayJwt(TokenSource::K8sServiceAccount), + source: TokenSource::K8sServiceAccount, }); } @@ -407,14 +540,13 @@ async fn acquire_k8s_sandbox_token( .trim() .to_string(); info!(endpoint = %endpoint, "exchanging K8s ServiceAccount token for sandbox JWT"); - // The bootstrap exchange uses a one-off interceptor pinned to the - // SA token; the resulting gateway JWT becomes the value in the - // shared `TOKEN_SLOT` once `connect_channel` returns. - let bootstrap_slot: TokenSlot = Arc::new(RwLock::new( + // The bootstrap exchange uses a one-off interceptor pinned to the SA + // token; the caller installs the resulting gateway JWT. + let bootstrap_slot: TokenSlot = Arc::new(RwLock::new(Some( AsciiMetadataValue::try_from(format!("Bearer {sa_token}")) .into_diagnostic() .wrap_err("SA token contained characters not valid for a header value")?, - )); + ))); let interceptor = AuthInterceptor::new(bootstrap_slot); let bootstrap = InterceptedService::new(plain_channel.clone(), interceptor); let mut client = OpenShellClient::new(bootstrap); @@ -428,8 +560,8 @@ async fn acquire_k8s_sandbox_token( /// Build an authenticated channel for direct external use (e.g. the /// long-lived `supervisor_session` control stream). -pub async fn connect_channel_pub(endpoint: &str) -> Result { - connect_channel(endpoint).await +pub async fn connect_channel_pub(gateway: &GatewayEndpoint) -> Result { + connect_channel(gateway).await } /// Report installed provider state for the current authenticated supervisor session. @@ -437,11 +569,11 @@ pub async fn connect_channel_pub(endpoint: &str) -> Result { /// The observation must carry the session ID returned by `ConnectSupervisor`. /// Reconnects start a new report sequence; retries preserve the complete report. pub async fn report_provider_readiness( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, observation: crate::proto::ProviderReadinessObservation, ) -> Result { - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; client .report_provider_readiness(crate::proto::ReportProviderReadinessRequest { sandbox_id: sandbox_id.to_string(), @@ -452,18 +584,34 @@ pub async fn report_provider_readiness( .into_diagnostic() } -/// Background task that renews the sandbox JWT at ~80% of its remaining -/// lifetime. The new token replaces the value in [`TOKEN_SLOT`], so all -/// in-flight and future clients pick it up on their next request. The -/// loop never panics: every failure is logged and re-attempted after a -/// bounded backoff. -async fn refresh_token_loop( +/// Slots and transport retained by one sandbox instance's renewal task. It +/// holds no reference to [`GatewayCredentials`], so dropping them still stops +/// it. +struct RenewalTarget { channel: AuthedChannel, - slot: TokenSlot, + bearer: TokenSlot, + #[cfg(feature = "jwt")] + sandbox_bearer: Option, source: TokenSource, endpoint: String, plain_channel: Channel, -) { +} + +/// Background task that renews the sandbox JWT at ~80% of its remaining +/// lifetime. The new token replaces the value in the sandbox instance's slot, +/// so all in-flight and future clients of that instance pick it up on their +/// next request. The loop never panics: every failure is logged and +/// re-attempted after a bounded backoff. +async fn refresh_token_loop(target: RenewalTarget) { + let RenewalTarget { + channel, + bearer: slot, + #[cfg(feature = "jwt")] + sandbox_bearer, + source, + endpoint, + plain_channel, + } = target; let mut client = OpenShellClient::new(channel); loop { let Some(sleep) = compute_refresh_delay(&slot) else { @@ -495,11 +643,17 @@ async fn refresh_token_loop( } }; - *slot - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = gateway_bearer; + { + let mut current = slot + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + return; + } + *current = Some(gateway_bearer); + } #[cfg(feature = "jwt")] - if let Some(sandbox_slot) = SANDBOX_BEARER_SLOT.get() + if let Some(sandbox_slot) = sandbox_bearer.as_ref() && let Err(error) = sandbox_slot.update( sandbox_refresh.token, sandbox_refresh.expires_at, @@ -523,8 +677,10 @@ async fn refresh_token_loop( Ok(new_token) => { match AsciiMetadataValue::try_from(format!("Bearer {new_token}")) { Ok(value) => { - if let Ok(mut guard) = slot.write() { - *guard = value; + if let Ok(mut guard) = slot.write() + && guard.is_some() + { + *guard = Some(value); info!( "rebootstrapped gateway sandbox JWT after refresh authentication failure" ); @@ -585,6 +741,7 @@ fn authenticated_service_names( async fn refresh_extension_credentials_with_client( client: &mut OpenShellClient, + credentials: &GatewayCredentials, store: &ExtensionCredentialStore, services: &[crate::proto::SupervisorMiddlewareService], ) -> Result> { @@ -645,9 +802,9 @@ async fn refresh_extension_credentials_with_client( // Commit only after the gateway, Sandbox Protocol, and extension // credentials have all been parsed and validated. The remaining updates // repeat those validations but cannot fail for the validated inputs. - install_validated_token_slot(gateway_bearer); + credentials.install_gateway_bearer(gateway_bearer); #[cfg(feature = "jwt")] - if let Some(sandbox_slot) = SANDBOX_BEARER_SLOT.get() { + if let Some(sandbox_slot) = credentials.inner.sandbox_bearer.as_ref() { match sandbox_slot.update( sandbox_refresh.token, sandbox_refresh.expires_at, @@ -679,11 +836,14 @@ async fn refresh_extension_credentials_with_client( /// non-expiring session credential and needs no periodic refresh. If the token /// can't be parsed (for example, an opaque bootstrap bearer), default to 6 h. fn compute_refresh_delay(slot: &TokenSlot) -> Option { - let token = slot - .read() - .ok() - .and_then(|v| v.to_str().ok().map(str::to_string)) - .unwrap_or_default(); + let token = { + let current = slot + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // Revoked instance credentials have nothing to renew. + current.as_ref()?.to_str().ok().map(str::to_string) + } + .unwrap_or_default(); let now_ms = i64::try_from( SystemTime::now() .duration_since(UNIX_EPOCH) @@ -819,7 +979,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload_json); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expiring token needs refresh"); // 800 s baseline + up to 10 % jitter → 800..=880 s, with some slack // for the 1-second resolution of the exp claim. @@ -840,7 +1000,7 @@ mod auth_tests { base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#)); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expired token needs refresh"); assert!((1..60).contains(&delay.as_secs())); } @@ -851,7 +1011,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"exp":0}"#); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); assert_eq!(compute_refresh_delay(&slot), None); } @@ -867,7 +1027,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload_json); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expiring token needs refresh"); assert!( delay.as_secs() < 30, @@ -906,9 +1066,9 @@ mod workspace_tests { } } -/// Connect to the `OpenShell` server. -async fn connect(endpoint: &str) -> Result> { - let channel = connect_channel(endpoint).await?; +/// Connect to the `OpenShell` server with one sandbox instance's credentials. +async fn connect(gateway: &GatewayEndpoint) -> Result> { + let channel = connect_channel(gateway).await?; Ok(OpenShellClient::new(channel)) } @@ -918,12 +1078,12 @@ async fn connect(endpoint: &str) -> Result> { /// or `Ok(None)` when the sandbox was created without a policy (the sandbox /// should discover one from disk or use the restrictive default). pub async fn fetch_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_name: &str, ) -> Result> { - debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to OpenShell server"); + debug!(endpoint = %gateway.url(), sandbox_name = %sandbox_name, "Connecting to OpenShell server"); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; debug!("Connected, fetching sandbox policy"); @@ -942,12 +1102,12 @@ pub async fn fetch_policy( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn fetch_settings_snapshot( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_name: &str, ) -> Result { let status = ClientSpanStatus::current(); - debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot"); - let mut client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot"); + let mut client = connect(gateway).await?; status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox_name, None).await) } @@ -1014,18 +1174,18 @@ async fn sync_policy_with_client( /// Performs the full discovery flow (fetch → sync → re-fetch) over one /// channel instead of establishing three separate connections. pub async fn discover_and_sync_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, discovered_policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result { debug!( - endpoint = %endpoint, + endpoint = %gateway.url(), sandbox = %sandbox, "Syncing discovered policy and re-fetching canonical version" ); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; // Sync the discovered policy to the gateway. sync_policy_with_client(&mut client, sandbox, discovered_policy, workspace).await?; @@ -1044,13 +1204,13 @@ pub async fn discover_and_sync_policy( /// Used by the supervisor to push baseline-path-enriched policies so the /// gateway stores the effective policy users see via `openshell sandbox get`. pub async fn sync_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result<()> { - debug!(endpoint = %endpoint, sandbox = %sandbox, "Syncing enriched policy to gateway"); - let mut client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), sandbox = %sandbox, "Syncing enriched policy to gateway"); + let mut client = connect(gateway).await?; sync_policy_with_client(&mut client, sandbox, policy, workspace).await } @@ -1061,13 +1221,13 @@ pub async fn sync_policy( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn sync_policy_and_fetch_snapshot( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result { let status = ClientSpanStatus::current(); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; sync_policy_with_client(&mut client, sandbox, policy, workspace).await?; status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await) } @@ -1080,7 +1240,7 @@ pub async fn sync_policy_and_fetch_snapshot( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn report_sandbox_configuration( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, instance_id: &str, snapshot: Option<&SettingsPollResult>, @@ -1088,7 +1248,7 @@ pub async fn report_sandbox_configuration( error: &str, ) -> Result<()> { let status = ClientSpanStatus::current(); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; client .report_sandbox_configuration(crate::proto::ReportSandboxConfigurationRequest { sandbox_id: sandbox_id.to_string(), @@ -1118,10 +1278,10 @@ pub async fn report_sandbox_configuration( /// environment represents a sandbox without provider credentials. Transport /// failure returns an error so callers can revoke credentials and retry. pub async fn fetch_provider_environment( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, ) -> Result { - fetch_provider_environment_with_stable_placeholders(endpoint, sandbox_id, false).await + fetch_provider_environment_with_stable_placeholders(gateway, sandbox_id, false).await } /// Fetch a snapshot without downgrading previously activated external stable credentials. @@ -1135,14 +1295,14 @@ pub async fn fetch_provider_environment( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn fetch_provider_environment_with_stable_placeholders( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { let status = ClientSpanStatus::current(); - debug!(endpoint = %endpoint, sandbox_id = %sandbox_id, "Fetching provider environment"); + debug!(endpoint = %gateway.url(), sandbox_id = %sandbox_id, "Fetching provider environment"); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; let response = client .get_sandbox_provider_environment(GetSandboxProviderEnvironmentRequest { @@ -1366,7 +1526,7 @@ mod provider_environment_tests { fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn exchange_provider_subject_token( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, provider: &str, credential_key: &str, @@ -1374,7 +1534,7 @@ pub async fn exchange_provider_subject_token( request_id: Option<&str>, ) -> std::result::Result { let status = ClientSpanStatus::current(); - let endpoint = token_grant_gateway_endpoint(endpoint)?; + let endpoint = token_grant_gateway_endpoint(gateway.url())?; if sandbox_id.is_empty() || provider.is_empty() || credential_key.is_empty() { return Err(gateway_grant_failure(TokenGrantFailureKind::Configuration)); } @@ -1388,7 +1548,7 @@ pub async fn exchange_provider_subject_token( // Discard those sources at the grant boundary; never diagnose from prose. // The shared channel bounds the connection attempt; the gateway bounds its // token-service calls, and a gateway deadline arrives as DeadlineExceeded. - let mut client = connect(endpoint.as_str()) + let mut client = connect(&gateway.with_url(endpoint.as_str())) .await .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Unclassified))?; let request = provider_subject_token_exchange_request( @@ -1559,6 +1719,9 @@ mod token_grant_exchange_tests { #[derive(Clone)] pub struct CachedOpenShellClient { client: OpenShellClient, + /// Sandbox instance gateway credentials rotated alongside extension + /// credentials. + credentials: GatewayCredentials, workspace: Arc>, /// Extension credentials for this supervisor. Cloning the client shares /// the store, so the middleware registry and the polling loop that rotates @@ -1887,8 +2050,8 @@ pub struct ProviderSubjectTokenExchangeResult { } impl CachedOpenShellClient { - pub async fn connect(endpoint: &str) -> Result { - Self::connect_with_credentials(endpoint, ExtensionCredentialStore::new()).await + pub async fn connect(gateway: &GatewayEndpoint) -> Result { + Self::connect_with_credentials(gateway, ExtensionCredentialStore::new()).await } /// Connect while sharing an existing credential store. @@ -1898,13 +2061,14 @@ impl CachedOpenShellClient { /// credentials handed to the middleware registry are not the ones the loop /// rotates. pub async fn connect_with_credentials( - endpoint: &str, + gateway: &GatewayEndpoint, extension_credentials: ExtensionCredentialStore, ) -> Result { - debug!(endpoint = %endpoint, "Connecting openshell gRPC client for policy polling"); - let client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), "Connecting openshell gRPC client for policy polling"); + let client = connect(gateway).await?; Ok(Self { client, + credentials: gateway.credentials.clone(), workspace: Arc::new(tokio::sync::OnceCell::new()), extension_credentials, }) @@ -1972,6 +2136,7 @@ impl CachedOpenShellClient { let mut client = self.client.clone(); refresh_extension_credentials_with_client( &mut client, + &self.credentials, &self.extension_credentials, services, ) diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index c7a966a32b..2772f1a69c 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -870,7 +870,7 @@ async fn run_lifecycle( binary_path: host_proxy_binary_path(&sandbox_config), sandbox_id: Some(sandbox_id.clone()), sandbox_name: Some(sandbox_name.clone()), - openshell_endpoint: None, + gateway: None, provider_credentials: None, agent_proposals: openshell_core::proposals::AgentProposals::default(), denial_tx: None, diff --git a/crates/openshell-ocsf/src/builders/api_activity.rs b/crates/openshell-ocsf/src/builders/api_activity.rs index fe88434804..4740b05053 100644 --- a/crates/openshell-ocsf/src/builders/api_activity.rs +++ b/crates/openshell-ocsf/src/builders/api_activity.rs @@ -3,7 +3,7 @@ //! Builder for API Activity [6003] events. -use crate::builders::{EventContext, EventOrigin}; +use crate::builders::{ContextRef, EventOrigin}; use crate::enums::{ActionId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{ApiActivityEvent, OcsfEvent}; @@ -28,7 +28,7 @@ fn sanitize_model_name(name: &str) -> String { /// Builder for API Activity [6003] events. pub struct ApiActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, message: Option, @@ -49,9 +49,9 @@ pub struct ApiActivityBuilder<'a> { impl<'a> ApiActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext, operation: impl Into) -> Self { + pub fn new(ctx: impl Into>, operation: impl Into) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, message: None, diff --git a/crates/openshell-ocsf/src/builders/authentication.rs b/crates/openshell-ocsf/src/builders/authentication.rs index a12cf2a5d0..492bdf65b7 100644 --- a/crates/openshell-ocsf/src/builders/authentication.rs +++ b/crates/openshell-ocsf/src/builders/authentication.rs @@ -3,7 +3,7 @@ //! Builder for Authentication [3002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, AuthProtocolId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{AuthenticationEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Endpoint, HttpRequest, Service, User}; /// Builder for Authentication [3002] Logon events. pub struct AuthenticationBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, user: User, service: Service, auth_protocol: Option, @@ -29,9 +29,9 @@ pub struct AuthenticationBuilder<'a> { impl<'a> AuthenticationBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext, user: User, service: Service) -> Self { + pub fn new(ctx: impl Into>, user: User, service: Service) -> Self { Self { - ctx, + ctx: ctx.into(), user, service, auth_protocol: None, diff --git a/crates/openshell-ocsf/src/builders/base.rs b/crates/openshell-ocsf/src/builders/base.rs index c2edb5b169..c08f1aa0aa 100644 --- a/crates/openshell-ocsf/src/builders/base.rs +++ b/crates/openshell-ocsf/src/builders/base.rs @@ -3,14 +3,14 @@ //! Builder for Base Event [0]. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{BaseEvent, OcsfEvent}; /// Builder for Base Event [0] — events without a specific OCSF class. pub struct BaseEventBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, message: Option, @@ -20,9 +20,9 @@ pub struct BaseEventBuilder<'a> { impl<'a> BaseEventBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, message: None, diff --git a/crates/openshell-ocsf/src/builders/config.rs b/crates/openshell-ocsf/src/builders/config.rs index 1bc415b612..906470408d 100644 --- a/crates/openshell-ocsf/src/builders/config.rs +++ b/crates/openshell-ocsf/src/builders/config.rs @@ -3,14 +3,14 @@ //! Builder for Device Config State Change [5019] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{SecurityLevelId, SeverityId, StateId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{DeviceConfigStateChangeEvent, OcsfEvent}; /// Builder for Device Config State Change [5019] events. pub struct ConfigStateChangeBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, state_id: Option, @@ -23,9 +23,9 @@ pub struct ConfigStateChangeBuilder<'a> { impl<'a> ConfigStateChangeBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, state_id: None, diff --git a/crates/openshell-ocsf/src/builders/finding.rs b/crates/openshell-ocsf/src/builders/finding.rs index aea063a527..ccb45d29a8 100644 --- a/crates/openshell-ocsf/src/builders/finding.rs +++ b/crates/openshell-ocsf/src/builders/finding.rs @@ -3,7 +3,7 @@ //! Builder for Detection Finding [2004] events. -use crate::builders::{EventContext, EventOrigin}; +use crate::builders::{ContextRef, EventOrigin}; use crate::enums::{ActionId, ActivityId, ConfidenceId, DispositionId, RiskLevelId, SeverityId}; use crate::events::base_event::BaseEventData; use crate::events::{DetectionFindingEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Attack, Evidence, FindingInfo, Remediation}; /// Builder for Detection Finding [2004] events. pub struct DetectionFindingBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, action: Option, @@ -31,9 +31,9 @@ pub struct DetectionFindingBuilder<'a> { impl<'a> DetectionFindingBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Open, severity: SeverityId::Medium, action: None, diff --git a/crates/openshell-ocsf/src/builders/http.rs b/crates/openshell-ocsf/src/builders/http.rs index b027421bd0..86a22f63ac 100644 --- a/crates/openshell-ocsf/src/builders/http.rs +++ b/crates/openshell-ocsf/src/builders/http.rs @@ -3,7 +3,7 @@ //! Builder for HTTP Activity [4002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{HttpActivityEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Endpoint, FirewallRule, HttpRequest, HttpResponse}; /// Builder for HTTP Activity [4002] events. pub struct HttpActivityBuilder<'a, Context = MissingHttpContext> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, action: Option, disposition: Option, @@ -56,9 +56,9 @@ impl<'a> HttpActivityBuilder<'a, MissingHttpContext> { /// HttpActivityBuilder::new(&ctx).build(); /// ``` #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, action: None, disposition: None, diff --git a/crates/openshell-ocsf/src/builders/lifecycle.rs b/crates/openshell-ocsf/src/builders/lifecycle.rs index 8fc648ee37..b425b3e79b 100644 --- a/crates/openshell-ocsf/src/builders/lifecycle.rs +++ b/crates/openshell-ocsf/src/builders/lifecycle.rs @@ -3,7 +3,7 @@ //! Builder for Application Lifecycle [6002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActivityId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{ApplicationLifecycleEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::Product; /// Builder for Application Lifecycle [6002] events. pub struct AppLifecycleBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, status: Option, @@ -20,9 +20,9 @@ pub struct AppLifecycleBuilder<'a> { impl<'a> AppLifecycleBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, diff --git a/crates/openshell-ocsf/src/builders/mod.rs b/crates/openshell-ocsf/src/builders/mod.rs index 96c966b732..7026979860 100644 --- a/crates/openshell-ocsf/src/builders/mod.rs +++ b/crates/openshell-ocsf/src/builders/mod.rs @@ -157,6 +157,7 @@ pub use process::ProcessActivityBuilder; pub use ssh::SshActivityBuilder; use std::net::IpAddr; +use std::sync::Arc; use crate::OCSF_VERSION; use crate::enums::StatusId; @@ -199,6 +200,46 @@ pub struct EventContext { pub origin: EventOrigin, } +/// Event context borrowed from the caller or shared by a hosted sandbox +/// instance. +/// +/// Builders accept either form, so events attributed to a hosted sandbox +/// instance need not leak a `'static` context for each instance. +#[derive(Debug, Clone)] +pub enum ContextRef<'a> { + Borrowed(&'a EventContext), + Shared(Arc), +} + +impl std::ops::Deref for ContextRef<'_> { + type Target = EventContext; + + fn deref(&self) -> &EventContext { + match self { + Self::Borrowed(context) => context, + Self::Shared(context) => context, + } + } +} + +impl<'a> From<&'a EventContext> for ContextRef<'a> { + fn from(context: &'a EventContext) -> Self { + Self::Borrowed(context) + } +} + +impl From> for ContextRef<'_> { + fn from(context: Arc) -> Self { + Self::Shared(context) + } +} + +impl<'a> From<&'a ContextRef<'_>> for ContextRef<'a> { + fn from(context: &'a ContextRef<'_>) -> Self { + Self::Borrowed(context) + } +} + impl EventContext { /// Build the OCSF `Metadata` object for any event. #[must_use] diff --git a/crates/openshell-ocsf/src/builders/network.rs b/crates/openshell-ocsf/src/builders/network.rs index 09f4c4ff0d..7907abe4a6 100644 --- a/crates/openshell-ocsf/src/builders/network.rs +++ b/crates/openshell-ocsf/src/builders/network.rs @@ -3,7 +3,7 @@ //! Builder for Network Activity [4001] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{NetworkActivityEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, ConnectionInfo, Endpoint, FirewallRule}; /// Builder for Network Activity [4001] events. pub struct NetworkActivityBuilder<'a, EndpointState = MissingNetworkEndpoint> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, activity_name: Option, action: Option, @@ -58,9 +58,9 @@ impl<'a> NetworkActivityBuilder<'a, MissingNetworkEndpoint> { /// NetworkActivityBuilder::new(&ctx).build(); /// ``` #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, activity_name: None, action: None, diff --git a/crates/openshell-ocsf/src/builders/process.rs b/crates/openshell-ocsf/src/builders/process.rs index 6e06e6c6e7..5acdb75f01 100644 --- a/crates/openshell-ocsf/src/builders/process.rs +++ b/crates/openshell-ocsf/src/builders/process.rs @@ -3,7 +3,7 @@ //! Builder for Process Activity [1007] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, LaunchTypeId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{OcsfEvent, ProcessActivityEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Process}; /// Builder for Process Activity [1007] events. pub struct ProcessActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, status: Option, @@ -26,9 +26,9 @@ pub struct ProcessActivityBuilder<'a> { impl<'a> ProcessActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, diff --git a/crates/openshell-ocsf/src/builders/ssh.rs b/crates/openshell-ocsf/src/builders/ssh.rs index 36383ae9f4..2777bbd05b 100644 --- a/crates/openshell-ocsf/src/builders/ssh.rs +++ b/crates/openshell-ocsf/src/builders/ssh.rs @@ -3,7 +3,7 @@ //! Builder for SSH Activity [4007] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, AuthTypeId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{OcsfEvent, SshActivityEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Endpoint}; /// Builder for SSH Activity [4007] events. pub struct SshActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, action: Option, disposition: Option, @@ -28,9 +28,9 @@ pub struct SshActivityBuilder<'a> { impl<'a> SshActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, action: None, disposition: None, diff --git a/crates/openshell-ocsf/src/ctx.rs b/crates/openshell-ocsf/src/ctx.rs index 7e1c4c7796..726d4ad0a9 100644 --- a/crates/openshell-ocsf/src/ctx.rs +++ b/crates/openshell-ocsf/src/ctx.rs @@ -1,15 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process-wide [`EventContext`] singleton. +//! Process-wide and sandbox-instance-bound [`EventContext`]. //! -//! Initialised once via [`set_ctx`] during sandbox start; read by every event -//! builder via [`ctx`]. Falls back to a default context when the singleton has -//! not been set (e.g. unit tests that exercise builders without booting the -//! sandbox). +//! A dedicated supervisor initialises the process-wide context once via +//! [`set_ctx`]. A process hosting several sandbox instances instead binds +//! each instance's context to the threads of that instance's runtime with +//! [`bind_thread_ctx`]; a bound context takes precedence. [`ctx`] falls back +//! to a default context when neither is set (e.g. unit tests that exercise +//! builders without booting the sandbox). -use crate::{EventContext, EventOrigin}; -use std::sync::{LazyLock, OnceLock}; +use crate::{ContextRef, EventContext, EventOrigin}; +use std::cell::RefCell; +use std::sync::{Arc, LazyLock, OnceLock}; static OCSF_CTX: OnceLock = OnceLock::new(); @@ -24,6 +27,11 @@ static OCSF_CTX_FALLBACK: LazyLock = LazyLock::new(|| EventContext origin: EventOrigin::Supervisor, }); +std::thread_local! { + /// Context of the hosted sandbox instance whose runtime owns this thread. + static THREAD_CTX: RefCell>> = const { RefCell::new(None) }; +} + /// Initialise the process-wide OCSF sandbox context. /// /// Returns `false` if the context was already set; the caller may log and @@ -32,11 +40,22 @@ pub fn set_ctx(ctx: EventContext) -> bool { OCSF_CTX.set(ctx).is_ok() } -/// Return a reference to the process-wide [`EventContext`]. +/// Attribute events emitted on the calling thread to one hosted sandbox +/// instance. +/// +/// Instance runtimes bind every thread they start; `None` unbinds it. +pub fn bind_thread_ctx(ctx: Option>) { + THREAD_CTX.with(|current| *current.borrow_mut() = ctx); +} + +/// Return the context for events emitted on the calling thread. /// -/// Falls back to a default context if [`set_ctx`] has not been called (e.g. -/// during unit tests that exercise individual builders). +/// An instance-bound context wins over the process-wide one. Falls back to a +/// default context if neither is set. #[must_use] -pub fn ctx() -> &'static EventContext { - OCSF_CTX.get().unwrap_or(&OCSF_CTX_FALLBACK) +pub fn ctx() -> ContextRef<'static> { + if let Some(bound) = THREAD_CTX.with(|current| current.borrow().clone()) { + return ContextRef::Shared(bound); + } + ContextRef::Borrowed(OCSF_CTX.get().unwrap_or(&OCSF_CTX_FALLBACK)) } diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index b9878eb68a..310cd9afe4 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -68,12 +68,13 @@ pub use format::downgrade::OcsfSchemaVersion; // --- Builders --- pub use builders::{ ApiActivityBuilder, AppLifecycleBuilder, AuthenticationBuilder, BaseEventBuilder, - ConfigStateChangeBuilder, DetectionFindingBuilder, EventContext, EventOrigin, + ConfigStateChangeBuilder, ContextRef, DetectionFindingBuilder, EventContext, EventOrigin, HttpActivityBuilder, NetworkActivityBuilder, ProcessActivityBuilder, SshActivityBuilder, }; // --- Tracing layers --- pub use tracing_layers::{ - OCSF_TARGET, OcsfJsonlLayer, OcsfShorthandLayer, clear_current_event, clone_current_event, - emit_ocsf_event, emit_ocsf_event_routed, set_current_event, + OCSF_TARGET, OcsfJsonSettings, OcsfJsonlLayer, OcsfShorthandLayer, bind_thread_json_settings, + clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, + set_current_event, }; diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index a0746b8621..40484ce43b 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -16,6 +16,28 @@ use tracing_subscriber::layer::Context; use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; +/// OCSF JSON output settings owned by one hosted sandbox instance. +/// +/// Instances apply their own `ocsf_json_enabled` and schema-version settings +/// to these values. Binding them to the instance's runtime threads makes every +/// [`OcsfJsonlLayer`] use them for events those threads emit. +#[derive(Clone, Debug, Default)] +pub struct OcsfJsonSettings { + pub enabled: Arc, + pub target_version: Arc>, +} + +std::thread_local! { + static THREAD_JSON_SETTINGS: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// Apply one hosted sandbox instance's JSON settings to events emitted on this +/// thread. +pub fn bind_thread_json_settings(settings: Option) { + THREAD_JSON_SETTINGS.with(|current| *current.borrow_mut() = settings); +} + /// A tracing `Layer` that intercepts OCSF events and writes JSONL output. /// /// Only events with `target: "ocsf"` are processed; non-OCSF events are ignored. @@ -90,8 +112,15 @@ where return; } + // A hosted sandbox instance's settings replace the process-wide ones. + let instance = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); + let (enabled, target_version) = instance.as_ref().map_or( + (self.enabled.as_ref(), self.target_version.as_ref()), + |settings| (Some(&settings.enabled), Some(&settings.target_version)), + ); + // If an enabled flag is set and it reads `false`, skip writing. - if let Some(ref flag) = self.enabled + if let Some(flag) = enabled && !flag.load(Ordering::Relaxed) { return; @@ -100,7 +129,7 @@ where if let Some(ocsf_event) = clone_current_event() && let Ok(mut w) = self.writer.lock() { - let line = if let Some(ref target) = self.target_version + let line = if let Some(target) = target_version && let Ok(version) = target.lock() && !version.is_empty() { diff --git a/crates/openshell-ocsf/src/tracing_layers/mod.rs b/crates/openshell-ocsf/src/tracing_layers/mod.rs index b57ba1364a..808051b061 100644 --- a/crates/openshell-ocsf/src/tracing_layers/mod.rs +++ b/crates/openshell-ocsf/src/tracing_layers/mod.rs @@ -15,5 +15,5 @@ pub use event_bridge::{ OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, set_current_event, }; -pub use jsonl_layer::OcsfJsonlLayer; +pub use jsonl_layer::{OcsfJsonSettings, OcsfJsonlLayer, bind_thread_json_settings}; pub use shorthand_layer::OcsfShorthandLayer; diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index b10c300833..914bbea861 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -23,7 +23,7 @@ pub use ocsf::{OcsfOtlp, OcsfOtlpLayer}; pub use propagation::{ HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV, TRACESTATE_ENV, TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment, - set_parent_from_environment, trace_context_environment, + set_parent_from_environment, set_parent_from_trace_context, trace_context_environment, }; use opentelemetry::KeyValue; diff --git a/crates/openshell-otel/src/propagation.rs b/crates/openshell-otel/src/propagation.rs index 764e343146..02f2bdb120 100644 --- a/crates/openshell-otel/src/propagation.rs +++ b/crates/openshell-otel/src/propagation.rs @@ -114,17 +114,27 @@ pub fn current_trace_context_environment() -> Vec<(&'static str, String)> { /// /// Leaves `span` unchanged when the environment carries no valid context. pub fn set_parent_from_environment(span: &tracing::Span) { - let carrier = [ - ("traceparent", TRACEPARENT_ENV), - ("tracestate", TRACESTATE_ENV), - ] - .into_iter() - .filter_map(|(field, name)| { - std::env::var(name) - .ok() - .map(|value| (field.to_string(), value)) - }) - .collect::>(); + set_parent_from_trace_context( + span, + std::env::var(TRACEPARENT_ENV).ok().as_deref(), + std::env::var(TRACESTATE_ENV).ok().as_deref(), + ); +} + +/// Parent `span` under an explicit W3C `traceparent` and `tracestate`. +/// +/// A process supervising several sandboxes receives each launch's context +/// with its sandbox instance instead of through process environment. Leaves +/// `span` unchanged when the values carry no valid context. +pub fn set_parent_from_trace_context( + span: &tracing::Span, + traceparent: Option<&str>, + tracestate: Option<&str>, +) { + let carrier = [("traceparent", traceparent), ("tracestate", tracestate)] + .into_iter() + .filter_map(|(field, value)| value.map(|value| (field.to_string(), value.to_string()))) + .collect::>(); set_parent_from_carrier(span, &carrier); } diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index d7ceb67024..45639c4c00 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -21,7 +21,7 @@ use openshell_ocsf::{ use crate::process::{ProcessHandle, ProcessStatus, ResolvedWorkspace}; -fn ocsf_ctx() -> &'static openshell_ocsf::EventContext { +fn ocsf_ctx() -> openshell_ocsf::ContextRef<'static> { openshell_ocsf::ctx::ctx() } diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 30f404752a..7ce403a15e 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -14,3 +14,15 @@ allocator implementation belongs to the VM crate. The VM dependency disables default features, so this instrumentation does not require the compute runtime or Unix-only sandbox library. It is a development dependency only. + +## Sandbox instance inputs + +`run_networking` takes a `SandboxInstanceNetwork` with the sandbox identity, +the sandbox instance's `GatewayEndpoint`, the SPIFFE Workload API socket for +provider token grants, the `policy.local` denial source, and proxy CA files. +None of these are read from the process environment, so one process can run +several sandbox instances. Each instance owns its provider token cache and +HTTP client; a token acquired for one sandbox is never served to another. +Without a Workload API socket, token grants fail closed. A process hosting +several instances serves `/v1/denials` from a per-instance +`SandboxInstanceDenialLog` instead of the shared shorthand log files. diff --git a/crates/openshell-supervisor-network/src/host.rs b/crates/openshell-supervisor-network/src/host.rs index b07bc0f537..45b21127c4 100644 --- a/crates/openshell-supervisor-network/src/host.rs +++ b/crates/openshell-supervisor-network/src/host.rs @@ -48,7 +48,8 @@ pub struct HostProxyConfig { pub binary_path: PathBuf, pub sandbox_id: Option, pub sandbox_name: Option, - pub openshell_endpoint: Option, + /// Gateway access with this sandbox's credentials. + pub gateway: Option, pub provider_credentials: Option, /// Shared feature state for the policy.local agent proposal surface. pub agent_proposals: AgentProposals, @@ -99,7 +100,7 @@ pub async fn start_host_proxy(config: HostProxyConfig) -> Result Pin> + Send + 'a>>; } -/// Resolves provider tokens using the supervisor's authenticated grant flow. -#[derive(Default)] -pub struct SpiffeTokenGrantResolver; +/// Resolves provider tokens through one sandbox instance's authenticated grant flow. +pub struct SpiffeTokenGrantResolver { + grants: crate::token_grant::SandboxInstanceTokenGrants, +} + +impl SpiffeTokenGrantResolver { + #[must_use] + pub fn new(grants: crate::token_grant::SandboxInstanceTokenGrants) -> Self { + Self { grants } + } +} impl TokenGrantResolver for SpiffeTokenGrantResolver { fn obtain<'a>( @@ -63,8 +71,8 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { request: TokenGrantRequest<'a>, ) -> Pin> + Send + 'a>> { Box::pin(async move { - crate::token_grant::obtain_provider_token( - crate::token_grant::ObtainProviderTokenRequest { + self.grants + .obtain_provider_token(crate::token_grant::ObtainProviderTokenRequest { provider_name: request.provider_key, token_endpoint: request.token_endpoint, jwt_svid_audience: request.jwt_svid_audience, @@ -75,18 +83,12 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { grant_type: request.grant_type, requested_token_type: request.requested_token_type, request_id: request.request_id, - }, - ) - .await + }) + .await }) } } -/// Construct the production SPIFFE-backed resolver. -pub fn default_resolver() -> Arc { - Arc::new(SpiffeTokenGrantResolver) -} - /// A refused request: the grant failure and the credential whose grant failed. /// /// The credential is the `provider:credential` name pair of the selected diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index c3bb88b4ed..033f492aa7 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -3,7 +3,9 @@ //! Sandbox-local policy advisor HTTP API. +use crate::l7::token_grant_injection::TokenGrantResolver; use miette::{IntoDiagnostic, Result}; +use openshell_core::grpc_client::GatewayEndpoint; use openshell_core::proposals::AgentProposals; use openshell_core::proto::{ L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, PolicyChunk, @@ -82,50 +84,153 @@ const SHORTHAND_LOG_PREFIX: &str = "openshell."; /// log entry (very long URL path, etc.) cannot blow up the response. const MAX_DENIAL_LINE_BYTES: usize = 4096; -#[derive(Debug)] +/// Most recent denial lines retained for one hosted sandbox instance. +const MAX_INSTANCE_DENIAL_LINES: usize = 256; + +/// Where `/v1/denials` reads this sandbox's recent denials. +#[derive(Clone, Debug)] +pub enum DenialLogSource { + /// Shorthand log files written by a supervisor process that serves one + /// sandbox. + ShorthandFiles(PathBuf), + /// Denials recorded for this sandbox instance alone. A process hosting several + /// sandboxes cannot expose its shared log files to any one of them. + SandboxInstance(SandboxInstanceDenialLog), +} + +impl Default for DenialLogSource { + fn default() -> Self { + Self::ShorthandFiles(PathBuf::from(LOG_DIR)) + } +} + +/// Bounded recent OCSF denial lines recorded for one hosted sandbox instance. +#[derive(Clone, Default)] +pub struct SandboxInstanceDenialLog { + lines: Arc>>, +} + +impl std::fmt::Debug for SandboxInstanceDenialLog { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let retained = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len(); + formatter + .debug_struct("SandboxInstanceDenialLog") + .field("retained", &retained) + .finish() + } +} + +impl SandboxInstanceDenialLog { + /// Retain `line` when it is a rendered OCSF denial; ignore anything else. + pub fn record(&self, line: &str) { + let line = line.trim_end(); + if !is_ocsf_denial_line(line) { + return; + } + let mut lines = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if lines.len() == MAX_INSTANCE_DENIAL_LINES { + lines.pop_front(); + } + lines.push_back(line.to_string()); + } + + fn recent(&self, limit: usize) -> Vec { + self.lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .rev() + .take(limit) + .map(|line| surfaced_denial_line(line)) + .collect() + } +} + pub struct PolicyLocalContext { current_policy: Arc>>>, agent_proposals: AgentProposals, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, - shorthand_log_dir: PathBuf, + denials: DenialLogSource, + token_grants: Option>, workspace_rx: tokio::sync::watch::Receiver, } +impl std::fmt::Debug for PolicyLocalContext { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PolicyLocalContext") + .field("gateway", &self.gateway) + .field("sandbox_name", &self.sandbox_name) + .field("denials", &self.denials) + .field("token_grants", &self.token_grants.is_some()) + .finish_non_exhaustive() + } +} + impl PolicyLocalContext { pub fn new( current_policy: Option, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, ) -> Self { - Self::with_log_dir( - current_policy, - gateway_endpoint, - sandbox_name, + Self { + current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))), agent_proposals, - PathBuf::from(LOG_DIR), + gateway, + sandbox_name, + denials: DenialLogSource::default(), + token_grants: None, workspace_rx, - ) + } } + #[cfg(test)] fn with_log_dir( current_policy: Option, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, agent_proposals: AgentProposals, shorthand_log_dir: PathBuf, workspace_rx: tokio::sync::watch::Receiver, ) -> Self { - Self { - current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))), - agent_proposals, - gateway_endpoint, + Self::new( + current_policy, + gateway, sandbox_name, - shorthand_log_dir, + agent_proposals, workspace_rx, - } + ) + .with_denial_log(DenialLogSource::ShorthandFiles(shorthand_log_dir)) + } + + /// Serve `/v1/denials` from `denials` instead of the process log files. + #[must_use] + pub fn with_denial_log(mut self, denials: DenialLogSource) -> Self { + self.denials = denials; + self + } + + /// Acquire provider token grants for this sandbox's HTTP relays. + #[must_use] + pub fn with_token_grants(mut self, resolver: Arc) -> Self { + self.token_grants = Some(resolver); + self + } + + /// The sandbox instance's provider token grant resolver, absent when grants are + /// unavailable and must fail closed. + pub(crate) fn token_grant_resolver(&self) -> Option> { + self.token_grants.clone() } pub async fn set_current_policy(&self, policy: ProtoSandboxPolicy) { @@ -318,7 +423,18 @@ async fn recent_denials_response( query: &str, ) -> (u16, serde_json::Value) { let limit = parse_last_query(query).unwrap_or(DEFAULT_DENIALS_LIMIT); - let log_dir = ctx.shorthand_log_dir.clone(); + let log_dir = match &ctx.denials { + DenialLogSource::ShorthandFiles(log_dir) => log_dir.clone(), + DenialLogSource::SandboxInstance(log) => { + return ( + 200, + serde_json::json!({ + "denials": log.recent(limit), + "log_available": true, + }), + ); + } + }; // Distinguish "shorthand log exists and no denials happened" from "no log // file yet, so we have nothing to read." Without this flag the agent sees @@ -398,9 +514,7 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec { // proxy.rs FORWARD path so the on-disk shorthand log itself is // clean — tracked separately). Redact first so truncation cannot // slice mid-secret. - let redacted = redact_query_strings(line); - let surfaced = truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES); - lines.push(surfaced); + lines.push(surfaced_denial_line(line)); if lines.len() >= limit { return lines; } @@ -409,6 +523,11 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec { lines } +fn surfaced_denial_line(line: &str) -> String { + let redacted = redact_query_strings(line); + truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES) +} + /// Replace any `?` substring with `?[redacted]` to keep query-string /// secrets out of the agent's view. Walks per Unicode scalar value so multi-byte /// content is safe. A query is everything from `?` until the next whitespace or @@ -486,7 +605,7 @@ fn collect_shorthand_log_files(log_dir: &Path, max_files: usize) -> std::io::Res } async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_json::Value) { - let Some(endpoint) = ctx.gateway_endpoint.as_deref() else { + let Some(gateway) = ctx.gateway.as_ref() else { return ( 503, serde_json::json!({ @@ -526,7 +645,7 @@ async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_j Err(error) => return (400, error_payload("invalid_proposal", error)), }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await { Ok(client) => { client.set_workspace(workspace); client @@ -956,7 +1075,7 @@ struct LookupSession<'a> { async fn open_lookup_session( ctx: &PolicyLocalContext, ) -> std::result::Result, (u16, serde_json::Value)> { - let endpoint = ctx.gateway_endpoint.as_deref().ok_or_else(|| { + let gateway = ctx.gateway.as_ref().ok_or_else(|| { ( 503, error_payload( @@ -989,7 +1108,7 @@ async fn open_lookup_session( ), )); } - let client = openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint) + let client = openshell_core::grpc_client::CachedOpenShellClient::connect(gateway) .await .map_err(|e| (502, error_payload("gateway_connect_failed", e.to_string())))?; client.set_workspace(workspace); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index ffdb4674d1..bd93c78c57 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1227,11 +1227,11 @@ async fn handle_transparent_tcp_connection( &decision, None, None, - agent_proposals, // The transparent TCP path carries no PolicyLocalContext, so no // workspace is available here; matches the CONNECT path default when // policy-local context is absent. - String::new(), + None, + agent_proposals, relay::RelaySignals { activity: activity_tx.clone(), endpoint_observation: None, @@ -3019,16 +3019,12 @@ async fn handle_mediated_connection( // gate needs it) and drives the raw-tunnel branch below. // Build request-processing context shared by CONNECT and forward HTTP. - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); let mut ctx = relay::http_context( &decision, provider_credentials, secret_resolver.clone(), + policy_local_ctx.as_deref(), agent_proposals, - workspace, relay::RelaySignals { activity: activity_tx.clone(), endpoint_observation: endpoint_observation_tx, @@ -5673,16 +5669,12 @@ async fn handle_forward_proxy( .map_or_else(std::collections::HashMap::new, |query| { crate::l7::rest::parse_query_params(query).unwrap_or_default() }); - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); let mut l7_ctx = relay::http_context( &decision, provider_credentials, secret_resolver.clone(), + policy_local_ctx.as_deref(), agent_proposals, - workspace, relay::RelaySignals { activity: activity_tx.cloned(), endpoint_observation: endpoint_observation_tx, diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index dad4b12633..3e722e3b50 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -47,8 +47,8 @@ pub(super) fn http_context( decision: &EgressDecision, provider_credentials: Option, secret_resolver: Option>, + policy_local: Option<&crate::policy_local::PolicyLocalContext>, agent_proposals: openshell_core::proposals::AgentProposals, - workspace: String, signals: RelaySignals, ) -> L7EvalContext { // Provider-backed credentials must be acquired from the live state for @@ -58,9 +58,14 @@ pub(super) fn http_context( .is_none() .then_some(secret_resolver) .flatten(); + // Grants use the owning sandbox instance's resolver; without one they fail closed. let token_grant_resolver = provider_credentials .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()); + .and(policy_local) + .and_then(crate::policy_local::PolicyLocalContext::token_grant_resolver); + let workspace = policy_local + .map(crate::policy_local::PolicyLocalContext::workspace) + .unwrap_or_default(); let policy_name = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone().unwrap_or_default(), NetworkAction::Deny { .. } => String::new(), diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index a7647d39ad..340b3984c5 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -35,10 +35,13 @@ use crate::l7::tls::{ CertCache, ProxyTlsState, SandboxCa, build_upstream_client_config, read_system_ca_bundle, write_ca_files, }; +use crate::l7::token_grant_injection::SpiffeTokenGrantResolver; use crate::opa::OpaEngine; -use crate::policy_local::PolicyLocalContext; +use crate::policy_local::{DenialLogSource, PolicyLocalContext}; use crate::proxy::ProxyHandle; +use crate::token_grant::SandboxInstanceTokenGrants; use openshell_core::endpoint_status::EndpointObservationSender; +use openshell_core::grpc_client::GatewayEndpoint; /// Concrete listener options supplied by trusted supervisor composition. /// This is not an isolation-backend capability or a serialized launch contract. @@ -189,6 +192,52 @@ fn append_additional_ca_bundle( Ok(()) } +/// Identity and authority of the one sandbox a [`run_networking`] call serves. +/// +/// A process hosting several sandboxes builds one per sandbox instance; no value +/// is read from the process environment, so instances cannot share gateway +/// credentials, provider token grants, denial history, or proxy CA files. +#[derive(Debug, Default)] +pub struct SandboxInstanceNetwork { + pub sandbox_id: Option, + pub sandbox_name: Option, + /// Gateway access with this instance's credentials. + pub gateway: Option, + /// SPIFFE Workload API socket for provider token grants. Without one, + /// grants fail closed. + pub provider_workload_api_socket: Option, + /// Source served by `policy.local` `/v1/denials`. + pub denial_log: DenialLogSource, + /// Proxy interception CA and trust-file directory. + pub proxy_tls: ProxyTlsFiles, +} + +/// Provisioned proxy CA files and the directory that receives trust files. +/// +/// Without a provisioned CA the proxy generates an ephemeral one. Without a +/// directory it writes to the default container TLS root. +#[derive(Clone, Debug, Default)] +pub struct ProxyTlsFiles { + pub ca_certificate: Option, + pub ca_private_key: Option, + pub directory: Option, +} + +impl ProxyTlsFiles { + /// Files configured for a single-sandbox supervisor through its environment. + #[must_use] + pub fn from_environment() -> Self { + Self { + ca_certificate: std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT) + .map(std::path::PathBuf::from), + ca_private_key: std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY) + .map(std::path::PathBuf::from), + directory: std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR) + .map(std::path::PathBuf::from), + } + } +} + /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. /// @@ -235,9 +284,7 @@ pub async fn run_networking( entrypoint_pid: Arc, process_enabled: bool, provider_credentials: &ProviderCredentialState, - sandbox_id: Option<&str>, - sandbox_name: Option<&str>, - openshell_endpoint: Option<&str>, + instance: SandboxInstanceNetwork, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -245,7 +292,6 @@ pub async fn run_networking( workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, additional_ca_bundle: Option<&str>, - proxy_tls_dir: Option<&std::path::Path>, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, network_mediation_source: Option>, @@ -254,15 +300,30 @@ pub async fn run_networking( // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. - let policy_local_ctx = Arc::new(PolicyLocalContext::new( - retained_proto.cloned(), - openshell_endpoint.map(str::to_string), - sandbox_name - .map(str::to_string) - .or_else(|| sandbox_id.map(str::to_string)), - agent_proposals.clone(), - workspace_rx, + let SandboxInstanceNetwork { + sandbox_id, + sandbox_name, + gateway, + provider_workload_api_socket, + denial_log, + proxy_tls, + } = instance; + let token_grants = SpiffeTokenGrantResolver::new(SandboxInstanceTokenGrants::new( + gateway.clone(), + sandbox_id.clone(), + provider_workload_api_socket, )); + let policy_local_ctx = Arc::new( + PolicyLocalContext::new( + retained_proto.cloned(), + gateway, + sandbox_name.or(sandbox_id), + agent_proposals.clone(), + workspace_rx, + ) + .with_denial_log(denial_log) + .with_token_grants(Arc::new(token_grants)), + ); // Readiness signal for the proxy accept loop: the proxy binds the TCP // listener immediately (so the OS backlog queues early SYN packets) but @@ -370,14 +431,10 @@ pub async fn run_networking( // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - let configured_ca = match ( - std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), - std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), - ) { - (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( - std::path::Path::new(&certificate), - std::path::Path::new(&private_key), - )?), + let configured_ca = match (&proxy_tls.ca_certificate, &proxy_tls.ca_private_key) { + (Some(certificate), Some(private_key)) => { + Some(SandboxCa::load_from_paths(certificate, private_key)?) + } (None, None) => None, _ => { return Err(miette::miette!( @@ -390,14 +447,9 @@ pub async fn run_networking( let durable_ca = configured_ca.is_some(); match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { - let configured_tls_dir = - std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR) - .map(std::path::PathBuf::from); - let tls_dir = proxy_tls_dir - .or(configured_tls_dir.as_deref()) - .unwrap_or_else(|| { - std::path::Path::new(openshell_core::container_paths::TLS_ROOT) - }); + let tls_dir = proxy_tls.directory.as_deref().unwrap_or_else(|| { + std::path::Path::new(openshell_core::container_paths::TLS_ROOT) + }); let mut system_ca_bundle = read_system_ca_bundle(); // A TLS-intercepting corporate proxy (issue #1792) re-signs // tunneled server certificates with the corporate CA, so the diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index e165754105..83d36d36b3 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -27,32 +27,30 @@ //! - `scopes` — `OAuth2` scopes to request (optional) //! - `cache_ttl` — Optional exact cache override; zero disables caching //! -//! ## Environment +//! ## Sandbox instance scope //! -//! Requires `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to be set (path to -//! the SPIFFE Workload API socket). +//! Each sandbox instance owns a [`SandboxInstanceTokenGrants`]: its token cache, +//! HTTP client, gateway credentials, sandbox ID, and SPIFFE Workload API +//! socket. A token is never served to another sandbox. Without a configured +//! Workload API socket, grants fail closed with a configuration failure. use std::collections::HashMap; use std::future::Future; -use std::sync::{Arc, LazyLock, RwLock}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use openshell_core::grpc_client::GatewayEndpoint; use openshell_core::oauth::{ self, ACCESS_TOKEN_TYPE, OAuthTokenResponse, TokenExchangeParams, TokenGrantClient, TokenGrantParams, effective_client_assertion_type, effective_token_type, }; use openshell_core::proto::ProviderCredentialTokenGrantType; -use openshell_core::sandbox_env; use openshell_core::token_grant_failure::{ TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, }; use spiffe::{WorkloadApiClient, WorkloadApiError}; -/// Token cache shared across all provider token grants. -static TOKEN_CACHE: LazyLock = LazyLock::new(TokenCache::new); -// Initialization failures stay bounded and typed; credential acquisition must not panic. -static TOKEN_GRANT_HTTP_CLIENT: LazyLock> = - LazyLock::new(TokenGrantClient::new); const DEFAULT_TOKEN_CACHE_TTL_SECONDS: i64 = 300; const TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; const MAX_TOKEN_EXPIRES_IN_SECONDS: i64 = 3600; @@ -102,33 +100,154 @@ impl TokenCache { } } -/// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. +/// Provider token grants owned by one supervised sandbox. /// -/// This function fetches the sandbox's SPIFFE JWT-SVID from the local SPIRE -/// agent, then exchanges it for an access token with a POST request to the provider's -/// token endpoint with the JWT client assertion grant flow (RFC 7523). -/// -/// Tokens are cached per provider name with TTL. Subsequent calls return the -/// cached token if it has not expired. -/// -/// # Arguments -/// -/// * `provider_name` — Unique provider identifier (used as cache key) -/// * `token_endpoint` — `OAuth2` token service URL -/// * `jwt_svid_audience` — Optional audience to request when fetching the JWT-SVID -/// * `client_assertion_type` — Optional `OAuth2` client assertion type -/// * `audience` — Resource audience to request in the token request -/// * `scopes` — `OAuth2` scopes to request (may be empty) -/// * `cache_ttl_override` — Exact cache TTL; absence uses `expires_in` and zero disables caching -/// -/// # Errors -/// -/// Returns an error if: -/// - SPIFFE Workload API socket is not configured -/// - SPIRE agent is unreachable -/// - JWT-SVID fetch fails -/// - Token service request fails -/// - Token response is invalid +/// The cache, HTTP connection pool, gateway credentials, sandbox ID, and +/// workload identity socket all belong to this sandbox instance, so a token acquired +/// for one sandbox is never served to another. +pub struct SandboxInstanceTokenGrants { + cache: TokenCache, + // Initialization failures stay bounded and typed; acquisition must not panic. + http_client: Result, + gateway: Option, + sandbox_id: Option, + workload_api_socket: Option, +} + +impl SandboxInstanceTokenGrants { + /// Grants authenticated as `sandbox_id` through `gateway`. A missing + /// Workload API socket makes every grant fail closed. + #[must_use] + pub fn new( + gateway: Option, + sandbox_id: Option, + workload_api_socket: Option, + ) -> Self { + Self { + cache: TokenCache::new(), + http_client: TokenGrantClient::new(), + gateway, + sandbox_id: sandbox_id.filter(|value| !value.trim().is_empty()), + workload_api_socket: workload_api_socket.filter(|path| !path.as_os_str().is_empty()), + } + } + + /// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. + /// + /// This fetches the sandbox's SPIFFE JWT-SVID from the instance's Workload + /// API socket, then exchanges it for an access token with a POST request to + /// the provider's token endpoint with the JWT client assertion grant flow + /// (RFC 7523). + /// + /// Tokens are cached per provider name with TTL in this instance's cache. + /// Subsequent calls return the cached token if it has not expired. + /// + /// # Errors + /// + /// Returns an error if: + /// - SPIFFE Workload API socket is not configured + /// - SPIRE agent is unreachable + /// - JWT-SVID fetch fails + /// - Token service request fails + /// - Token response is invalid + pub async fn obtain_provider_token( + &self, + request: ObtainProviderTokenRequest<'_>, + ) -> Result { + let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) + .map_err(|_| configuration_failure())?; + obtain_provider_token_with_grant( + ObtainProviderTokenInput { + cache: &self.cache, + provider_name: request.provider_name, + token_endpoint: request.token_endpoint, + jwt_svid_audience: request.jwt_svid_audience, + client_assertion_type: request.client_assertion_type, + audience: request.audience, + scopes: request.scopes, + cache_ttl_override: request.cache_ttl_override, + grant_type, + requested_token_type: request.requested_token_type, + }, + |jwt_audience| async move { + // Resolve local exchange prerequisites before fetching any workload credential. + let exchange_context = match grant_type { + ProviderCredentialTokenGrantType::ClientCredentials + | ProviderCredentialTokenGrantType::Unspecified => None, + ProviderCredentialTokenGrantType::TokenExchange => { + let (provider, credential_key) = + parse_provider_credential_key(request.provider_name)?; + let gateway = self.gateway.as_ref().ok_or_else(configuration_failure)?; + openshell_core::grpc_client::token_grant_gateway_endpoint(gateway.url())?; + let sandbox_id = self + .sandbox_id + .as_deref() + .ok_or_else(configuration_failure)?; + Some((provider, credential_key, gateway, sandbox_id)) + } + }; + let http_client = self.http_client.as_ref().map_err(|failure| *failure)?; + let socket_path = self + .workload_api_socket + .as_deref() + .ok_or_else(configuration_failure)?; + // Fetch JWT-SVID with authorization server as audience + // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm + let jwt_svid = fetch_jwt_svid_from_socket(socket_path, &jwt_audience).await?; + + match exchange_context { + None => { + // Perform OAuth2 JWT client assertion grant. The audience + // parameter in the token request specifies the resource server. + perform_token_grant( + http_client, + request.token_endpoint, + &jwt_svid, + request.client_assertion_type, + request.audience, + request.scopes, + request.request_id, + ) + .await + } + Some((provider, credential_key, gateway, sandbox_id)) => { + let intermediate = + openshell_core::grpc_client::exchange_provider_subject_token( + gateway, + sandbox_id, + provider, + credential_key, + &jwt_svid, + Some(request.request_id), + ) + .await?; + validate_access_token(&intermediate.access_token).map_err(|failure| { + failure.at_stage(TokenGrantStage::IntermediateExchange) + })?; + perform_token_exchange( + http_client, + request.token_endpoint, + &TokenExchangeParams { + client_assertion: &jwt_svid, + client_assertion_type: request.client_assertion_type, + subject_token: &intermediate.access_token, + subject_token_type: ACCESS_TOKEN_TYPE, + audience: request.audience, + scopes: request.scopes, + requested_token_type: request.requested_token_type, + }, + request.request_id, + ) + .await + } + } + }, + ) + .await + } +} + +/// One provider token request; see [`SandboxInstanceTokenGrants::obtain_provider_token`]. pub struct ObtainProviderTokenRequest<'a> { pub provider_name: &'a str, pub token_endpoint: &'a str, @@ -144,96 +263,6 @@ pub struct ObtainProviderTokenRequest<'a> { pub request_id: &'a str, } -/// Obtain and cache a provider token, retaining only bounded grant failures. -pub async fn obtain_provider_token( - request: ObtainProviderTokenRequest<'_>, -) -> Result { - let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) - .map_err(|_| configuration_failure())?; - obtain_provider_token_with_grant( - ObtainProviderTokenInput { - cache: &TOKEN_CACHE, - provider_name: request.provider_name, - token_endpoint: request.token_endpoint, - jwt_svid_audience: request.jwt_svid_audience, - client_assertion_type: request.client_assertion_type, - audience: request.audience, - scopes: request.scopes, - cache_ttl_override: request.cache_ttl_override, - grant_type, - requested_token_type: request.requested_token_type, - }, - |jwt_audience| async move { - // Resolve local exchange prerequisites before fetching any workload credential. - let exchange_context = match grant_type { - ProviderCredentialTokenGrantType::ClientCredentials - | ProviderCredentialTokenGrantType::Unspecified => None, - ProviderCredentialTokenGrantType::TokenExchange => { - let (provider, credential_key) = - parse_provider_credential_key(request.provider_name)?; - let endpoint = supervisor_gateway_endpoint_from_env()?; - openshell_core::grpc_client::token_grant_gateway_endpoint(&endpoint)?; - Some(( - provider, - credential_key, - endpoint, - supervisor_sandbox_id_from_env()?, - )) - } - }; - // Fetch JWT-SVID with authorization server as audience - // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm - let jwt_svid = fetch_jwt_svid_for_token_grant(&jwt_audience).await?; - - match exchange_context { - None => { - // Perform OAuth2 JWT client assertion grant. The audience - // parameter in the token request specifies the resource server. - perform_token_grant( - request.token_endpoint, - &jwt_svid, - request.client_assertion_type, - request.audience, - request.scopes, - request.request_id, - ) - .await - } - Some((provider, credential_key, endpoint, sandbox_id)) => { - let intermediate = - openshell_core::grpc_client::exchange_provider_subject_token( - &endpoint, - &sandbox_id, - provider, - credential_key, - &jwt_svid, - Some(request.request_id), - ) - .await?; - validate_access_token(&intermediate.access_token).map_err(|failure| { - failure.at_stage(TokenGrantStage::IntermediateExchange) - })?; - perform_token_exchange( - request.token_endpoint, - &TokenExchangeParams { - client_assertion: &jwt_svid, - client_assertion_type: request.client_assertion_type, - subject_token: &intermediate.access_token, - subject_token_type: ACCESS_TOKEN_TYPE, - audience: request.audience, - scopes: request.scopes, - requested_token_type: request.requested_token_type, - }, - request.request_id, - ) - .await - } - } - }, - ) - .await -} - struct ObtainProviderTokenInput<'a> { cache: &'a TokenCache, provider_name: &'a str, @@ -297,21 +326,14 @@ where Ok(token_response.access_token) } -/// Fetch JWT-SVID from SPIRE agent for token grant authentication. +/// Fetch a JWT-SVID from the SPIRE agent for token grant authentication. /// -/// This function connects to the local SPIRE agent via the Workload API and -/// requests a JWT-SVID with the specified audience. The JWT-SVID is used as -/// the client assertion in the `OAuth2` grant request. -async fn fetch_jwt_svid_for_token_grant(audience: &str) -> Result { - let socket_path = provider_spiffe_workload_api_socket_from_env()?; - fetch_jwt_svid_from_socket(&socket_path, audience).await -} - +/// The JWT-SVID is used as the client assertion in the `OAuth2` grant request. async fn fetch_jwt_svid_from_socket( - socket_path: &str, + socket_path: &std::path::Path, audience: &str, ) -> Result { - let endpoint = openshell_core::spiffe::workload_api_endpoint(std::path::Path::new(socket_path)); + let endpoint = openshell_core::spiffe::workload_api_endpoint(socket_path); // Connect to SPIRE agent let client = WorkloadApiClient::connect_to(&endpoint) @@ -357,14 +379,8 @@ fn configuration_failure() -> TokenGrantFailure { ) } -fn provider_spiffe_workload_api_socket_from_env() -> Result { - std::env::var(sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - async fn perform_token_grant( + http_client: &TokenGrantClient, token_endpoint: &str, jwt_svid: &str, client_assertion_type: &str, @@ -373,9 +389,7 @@ async fn perform_token_grant( request_id: &str, ) -> Result { oauth::post_oauth_token_grant( - TOKEN_GRANT_HTTP_CLIENT - .as_ref() - .map_err(|failure| *failure)?, + http_client, token_endpoint, &TokenGrantParams { client_assertion: jwt_svid, @@ -389,19 +403,12 @@ async fn perform_token_grant( } async fn perform_token_exchange( + http_client: &TokenGrantClient, token_endpoint: &str, params: &TokenExchangeParams<'_>, request_id: &str, ) -> Result { - oauth::post_oauth_token_exchange( - TOKEN_GRANT_HTTP_CLIENT - .as_ref() - .map_err(|failure| *failure)?, - token_endpoint, - params, - Some(request_id), - ) - .await + oauth::post_oauth_token_exchange(http_client, token_endpoint, params, Some(request_id)).await } pub use oauth::validate_access_token; @@ -457,20 +464,6 @@ fn effective_jwt_svid_audience(token_endpoint: &str, jwt_svid_audience: &str) -> } } -fn supervisor_gateway_endpoint_from_env() -> Result { - std::env::var(sandbox_env::ENDPOINT) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - -fn supervisor_sandbox_id_from_env() -> Result { - std::env::var(sandbox_env::SANDBOX_ID) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - fn parse_provider_credential_key(key: &str) -> Result<(&str, &str), TokenGrantFailure> { openshell_core::dynamic_credential_key::credential_identity(key) .and_then(|identity| identity.split_once(':')) @@ -670,6 +663,10 @@ mod tests { const TEST_REQUEST_ID: &str = "4f1c2b3a-9d8e-4c7b-a6f5-e4d3c2b1a098"; + fn test_http_client() -> TokenGrantClient { + TokenGrantClient::new().expect("token grant HTTP client") + } + #[derive(Debug)] struct CapturedTokenRequest { request_line: String, @@ -1415,6 +1412,7 @@ mod tests { let scopes = vec!["read".to_string(), "write".to_string()]; let response = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "", @@ -1475,6 +1473,7 @@ mod tests { token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; let response = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "urn:ietf:params:oauth:client-assertion-type:jwt-spiffe", @@ -1504,9 +1503,17 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("malformed access token should fail closed"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("malformed access token should fail closed"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(200)); @@ -1516,9 +1523,17 @@ mod tests { async fn perform_token_grant_does_not_follow_redirects() { let (endpoint, request) = token_endpoint_redirect_once("http://127.0.0.1:1/stolen").await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("redirect response should fail closed"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("redirect response should fail closed"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(302)); @@ -1529,10 +1544,17 @@ mod tests { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; - let response = - perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect("token grant should succeed without audience or scopes"); + let response = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect("token grant should succeed without audience or scopes"); let request = request.await.expect("token endpoint task should finish"); assert_eq!(response.access_token, "access-123"); @@ -1553,6 +1575,7 @@ mod tests { .await; let err = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "", @@ -1580,9 +1603,17 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("token grant should fail on server error"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("token grant should fail on server error"); let _request = request.await.expect("token endpoint task should finish"); let message = err.to_string(); @@ -1597,9 +1628,17 @@ mod tests { async fn perform_token_grant_reports_malformed_success_json() { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":42"#).await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("malformed token response should fail"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("malformed token response should fail"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(200)); diff --git a/crates/openshell-supervisor-process/src/debug_rpc.rs b/crates/openshell-supervisor-process/src/debug_rpc.rs index 04cd1e9ba7..b4370c8954 100644 --- a/crates/openshell-supervisor-process/src/debug_rpc.rs +++ b/crates/openshell-supervisor-process/src/debug_rpc.rs @@ -24,7 +24,9 @@ use openshell_core::proto::{ }; use sha2::{Digest, Sha256}; -use openshell_core::grpc_client::{AuthedChannel, connect_channel_pub}; +use openshell_core::grpc_client::{ + AuthedChannel, GatewayCredentials, GatewayEndpoint, connect_channel_pub, +}; /// Entry point for the `debug-rpc` subcommand. Returns the process exit /// code; `main` propagates it. @@ -66,7 +68,8 @@ async fn open_client() -> Result> { let endpoint = std::env::var(openshell_core::sandbox_env::ENDPOINT) .into_diagnostic() .wrap_err("OPENSHELL_ENDPOINT must be set")?; - let channel = connect_channel_pub(&endpoint).await?; + let credentials = GatewayCredentials::from_environment(&endpoint).await?; + let channel = connect_channel_pub(&GatewayEndpoint::new(endpoint, credentials)).await?; Ok(OpenShellClient::new(channel)) } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 3529683403..32e3fd4433 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -16,7 +16,7 @@ use miette::Result; use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit}; #[cfg(unix)] -fn ocsf_ctx() -> &'static openshell_ocsf::EventContext { +fn ocsf_ctx() -> openshell_ocsf::ContextRef<'static> { openshell_ocsf::ctx::ctx() } @@ -82,7 +82,7 @@ impl Drop for BoundaryAccess { #[allow(clippy::too_many_arguments)] pub async fn start_boundary_access( sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, + gateway: Option<&openshell_core::grpc_client::GatewayEndpoint>, ssh_socket_path: Option<&str>, shared_ssh_socket: bool, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -116,10 +116,10 @@ pub async fn start_boundary_access( // Gateway authentication, forwarding, and readiness do not depend on SSH // or the host OS. Acceptance remains false until the gateway authenticates // this supervisor, and reconnects retain main's retry behavior. - let (session_task, session_readiness) = match (openshell_endpoint, sandbox_id) { - (Some(endpoint), Some(id)) => { + let (session_task, session_readiness) = match (gateway, sandbox_id) { + (Some(gateway), Some(id)) => { let (task, accepted) = crate::supervisor_session::spawn_with_readiness( - endpoint.to_string(), + gateway.clone(), id.to_string(), ssh_socket_path.unwrap_or_default(), port_forward, @@ -223,7 +223,7 @@ async fn start_optional_ssh( /// Report the canonical process exit until the gateway acknowledges it. pub async fn report_main_process_exit( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, exit_code: i32, @@ -231,7 +231,7 @@ pub async fn report_main_process_exit( let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::report_main_process_exit( - endpoint, + gateway, sandbox_id, instance_id, exit_code, @@ -249,11 +249,15 @@ pub async fn report_main_process_exit( } /// Finalize canonical process terminal delivery until acknowledged. -pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { +pub async fn finalize_main_process_exit( + gateway: &openshell_core::grpc_client::GatewayEndpoint, + sandbox_id: &str, + instance_id: &str, +) { let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::finalize_main_process_exit( - endpoint, + gateway, sandbox_id, instance_id, ) @@ -352,9 +356,14 @@ mod tests { #[tokio::test] async fn no_ssh_access_retains_main_attachment_and_session_readiness() { let boundary = Arc::new(AccessBoundary); + let gateway = openshell_core::grpc_client::GatewayEndpoint::new( + "http://127.0.0.1:1", + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .expect("test credentials"), + ); let access = start_boundary_access( Some("sandbox"), - Some("http://127.0.0.1:1"), + Some(&gateway), None, false, None, diff --git a/crates/openshell-supervisor-process/src/log_push.rs b/crates/openshell-supervisor-process/src/log_push.rs index d12061e82e..c2ee97760e 100644 --- a/crates/openshell-supervisor-process/src/log_push.rs +++ b/crates/openshell-supervisor-process/src/log_push.rs @@ -7,32 +7,79 @@ //! channel to a background task. The task batches lines and streams them to //! the server using the `PushSandboxLogs` client-streaming RPC. -use openshell_core::grpc_client::CachedOpenShellClient; +use openshell_core::grpc_client::{CachedOpenShellClient, GatewayEndpoint}; use openshell_core::proto::{PushSandboxLogsRequest, SandboxLogLine}; +use std::cell::RefCell; +use std::sync::Arc; use tokio::sync::mpsc; use tracing::{Event, Subscriber}; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; +/// One sandbox's push queue. Lines are labelled with this sandbox and sent +/// only with the credentials of the task that drains the queue. +#[derive(Clone)] +pub struct LogSink { + sandbox_id: Arc, + tx: mpsc::Sender, +} + +impl LogSink { + #[must_use] + pub fn new(sandbox_id: impl Into>, tx: mpsc::Sender) -> Self { + Self { + sandbox_id: sandbox_id.into(), + tx, + } + } +} + +std::thread_local! { + /// Sink of the hosted sandbox instance whose runtime owns this thread. + static THREAD_SINK: RefCell> = const { RefCell::new(None) }; +} + +/// Route events emitted on the calling thread to a hosted sandbox instance's +/// sink. +/// +/// Hosted instances bind every thread of their runtime. Unbound threads push +/// nothing, so an event cannot reach another sandbox's log stream. +pub fn bind_thread_sink(sink: Option) { + THREAD_SINK.with(|current| *current.borrow_mut() = sink); +} + +#[derive(Clone)] +enum Route { + /// The process supervises one sandbox; every event belongs to it. + Fixed(LogSink), + /// Events belong to the sandbox instance bound to the emitting thread, if any. + ThreadBound, +} + /// Tracing layer that pushes log events to the `OpenShell` server. /// /// Events are sent best-effort via `try_send` — if the channel is full the /// event is dropped. Logging must never block the sandbox. #[derive(Clone)] pub struct LogPushLayer { - sandbox_id: String, - tx: mpsc::Sender, + route: Route, max_level: tracing::Level, } impl LogPushLayer { pub fn new(sandbox_id: String, tx: mpsc::Sender) -> Self { + Self::with_route(Route::Fixed(LogSink::new(sandbox_id, tx))) + } + + /// Push each event to the sink bound to its emitting thread. + #[must_use] + pub fn thread_bound() -> Self { + Self::with_route(Route::ThreadBound) + } + + fn with_route(route: Route) -> Self { let max_level = parse_max_level(std::env::var("OPENSHELL_LOG_PUSH_LEVEL").ok().as_deref()); - Self { - sandbox_id, - tx, - max_level, - } + Self { route, max_level } } } @@ -50,62 +97,83 @@ impl Layer for LogPushLayer { if *meta.level() > self.max_level { return; } + match &self.route { + Route::Fixed(sink) => push_event(sink, event), + Route::ThreadBound => THREAD_SINK.with(|sink| { + if let Some(sink) = sink.borrow().as_ref() { + push_event(sink, event); + } + }), + } + } +} - // OCSF events carry their payload in a thread-local; extract the - // shorthand representation for the push message. Non-OCSF events - // use the original visitor-based extraction. - let (msg, fields) = if meta.target() == openshell_ocsf::OCSF_TARGET { - if let Some(ocsf_event) = openshell_ocsf::clone_current_event() { - ( - ocsf_event.format_shorthand(), - std::collections::HashMap::new(), - ) - } else { - return; - } +fn push_event(sink: &LogSink, event: &Event<'_>) { + let meta = event.metadata(); + // OCSF events carry their payload in a thread-local; extract the + // shorthand representation for the push message. Non-OCSF events + // use the original visitor-based extraction. + let (msg, fields) = if meta.target() == openshell_ocsf::OCSF_TARGET { + if let Some(ocsf_event) = openshell_ocsf::clone_current_event() { + ( + ocsf_event.format_shorthand(), + std::collections::HashMap::new(), + ) } else { - let mut visitor = LogVisitor::default(); - event.record(&mut visitor); - visitor.into_parts(meta.name()) - }; + return; + } + } else { + let mut visitor = LogVisitor::default(); + event.record(&mut visitor); + visitor.into_parts(meta.name()) + }; - let ts = openshell_core::time::now_ms(); + let ts = openshell_core::time::now_ms(); - let is_ocsf = meta.target() == openshell_ocsf::OCSF_TARGET; + let is_ocsf = meta.target() == openshell_ocsf::OCSF_TARGET; - let log = SandboxLogLine { - sandbox_id: self.sandbox_id.clone(), - event_time: openshell_core::time::timestamp_from_millis(ts).ok(), - level: if is_ocsf { - "OCSF".to_string() - } else { - meta.level().to_string() - }, - target: meta.target().to_string(), - message: msg, - source: "sandbox".to_string(), - fields, - }; + let log = SandboxLogLine { + sandbox_id: sink.sandbox_id.to_string(), + event_time: openshell_core::time::timestamp_from_millis(ts).ok(), + level: if is_ocsf { + "OCSF".to_string() + } else { + meta.level().to_string() + }, + target: meta.target().to_string(), + message: msg, + source: "sandbox".to_string(), + fields, + }; - // Best-effort: drop if the channel is full (don't block tracing). - let _ = self.tx.try_send(log); - } + // Best-effort: drop if the channel is full (don't block tracing). + let _ = sink.tx.try_send(log); } +/// Lines a sandbox's push queue holds before the layer starts dropping them. +pub const LOG_PUSH_QUEUE_LINES: usize = 1024; + /// Spawn a background task that batches and pushes log lines to the server. /// /// Returns the sender half of the channel (for the [`LogPushLayer`]) and the /// task handle. The task runs until the sender is dropped or the gRPC stream /// breaks. pub fn spawn_log_push_task( - endpoint: String, + gateway: GatewayEndpoint, sandbox_id: String, ) -> (mpsc::Sender, tokio::task::JoinHandle<()>) { - let (tx, rx) = mpsc::channel::(1024); - - let handle = tokio::spawn(run_push_loop(endpoint, sandbox_id, rx)); + let (tx, rx) = mpsc::channel::(LOG_PUSH_QUEUE_LINES); + (tx, start_log_push(gateway, sandbox_id, rx)) +} - (tx, handle) +/// Push lines from a queue created before the caller's runtime, with the +/// owning sandbox's gateway credentials. +pub fn start_log_push( + gateway: GatewayEndpoint, + sandbox_id: String, + lines: mpsc::Receiver, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(run_push_loop(gateway, sandbox_id, lines)) } /// Maximum backoff delay between reconnection attempts. @@ -114,7 +182,7 @@ const MAX_BACKOFF: tokio::time::Duration = tokio::time::Duration::from_secs(30); const INITIAL_BACKOFF: tokio::time::Duration = tokio::time::Duration::from_secs(1); async fn run_push_loop( - endpoint: String, + gateway: GatewayEndpoint, sandbox_id: String, mut rx: mpsc::Receiver, ) { @@ -127,7 +195,7 @@ async fn run_push_loop( attempt += 1; // --- Connect --- - let client = match CachedOpenShellClient::connect(&endpoint).await { + let client = match CachedOpenShellClient::connect(&gateway).await { Ok(c) => { if attempt > 1 { eprintln!("openshell: log push reconnected (attempt {attempt})"); @@ -339,8 +407,7 @@ mod tests { fn capture(capacity: usize, f: impl FnOnce()) -> Vec { let (tx, mut rx) = mpsc::channel::(capacity); let layer = LogPushLayer { - sandbox_id: "sb-test".to_string(), - tx, + route: Route::Fixed(LogSink::new("sb-test", tx)), max_level: tracing::Level::INFO, }; let subscriber = tracing_subscriber::registry().with(layer); diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 1f1c7767ff..71313e19c0 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -381,7 +381,7 @@ fn map_session_stream_message( /// The task runs for the lifetime of the sandbox process, reconnecting with /// exponential backoff on failures. pub fn spawn( - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -390,7 +390,7 @@ pub fn spawn( runtime: SessionRuntimeContext, ) -> tokio::task::JoinHandle<()> { spawn_with_readiness( - endpoint, + gateway, sandbox_id, ssh_socket_path, port_forward, @@ -403,7 +403,7 @@ pub fn spawn( /// Spawn the supervisor session and expose when the gateway has accepted it. pub fn spawn_with_readiness( - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -413,7 +413,7 @@ pub fn spawn_with_readiness( ) -> (tokio::task::JoinHandle<()>, watch::Receiver) { let (ready_tx, ready_rx) = watch::channel(false); let config = SessionConfig { - endpoint, + gateway, sandbox_id, ssh_socket_path, port_forward, @@ -429,7 +429,7 @@ pub fn spawn_with_readiness( } struct SessionConfig { - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -453,7 +453,7 @@ async fn run_session_loop(config: SessionConfig) { // dial `target` for the next attempt and mark it as redirected so that // replica serves us rather than redirecting again. Any failure sends us // back to the configured address, which load-balances across replicas. - let mut target = config.endpoint.clone(); + let mut target = config.gateway.clone(); let mut redirected = false; loop { @@ -466,8 +466,11 @@ async fn run_session_loop(config: SessionConfig) { match result { Ok(SessionOutcome::Closed) => { config.ready_tx.send_replace(false); - let event = - session_closed_event(openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); + let event = session_closed_event( + &openshell_ocsf::ctx::ctx(), + target.url(), + &config.sandbox_id, + ); ocsf_emit!(event); break; } @@ -482,23 +485,23 @@ async fn run_session_loop(config: SessionConfig) { peer_endpoint = %peer_endpoint, "supervisor session: following gateway redirect" ); - target = peer_endpoint; + target = config.gateway.with_url(peer_endpoint); redirected = true; // No backoff: this is an expected handoff, not a failure. } Err(e) => { let accepted = config.ready_tx.send_replace(false); let event = session_failed_event( - openshell_ocsf::ctx::ctx(), - &target, + &openshell_ocsf::ctx::ctx(), + target.url(), attempt, &e.to_string(), ); ocsf_emit!(event); // Fall back to the configured gateway address so a redirect // to a replica that is going away cannot strand the sandbox. - let failed_redirect_target = target != config.endpoint && !accepted; - target.clone_from(&config.endpoint); + let failed_redirect_target = target.url() != config.gateway.url() && !accepted; + target.clone_from(&config.gateway); redirected = redirect_survives_failure(redirected, accepted); if let Some(delay) = backoff.on_failure(accepted, failed_redirect_target) { tokio::time::sleep(delay).await; @@ -580,7 +583,7 @@ enum SessionOutcome { async fn run_single_session( config: &SessionConfig, - target: &str, + target: &grpc_client::GatewayEndpoint, redirected: bool, connection_epoch: u64, ) -> Result> { @@ -667,8 +670,8 @@ async fn run_single_session( updates.send_replace(Some(accepted.session_id.clone())); } let event = session_established_event( - openshell_ocsf::ctx::ctx(), - target, + &openshell_ocsf::ctx::ctx(), + target.url(), &accepted.session_id, u32::try_from(heartbeat_secs).unwrap_or(u32::MAX), ); @@ -807,12 +810,12 @@ async fn apply_update( /// Report the canonical process result and wait for durable handling. pub async fn report_main_process_exit( - endpoint: &str, + gateway: &grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, exit_code: i32, ) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) + let channel = grpc_client::connect_channel_pub(gateway) .await .map_err(|error| format!("connect failed: {error}"))?; let mut client = OpenShellClient::new(channel); @@ -844,11 +847,11 @@ pub(crate) async fn test_bridge_ssh_relay( /// Confirm terminal delivery and permit ephemeral cleanup. pub async fn finalize_main_process_exit( - endpoint: &str, + gateway: &grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, ) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) + let channel = grpc_client::connect_channel_pub(gateway) .await .map_err(|error| format!("connect failed: {error}"))?; let mut client = OpenShellClient::new(channel); @@ -904,7 +907,8 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); - let event = relay_open_event(openshell_ocsf::ctx::ctx(), &relay_open, &ssh_socket_path); + let event = + relay_open_event(&openshell_ocsf::ctx::ctx(), &relay_open, &ssh_socket_path); ocsf_emit!(event); tokio::spawn(async move { @@ -922,7 +926,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< { Ok(()) => { let event = relay_closed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &event_open, &ssh_socket_path, ); @@ -930,7 +934,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< } Err(e) => { let event = relay_failed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &event_open, &ssh_socket_path, &e.to_string(), @@ -948,7 +952,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< } Some(gateway_message::Payload::RelayClose(close)) => { let event = relay_close_from_gateway_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &close.channel_id, &close.reason, ); diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index 9e2e0ea8ad..a42c370404 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -58,6 +58,11 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } tonic-prost = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } tokio = { workspace = true, features = ["test-util"] } futures = { workspace = true } temp-env = "0.3" diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 8108c6ea03..ac2f3ed888 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -4,7 +4,7 @@ The supervisor loads and reconciles policy, maintains provider credentials, appl ## Backend startup -The public `run_sandbox` entry point selects the OpenShell Sandbox Protocol backend and collects its startup inputs into a public `SandboxRunConfig`. Shared startup receives that config and the trusted backend setup separately. The `backend_setup` module owns that backend's launch-data decoder, workload policy discovery, and client construction. Descriptor contents cannot select an implementation. +The supervisor binary selects a backend setup and collects its launch inputs into a public `SandboxInstanceConfig`. Shared startup receives that config and the trusted backend setup separately. The `backend_setup` module owns that backend's launch-data decoder, workload policy discovery, and client construction. Descriptor contents cannot select an implementation. Shared startup checks the admitted backend name before passing the opaque payload to its decoder. It then compares the decoded sandbox, session, and runtime generation with the trusted launch inputs before installing credentials or discovering workload policy. A mismatch stops startup. @@ -14,7 +14,7 @@ The supervisor admits policy and prepares credentials before constructing and at The client receives the supervisor's live provider state, bearer-token slot, and CA-path slot. Provider refresh, token rotation, and later CA publication must remain visible through those shared handles. Startup does not create independent copies of their current values. -`backend_setup`, `SandboxRunConfig`, and `run_sandbox_with_backend` are public so the supervisor binary can select a registered backend. External backends should implement the `IsolationBackend` and `SandboxProtocol` services rather than link this crate. +`backend_setup`, `SandboxInstanceConfig`, `SandboxInstanceBindings`, and `run_sandbox_with_backend` are public so the supervisor binary can select a registered backend. External backends should implement the `IsolationBackend` and `SandboxProtocol` services rather than link this crate. The startup result separates the backend transport payload from optional authenticated CONNECT listener settings. Shared networking starts the listener @@ -22,6 +22,53 @@ only after boundary attachment and confirmation; the supervisor owns policy evaluation, live credentials, and listener lifetime. These settings do not add a proxy hook to the isolation interface or the shared Sandbox Protocol descriptor. +## Sandbox instance ownership + +A sandbox instance is one running generation of a stable Sandbox resource. +Gateway connections, and the gateway supervisor sessions carried over them, +can reconnect within an instance without creating another one; the generation +remains the fencing identifier. + +`run_sandbox_instance` supervises one sandbox instance from three inputs. +`SandboxInstanceConfig` is the single launch configuration: identity, command, +policy inputs, auth bundle, backend descriptor, and file locations. +`SandboxInstanceBindings` is the live state its owner prepares once: the +`GatewayCredentials` issued from the auth bundle, the OCSF context and JSON +settings, the log sink, and the denial history served by `policy.local`. +`SandboxInstanceControls` carries the owner's stop signal and readiness. +Everything else, including credential renewal, policy, provider credentials and +token grants, middleware credentials, the backend connection, and the access +plane, is created by the instance. Gateway clients receive a `GatewayEndpoint`, +which pairs the URL with that instance's `GatewayCredentials`; nothing is +installed process-wide. + +The dedicated process wrapper keeps the existing process contract. The binary +loads the launch configuration from its arguments and environment and prepares +`SandboxInstanceBindings::dedicated`, sharing one `GatewayCredentials` with log +push and the OCSF JSON settings with its logging layers, so renewal by either +gateway client keeps both current. `run_sandbox_with_backend` sets the +process-wide OCSF context, binds the readiness socket and port to instance +readiness, and stops the instance on SIGTERM or SIGINT. + +`sandbox_instance_host::SandboxInstanceHost` runs several sandbox instances in +one process. Each instance gets its own Tokio runtime; every thread of that +runtime, and the thread driving it, is bound to the instance's +`SandboxInstanceBindings`: OCSF context, OCSF JSON settings, log sink, and +denial log. Spawned tasks, blocking work, +callbacks, reconnects, and renewal therefore inherit the instance's identity, +and threads outside an instance carry none and push no logs. The OCSF context +also supplies the sandbox identity in middleware requests. + +Removing an instance stops its workload, shuts down only its runtime, and +revokes its gateway credentials; the key stays registered until the +instance's thread has finished. An instance that panics reports a failure and +is revoked the same way. An instance that is still starting is cancelled at +once; the sandbox runtime then loses supervision and fails closed, as it does +when a dedicated supervisor exits. A hosted instance without an explicit proxy +trust directory receives a private one for its lifetime. Hosted processes must +install `LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to +`SandboxInstanceDenialWriter`. + ## Console logging Diagnostics and OCSF shorthand share a bounded, nonblocking stderr writer. With `ocsf_json_enabled=true`, the same writer also receives timestamped `OCSF-JSON` records. Each formatter submits a complete line in one write so concurrent producers cannot interleave records in the queue. The 1,024-line queue drops new lines when full rather than waiting for stderr. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index c291b42ef0..b4704bf623 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -680,31 +680,42 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .require_networking .store(true, Ordering::SeqCst); let startup = |await_main_process_attachment| { + let config = crate::SandboxInstanceConfig { + command: vec!["test-agent".into(), "test-argument".into()], + workdir: Some("/test-workspace".into()), + timeout_secs: 15, + interactive: false, + await_main_process_attachment, + sandbox_id: Some(setup.sandbox_id.clone()), + sandbox: None, + container_image: String::new(), + openshell_endpoint: None, + policy_rules: Some(rules.to_string_lossy().into_owned()), + policy_data: Some(data.to_string_lossy().into_owned()), + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: TestSetup::descriptor(), + auth_bundle: setup.auth.clone(), + admitted_isolation_backend: Some(TEST_BACKEND.into()), + main_exit_marker: Some(marker.clone()), + trace_context: None, + provider_workload_api_socket: None, + // The binary loads the proxy trust directory from its environment. + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + }; + let bindings = crate::SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), + openshell_ocsf::OcsfJsonSettings::default(), + ); Box::pin(crate::run_sandbox_with_backend( &setup, - crate::SandboxRunConfig { - command: vec!["test-agent".into(), "test-argument".into()], - workdir: Some("/test-workspace".into()), - timeout_secs: 15, - interactive: false, - await_main_process_attachment, - sandbox_id: Some(setup.sandbox_id.clone()), - sandbox: None, - openshell_endpoint: None, - policy_rules: Some(rules.to_string_lossy().into_owned()), - policy_data: Some(data.to_string_lossy().into_owned()), - ssh_socket_path: None, - health_socket_path: Some(readiness.clone()), - health_port: Some(health_port), - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(Mutex::new(String::new())), - upstream_proxy_args: UpstreamProxyArgs::default(), - additional_ca_bundle: None, - backend_descriptor: TestSetup::descriptor(), - auth_bundle: setup.auth.clone(), - admitted_isolation_backend: Some(TEST_BACKEND.into()), - main_exit_marker: Some(marker.clone()), - }, + config, + bindings, + Some(readiness.clone()), + Some(health_port), )) }; diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 62e7ffb6d4..1a83fd0887 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -738,7 +738,7 @@ impl ConfigRuntime { ); let provider = match client .fetch_provider_environment( - &ctx.endpoint, + &ctx.gateway, &ctx.sandbox_id, ctx.provider_credentials.requires_stable_placeholders(), ) @@ -1159,7 +1159,7 @@ impl ConfigRuntime { pub async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - &ctx.endpoint, + &ctx.gateway, ctx.extension_credentials.clone(), ) .await?; @@ -1198,7 +1198,7 @@ async fn report_runtime_configuration( ); } openshell_core::grpc_client::report_sandbox_configuration( - &ctx.endpoint, + &ctx.gateway, &ctx.sandbox_id, instance_id, Some(snapshot), @@ -1618,6 +1618,7 @@ mod tests { use super::*; use crate::tests::{ proto_policy_fixture, proto_tcp_policy_fixture, settings_poll_result, startup_provider, + test_gateway, }; use crate::*; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1779,7 +1780,7 @@ mod tests { async fn fetch_provider_environment( &self, - _endpoint: &str, + _gateway: &openshell_core::grpc_client::GatewayEndpoint, _sandbox_id: &str, _requires_stable_placeholders: bool, ) -> Result { @@ -1826,12 +1827,12 @@ mod tests { async fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { self.inner - .fetch_provider_environment(endpoint, sandbox_id, requires_stable_placeholders) + .fetch_provider_environment(gateway, sandbox_id, requires_stable_placeholders) .await } @@ -2074,7 +2075,7 @@ mod tests { async fn fetch_provider_environment( &self, - _endpoint: &str, + _gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { @@ -2212,12 +2213,12 @@ mod tests { async fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { self.inner - .fetch_provider_environment(endpoint, sandbox_id, requires_stable_placeholders) + .fetch_provider_environment(gateway, sandbox_id, requires_stable_placeholders) .await } @@ -2709,7 +2710,7 @@ mod tests { ); } PolicyPollLoopContext { - endpoint: String::new(), + gateway: test_gateway(), sandbox_id: "sandbox-test".to_string(), sandbox: "sandbox-test-name".to_string(), opa_engine, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index d78c4c04b9..699880f1ad 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -11,7 +11,7 @@ use openshell_core::proto::{ UpdateConfigResponse, gateway_message, supervisor_message, }; use openshell_isolation_interface::contract::{BackendError, ResolvedWorkloadIdentity}; -use std::sync::{Mutex, atomic::Ordering}; +use std::sync::atomic::Ordering; use tonic::codegen::tokio_stream::{StreamExt as _, wrappers::TcpListenerStream}; use tonic::codegen::{Body, BoxFuture, BoxStream, Bytes, Service, StdError, http}; @@ -277,36 +277,41 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl session_id, probe: probe.clone(), }; - let startup = run_sandbox_with_backend( - &setup, - SandboxRunConfig { - command: vec!["true".into()], - workdir: None, - timeout_secs: 0, - interactive: false, - await_main_process_attachment: false, - sandbox_id: Some("sb-legacy-restart".into()), - sandbox: Some("legacy-restart".into()), - openshell_endpoint: Some(endpoint), - policy_rules: None, - policy_data: None, - ssh_socket_path: None, - health_socket_path: None, - health_port: None, - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(Mutex::new(String::new())), - upstream_proxy_args: - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), - additional_ca_bundle: None, - backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { - backend_name: "test-held".into(), - payload: Vec::new(), - }, - auth_bundle, - admitted_isolation_backend: Some("test-held".into()), - main_exit_marker: None, + let config = SandboxInstanceConfig { + command: vec!["true".into()], + workdir: None, + timeout_secs: 0, + interactive: false, + await_main_process_attachment: false, + sandbox_id: Some("sb-legacy-restart".into()), + sandbox: Some("legacy-restart".into()), + container_image: String::new(), + openshell_endpoint: Some(endpoint), + policy_rules: None, + policy_data: None, + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { + backend_name: "test-held".into(), + payload: Vec::new(), }, + auth_bundle, + admitted_isolation_backend: Some("test-held".into()), + main_exit_marker: None, + trace_context: None, + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + }; + let bindings = SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) + .unwrap(), + openshell_ocsf::OcsfJsonSettings::default(), ); + let startup = run_sandbox_with_backend(&setup, config, bindings, None, None); tokio::pin!(startup); let result = tokio::select! { () = probe.ready_event.notified() => None, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 9628d811a0..39699751e2 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -24,6 +24,7 @@ mod denial_aggregator; mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; +pub mod sandbox_instance_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -66,7 +67,7 @@ use openshell_ocsf::{ /// /// The singleton lives in `openshell-ocsf` so both supervisor leaves can /// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +/// `run_sandbox_with_backend()` startup via `openshell_ocsf::ctx::set_ctx`. pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; /// The sandbox rejected the image working directory while starting the agent. @@ -387,7 +388,10 @@ use provider_readiness::{EnvironmentIdentity, Tracker as ProviderReadinessTracke use tokio::sync::mpsc::UnboundedSender; use tokio::time::timeout; -fn shared_ssh_socket_from_env() -> bool { +/// Whether a dedicated supervisor's environment grants the SSH socket to its +/// group rather than its owner. +#[must_use] +pub fn shared_ssh_socket_from_env() -> bool { std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) .is_ok_and(|value| shared_ssh_socket_value(&value)) } @@ -601,6 +605,8 @@ pub async fn run_network_proxy( ); let (_, workspace_rx) = tokio::sync::watch::channel(String::new()); let tls_dir = prepare_network_proxy_tls_dir(tls_dir)?; + let mut proxy_tls = openshell_supervisor_network::run::ProxyTlsFiles::from_environment(); + proxy_tls.directory = Some(tls_dir.path.clone()); let mut networking = openshell_supervisor_network::run::run_networking( &policy, None, @@ -609,9 +615,11 @@ pub async fn run_network_proxy( Arc::new(AtomicU32::new(0)), false, &provider_credentials, - None, - Some("network-proxy"), - None, + openshell_supervisor_network::run::SandboxInstanceNetwork { + sandbox_name: Some("network-proxy".to_string()), + proxy_tls, + ..Default::default() + }, None, None, None, @@ -619,7 +627,6 @@ pub async fn run_network_proxy( workspace_rx, &upstream_proxy_args, additional_ca_bundle.as_deref(), - Some(&tls_dir.path), None, #[cfg(target_os = "linux")] None, @@ -657,72 +664,114 @@ pub async fn run_network_proxy( } } -/// Run a command in the sandbox. +/// Supervise the one sandbox instance this process was launched for. +/// +/// This is the dedicated process wrapper. It sets the process-wide OCSF +/// context from `bindings`, publishes instance readiness on the health socket +/// and port, and stops the instance on SIGTERM or SIGINT; +/// [`run_sandbox_instance`] owns everything else. The caller loads `config` +/// from the process's arguments and environment and prepares `bindings` once, +/// sharing their gateway credentials and OCSF JSON settings with the +/// process's log push and logging layers. +/// +/// Trusted composition chooses the setup before shared admission, policy, and +/// lifecycle handling. Payload contents never select a backend implementation. /// /// # Errors /// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::implicit_hasher, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, +/// Returns an error if a readiness endpoint cannot be prepared or the +/// instance fails to start or supervise its workload. +pub async fn run_sandbox_with_backend( + backend_setup: &dyn backend_setup::BackendSetup, + config: SandboxInstanceConfig, + bindings: SandboxInstanceBindings, health_socket_path: Option, health_port: Option, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, - additional_ca_bundle: Option, - backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, - auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - admitted_isolation_backend: Option, - main_exit_marker: Option, ) -> Result { - // Shared startup retains policy and networking state; box it to keep callers' futures small. - Box::pin(run_sandbox_with_backend( - &backend_setup::OpenShellBackendSetup, - SandboxRunConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - health_socket_path, - health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - admitted_isolation_backend, - main_exit_marker, + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; the boundary mediation source carries + // workload-side connection metadata. + if !openshell_ocsf::ctx::set_ctx((*bindings.event_context).clone()) { + debug!("OCSF context already initialized, keeping existing"); + } + + let (readiness, instance_readiness) = tokio::sync::watch::channel(false); + let control_readiness = health_socket_path + .map(ReadinessEndpoint::Unix) + .into_iter() + .chain(health_port.map(ReadinessEndpoint::Tcp)) + .map(|endpoint| ControlReadiness::start(endpoint, Some(instance_readiness.clone()))) + .collect::>>()?; + let result = Box::pin(run_sandbox_instance( + backend_setup, + config, + bindings, + SandboxInstanceControls { + shutdown: Box::pin(wait_for_control_shutdown_signal()), + readiness, }, )) - .await + .await; + drop(control_readiness); + result } -/// Trusted inputs for one supervisor launch. -pub struct SandboxRunConfig { +/// OCSF identity of one supervised sandbox. +#[must_use] +pub fn sandbox_instance_event_context( + sandbox_id: Option<&str>, + sandbox: Option<&str>, + container_image: String, +) -> EventContext { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + EventContext { + sandbox_id: sandbox_id.unwrap_or_default().to_string(), + sandbox_name: sandbox.unwrap_or_default().to_string(), + container_image, + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + origin: openshell_ocsf::EventOrigin::Supervisor, + } +} + +/// W3C trace context of the operation that launched a sandbox generation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TraceContext { + pub traceparent: String, + pub tracestate: Option, +} + +impl TraceContext { + /// The context a driver passed to a dedicated supervisor process. + #[must_use] + pub fn from_environment() -> Option { + let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) + .ok() + .filter(|value| !value.is_empty())?; + Some(Self { + traceparent, + tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) + .ok() + .filter(|value| !value.is_empty()), + }) + } +} + +/// Trusted launch inputs of one supervised sandbox instance. +/// +/// A sandbox instance is one running generation of a stable Sandbox resource; +/// gateway connections may reconnect within it without creating another. +/// Nothing in an instance reads process-wide sandbox state: identity, policy +/// inputs, and file locations come from here, and live per-instance state +/// from its [`SandboxInstanceBindings`], so one process can host several +/// instances. +pub struct SandboxInstanceConfig { pub command: Vec, pub workdir: Option, pub timeout_secs: u64, @@ -730,30 +779,151 @@ pub struct SandboxRunConfig { pub await_main_process_attachment: bool, pub sandbox_id: Option, pub sandbox: Option, + /// Workload image recorded in this instance's OCSF events. + pub container_image: String, pub openshell_endpoint: Option, pub policy_rules: Option, pub policy_data: Option, pub ssh_socket_path: Option, - pub health_socket_path: Option, - pub health_port: Option, - pub ocsf_enabled: Arc, - pub ocsf_schema_version: Arc>, + /// Grant the SSH socket to the supervisor's group rather than its owner. + pub shared_ssh_socket: bool, pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, pub additional_ca_bundle: Option, pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, pub admitted_isolation_backend: Option, pub main_exit_marker: Option, + /// Parent of the startup span. + pub trace_context: Option, + /// SPIFFE Workload API socket for provider token grants; absent fails closed. + pub provider_workload_api_socket: Option, + pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, } +impl SandboxInstanceConfig { + /// OCSF identity of this sandbox instance. + #[must_use] + pub fn event_context(&self) -> EventContext { + sandbox_instance_event_context( + self.sandbox_id.as_deref(), + self.sandbox.as_deref(), + self.container_image.clone(), + ) + } +} + +/// Live state bound to one sandbox instance, prepared once by its owner. +/// +/// A dedicated supervisor shares these with its process-wide logging and log +/// push; a hosted instance binds them to every thread of its runtime. Every +/// gateway client of the instance, including log push and credential renewal, +/// uses the same `gateway_credentials`, so renewal by one keeps all current. +#[derive(Clone)] +pub struct SandboxInstanceBindings { + /// Gateway credentials issued from the instance's auth bundle. + pub gateway_credentials: openshell_core::grpc_client::GatewayCredentials, + pub event_context: Arc, + /// OCSF JSON output settings this instance's gateway configuration controls. + pub ocsf_json: openshell_ocsf::OcsfJsonSettings, + /// Log push destination of a hosted instance's threads. A dedicated + /// process pushes through its process-wide layer instead. + pub log_sink: Option, + /// Source of `policy.local` recent denials. + pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, +} + +impl SandboxInstanceBindings { + /// Bindings of a process dedicated to the instance `config` launches. Its + /// process-wide log push and shorthand log files serve the instance. + #[must_use] + pub fn dedicated( + config: &SandboxInstanceConfig, + gateway_credentials: openshell_core::grpc_client::GatewayCredentials, + ocsf_json: openshell_ocsf::OcsfJsonSettings, + ) -> Self { + Self { + gateway_credentials, + event_context: Arc::new(config.event_context()), + ocsf_json, + log_sink: None, + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + } + } +} + +/// How the owner of a sandbox instance stops it and observes its readiness. +pub struct SandboxInstanceControls { + /// Completes when the owner asks the instance to stop its workload. It is + /// first polled once the workload has started. + pub shutdown: Pin + Send>>, + /// True while the gateway has accepted this instance's supervision and its + /// access plane is retained; false otherwise. + pub readiness: tokio::sync::watch::Sender, +} + +/// Mirrors gateway session acceptance into the owner's readiness until the +/// access plane is released. +struct SandboxInstanceReadiness { + readiness: tokio::sync::watch::Sender, + forward: Option>, +} + +impl SandboxInstanceReadiness { + fn start( + readiness: tokio::sync::watch::Sender, + accepted: Option>, + ) -> Self { + // Without a gateway session, an attached access plane is ready. + let Some(mut accepted) = accepted else { + readiness.send_replace(true); + return Self { + readiness, + forward: None, + }; + }; + let sender = readiness.clone(); + let forward = tokio::spawn(async move { + loop { + sender.send_replace(*accepted.borrow_and_update()); + if accepted.changed().await.is_err() { + sender.send_replace(false); + return; + } + } + }); + Self { + readiness, + forward: Some(forward), + } + } + + fn release(&mut self) { + if let Some(forward) = self.forward.take() { + forward.abort(); + } + self.readiness.send_replace(false); + } +} + +impl Drop for SandboxInstanceReadiness { + fn drop(&mut self) { + self.release(); + } +} + +/// Supervise one sandbox instance until its workload exits or the owner +/// stops it. +/// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. #[allow(clippy::similar_names)] -pub async fn run_sandbox_with_backend( +pub async fn run_sandbox_instance( backend_setup: &dyn backend_setup::BackendSetup, - config: SandboxRunConfig, + config: SandboxInstanceConfig, + bindings: SandboxInstanceBindings, + controls: SandboxInstanceControls, ) -> Result { - let SandboxRunConfig { + let SandboxInstanceConfig { command, workdir, timeout_secs, @@ -761,21 +931,34 @@ pub async fn run_sandbox_with_backend( await_main_process_attachment, sandbox_id, sandbox, + container_image: _, openshell_endpoint, policy_rules, policy_data, ssh_socket_path, - health_socket_path, - health_port, - ocsf_enabled, - ocsf_schema_version, + shared_ssh_socket, upstream_proxy_args, additional_ca_bundle, backend_descriptor, auth_bundle, admitted_isolation_backend, main_exit_marker, + trace_context, + provider_workload_api_socket, + proxy_tls, } = config; + // The owner binds the event context and log sink to the instance's + // threads; the instance itself uses the rest. + let SandboxInstanceBindings { + gateway_credentials, + ocsf_json, + denial_log, + .. + } = bindings; + let SandboxInstanceControls { + shutdown: shutdown_requested, + readiness, + } = controls; // The current client attachment adapter is SSH; retaining a process's I/O // internally does not make it reachable by a foreground client. if await_main_process_attachment && (!cfg!(unix) || ssh_socket_path.is_none()) { @@ -791,33 +974,10 @@ pub async fn run_sandbox_with_backend( |(program, args)| (program.clone(), args.to_vec()), ); - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; the boundary mediation source carries - // workload-side connection metadata. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(EventContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - origin: openshell_ocsf::EventOrigin::Supervisor, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - // Extension credentials are owned by this supervisor and shared by every - // gateway connection it opens, so the middleware registry's bearer slots - // and the policy poll loop that rotates them stay the same objects. + // Extension credentials are owned by this sandbox instance and shared + // by every gateway connection it opens, so the middleware registry's + // bearer slots and the policy poll loop that rotates them stay the same + // objects. let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); let selected_backend = backend_setup::SelectedBackend::select( @@ -827,7 +987,13 @@ pub async fn run_sandbox_with_backend( sandbox_id.as_deref(), &auth_bundle, )?; - let sandbox_bearer = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth_bundle)?; + let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { + miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") + })?; + // Every gateway client of this instance uses its own credentials. + let gateway = openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new(endpoint, gateway_credentials.clone()) + }); #[cfg(unix)] let ssh_host_key = if ssh_socket_path.is_some() { Some(openshell_supervisor_process::ssh::parse_host_key( @@ -852,7 +1018,13 @@ pub async fn run_sandbox_with_backend( sandbox.id = sandbox_id.as_deref().unwrap_or_default(), otel.status_code = tracing::field::Empty, ); - openshell_otel::set_parent_from_environment(&startup); + if let Some(trace_context) = &trace_context { + openshell_otel::set_parent_from_trace_context( + &startup, + Some(&trace_context.traceparent), + trace_context.tracestate.as_deref(), + ); + } let startup_status = startup.in_scope(openshell_otel::ErrorStatusGuard::current); let (image_yaml, invalid_image) = selected_backend .discover_policy(sandbox_bearer.clone()) @@ -872,7 +1044,6 @@ pub async fn run_sandbox_with_backend( let vm_policy_identity = selected_backend.vm_policy_identity(); // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); let ( policy, @@ -886,7 +1057,7 @@ pub async fn run_sandbox_with_backend( ) = load_policy_with_gateway( sandbox_id.clone(), sandbox.clone(), - openshell_endpoint.clone(), + gateway.clone(), policy_rules, policy_data, &extension_credentials, @@ -894,7 +1065,7 @@ pub async fn run_sandbox_with_backend( vm_policy_identity, Some(image_discovery), &RemoteStartupGateway { - endpoint: openshell_endpoint.clone().unwrap_or_default(), + gateway: gateway.clone(), }, ) .instrument(tracing::info_span!(parent: &startup, "supervisor.policy.load")) @@ -915,8 +1086,8 @@ pub async fn run_sandbox_with_backend( // Fetch provider environment variables from the server. // This is done after loading the policy so the sandbox can still start // even if provider env fetch fails (graceful degradation). - let environment = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id) + let environment = if let (Some(id), Some(gateway)) = (&sandbox_id, &gateway) { + match openshell_core::grpc_client::fetch_provider_environment(gateway, id) .instrument(startup.clone()) .await { @@ -1109,9 +1280,14 @@ pub async fn run_sandbox_with_backend( // unnecessarily delay DNS and network readiness. false, &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), + openshell_supervisor_network::run::SandboxInstanceNetwork { + sandbox_id: sandbox_id.clone(), + sandbox_name: sandbox_name_for_agg.clone(), + gateway: gateway.clone(), + provider_workload_api_socket, + denial_log, + proxy_tls, + }, denial_tx, activity_tx, endpoint_observation_tx.clone(), @@ -1119,7 +1295,6 @@ pub async fn run_sandbox_with_backend( workspace_rx.clone(), &upstream_proxy_args, additional_ca_bundle.as_deref(), - None, remote_host_gateway_ip, #[cfg(target_os = "linux")] None, @@ -1142,14 +1317,14 @@ pub async fn run_sandbox_with_backend( // Spawn the denial-aggregator flush task. The aggregator drains proxy // denial events, batches them, and ships summaries to the gateway via // `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + if let (Some(rx), Some(gateway)) = (denial_rx, gateway.as_ref()) { // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall // back to the ID when the name isn't set. let agg_name = sandbox_name_for_agg .clone() .or_else(|| sandbox_id.clone()) .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); + let agg_gateway = gateway.clone(); let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") .ok() .and_then(|v| v.parse().ok()) @@ -1163,12 +1338,12 @@ pub async fn run_sandbox_with_backend( aggregator .run( |summaries| { - let endpoint = agg_endpoint.clone(); + let gateway = agg_gateway.clone(); let sandbox_name = agg_name.clone(); let workspace = denial_workspace_rx.borrow().clone(); async move { if let Err(e) = flush_proposals_to_gateway( - &endpoint, + &gateway, &sandbox_name, &workspace, summaries, @@ -1188,12 +1363,12 @@ pub async fn run_sandbox_with_backend( // Spawn the activity-aggregator flush task. The aggregator drains // anonymous activity events from the proxy, sanitizes deny groups, // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + if let (Some(rx), Some(gateway)) = (activity_rx, gateway.as_ref()) { let agg_name = sandbox_name_for_agg .clone() .or_else(|| sandbox_id.clone()) .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); + let agg_gateway = gateway.clone(); let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") .ok() @@ -1208,12 +1383,12 @@ pub async fn run_sandbox_with_backend( aggregator .run( move |summary| { - let endpoint = agg_endpoint.clone(); + let gateway = agg_gateway.clone(); let sandbox_name = agg_name.clone(); let workspace = activity_workspace_rx.borrow().clone(); async move { if let Err(e) = flush_activity_to_gateway( - &endpoint, + &gateway, &sandbox_name, &workspace, summary, @@ -1231,18 +1406,18 @@ pub async fn run_sandbox_with_backend( } // Spawn background policy poll task (gRPC mode only). - if let (Some(id), Some(sandbox), Some(endpoint), Some(engine)) = ( + if let (Some(id), Some(sandbox), Some(gateway), Some(engine)) = ( sandbox_id.as_deref(), sandbox.as_deref(), - openshell_endpoint.as_deref(), + gateway.as_ref(), opa_engine.as_ref(), ) { let poll_id = id.to_string(); let poll_sandbox = sandbox.to_string(); - let poll_endpoint = endpoint.to_string(); + let poll_gateway = gateway.clone(); let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_ocsf_schema_version = ocsf_schema_version.clone(); + let poll_ocsf_enabled = ocsf_json.enabled.clone(); + let poll_ocsf_schema_version = ocsf_json.target_version.clone(); let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); @@ -1255,7 +1430,7 @@ pub async fn run_sandbox_with_backend( .and_then(|v| v.parse().ok()) .unwrap_or(10); let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, + gateway: poll_gateway, sandbox_id: poll_id, sandbox: poll_sandbox, opa_engine: poll_engine, @@ -1326,9 +1501,9 @@ pub async fn run_sandbox_with_backend( let agent = running.agent(); let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( sandbox_id.as_deref(), - openshell_endpoint.as_deref(), + gateway.as_ref(), ssh_socket_path.as_deref(), - shared_ssh_socket_from_env(), + shared_ssh_socket, networking .as_ref() .and_then(|runtime| runtime.ca_file_paths.clone()), @@ -1345,29 +1520,23 @@ pub async fn run_sandbox_with_backend( info!(backend = %backend_name, "Control-mode access plane started"); startup_status.finish(Ok::<_, ()>(())).ok(); drop(startup); - let _provider_reporter = - sandbox_id - .as_ref() - .zip(openshell_endpoint.as_ref()) - .map(|(id, endpoint)| { - provider_readiness.start_reporter( - endpoint.clone(), - id.clone(), - provider_credentials.clone(), - supervisor_session_id.clone(), - running.exec(), - ) - }); - let mut control_readiness = health_socket_path - .map(ReadinessEndpoint::Unix) - .into_iter() - .chain(health_port.map(ReadinessEndpoint::Tcp)) - .map(|endpoint| ControlReadiness::start(endpoint, boundary_access.session_readiness())) - .collect::>>()?; + let _provider_reporter = sandbox_id + .as_ref() + .zip(gateway.as_ref()) + .map(|(id, gateway)| { + provider_readiness.start_reporter( + gateway.clone(), + id.clone(), + provider_credentials.clone(), + supervisor_session_id.clone(), + running.exec(), + ) + }); + let mut instance_readiness = + SandboxInstanceReadiness::start(readiness, boundary_access.session_readiness()); let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); - let shutdown_requested = wait_for_control_shutdown_signal(); - tokio::pin!(shutdown_requested); + let mut shutdown_requested = shutdown_requested; let wait = async move { wait_agent .wait() @@ -1417,7 +1586,7 @@ pub async fn run_sandbox_with_backend( } }; if !retain_access { - control_readiness.clear(); + instance_readiness.release(); } boundary_access .publish_main_exit(exit_code, await_main_process_attachment) @@ -1431,11 +1600,10 @@ pub async fn run_sandbox_with_backend( .wrap_err("persist canonical-process completion marker")?; } if !completion_cancelled - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { let report = openshell_supervisor_process::delegated::report_main_process_exit( - endpoint, + gateway, id, &instance_id, exit_code, @@ -1449,11 +1617,10 @@ pub async fn run_sandbox_with_backend( completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; } if !completion_cancelled - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( - endpoint, + gateway, id, &instance_id, ); @@ -1462,13 +1629,13 @@ pub async fn run_sandbox_with_backend( } if completion_cancelled { retain_access = false; - control_readiness.clear(); + instance_readiness.release(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } - drop(control_readiness); + drop(instance_readiness); drop(running); drop(boundary_access); exit_code @@ -1521,7 +1688,7 @@ fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io:: /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_name: &str, workspace: &str, summaries: Vec, @@ -1529,7 +1696,7 @@ async fn flush_proposals_to_gateway( use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::{DenialSummary, L7RequestSample}; - let client = CachedOpenShellClient::connect(endpoint).await?; + let client = CachedOpenShellClient::connect(gateway).await?; client.set_workspace(workspace.to_string()); let proto_summaries: Vec = summaries @@ -1591,7 +1758,7 @@ async fn flush_proposals_to_gateway( /// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. async fn flush_activity_to_gateway( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_name: &str, workspace: &str, summary: activity_aggregator::FlushableActivitySummary, @@ -1599,7 +1766,7 @@ async fn flush_activity_to_gateway( use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - let client = CachedOpenShellClient::connect(endpoint).await?; + let client = CachedOpenShellClient::connect(gateway).await?; client.set_workspace(workspace.to_string()); let proto_summary = NetworkActivitySummary { @@ -2153,7 +2320,15 @@ trait StartupGateway: Send + Sync { } struct RemoteStartupGateway { - endpoint: String, + gateway: Option, +} + +impl RemoteStartupGateway { + fn gateway(&self) -> Result<&openshell_core::grpc_client::GatewayEndpoint> { + self.gateway + .as_ref() + .ok_or_else(|| miette::miette!("supervisor has no gateway endpoint")) + } } impl StartupGateway for RemoteStartupGateway { @@ -2161,13 +2336,13 @@ impl StartupGateway for RemoteStartupGateway { &self, sandbox: &str, ) -> Result { - openshell_core::grpc_client::fetch_settings_snapshot(&self.endpoint, sandbox).await + openshell_core::grpc_client::fetch_settings_snapshot(self.gateway()?, sandbox).await } async fn provider( &self, id: &str, ) -> Result { - openshell_core::grpc_client::fetch_provider_environment(&self.endpoint, id).await + openshell_core::grpc_client::fetch_provider_environment(self.gateway()?, id).await } async fn sync( &self, @@ -2176,7 +2351,7 @@ impl StartupGateway for RemoteStartupGateway { workspace: &str, ) -> Result { openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - &self.endpoint, + self.gateway()?, sandbox, policy, workspace, @@ -2192,7 +2367,7 @@ impl StartupGateway for RemoteStartupGateway { error: &str, ) -> Result<()> { openshell_core::grpc_client::report_sandbox_configuration( - &self.endpoint, + self.gateway()?, id, instance_id, snapshot, @@ -2291,7 +2466,7 @@ impl CapturedProviderEnvironment { async fn load_policy( sandbox_id: Option, sandbox: Option, - openshell_endpoint: Option, + openshell_gateway: Option, policy_rules: Option, policy_data: Option, extension_credentials: &openshell_extension_core::ExtensionCredentialStore, @@ -2309,7 +2484,7 @@ async fn load_policy( load_policy_with_gateway( sandbox_id, sandbox, - openshell_endpoint.clone(), + openshell_gateway.clone(), policy_rules, policy_data, extension_credentials, @@ -2317,7 +2492,7 @@ async fn load_policy( None, None, &RemoteStartupGateway { - endpoint: openshell_endpoint.unwrap_or_default(), + gateway: openshell_gateway, }, ) .await @@ -2330,7 +2505,7 @@ async fn load_policy( async fn load_policy_with_gateway( sandbox_id: Option, sandbox: Option, - openshell_endpoint: Option, + openshell_gateway: Option, policy_rules: Option, policy_data: Option, extension_credentials: &openshell_extension_core::ExtensionCredentialStore, @@ -2351,7 +2526,7 @@ async fn load_policy_with_gateway( use openshell_core::proto::ConfigurationAdmissionState; // File mode: load OPA engine from rego rules + YAML data (dev override) if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - if sandbox_id.is_some() && openshell_endpoint.is_some() { + if sandbox_id.is_some() && openshell_gateway.is_some() { return Err(miette::miette!( "Local policy overrides cannot be combined with gateway-managed activation; replace the sandbox policy through the gateway" )); @@ -2417,12 +2592,11 @@ async fn load_policy_with_gateway( } // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(sandbox), Some(endpoint)) = (&sandbox_id, &sandbox, &openshell_endpoint) - { + if let (Some(id), Some(sandbox), Some(endpoint)) = (&sandbox_id, &sandbox, &openshell_gateway) { info!( sandbox_id = %id, sandbox = %sandbox, - endpoint = %endpoint, + endpoint = %endpoint.url(), "Fetching sandbox policy via gRPC" ); let instance_id = uuid::Uuid::new_v4().to_string(); @@ -3511,14 +3685,14 @@ trait PolicyGatewayClient: Clone + Send + Sync + 'static { /// Refresh credentials while preserving the activated delivery requirement. fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> impl Future> + Send { async move { openshell_core::grpc_client::fetch_provider_environment_with_stable_placeholders( - endpoint, + gateway, sandbox_id, requires_stable_placeholders, ) @@ -3706,7 +3880,7 @@ fn enqueue_policy_status(sender: &UnboundedSender, update: P /// and preserves the original construction error as the reported message. A /// delivery failure here is swallowed so it can never mask that error. async fn report_initial_policy_failure( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, revision: Option<&LoadedPolicyRevision>, error: &miette::Report, @@ -3717,7 +3891,7 @@ async fn report_initial_policy_failure( }) else { return; }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await { Ok(client) => client, Err(e) => { warn!(error = %e, "Failed to connect to report initial policy failure"); @@ -3749,7 +3923,7 @@ async fn report_initial_policy_failure( /// When the entrypoint PID is available, policy reloads include symlink /// resolution for binary paths via the container filesystem. struct PolicyPollLoopContext { - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, /// Canonical sandbox reference used by name-scoped configuration APIs. sandbox: String, @@ -4141,6 +4315,14 @@ fn emit_policy_validation_failure( )] mod tests { use super::*; + + pub fn test_gateway() -> openshell_core::grpc_client::GatewayEndpoint { + openshell_core::grpc_client::GatewayEndpoint::new( + "http://unused.invalid", + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .expect("test gateway credentials"), + ) + } use std::sync::atomic::{AtomicUsize, Ordering}; /// Admission must reject this input before any backend decodes its payload @@ -4165,29 +4347,44 @@ mod tests { ssh_host_private_key: None, ocsf_otlp_headers: None, }; - let error = run_sandbox( - vec!["true".to_string()], - None, - 0, - false, - false, - Some("sandbox-1".to_string()), - None, - None, - None, - None, - None, - None, - None, - Arc::new(AtomicBool::new(false)), - Arc::new(std::sync::Mutex::new(String::new())), - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), - None, - openshell_isolation_interface::contract::BackendDescriptor { + let config = SandboxInstanceConfig { + command: vec!["true".to_string()], + workdir: None, + timeout_secs: 0, + interactive: false, + await_main_process_attachment: false, + sandbox_id: Some("sandbox-1".to_string()), + sandbox: None, + container_image: String::new(), + openshell_endpoint: None, + policy_rules: None, + policy_data: None, + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { backend_name: "in-process-test".to_string(), payload: b"in-process-only-launch-data".to_vec(), }, auth_bundle, + admitted_isolation_backend: None, + main_exit_marker: None, + trace_context: None, + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + }; + let bindings = SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) + .expect("credentials"), + openshell_ocsf::OcsfJsonSettings::default(), + ); + let error = run_sandbox_with_backend( + &backend_setup::OpenShellBackendSetup, + config, + bindings, None, None, ) @@ -4745,7 +4942,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".into()), Some("sandbox".into()), - Some("http://unused.invalid".into()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -4884,7 +5081,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -4961,7 +5158,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5007,7 +5204,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5340,7 +5537,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5489,7 +5686,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5815,7 +6012,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5910,7 +6107,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 0442fd10a2..b222c1c196 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -413,11 +413,10 @@ fn main() -> Result<()> { let setup = backend_setup(&args, &admitted_backend)?; let descriptor = backend_descriptor(&args, &admitted_backend)?; let auth = auth_bundle(&args)?; - // Install the driver-provisioned session before starting log push or - // any other gateway client. `run_sandbox` obtains the same Sandbox - // Protocol bearer slot after validating the descriptor binding. - let _ = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth)?; - Some((setup, descriptor, auth, admitted_backend)) + // Log push and supervision share these credentials, so either one's + // renewal keeps the other's gateway token current. + let credentials = openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth)?; + Some((setup, descriptor, auth, admitted_backend, credentials)) } else { None }; @@ -443,7 +442,7 @@ fn main() -> Result<()> { }); let ocsf_headers = isolation_inputs .as_ref() - .and_then(|(_, _, auth, _)| auth.ocsf_otlp_headers.as_ref()) + .and_then(|(_, _, auth, _, _)| auth.ocsf_otlp_headers.as_ref()) .map(openshell_core::jwt::SecretOtlpHeaders::expose_secret); let (ocsf_export, ocsf_setup_error) = runtime.block_on(async { start_ocsf_export( @@ -457,11 +456,18 @@ fn main() -> Result<()> { let exit_code = runtime.block_on(async move { openshell_crypto::tls::ensure_default_provider(); - let log_push_state = if args.role == SupervisorRole::IsolationBackend - && let (Some(sandbox_id), Some(endpoint)) = (&args.sandbox_id, &args.openshell_endpoint) - { + let log_push_state = if let (Some(sandbox_id), Some(endpoint), Some(credentials)) = ( + &args.sandbox_id, + &args.openshell_endpoint, + isolation_inputs + .as_ref() + .map(|(_, _, _, _, credentials)| credentials), + ) { let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), + openshell_core::grpc_client::GatewayEndpoint::new( + endpoint.as_str(), + credentials.clone(), + ), sandbox_id.clone(), ); let layer = @@ -538,38 +544,62 @@ fn main() -> Result<()> { match args.role { SupervisorRole::IsolationBackend => { info!(command = ?command, "Starting sandbox supervision"); - let Some((backend_setup, backend_descriptor, auth_bundle, admitted_backend)) = - isolation_inputs + let Some(( + backend_setup, + backend_descriptor, + auth_bundle, + admitted_backend, + gateway_credentials, + )) = isolation_inputs else { return Err(miette::miette!( "isolation-backend role started without validated runtime inputs" )); }; + let config = openshell_supervisor::SandboxInstanceConfig { + command, + workdir, + timeout_secs: args.timeout, + interactive, + await_main_process_attachment, + sandbox_id: args.sandbox_id, + sandbox: args.sandbox, + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + openshell_endpoint: args.openshell_endpoint, + policy_rules: args.policy_rules, + policy_data: args.policy_data, + ssh_socket_path: args.ssh_socket_path, + shared_ssh_socket: openshell_supervisor::shared_ssh_socket_from_env(), + upstream_proxy_args, + additional_ca_bundle: args.additional_ca_bundle, + backend_descriptor, + auth_bundle, + admitted_isolation_backend: Some(admitted_backend), + main_exit_marker: args.main_exit_marker, + trace_context: openshell_supervisor::TraceContext::from_environment(), + provider_workload_api_socket: std::env::var_os( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ) + .filter(|path| !path.is_empty()) + .map(PathBuf::from), + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + }; + // Log push already shares these credentials and the logging + // layers these OCSF JSON settings. + let bindings = openshell_supervisor::SandboxInstanceBindings::dedicated( + &config, + gateway_credentials, + openshell_ocsf::OcsfJsonSettings { + enabled: ocsf_enabled, + target_version: ocsf_schema_version, + }, + ); let result = Box::pin(openshell_supervisor::run_sandbox_with_backend( backend_setup.as_ref(), - openshell_supervisor::SandboxRunConfig { - command, - workdir, - timeout_secs: args.timeout, - interactive, - await_main_process_attachment, - sandbox_id: args.sandbox_id, - sandbox: args.sandbox, - openshell_endpoint: args.openshell_endpoint, - policy_rules: args.policy_rules, - policy_data: args.policy_data, - ssh_socket_path: args.ssh_socket_path, - health_socket_path: args.health_socket_path, - health_port: args.health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle: args.additional_ca_bundle, - backend_descriptor, - auth_bundle, - admitted_isolation_backend: Some(admitted_backend), - main_exit_marker: args.main_exit_marker, - }, + config, + bindings, + args.health_socket_path, + args.health_port, )) .await; supervisor_exit(result) diff --git a/crates/openshell-supervisor/src/provider_readiness.rs b/crates/openshell-supervisor/src/provider_readiness.rs index af883dc265..347f7ca3e9 100644 --- a/crates/openshell-supervisor/src/provider_readiness.rs +++ b/crates/openshell-supervisor/src/provider_readiness.rs @@ -283,7 +283,7 @@ impl Tracker { /// Report while the owner holds this handle; reconnects use the accepted session. pub(crate) fn start_reporter( &self, - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, credentials: ProviderCredentialState, sessions: watch::Receiver>, @@ -297,7 +297,7 @@ impl Tracker { sessions, boundary, GatewayReporter { - endpoint, + gateway, sandbox_id, }, ) @@ -399,7 +399,7 @@ trait ReportClient: Send + Sync { } struct GatewayReporter { - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, } @@ -409,7 +409,7 @@ impl ReportClient for GatewayReporter { observation: ProviderReadinessObservation, ) -> miette::Result { openshell_core::grpc_client::report_provider_readiness( - &self.endpoint, + &self.gateway, &self.sandbox_id, observation, ) diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs new file mode 100644 index 0000000000..65a8b27534 --- /dev/null +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -0,0 +1,954 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Several sandbox instances in one supervisor process. +//! +//! A sandbox instance is one running generation of a stable Sandbox +//! resource. Gateway transport connections, and the gateway supervisor +//! sessions carried over them, may reconnect within one instance without +//! creating another. +//! +//! Each instance runs on its own Tokio runtime. Every thread that runtime +//! starts, and the thread driving it, is bound to the instance's +//! [`SandboxInstanceBindings`]: OCSF context, OCSF JSON settings, log sink, and +//! denial log, so tasks, callbacks, reconnects, and credential renewal that the +//! instance spawns inherit its identity. Unbound threads carry no sandbox +//! identity and push no logs. Gateway credentials are never bound to threads; +//! each instance passes its own explicitly. Stopping an instance shuts down +//! only its runtime and revokes its gateway credentials. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use miette::Result; +use openshell_core::grpc_client::GatewayCredentials; +use openshell_ocsf::OcsfJsonSettings; +use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; +use openshell_supervisor_process::log_push::LogSink; +use tokio::sync::watch; + +use crate::{SandboxInstanceBindings, SandboxInstanceControls}; + +/// Worker threads in each sandbox instance runtime. +const INSTANCE_WORKER_THREADS: usize = 2; +/// Bound on joining an instance thread after its runtime is told to stop. +const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Bind the calling thread to one instance's bindings. +fn bind_thread(bindings: &SandboxInstanceBindings) { + openshell_ocsf::ctx::bind_thread_ctx(Some(bindings.event_context.clone())); + openshell_ocsf::bind_thread_json_settings(Some(bindings.ocsf_json.clone())); + openshell_supervisor_process::log_push::bind_thread_sink(bindings.log_sink.clone()); + bind_thread_denial_log(match &bindings.denial_log { + DenialLogSource::SandboxInstance(log) => Some(log.clone()), + DenialLogSource::ShorthandFiles(_) => None, + }); +} + +fn unbind_thread() { + openshell_ocsf::ctx::bind_thread_ctx(None); + openshell_ocsf::bind_thread_json_settings(None); + openshell_supervisor_process::log_push::bind_thread_sink(None); + bind_thread_denial_log(None); +} + +std::thread_local! { + static THREAD_DENIAL_LOG: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +fn bind_thread_denial_log(log: Option) { + THREAD_DENIAL_LOG.with(|current| *current.borrow_mut() = log); +} + +/// Shorthand-log destination for hosted sandbox instances' recent denials. +/// +/// Each line goes to the denial log of the instance bound to the emitting +/// thread. Lines from unbound threads are dropped, so no instance can read +/// another's denials through `policy.local`. +#[derive(Debug, Default, Clone, Copy)] +pub struct SandboxInstanceDenialWriter; + +impl std::io::Write for SandboxInstanceDenialWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + THREAD_DENIAL_LOG.with(|log| { + if let Some(log) = log.borrow().as_ref() { + for line in String::from_utf8_lossy(bytes).lines() { + log.record(line); + } + } + }); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Observable lifecycle of one hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SandboxInstancePhase { + /// Startup has not yet produced an accepted gateway session. + Starting, + /// The workload started and the gateway accepted supervision at least once. + Supervising, + /// The owner asked the instance to stop. + Stopping, + /// Supervision ended with the workload's exit status. + Exited(i32), + /// Startup or supervision failed; the workload is no longer supervised. + Failed(String), +} + +impl SandboxInstancePhase { + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited(_) | Self::Failed(_)) + } +} + +/// Current state of one hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SandboxInstanceStatus { + pub phase: SandboxInstancePhase, + /// Gateway supervision is currently accepted for this instance. + pub ready: bool, +} + +/// Body of a hosted sandbox instance, constructed on its own runtime. +pub type SandboxInstanceBody = Box< + dyn FnOnce(SandboxInstanceControls) -> Pin>>> + + Send + + 'static, +>; + +/// The hosted sandbox instance registration was rejected. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum SandboxInstanceHostError { + #[error("a sandbox instance with this key is already hosted")] + Duplicate, + #[error("start sandbox instance thread: {0}")] + Thread(String), +} + +struct HostedSandboxInstance { + /// Distinguishes this instance from a later one that reuses its key. + serial: u64, + stop: watch::Sender, + abort: watch::Sender, + phase: watch::Receiver, + ready: watch::Receiver, + thread: Option>, + /// Final status of the removal in progress, shared by every remover. + removal: Option>>, +} + +impl HostedSandboxInstance { + fn status(&self) -> SandboxInstanceStatus { + let phase = self.phase.borrow().clone(); + // The runner always reports a terminal phase, even while unwinding; + // a closed channel without one still means supervision has ended. + if !phase.is_terminal() && self.phase.has_changed().is_err() { + return SandboxInstanceStatus { + phase: SandboxInstancePhase::Failed( + "sandbox instance thread ended unexpectedly".to_string(), + ), + ready: false, + }; + } + SandboxInstanceStatus { + phase, + ready: *self.ready.borrow(), + } + } +} + +type Instances = Arc>>; + +fn lock(instances: &Instances) -> MutexGuard<'_, HashMap> { + instances + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Runs and stops sandbox instances, each on its own bound runtime. +#[derive(Default)] +pub struct SandboxInstanceHost { + instances: Instances, + next_serial: AtomicU64, +} + +impl SandboxInstanceHost { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Start `body` on a new runtime bound to `bindings`. + /// + /// The bindings' gateway credentials are revoked once the instance's + /// runtime has stopped, so nothing can reuse them after the instance ends. + /// + /// # Errors + /// + /// Returns [`SandboxInstanceHostError::Duplicate`] when `key` is already hosted, + /// including an instance that ended but has not been removed. + pub fn start( + &self, + key: &str, + bindings: SandboxInstanceBindings, + body: SandboxInstanceBody, + ) -> std::result::Result<(), SandboxInstanceHostError> { + let mut instances = lock(&self.instances); + if instances.contains_key(key) { + return Err(SandboxInstanceHostError::Duplicate); + } + let (stop, stop_requested) = watch::channel(false); + let (abort, abort_requested) = watch::channel(false); + let (phase_tx, phase) = watch::channel(SandboxInstancePhase::Starting); + let (ready_tx, ready) = watch::channel(false); + let thread = std::thread::Builder::new() + .name("openshell-instance".to_string()) + .spawn(move || { + run_hosted_sandbox_instance( + &bindings, + body, + SandboxInstanceSignals { + stop_requested, + abort_requested, + phase: phase_tx, + ready: ready_tx, + }, + ); + }) + .map_err(|error| SandboxInstanceHostError::Thread(error.to_string()))?; + instances.insert( + key.to_string(), + HostedSandboxInstance { + serial: self.next_serial.fetch_add(1, Ordering::Relaxed), + stop, + abort, + phase, + ready, + thread: Some(thread), + removal: None, + }, + ); + Ok(()) + } + + /// Status of a hosted sandbox instance, including one that ended but has + /// not been removed. + #[must_use] + pub fn status(&self, key: &str) -> Option { + lock(&self.instances) + .get(key) + .map(HostedSandboxInstance::status) + } + + /// Keys of every hosted sandbox instance. + #[must_use] + pub fn keys(&self) -> Vec { + lock(&self.instances).keys().cloned().collect() + } + + /// Stop one sandbox instance and forget it. Other instances keep running. + /// + /// A supervising instance terminates its workload gracefully within + /// `grace`; one still starting is cancelled at once, because startup has + /// no safe intermediate state to wind down. Removal runs on its own task, + /// so it completes even when the caller stops waiting, and concurrent + /// callers share it and its result. The key stays registered until the + /// instance's thread has finished, so the same key cannot start a second + /// owner while this one is stopping. Returns the final status, or `None` + /// when `key` is not hosted. + pub async fn remove(&self, key: &str, grace: Duration) -> Option { + let (mut removed, phase) = self.begin_removal(key, grace)?; + // The removal task ends without a report only with its runtime. + removed + .wait_for(Option::is_some) + .await + .map_or_else(|_| Some(terminal_status(&phase)), |status| status.clone()) + } + + fn begin_removal( + &self, + key: &str, + grace: Duration, + ) -> Option<( + watch::Receiver>, + watch::Receiver, + )> { + let mut instances = lock(&self.instances); + let instance = instances.get_mut(key)?; + if let Some(removed) = &instance.removal { + return Some((removed.clone(), instance.phase.clone())); + } + instance.stop.send_replace(true); + if *instance.phase.borrow() == SandboxInstancePhase::Starting { + instance.abort.send_replace(true); + } + let (report, removed) = watch::channel(None); + instance.removal = Some(removed.clone()); + tokio::spawn( + Removal { + instances: self.instances.clone(), + key: key.to_string(), + serial: instance.serial, + phase: instance.phase.clone(), + thread: instance.thread.take(), + grace, + report, + } + .run(), + ); + Some((removed, instance.phase.clone())) + } +} + +fn terminal_status(phase: &watch::Receiver) -> SandboxInstanceStatus { + SandboxInstanceStatus { + phase: phase.borrow().clone(), + ready: false, + } +} + +/// The single owner of one sandbox instance's removal. +struct Removal { + instances: Instances, + key: String, + serial: u64, + phase: watch::Receiver, + thread: Option>, + grace: Duration, + report: watch::Sender>, +} + +impl Removal { + async fn run(mut self) { + let finished = tokio::time::timeout( + self.grace, + self.phase.wait_for(SandboxInstancePhase::is_terminal), + ) + .await + .is_ok(); + if !finished + && let Some(instance) = lock(&self.instances) + .get(&self.key) + .filter(|instance| instance.serial == self.serial) + { + instance.abort.send_replace(true); + } + if let Some(thread) = self.thread.take() { + let _ = tokio::task::spawn_blocking(move || thread.join()).await; + } + let removed = { + let mut instances = lock(&self.instances); + match instances.get(&self.key) { + Some(instance) if instance.serial == self.serial => instances.remove(&self.key), + _ => None, + } + }; + let status = removed.map_or_else( + || terminal_status(&self.phase), + |instance| instance.status(), + ); + self.report.send_replace(Some(status)); + } +} + +struct SandboxInstanceSignals { + stop_requested: watch::Receiver, + abort_requested: watch::Receiver, + phase: watch::Sender, + ready: watch::Sender, +} + +/// Reports a hosted sandbox instance's end and revokes its credentials, +/// including when the instance thread unwinds. +struct SandboxInstanceExit { + credentials: GatewayCredentials, + phase: watch::Sender, + ready: watch::Sender, + outcome: Option, +} + +impl Drop for SandboxInstanceExit { + fn drop(&mut self) { + self.credentials.revoke(); + self.ready.send_replace(false); + let outcome = self.outcome.take().unwrap_or_else(|| { + SandboxInstancePhase::Failed( + "sandbox instance thread ended without reporting a result".to_string(), + ) + }); + self.phase.send_replace(outcome); + unbind_thread(); + } +} + +fn run_hosted_sandbox_instance( + bindings: &SandboxInstanceBindings, + body: SandboxInstanceBody, + signals: SandboxInstanceSignals, +) { + let SandboxInstanceSignals { + stop_requested, + mut abort_requested, + phase, + ready, + } = signals; + let mut exit = SandboxInstanceExit { + credentials: bindings.gateway_credentials.clone(), + phase: phase.clone(), + ready: ready.clone(), + outcome: None, + }; + bind_thread(bindings); + let thread_bindings = bindings.clone(); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(INSTANCE_WORKER_THREADS) + .thread_name("openshell-instance-worker") + .enable_all() + .on_thread_start(move || bind_thread(&thread_bindings)) + .on_thread_stop(unbind_thread) + .build(); + let outcome = match runtime { + Ok(runtime) => { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runtime.block_on(async { + let (readiness, mut accepted) = watch::channel(false); + let mirror_phase = phase.clone(); + let mirror_ready = ready.clone(); + let mirror = tokio::spawn(async move { + loop { + let is_ready = *accepted.borrow_and_update(); + mirror_ready.send_replace(is_ready); + if is_ready { + mirror_phase.send_if_modified(|current| { + let promote = *current == SandboxInstancePhase::Starting; + if promote { + *current = SandboxInstancePhase::Supervising; + } + promote + }); + } + if accepted.changed().await.is_err() { + mirror_ready.send_replace(false); + return; + } + } + }); + let mut stop = stop_requested.clone(); + let stop_phase = phase.clone(); + let shutdown = Box::pin(async move { + let _ = stop.wait_for(|requested| *requested).await; + stop_phase.send_replace(SandboxInstancePhase::Stopping); + }); + let instance = body(SandboxInstanceControls { + shutdown, + readiness, + }); + let outcome = tokio::select! { + result = instance => result, + _ = abort_requested.wait_for(|requested| *requested) => { + Err(miette::miette!("sandbox instance was cancelled before it finished")) + } + }; + mirror.abort(); + outcome + }) + })); + runtime.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); + outcome.unwrap_or_else(|_| Err(miette::miette!("sandbox instance panicked"))) + } + Err(error) => Err(miette::miette!("build sandbox instance runtime: {error}")), + }; + exit.outcome = Some(match outcome { + Ok(code) => SandboxInstancePhase::Exited(code), + Err(error) => SandboxInstancePhase::Failed(error.to_string()), + }); +} + +/// Trusted inputs for one sandbox instance hosted with others. +pub struct SandboxInstanceSpec { + /// Unique key for this generation within the host. + pub key: String, + pub backend_setup: Box, + pub config: crate::SandboxInstanceConfig, +} + +impl SandboxInstanceHost { + /// Start a sandbox instance with its own OCSF identity, OCSF JSON + /// settings, log push stream, denial log, and gateway credentials. + /// + /// # Errors + /// + /// Fails when the launch credentials are invalid or the key is hosted. + pub fn start_sandbox_instance(&self, spec: SandboxInstanceSpec) -> Result<()> { + let SandboxInstanceSpec { + key, + backend_setup, + mut config, + } = spec; + // Instances must never share proxy trust files; give one a private + // directory that lives exactly as long as the instance. + let private_tls_dir = if config.proxy_tls.directory.is_none() { + let prepared = crate::prepare_network_proxy_tls_dir(None)?; + config.proxy_tls.directory = Some(prepared.path.clone()); + Some(prepared) + } else { + None + }; + // Log push, supervision, and renewal share these credentials. + let gateway_credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; + let log_push = config + .sandbox_id + .clone() + .zip(config.openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new( + endpoint, + gateway_credentials.clone(), + ) + })); + let (log_tx, log_rx) = tokio::sync::mpsc::channel( + openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, + ); + let bindings = SandboxInstanceBindings { + gateway_credentials, + event_context: Arc::new(config.event_context()), + ocsf_json: OcsfJsonSettings::default(), + log_sink: log_push + .as_ref() + .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), + }; + let instance_bindings = bindings.clone(); + let body: SandboxInstanceBody = Box::new(move |controls| { + Box::pin(async move { + let _private_tls_dir = private_tls_dir; + let _log_push = log_push.map(|(sandbox_id, gateway)| { + openshell_supervisor_process::log_push::start_log_push( + gateway, sandbox_id, log_rx, + ) + }); + Box::pin(crate::run_sandbox_instance( + backend_setup.as_ref(), + config, + instance_bindings, + controls, + )) + .await + }) + }); + self.start(&key, bindings, body) + .map_err(|error| miette::miette!("{error}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use http_body_util::BodyExt as _; + use openshell_core::grpc_client::GatewayEndpoint; + use openshell_core::proto::SandboxLogLine; + use tokio::sync::mpsc; + use tracing_subscriber::layer::SubscriberExt as _; + + type Captured = Arc)>>>; + + /// Records each gRPC request's bearer and body, then answers `UNIMPLEMENTED`. + async fn capture_gateway() -> (String, Captured) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let captured = Captured::default(); + let sink = captured.clone(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let sink = sink.clone(); + tokio::spawn(async move { + let service = hyper::service::service_fn( + move |request: http::Request| { + let sink = sink.clone(); + async move { + let authorization = request + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let body = request + .into_body() + .collect() + .await + .map(|body| body.to_bytes().to_vec()) + .unwrap_or_default(); + sink.lock().unwrap().push((authorization, body)); + http::Response::builder() + .header("content-type", "application/grpc") + .header("grpc-status", "12") + .body(http_body_util::Empty::::new()) + } + }, + ); + let _ = hyper::server::conn::http2::Builder::new( + hyper_util::rt::TokioExecutor::new(), + ) + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + }); + } + }); + (format!("http://{address}"), captured) + } + + fn bearers_for(captured: &Captured, sandbox: &str) -> Vec { + captured + .lock() + .unwrap() + .iter() + .filter(|(_, body)| { + body.windows(sandbox.len()) + .any(|bytes| bytes == sandbox.as_bytes()) + }) + .map(|(authorization, _)| authorization.clone()) + .collect() + } + + struct Instance { + identity: tokio::sync::oneshot::Receiver, + logs: mpsc::Receiver, + call_again: mpsc::Sender<()>, + alive: mpsc::Receiver<()>, + credentials: GatewayCredentials, + } + + fn start_instance( + host: &SandboxInstanceHost, + gateway_url: &str, + sandbox: &'static str, + ) -> Instance { + let credentials = + GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); + let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); + let (log_tx, logs) = mpsc::channel(16); + let bindings = SandboxInstanceBindings { + gateway_credentials: credentials.clone(), + event_context: Arc::new(crate::sandbox_instance_event_context( + Some(sandbox), + Some(sandbox), + String::new(), + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: Some(LogSink::new(sandbox, log_tx)), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), + }; + let (identity_tx, identity) = tokio::sync::oneshot::channel(); + let (call_again, mut calls) = mpsc::channel(1); + let (alive_tx, alive) = mpsc::channel::<()>(1); + let body: SandboxInstanceBody = Box::new(move |controls| { + Box::pin(async move { + let SandboxInstanceControls { + shutdown, + readiness, + } = controls; + // Nested tasks and the blocking pool inherit the instance identity. + let observed = tokio::spawn(async { + tokio::task::spawn_blocking(|| openshell_ocsf::ctx::ctx().sandbox_id.clone()) + .await + .unwrap() + }) + .await + .unwrap(); + let _ = identity_tx.send(observed); + let _ = + openshell_core::grpc_client::fetch_settings_snapshot(&gateway, sandbox).await; + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(openshell_supervisor_process::log_push::LogPushLayer::thread_bound()), + ); + tokio::spawn(async move { + tracing::dispatcher::with_default(&dispatch, || { + tracing::info!(target: "instance_test", "emitted by {sandbox}"); + }); + }) + .await + .unwrap(); + // A long-lived task the instance never stops itself. + tokio::spawn(async move { + let _alive = alive_tx; + std::future::pending::<()>().await; + }); + readiness.send_replace(true); + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => return Ok(0), + Some(()) = calls.recv() => { + let _ = openshell_core::grpc_client::fetch_settings_snapshot( + &gateway, sandbox, + ) + .await; + } + } + } + }) + }); + host.start(sandbox, bindings, body).unwrap(); + Instance { + identity, + logs, + call_again, + alive, + credentials, + } + } + + async fn wait_supervising(host: &SandboxInstanceHost, key: &str) { + tokio::time::timeout(Duration::from_secs(10), async { + while host.status(key).map(|status| status.phase) + != Some(SandboxInstancePhase::Supervising) + { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("instance reaches supervision"); + } + + fn unbound() -> SandboxInstanceBindings { + SandboxInstanceBindings { + gateway_credentials: GatewayCredentials::from_gateway_token("unbound-token").unwrap(), + event_context: Arc::new(crate::sandbox_instance_event_context( + None, + None, + String::new(), + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: None, + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn stopping_and_panicking_instances_keep_their_key_and_fail_closed() { + let host = Arc::new(SandboxInstanceHost::new()); + let body: SandboxInstanceBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + // Winding down takes time; the key must stay taken meanwhile. + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(0) + }) + }); + host.start("generation", unbound(), body).unwrap(); + wait_supervising(&host, "generation").await; + let removal = tokio::spawn({ + let host = host.clone(); + async move { host.remove("generation", Duration::from_secs(10)).await } + }); + tokio::time::timeout(Duration::from_secs(10), async { + while host.status("generation").map(|status| status.phase) + != Some(SandboxInstancePhase::Stopping) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("instance observes the stop request"); + let replacement: SandboxInstanceBody = Box::new(|_| Box::pin(async { Ok(0) })); + assert_eq!( + host.start("generation", unbound(), replacement), + Err(SandboxInstanceHostError::Duplicate) + ); + let stopped = removal.await.unwrap().expect("instance was hosted"); + assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); + assert!(host.status("generation").is_none()); + + let credentials = GatewayCredentials::from_gateway_token("panicking-token").unwrap(); + let panicking: SandboxInstanceBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + tokio::task::yield_now().await; + panic!("hosted sandbox instance failure"); + }) + }); + host.start( + "panicking", + SandboxInstanceBindings { + gateway_credentials: credentials.clone(), + ..unbound() + }, + panicking, + ) + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(status) = host.status("panicking") + && status.phase.is_terminal() + { + return status; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("panicking instance reports an end"); + assert_eq!( + status, + SandboxInstanceStatus { + phase: SandboxInstancePhase::Failed("sandbox instance panicked".to_string()), + ready: false, + } + ); + let revoked = GatewayEndpoint::new("http://127.0.0.1:9", credentials); + let error = openshell_core::grpc_client::fetch_settings_snapshot(&revoked, "panicking") + .await + .unwrap_err(); + assert!(error.to_string().contains("revoked"), "{error}"); + assert!( + host.remove("panicking", Duration::from_secs(1)) + .await + .is_some() + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn removal_outlives_its_caller_and_reports_to_every_remover() { + let host = Arc::new(SandboxInstanceHost::new()); + let winding_down = |code| -> SandboxInstanceBody { + Box::new(move |controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(code) + }) + }) + }; + host.start("generation", unbound(), winding_down(3)) + .unwrap(); + wait_supervising(&host, "generation").await; + // A caller that gives up, such as an expired RPC, strands nothing. + assert!( + tokio::time::timeout( + Duration::from_millis(20), + host.remove("generation", Duration::from_secs(10)) + ) + .await + .is_err() + ); + let removers = [(); 2].map(|()| { + let host = host.clone(); + tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) + }); + for remover in removers { + let status = remover.await.unwrap().expect("instance was hosted"); + assert_eq!(status.phase, SandboxInstancePhase::Exited(3)); + } + assert!(host.status("generation").is_none()); + + // The key is free again, and the earlier removal cannot touch its new owner. + host.start("generation", unbound(), winding_down(4)) + .unwrap(); + wait_supervising(&host, "generation").await; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + host.status("generation").map(|status| status.phase), + Some(SandboxInstancePhase::Supervising) + ); + let status = host.remove("generation", Duration::from_secs(10)).await; + assert_eq!( + status.map(|status| status.phase), + Some(SandboxInstancePhase::Exited(4)) + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn hosted_instances_keep_identity_credentials_and_lifecycle_separate() { + let (gateway_url, captured) = capture_gateway().await; + let host = SandboxInstanceHost::new(); + let mut alpha = start_instance(&host, &gateway_url, "sandbox-alpha"); + let mut beta = start_instance(&host, &gateway_url, "sandbox-beta"); + wait_supervising(&host, "sandbox-alpha").await; + wait_supervising(&host, "sandbox-beta").await; + assert!(matches!( + host.start( + "sandbox-alpha", + unbound(), + Box::new(|_| Box::pin(async { Ok(0) })), + ), + Err(SandboxInstanceHostError::Duplicate) + )); + + assert_eq!((&mut alpha.identity).await.unwrap(), "sandbox-alpha"); + assert_eq!((&mut beta.identity).await.unwrap(), "sandbox-beta"); + // Other tests in this process may set the process-wide context; a + // thread outside every instance must still carry no instance identity. + let outside = openshell_ocsf::ctx::ctx().sandbox_id.clone(); + assert!(!["sandbox-alpha", "sandbox-beta"].contains(&outside.as_str())); + assert_eq!( + bearers_for(&captured, "sandbox-alpha"), + ["Bearer token-sandbox-alpha"] + ); + assert_eq!( + bearers_for(&captured, "sandbox-beta"), + ["Bearer token-sandbox-beta"] + ); + for (instance, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { + let line = instance.logs.try_recv().expect("instance log line"); + assert_eq!(line.sandbox_id, sandbox); + assert_eq!(line.message, format!("emitted by {sandbox}")); + assert!(instance.logs.try_recv().is_err()); + } + + let stopped = host + .remove("sandbox-alpha", Duration::from_secs(10)) + .await + .expect("alpha was hosted"); + assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); + assert!(!stopped.ready); + assert!(host.status("sandbox-alpha").is_none()); + assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); + let revoked = GatewayEndpoint::new(gateway_url.as_str(), alpha.credentials.clone()); + let error = openshell_core::grpc_client::fetch_settings_snapshot(&revoked, "sandbox-alpha") + .await + .unwrap_err(); + assert!(error.to_string().contains("revoked"), "{error}"); + + beta.call_again.send(()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while bearers_for(&captured, "sandbox-beta").len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("beta keeps calling the gateway"); + assert_eq!( + bearers_for(&captured, "sandbox-beta"), + ["Bearer token-sandbox-beta", "Bearer token-sandbox-beta"] + ); + assert_eq!( + host.status("sandbox-beta").map(|status| status.phase), + Some(SandboxInstancePhase::Supervising) + ); + assert!( + beta.alive + .try_recv() + .is_err_and(|error| error == mpsc::error::TryRecvError::Empty) + ); + + let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; + assert_eq!( + stopped.map(|status| status.phase), + Some(SandboxInstancePhase::Exited(0)) + ); + } +}