From 76d5b4a0569da2278b3672c68e31b5cf9dab9b62 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 17:00:24 -0700 Subject: [PATCH 1/6] refactor(ocsf): attribute events to session-bound contexts A supervisor process that hosts several sandboxes cannot use one process-wide OCSF context: every event would carry the wrong sandbox identity, and middleware requests read their sandbox identity from the same context. Builders now accept a ContextRef, either a borrowed context or a context shared by a hosted session, so per-session contexts need not be leaked. ctx() returns the context bound to the calling thread when present and falls back to the process-wide context otherwise. OCSF JSON output settings can likewise be bound per thread, and OpenTelemetry can parent a span from an explicit W3C trace context instead of process environment. A dedicated supervisor keeps using the process-wide context. Signed-off-by: Drew Newberry --- .../src/builders/api_activity.rs | 8 ++-- .../src/builders/authentication.rs | 8 ++-- crates/openshell-ocsf/src/builders/base.rs | 8 ++-- crates/openshell-ocsf/src/builders/config.rs | 8 ++-- crates/openshell-ocsf/src/builders/finding.rs | 8 ++-- crates/openshell-ocsf/src/builders/http.rs | 8 ++-- .../openshell-ocsf/src/builders/lifecycle.rs | 8 ++-- crates/openshell-ocsf/src/builders/mod.rs | 40 ++++++++++++++++++ crates/openshell-ocsf/src/builders/network.rs | 8 ++-- crates/openshell-ocsf/src/builders/process.rs | 8 ++-- crates/openshell-ocsf/src/builders/ssh.rs | 8 ++-- crates/openshell-ocsf/src/ctx.rs | 42 +++++++++++++------ crates/openshell-ocsf/src/lib.rs | 7 ++-- .../src/tracing_layers/jsonl_layer.rs | 32 +++++++++++++- .../openshell-ocsf/src/tracing_layers/mod.rs | 2 +- crates/openshell-otel/src/lib.rs | 2 +- crates/openshell-otel/src/propagation.rs | 32 +++++++++----- crates/openshell-sandbox/src/delegated.rs | 2 +- .../src/l7/middleware.rs | 4 +- .../src/l7/relay.rs | 4 +- .../src/delegated.rs | 2 +- .../src/supervisor_session.rs | 15 +++---- 22 files changed, 181 insertions(+), 83 deletions(-) diff --git a/crates/openshell-ocsf/src/builders/api_activity.rs b/crates/openshell-ocsf/src/builders/api_activity.rs index fe88434804..4740b05053 100644 --- a/crates/openshell-ocsf/src/builders/api_activity.rs +++ b/crates/openshell-ocsf/src/builders/api_activity.rs @@ -3,7 +3,7 @@ //! Builder for API Activity [6003] events. -use crate::builders::{EventContext, EventOrigin}; +use crate::builders::{ContextRef, EventOrigin}; use crate::enums::{ActionId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{ApiActivityEvent, OcsfEvent}; @@ -28,7 +28,7 @@ fn sanitize_model_name(name: &str) -> String { /// Builder for API Activity [6003] events. pub struct ApiActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, message: Option, @@ -49,9 +49,9 @@ pub struct ApiActivityBuilder<'a> { impl<'a> ApiActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext, operation: impl Into) -> Self { + pub fn new(ctx: impl Into>, operation: impl Into) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, message: None, diff --git a/crates/openshell-ocsf/src/builders/authentication.rs b/crates/openshell-ocsf/src/builders/authentication.rs index a12cf2a5d0..492bdf65b7 100644 --- a/crates/openshell-ocsf/src/builders/authentication.rs +++ b/crates/openshell-ocsf/src/builders/authentication.rs @@ -3,7 +3,7 @@ //! Builder for Authentication [3002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, AuthProtocolId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{AuthenticationEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Endpoint, HttpRequest, Service, User}; /// Builder for Authentication [3002] Logon events. pub struct AuthenticationBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, user: User, service: Service, auth_protocol: Option, @@ -29,9 +29,9 @@ pub struct AuthenticationBuilder<'a> { impl<'a> AuthenticationBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext, user: User, service: Service) -> Self { + pub fn new(ctx: impl Into>, user: User, service: Service) -> Self { Self { - ctx, + ctx: ctx.into(), user, service, auth_protocol: None, diff --git a/crates/openshell-ocsf/src/builders/base.rs b/crates/openshell-ocsf/src/builders/base.rs index c2edb5b169..c08f1aa0aa 100644 --- a/crates/openshell-ocsf/src/builders/base.rs +++ b/crates/openshell-ocsf/src/builders/base.rs @@ -3,14 +3,14 @@ //! Builder for Base Event [0]. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{BaseEvent, OcsfEvent}; /// Builder for Base Event [0] — events without a specific OCSF class. pub struct BaseEventBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, message: Option, @@ -20,9 +20,9 @@ pub struct BaseEventBuilder<'a> { impl<'a> BaseEventBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, message: None, diff --git a/crates/openshell-ocsf/src/builders/config.rs b/crates/openshell-ocsf/src/builders/config.rs index 1bc415b612..906470408d 100644 --- a/crates/openshell-ocsf/src/builders/config.rs +++ b/crates/openshell-ocsf/src/builders/config.rs @@ -3,14 +3,14 @@ //! Builder for Device Config State Change [5019] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{SecurityLevelId, SeverityId, StateId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{DeviceConfigStateChangeEvent, OcsfEvent}; /// Builder for Device Config State Change [5019] events. pub struct ConfigStateChangeBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, state_id: Option, @@ -23,9 +23,9 @@ pub struct ConfigStateChangeBuilder<'a> { impl<'a> ConfigStateChangeBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, state_id: None, diff --git a/crates/openshell-ocsf/src/builders/finding.rs b/crates/openshell-ocsf/src/builders/finding.rs index aea063a527..ccb45d29a8 100644 --- a/crates/openshell-ocsf/src/builders/finding.rs +++ b/crates/openshell-ocsf/src/builders/finding.rs @@ -3,7 +3,7 @@ //! Builder for Detection Finding [2004] events. -use crate::builders::{EventContext, EventOrigin}; +use crate::builders::{ContextRef, EventOrigin}; use crate::enums::{ActionId, ActivityId, ConfidenceId, DispositionId, RiskLevelId, SeverityId}; use crate::events::base_event::BaseEventData; use crate::events::{DetectionFindingEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Attack, Evidence, FindingInfo, Remediation}; /// Builder for Detection Finding [2004] events. pub struct DetectionFindingBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, action: Option, @@ -31,9 +31,9 @@ pub struct DetectionFindingBuilder<'a> { impl<'a> DetectionFindingBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Open, severity: SeverityId::Medium, action: None, diff --git a/crates/openshell-ocsf/src/builders/http.rs b/crates/openshell-ocsf/src/builders/http.rs index b027421bd0..86a22f63ac 100644 --- a/crates/openshell-ocsf/src/builders/http.rs +++ b/crates/openshell-ocsf/src/builders/http.rs @@ -3,7 +3,7 @@ //! Builder for HTTP Activity [4002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{HttpActivityEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Endpoint, FirewallRule, HttpRequest, HttpResponse}; /// Builder for HTTP Activity [4002] events. pub struct HttpActivityBuilder<'a, Context = MissingHttpContext> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, action: Option, disposition: Option, @@ -56,9 +56,9 @@ impl<'a> HttpActivityBuilder<'a, MissingHttpContext> { /// HttpActivityBuilder::new(&ctx).build(); /// ``` #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, action: None, disposition: None, diff --git a/crates/openshell-ocsf/src/builders/lifecycle.rs b/crates/openshell-ocsf/src/builders/lifecycle.rs index 8fc648ee37..b425b3e79b 100644 --- a/crates/openshell-ocsf/src/builders/lifecycle.rs +++ b/crates/openshell-ocsf/src/builders/lifecycle.rs @@ -3,7 +3,7 @@ //! Builder for Application Lifecycle [6002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActivityId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{ApplicationLifecycleEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::Product; /// Builder for Application Lifecycle [6002] events. pub struct AppLifecycleBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, status: Option, @@ -20,9 +20,9 @@ pub struct AppLifecycleBuilder<'a> { impl<'a> AppLifecycleBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, diff --git a/crates/openshell-ocsf/src/builders/mod.rs b/crates/openshell-ocsf/src/builders/mod.rs index 96c966b732..5c2219cc6d 100644 --- a/crates/openshell-ocsf/src/builders/mod.rs +++ b/crates/openshell-ocsf/src/builders/mod.rs @@ -157,6 +157,7 @@ pub use process::ProcessActivityBuilder; pub use ssh::SshActivityBuilder; use std::net::IpAddr; +use std::sync::Arc; use crate::OCSF_VERSION; use crate::enums::StatusId; @@ -199,6 +200,45 @@ pub struct EventContext { pub origin: EventOrigin, } +/// Event context borrowed from the caller or shared by a hosted session. +/// +/// Builders accept either form, so events attributed to a hosted supervisor +/// session need not leak a `'static` context for each session. +#[derive(Debug, Clone)] +pub enum ContextRef<'a> { + Borrowed(&'a EventContext), + Shared(Arc), +} + +impl std::ops::Deref for ContextRef<'_> { + type Target = EventContext; + + fn deref(&self) -> &EventContext { + match self { + Self::Borrowed(context) => context, + Self::Shared(context) => context, + } + } +} + +impl<'a> From<&'a EventContext> for ContextRef<'a> { + fn from(context: &'a EventContext) -> Self { + Self::Borrowed(context) + } +} + +impl From> for ContextRef<'_> { + fn from(context: Arc) -> Self { + Self::Shared(context) + } +} + +impl<'a> From<&'a ContextRef<'_>> for ContextRef<'a> { + fn from(context: &'a ContextRef<'_>) -> Self { + Self::Borrowed(context) + } +} + impl EventContext { /// Build the OCSF `Metadata` object for any event. #[must_use] diff --git a/crates/openshell-ocsf/src/builders/network.rs b/crates/openshell-ocsf/src/builders/network.rs index 09f4c4ff0d..7907abe4a6 100644 --- a/crates/openshell-ocsf/src/builders/network.rs +++ b/crates/openshell-ocsf/src/builders/network.rs @@ -3,7 +3,7 @@ //! Builder for Network Activity [4001] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{NetworkActivityEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, ConnectionInfo, Endpoint, FirewallRule}; /// Builder for Network Activity [4001] events. pub struct NetworkActivityBuilder<'a, EndpointState = MissingNetworkEndpoint> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, activity_name: Option, action: Option, @@ -58,9 +58,9 @@ impl<'a> NetworkActivityBuilder<'a, MissingNetworkEndpoint> { /// NetworkActivityBuilder::new(&ctx).build(); /// ``` #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, activity_name: None, action: None, diff --git a/crates/openshell-ocsf/src/builders/process.rs b/crates/openshell-ocsf/src/builders/process.rs index 6e06e6c6e7..5acdb75f01 100644 --- a/crates/openshell-ocsf/src/builders/process.rs +++ b/crates/openshell-ocsf/src/builders/process.rs @@ -3,7 +3,7 @@ //! Builder for Process Activity [1007] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, LaunchTypeId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{OcsfEvent, ProcessActivityEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Process}; /// Builder for Process Activity [1007] events. pub struct ProcessActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, status: Option, @@ -26,9 +26,9 @@ pub struct ProcessActivityBuilder<'a> { impl<'a> ProcessActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, diff --git a/crates/openshell-ocsf/src/builders/ssh.rs b/crates/openshell-ocsf/src/builders/ssh.rs index 36383ae9f4..2777bbd05b 100644 --- a/crates/openshell-ocsf/src/builders/ssh.rs +++ b/crates/openshell-ocsf/src/builders/ssh.rs @@ -3,7 +3,7 @@ //! Builder for SSH Activity [4007] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, AuthTypeId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{OcsfEvent, SshActivityEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Endpoint}; /// Builder for SSH Activity [4007] events. pub struct SshActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, action: Option, disposition: Option, @@ -28,9 +28,9 @@ pub struct SshActivityBuilder<'a> { impl<'a> SshActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, action: None, disposition: None, diff --git a/crates/openshell-ocsf/src/ctx.rs b/crates/openshell-ocsf/src/ctx.rs index 7e1c4c7796..9dd542a9aa 100644 --- a/crates/openshell-ocsf/src/ctx.rs +++ b/crates/openshell-ocsf/src/ctx.rs @@ -1,15 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process-wide [`EventContext`] singleton. +//! Process-wide and session-bound [`EventContext`]. //! -//! Initialised once via [`set_ctx`] during sandbox start; read by every event -//! builder via [`ctx`]. Falls back to a default context when the singleton has -//! not been set (e.g. unit tests that exercise builders without booting the -//! sandbox). +//! A dedicated supervisor initialises the process-wide context once via +//! [`set_ctx`]. A process hosting several supervisor sessions instead binds +//! each session's context to the threads of that session's runtime with +//! [`bind_thread_ctx`]; a bound context takes precedence. [`ctx`] falls back +//! to a default context when neither is set (e.g. unit tests that exercise +//! builders without booting the sandbox). -use crate::{EventContext, EventOrigin}; -use std::sync::{LazyLock, OnceLock}; +use crate::{ContextRef, EventContext, EventOrigin}; +use std::cell::RefCell; +use std::sync::{Arc, LazyLock, OnceLock}; static OCSF_CTX: OnceLock = OnceLock::new(); @@ -24,6 +27,11 @@ static OCSF_CTX_FALLBACK: LazyLock = LazyLock::new(|| EventContext origin: EventOrigin::Supervisor, }); +std::thread_local! { + /// Context of the hosted session whose runtime owns this thread. + static THREAD_CTX: RefCell>> = const { RefCell::new(None) }; +} + /// Initialise the process-wide OCSF sandbox context. /// /// Returns `false` if the context was already set; the caller may log and @@ -32,11 +40,21 @@ pub fn set_ctx(ctx: EventContext) -> bool { OCSF_CTX.set(ctx).is_ok() } -/// Return a reference to the process-wide [`EventContext`]. +/// Attribute events emitted on the calling thread to one hosted session. +/// +/// Session runtimes bind every thread they start; `None` unbinds it. +pub fn bind_thread_ctx(ctx: Option>) { + THREAD_CTX.with(|current| *current.borrow_mut() = ctx); +} + +/// Return the context for events emitted on the calling thread. /// -/// Falls back to a default context if [`set_ctx`] has not been called (e.g. -/// during unit tests that exercise individual builders). +/// A session-bound context wins over the process-wide one. Falls back to a +/// default context if neither is set. #[must_use] -pub fn ctx() -> &'static EventContext { - OCSF_CTX.get().unwrap_or(&OCSF_CTX_FALLBACK) +pub fn ctx() -> ContextRef<'static> { + if let Some(bound) = THREAD_CTX.with(|current| current.borrow().clone()) { + return ContextRef::Shared(bound); + } + ContextRef::Borrowed(OCSF_CTX.get().unwrap_or(&OCSF_CTX_FALLBACK)) } diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index b9878eb68a..310cd9afe4 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -68,12 +68,13 @@ pub use format::downgrade::OcsfSchemaVersion; // --- Builders --- pub use builders::{ ApiActivityBuilder, AppLifecycleBuilder, AuthenticationBuilder, BaseEventBuilder, - ConfigStateChangeBuilder, DetectionFindingBuilder, EventContext, EventOrigin, + ConfigStateChangeBuilder, ContextRef, DetectionFindingBuilder, EventContext, EventOrigin, HttpActivityBuilder, NetworkActivityBuilder, ProcessActivityBuilder, SshActivityBuilder, }; // --- Tracing layers --- pub use tracing_layers::{ - OCSF_TARGET, OcsfJsonlLayer, OcsfShorthandLayer, clear_current_event, clone_current_event, - emit_ocsf_event, emit_ocsf_event_routed, set_current_event, + OCSF_TARGET, OcsfJsonSettings, OcsfJsonlLayer, OcsfShorthandLayer, bind_thread_json_settings, + clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, + set_current_event, }; diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index a0746b8621..3f0936e841 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -16,6 +16,27 @@ use tracing_subscriber::layer::Context; use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; +/// OCSF JSON output settings owned by one hosted supervisor session. +/// +/// Sessions apply their own `ocsf_json_enabled` and schema-version settings +/// to these values. Binding them to the session's runtime threads makes every +/// [`OcsfJsonlLayer`] use them for events those threads emit. +#[derive(Clone, Debug, Default)] +pub struct OcsfJsonSettings { + pub enabled: Arc, + pub target_version: Arc>, +} + +std::thread_local! { + static THREAD_JSON_SETTINGS: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// Apply one hosted session's JSON settings to events emitted on this thread. +pub fn bind_thread_json_settings(settings: Option) { + THREAD_JSON_SETTINGS.with(|current| *current.borrow_mut() = settings); +} + /// A tracing `Layer` that intercepts OCSF events and writes JSONL output. /// /// Only events with `target: "ocsf"` are processed; non-OCSF events are ignored. @@ -90,8 +111,15 @@ where return; } + // A hosted session's settings replace the process-wide ones. + let session = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); + let (enabled, target_version) = session.as_ref().map_or( + (self.enabled.as_ref(), self.target_version.as_ref()), + |settings| (Some(&settings.enabled), Some(&settings.target_version)), + ); + // If an enabled flag is set and it reads `false`, skip writing. - if let Some(ref flag) = self.enabled + if let Some(flag) = enabled && !flag.load(Ordering::Relaxed) { return; @@ -100,7 +128,7 @@ where if let Some(ocsf_event) = clone_current_event() && let Ok(mut w) = self.writer.lock() { - let line = if let Some(ref target) = self.target_version + let line = if let Some(target) = target_version && let Ok(version) = target.lock() && !version.is_empty() { diff --git a/crates/openshell-ocsf/src/tracing_layers/mod.rs b/crates/openshell-ocsf/src/tracing_layers/mod.rs index b57ba1364a..808051b061 100644 --- a/crates/openshell-ocsf/src/tracing_layers/mod.rs +++ b/crates/openshell-ocsf/src/tracing_layers/mod.rs @@ -15,5 +15,5 @@ pub use event_bridge::{ OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, set_current_event, }; -pub use jsonl_layer::OcsfJsonlLayer; +pub use jsonl_layer::{OcsfJsonSettings, OcsfJsonlLayer, bind_thread_json_settings}; pub use shorthand_layer::OcsfShorthandLayer; diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index b10c300833..914bbea861 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -23,7 +23,7 @@ pub use ocsf::{OcsfOtlp, OcsfOtlpLayer}; pub use propagation::{ HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV, TRACESTATE_ENV, TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment, - set_parent_from_environment, trace_context_environment, + set_parent_from_environment, set_parent_from_trace_context, trace_context_environment, }; use opentelemetry::KeyValue; diff --git a/crates/openshell-otel/src/propagation.rs b/crates/openshell-otel/src/propagation.rs index 764e343146..82e5fa784d 100644 --- a/crates/openshell-otel/src/propagation.rs +++ b/crates/openshell-otel/src/propagation.rs @@ -114,17 +114,27 @@ pub fn current_trace_context_environment() -> Vec<(&'static str, String)> { /// /// Leaves `span` unchanged when the environment carries no valid context. pub fn set_parent_from_environment(span: &tracing::Span) { - let carrier = [ - ("traceparent", TRACEPARENT_ENV), - ("tracestate", TRACESTATE_ENV), - ] - .into_iter() - .filter_map(|(field, name)| { - std::env::var(name) - .ok() - .map(|value| (field.to_string(), value)) - }) - .collect::>(); + set_parent_from_trace_context( + span, + std::env::var(TRACEPARENT_ENV).ok().as_deref(), + std::env::var(TRACESTATE_ENV).ok().as_deref(), + ); +} + +/// Parent `span` under an explicit W3C `traceparent` and `tracestate`. +/// +/// A process supervising several sandboxes receives each launch's context +/// with its session instead of through process environment. Leaves `span` +/// unchanged when the values carry no valid context. +pub fn set_parent_from_trace_context( + span: &tracing::Span, + traceparent: Option<&str>, + tracestate: Option<&str>, +) { + let carrier = [("traceparent", traceparent), ("tracestate", tracestate)] + .into_iter() + .filter_map(|(field, value)| value.map(|value| (field.to_string(), value.to_string()))) + .collect::>(); set_parent_from_carrier(span, &carrier); } diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index d7ceb67024..45639c4c00 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -21,7 +21,7 @@ use openshell_ocsf::{ use crate::process::{ProcessHandle, ProcessStatus, ResolvedWorkspace}; -fn ocsf_ctx() -> &'static openshell_ocsf::EventContext { +fn ocsf_ctx() -> openshell_ocsf::ContextRef<'static> { openshell_ocsf::ctx::ctx() } diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 10fa59b36e..d2344958b7 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -568,7 +568,7 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id< // unresolved binding is handled before the body is read) and forward // the original request unchanged if the chain allows. let input = middleware_request_input_with_id( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), scheme, &req, ctx, @@ -615,7 +615,7 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id< let headers = safe_middleware_headers(&buffered.headers)?; let query = raw_query_from_request_headers(&buffered.headers)?; let input = middleware_request_input_with_id( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), scheme, &req, ctx, diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 512c9096d1..f3e9e34741 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -1748,7 +1748,7 @@ pub(crate) async fn websocket_middleware_preflight( let requested_subprotocols = crate::l7::rest::websocket_requested_subprotocols(&req.raw_header[..header_end])?; let input = websocket_preflight_input( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), ctx, req, scheme, @@ -7604,7 +7604,7 @@ network_policies: }; let input = middleware_request_input( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), "http", &req, &ctx, diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 3529683403..91e1fa5337 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -16,7 +16,7 @@ use miette::Result; use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit}; #[cfg(unix)] -fn ocsf_ctx() -> &'static openshell_ocsf::EventContext { +fn ocsf_ctx() -> openshell_ocsf::ContextRef<'static> { openshell_ocsf::ctx::ctx() } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 1f1c7767ff..77620a94d1 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -467,7 +467,7 @@ async fn run_session_loop(config: SessionConfig) { Ok(SessionOutcome::Closed) => { config.ready_tx.send_replace(false); let event = - session_closed_event(openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); + session_closed_event(&openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); ocsf_emit!(event); break; } @@ -489,7 +489,7 @@ async fn run_session_loop(config: SessionConfig) { Err(e) => { let accepted = config.ready_tx.send_replace(false); let event = session_failed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &target, attempt, &e.to_string(), @@ -667,7 +667,7 @@ async fn run_single_session( updates.send_replace(Some(accepted.session_id.clone())); } let event = session_established_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), target, &accepted.session_id, u32::try_from(heartbeat_secs).unwrap_or(u32::MAX), @@ -904,7 +904,8 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); - let event = relay_open_event(openshell_ocsf::ctx::ctx(), &relay_open, &ssh_socket_path); + let event = + relay_open_event(&openshell_ocsf::ctx::ctx(), &relay_open, &ssh_socket_path); ocsf_emit!(event); tokio::spawn(async move { @@ -922,7 +923,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< { Ok(()) => { let event = relay_closed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &event_open, &ssh_socket_path, ); @@ -930,7 +931,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< } Err(e) => { let event = relay_failed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &event_open, &ssh_socket_path, &e.to_string(), @@ -948,7 +949,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< } Some(gateway_message::Payload::RelayClose(close)) => { let event = relay_close_from_gateway_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &close.channel_id, &close.reason, ); From 0e7666de5bd270e9ede5b586e19b1424387ebbe6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 17:02:44 -0700 Subject: [PATCH 2/6] refactor(supervisor): scope gateway credentials and network state to a session The supervisor kept its gateway bearer, Sandbox Protocol bearer, and renewal task in process-wide slots, and the network supervisor read provider token grants, proxy CA files, and the gateway identity from process environment. A second sandbox in the same process would have replaced or rotated the first sandbox's credentials, reused its provider access tokens, and exposed its denials through policy.local. GatewayCredentials now holds one session's gateway and Sandbox Protocol credentials and their renewal; every gateway client receives a GatewayEndpoint that pairs the URL with them, including log push, ConnectSupervisor and its redirects, and extension-credential rotation. run_networking takes a NetworkSession with the sandbox identity, the session's gateway access, its Workload API socket, its policy.local denial source, and its proxy CA files. Provider token grants use a cache and HTTP client owned by the session and fail closed without a Workload API socket. run_session supervises one generation from an explicit SessionConfig and owner-provided SessionControls. run_sandbox_with_backend remains the dedicated entry point: it keeps the process-wide OCSF context, reads the same environment, binds readiness endpoints to session readiness, and stops on SIGTERM or SIGINT. The binary shares one set of credentials between log push and supervision, as before. Signed-off-by: Drew Newberry --- crates/openshell-core/src/grpc_client.rs | 512 ++++++++++++------ crates/openshell-driver-mxc/src/driver.rs | 2 +- crates/openshell-supervisor-network/README.md | 12 + .../openshell-supervisor-network/src/host.rs | 5 +- .../src/l7/token_grant_injection.rs | 28 +- .../src/policy_local.rs | 169 +++++- .../openshell-supervisor-network/src/proxy.rs | 16 +- .../src/proxy/relay.rs | 9 +- .../openshell-supervisor-network/src/run.rs | 110 +++- .../src/token_grant.rs | 411 +++++++------- .../src/debug_rpc.rs | 7 +- .../src/delegated.rs | 27 +- .../src/log_push.rs | 168 ++++-- .../src/supervisor_session.rs | 39 +- crates/openshell-supervisor/README.md | 18 + .../src/backend_setup/tests.rs | 1 + .../src/config_runtime.rs | 21 +- .../src/legacy_startup_regression_tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 503 +++++++++++++---- crates/openshell-supervisor/src/main.rs | 36 +- .../src/provider_readiness.rs | 8 +- 21 files changed, 1431 insertions(+), 672 deletions(-) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index a97deab7d6..80b20b0264 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -5,21 +5,23 @@ //! `OpenShell` server. //! //! Every request carries a sandbox bearer credential in the `Authorization` -//! header. The token is resolved at startup from one of three sources: +//! header. Each supervisor session owns its [`GatewayCredentials`], usually +//! built from the trusted launch bundle with +//! [`GatewayCredentials::from_auth_bundle`]. Diagnostic tools can instead +//! resolve a token with [`GatewayCredentials::from_environment`] from one of: //! //! 1. `OPENSHELL_SANDBOX_TOKEN` — raw JWT in the env (test harness path). -//! 2. `OPENSHELL_SANDBOX_TOKEN_FILE` — file containing the JWT (Docker / -//! Podman / VM drivers write this to a bundle file at sandbox-create -//! time). +//! 2. `OPENSHELL_SANDBOX_TOKEN_FILE` — file containing the JWT. //! 3. `OPENSHELL_K8S_SA_TOKEN_FILE` — projected `ServiceAccount` JWT; the -//! supervisor exchanges it for a gateway JWT via `IssueSandboxToken` -//! once at startup. +//! caller exchanges it for a gateway JWT via `IssueSandboxToken`. //! -//! The resolved bearer credential is held in process memory thereafter and -//! injected on every outbound call by [`AuthInterceptor`]. +//! Callers reach the gateway through a [`GatewayEndpoint`], which pairs the +//! URL with one session's credentials. No credential is installed +//! process-wide, so sessions hosted by one supervisor process cannot use or +//! rotate each other's identity. use std::collections::HashMap; -use std::sync::{Arc, OnceLock, RwLock}; +use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::endpoint_status::{EndpointResult, EndpointStatusSnapshot}; @@ -63,8 +65,8 @@ pub const MAX_SUPERVISOR_STREAM_MESSAGE_BYTES: usize = 9 * 1024 * 1024; /// Shared, refreshable Bearer header. All [`AuthInterceptor`] clones read /// the same slot, so the renewal task can replace the token in place without -/// rebuilding the channel. -type TokenSlot = Arc>; +/// rebuilding the channel. `None` means the owning session revoked it. +type TokenSlot = Arc>>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TokenSource { @@ -73,38 +75,222 @@ enum TokenSource { K8sServiceAccount, } -/// Process-wide token slot. Initialized by the first [`connect_channel`] -/// call and shared with every subsequent client and the renewal loop. -static TOKEN_SLOT: OnceLock = OnceLock::new(); +/// Gateway credentials owned by one supervisor session. +/// +/// Clients built from a [`GatewayEndpoint`] read only these slots, and the +/// renewal task started by the first connection updates only them. Dropping +/// the last clone or calling [`GatewayCredentials::revoke`] stops renewal. +#[derive(Clone)] +pub struct GatewayCredentials { + inner: Arc, +} -/// Refresh strategy used by the process-wide token slot. -static TOKEN_REFRESH_MODE: OnceLock = OnceLock::new(); +struct CredentialState { + bearer: TokenSlot, + source: TokenSource, + #[cfg(feature = "jwt")] + sandbox_bearer: Option, + renewal: std::sync::Mutex, +} -/// Serializes the first token acquisition. Several supervisor subsystems -/// connect during startup; without this guard they can all observe an empty -/// [`TOKEN_SLOT`] and perform duplicate K8s bootstrap exchanges. -static TOKEN_INIT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); +enum Renewal { + Idle, + Started(tokio::task::AbortHandle), + Revoked, +} -/// One-shot guard so the renewal loop spawns at most once per process. -static REFRESH_SPAWNED: OnceLock<()> = OnceLock::new(); +impl Drop for CredentialState { + fn drop(&mut self) { + let renewal = self + .renewal + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Renewal::Started(task) = renewal { + task.abort(); + } + } +} -#[cfg(feature = "jwt")] -static SANDBOX_BEARER_SLOT: OnceLock = OnceLock::new(); +impl std::fmt::Debug for GatewayCredentials { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GatewayCredentials") + .field("source", &self.inner.source) + .finish_non_exhaustive() + } +} -#[derive(Clone, Debug)] -enum RefreshMode { - GatewayJwt(TokenSource), +impl GatewayCredentials { + fn new(bearer: AsciiMetadataValue, source: TokenSource) -> Self { + Self { + inner: Arc::new(CredentialState { + bearer: Arc::new(RwLock::new(Some(bearer))), + source, + #[cfg(feature = "jwt")] + sandbox_bearer: None, + renewal: std::sync::Mutex::new(Renewal::Idle), + }), + } + } + + /// Credentials issued for one exact sandbox launch. Renewal also rotates + /// the bundle's Sandbox Protocol credential, returned by + /// [`GatewayCredentials::sandbox_bearer`]. + #[cfg(feature = "jwt")] + pub fn from_auth_bundle(bundle: &crate::jwt::SupervisorAuthBundle) -> Result { + let bearer = validate_gateway_bearer(bundle.gateway_token.expose_secret())?; + let sandbox_bearer = bundle + .sandbox_bearer_slot() + .into_diagnostic() + .wrap_err("invalid Sandbox Protocol credential")?; + Ok(Self { + inner: Arc::new(CredentialState { + bearer: Arc::new(RwLock::new(Some(bearer))), + source: TokenSource::File, + sandbox_bearer: Some(sandbox_bearer), + renewal: std::sync::Mutex::new(Renewal::Idle), + }), + }) + } + + /// Use a caller-supplied gateway JWT. Renewal cannot rebootstrap it. + pub fn from_gateway_token(token: &str) -> Result { + Ok(Self::new(validate_gateway_bearer(token)?, TokenSource::Env)) + } + + /// Resolve a token for diagnostic tools from the environment, exchanging a + /// projected `ServiceAccount` token when no gateway JWT is present. + pub async fn from_environment(endpoint: &str) -> Result { + let plain_channel = build_plain_channel(endpoint).await?; + let acquired = acquire_sandbox_token(endpoint, &plain_channel).await?; + Ok(Self::new( + validate_gateway_bearer(&acquired.token)?, + acquired.source, + )) + } + + /// The Sandbox Protocol credential renewed with these gateway credentials. + #[cfg(feature = "jwt")] + #[must_use] + pub fn sandbox_bearer(&self) -> Option { + self.inner.sandbox_bearer.clone() + } + + /// Clear both credentials and stop renewal. Existing clients fail their + /// next request locally instead of presenting a revoked identity. + pub fn revoke(&self) { + let mut renewal = self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Renewal::Started(task) = &*renewal { + task.abort(); + } + *renewal = Renewal::Revoked; + *self + .inner + .bearer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + #[cfg(feature = "jwt")] + if let Some(sandbox_bearer) = &self.inner.sandbox_bearer { + sandbox_bearer.clear(); + } + } + + /// Start the renewal task once for these credentials, on the caller's + /// runtime, so a hosted session's renewal stops with that session. + fn start_renewal(&self, channel: &AuthedChannel, plain_channel: &Channel, endpoint: &str) { + let mut renewal = self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !matches!(*renewal, Renewal::Idle) { + return; + } + let task = tokio::spawn(refresh_token_loop(RenewalTarget { + channel: channel.clone(), + bearer: self.inner.bearer.clone(), + #[cfg(feature = "jwt")] + sandbox_bearer: self.inner.sandbox_bearer.clone(), + source: self.inner.source, + endpoint: endpoint.to_string(), + plain_channel: plain_channel.clone(), + })); + *renewal = Renewal::Started(task.abort_handle()); + } + + fn is_revoked(&self) -> bool { + matches!( + *self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + Renewal::Revoked + ) + } + + fn install_gateway_bearer(&self, bearer: AsciiMetadataValue) { + let mut slot = self + .inner + .bearer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // A revoked session must not be revived by an in-flight response. + if slot.is_some() { + *slot = Some(bearer); + } + } +} + +/// The gateway URL a session uses, paired with that session's credentials. +#[derive(Clone)] +pub struct GatewayEndpoint { + url: Arc, + credentials: GatewayCredentials, +} + +impl GatewayEndpoint { + pub fn new(url: impl Into>, credentials: GatewayCredentials) -> Self { + Self { + url: url.into(), + credentials, + } + } + + #[must_use] + pub fn url(&self) -> &str { + &self.url + } + + #[must_use] + pub fn credentials(&self) -> &GatewayCredentials { + &self.credentials + } + + /// Reach another gateway address with the same session credentials. + #[must_use] + pub fn with_url(&self, url: impl Into>) -> Self { + Self::new(url, self.credentials.clone()) + } +} + +impl std::fmt::Debug for GatewayEndpoint { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GatewayEndpoint") + .field("url", &self.url) + .finish_non_exhaustive() + } } #[derive(Debug)] struct AcquiredToken { token: String, - refresh_mode: RefreshMode, -} - -fn install_token_slot(token: &str) -> Result { - let bearer = validate_gateway_bearer(token)?; - Ok(install_validated_token_slot(bearer)) + source: TokenSource, } fn validate_gateway_bearer(token: &str) -> Result { @@ -113,18 +299,6 @@ fn validate_gateway_bearer(token: &str) -> Result { .wrap_err("sandbox JWT contained characters not valid for a header value") } -fn install_validated_token_slot(bearer: AsciiMetadataValue) -> TokenSlot { - if let Some(existing) = TOKEN_SLOT.get() { - *existing - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = bearer; - return existing.clone(); - } - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); - let _ = TOKEN_SLOT.set(slot.clone()); - TOKEN_SLOT.get().cloned().unwrap_or(slot) -} - #[cfg(feature = "jwt")] struct ValidatedSandboxRefresh { token: crate::jwt::SecretJwt, @@ -159,22 +333,6 @@ fn validate_sandbox_refresh( }) } -/// Install the gateway-session credential supplied in trusted supervisor -/// launch state before any gateway client is constructed. -#[cfg(feature = "jwt")] -pub fn install_supervisor_auth_bundle( - bundle: &crate::jwt::SupervisorAuthBundle, -) -> Result { - install_token_slot(bundle.gateway_token.expose_secret())?; - let _ = TOKEN_REFRESH_MODE.set(RefreshMode::GatewayJwt(TokenSource::File)); - let slot = bundle - .sandbox_bearer_slot() - .into_diagnostic() - .wrap_err("invalid Sandbox Protocol credential")?; - let _ = SANDBOX_BEARER_SLOT.set(slot.clone()); - Ok(SANDBOX_BEARER_SLOT.get().cloned().unwrap_or(slot)) -} - /// gRPC interceptor that injects `authorization: Bearer ` on every /// outbound request. The token lives in a shared [`TokenSlot`] so the renewal /// task can replace it without rebuilding clients. @@ -197,8 +355,9 @@ impl tonic::service::Interceptor for AuthInterceptor { let bearer = self .bearer .read() - .expect("auth interceptor token slot poisoned") - .clone(); + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + .ok_or_else(|| Status::unauthenticated("gateway session credentials were revoked"))?; req.metadata_mut().insert("authorization", bearer); #[cfg(feature = "trace-context")] let req = @@ -299,56 +458,23 @@ impl Drop for ClientSpanStatus { } } -/// Build a Bearer-authenticated channel to the gateway. -/// -/// First call per process resolves the sandbox JWT via the three-step -/// lookup (env → file → K8s SA bootstrap exchange) and installs it into -/// the process-wide [`TOKEN_SLOT`]. Subsequent calls reuse the cached -/// slot — the renewal loop keeps the value fresh, so re-running the -/// bootstrap is both unnecessary and (on the K8s SA path) expensive -/// (one apiserver round-trip per call). The renewal loop itself is -/// spawned once per process via [`REFRESH_SPAWNED`]. -async fn connect_channel(endpoint: &str) -> Result { - let channel = build_plain_channel(endpoint).await?; - let (slot, refresh_mode) = token_slot(endpoint, &channel).await?; - let plain_channel = channel.clone(); - let intercepted = InterceptedService::new(channel, AuthInterceptor::new(slot.clone())); - if REFRESH_SPAWNED.set(()).is_ok() { - let RefreshMode::GatewayJwt(source) = refresh_mode; - let refresh_channel = intercepted.clone(); - let endpoint = endpoint.to_string(); - tokio::spawn(async move { - refresh_token_loop(refresh_channel, slot, source, endpoint, plain_channel).await; - }); +/// Build a Bearer-authenticated channel to the gateway with one session's +/// credentials. The first connection starts that session's renewal task. +async fn connect_channel(gateway: &GatewayEndpoint) -> Result { + if gateway.credentials.is_revoked() { + return Err(miette::miette!("gateway session credentials were revoked")); } + let channel = build_plain_channel(gateway.url()).await?; + let intercepted = InterceptedService::new( + channel.clone(), + AuthInterceptor::new(gateway.credentials.inner.bearer.clone()), + ); + gateway + .credentials + .start_renewal(&intercepted, &channel, gateway.url()); Ok(intercepted) } -async fn token_slot(endpoint: &str, plain_channel: &Channel) -> Result<(TokenSlot, RefreshMode)> { - if let Some(existing) = TOKEN_SLOT.get() { - let refresh_mode = TOKEN_REFRESH_MODE - .get() - .cloned() - .unwrap_or(RefreshMode::GatewayJwt(TokenSource::Env)); - return Ok((existing.clone(), refresh_mode)); - } - - let _guard = TOKEN_INIT_LOCK.lock().await; - - if let Some(existing) = TOKEN_SLOT.get() { - let refresh_mode = TOKEN_REFRESH_MODE - .get() - .cloned() - .unwrap_or(RefreshMode::GatewayJwt(TokenSource::Env)); - return Ok((existing.clone(), refresh_mode)); - } - - let acquired = acquire_sandbox_token(endpoint, plain_channel).await?; - let slot = install_token_slot(&acquired.token)?; - let _ = TOKEN_REFRESH_MODE.set(acquired.refresh_mode.clone()); - Ok((slot, acquired.refresh_mode)) -} - /// Resolve the sandbox JWT used to authenticate every outbound RPC. /// /// `endpoint` is logged on errors but never used for transport here; the @@ -362,7 +488,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul debug!(source = "env", "loaded sandbox token"); return Ok(AcquiredToken { token: t, - refresh_mode: RefreshMode::GatewayJwt(TokenSource::Env), + source: TokenSource::Env, }); } @@ -375,7 +501,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul debug!(source = "file", path = %path, "loaded sandbox token"); return Ok(AcquiredToken { token: contents.trim().to_string(), - refresh_mode: RefreshMode::GatewayJwt(TokenSource::File), + source: TokenSource::File, }); } @@ -384,7 +510,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul { return Ok(AcquiredToken { token: acquire_k8s_sandbox_token(endpoint, plain_channel, &sa_path).await?, - refresh_mode: RefreshMode::GatewayJwt(TokenSource::K8sServiceAccount), + source: TokenSource::K8sServiceAccount, }); } @@ -407,14 +533,13 @@ async fn acquire_k8s_sandbox_token( .trim() .to_string(); info!(endpoint = %endpoint, "exchanging K8s ServiceAccount token for sandbox JWT"); - // The bootstrap exchange uses a one-off interceptor pinned to the - // SA token; the resulting gateway JWT becomes the value in the - // shared `TOKEN_SLOT` once `connect_channel` returns. - let bootstrap_slot: TokenSlot = Arc::new(RwLock::new( + // The bootstrap exchange uses a one-off interceptor pinned to the SA + // token; the caller installs the resulting gateway JWT. + let bootstrap_slot: TokenSlot = Arc::new(RwLock::new(Some( AsciiMetadataValue::try_from(format!("Bearer {sa_token}")) .into_diagnostic() .wrap_err("SA token contained characters not valid for a header value")?, - )); + ))); let interceptor = AuthInterceptor::new(bootstrap_slot); let bootstrap = InterceptedService::new(plain_channel.clone(), interceptor); let mut client = OpenShellClient::new(bootstrap); @@ -428,8 +553,8 @@ async fn acquire_k8s_sandbox_token( /// Build an authenticated channel for direct external use (e.g. the /// long-lived `supervisor_session` control stream). -pub async fn connect_channel_pub(endpoint: &str) -> Result { - connect_channel(endpoint).await +pub async fn connect_channel_pub(gateway: &GatewayEndpoint) -> Result { + connect_channel(gateway).await } /// Report installed provider state for the current authenticated supervisor session. @@ -437,11 +562,11 @@ pub async fn connect_channel_pub(endpoint: &str) -> Result { /// The observation must carry the session ID returned by `ConnectSupervisor`. /// Reconnects start a new report sequence; retries preserve the complete report. pub async fn report_provider_readiness( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, observation: crate::proto::ProviderReadinessObservation, ) -> Result { - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; client .report_provider_readiness(crate::proto::ReportProviderReadinessRequest { sandbox_id: sandbox_id.to_string(), @@ -452,18 +577,33 @@ pub async fn report_provider_readiness( .into_diagnostic() } -/// Background task that renews the sandbox JWT at ~80% of its remaining -/// lifetime. The new token replaces the value in [`TOKEN_SLOT`], so all -/// in-flight and future clients pick it up on their next request. The -/// loop never panics: every failure is logged and re-attempted after a -/// bounded backoff. -async fn refresh_token_loop( +/// Slots and transport retained by one session's renewal task. It holds no +/// reference to [`GatewayCredentials`], so dropping them still stops it. +struct RenewalTarget { channel: AuthedChannel, - slot: TokenSlot, + bearer: TokenSlot, + #[cfg(feature = "jwt")] + sandbox_bearer: Option, source: TokenSource, endpoint: String, plain_channel: Channel, -) { +} + +/// Background task that renews the sandbox JWT at ~80% of its remaining +/// lifetime. The new token replaces the value in the session's slot, so all +/// in-flight and future clients of that session pick it up on their next +/// request. The loop never panics: every failure is logged and re-attempted +/// after a bounded backoff. +async fn refresh_token_loop(target: RenewalTarget) { + let RenewalTarget { + channel, + bearer: slot, + #[cfg(feature = "jwt")] + sandbox_bearer, + source, + endpoint, + plain_channel, + } = target; let mut client = OpenShellClient::new(channel); loop { let Some(sleep) = compute_refresh_delay(&slot) else { @@ -495,11 +635,17 @@ async fn refresh_token_loop( } }; - *slot - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = gateway_bearer; + { + let mut current = slot + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + return; + } + *current = Some(gateway_bearer); + } #[cfg(feature = "jwt")] - if let Some(sandbox_slot) = SANDBOX_BEARER_SLOT.get() + if let Some(sandbox_slot) = sandbox_bearer.as_ref() && let Err(error) = sandbox_slot.update( sandbox_refresh.token, sandbox_refresh.expires_at, @@ -523,8 +669,10 @@ async fn refresh_token_loop( Ok(new_token) => { match AsciiMetadataValue::try_from(format!("Bearer {new_token}")) { Ok(value) => { - if let Ok(mut guard) = slot.write() { - *guard = value; + if let Ok(mut guard) = slot.write() + && guard.is_some() + { + *guard = Some(value); info!( "rebootstrapped gateway sandbox JWT after refresh authentication failure" ); @@ -585,6 +733,7 @@ fn authenticated_service_names( async fn refresh_extension_credentials_with_client( client: &mut OpenShellClient, + credentials: &GatewayCredentials, store: &ExtensionCredentialStore, services: &[crate::proto::SupervisorMiddlewareService], ) -> Result> { @@ -645,9 +794,9 @@ async fn refresh_extension_credentials_with_client( // Commit only after the gateway, Sandbox Protocol, and extension // credentials have all been parsed and validated. The remaining updates // repeat those validations but cannot fail for the validated inputs. - install_validated_token_slot(gateway_bearer); + credentials.install_gateway_bearer(gateway_bearer); #[cfg(feature = "jwt")] - if let Some(sandbox_slot) = SANDBOX_BEARER_SLOT.get() { + if let Some(sandbox_slot) = credentials.inner.sandbox_bearer.as_ref() { match sandbox_slot.update( sandbox_refresh.token, sandbox_refresh.expires_at, @@ -679,11 +828,14 @@ async fn refresh_extension_credentials_with_client( /// non-expiring session credential and needs no periodic refresh. If the token /// can't be parsed (for example, an opaque bootstrap bearer), default to 6 h. fn compute_refresh_delay(slot: &TokenSlot) -> Option { - let token = slot - .read() - .ok() - .and_then(|v| v.to_str().ok().map(str::to_string)) - .unwrap_or_default(); + let token = { + let current = slot + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // A revoked session has nothing to renew. + current.as_ref()?.to_str().ok().map(str::to_string) + } + .unwrap_or_default(); let now_ms = i64::try_from( SystemTime::now() .duration_since(UNIX_EPOCH) @@ -819,7 +971,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload_json); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expiring token needs refresh"); // 800 s baseline + up to 10 % jitter → 800..=880 s, with some slack // for the 1-second resolution of the exp claim. @@ -840,7 +992,7 @@ mod auth_tests { base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#)); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expired token needs refresh"); assert!((1..60).contains(&delay.as_secs())); } @@ -851,7 +1003,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"exp":0}"#); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); assert_eq!(compute_refresh_delay(&slot), None); } @@ -867,7 +1019,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload_json); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expiring token needs refresh"); assert!( delay.as_secs() < 30, @@ -906,9 +1058,9 @@ mod workspace_tests { } } -/// Connect to the `OpenShell` server. -async fn connect(endpoint: &str) -> Result> { - let channel = connect_channel(endpoint).await?; +/// Connect to the `OpenShell` server with one session's credentials. +async fn connect(gateway: &GatewayEndpoint) -> Result> { + let channel = connect_channel(gateway).await?; Ok(OpenShellClient::new(channel)) } @@ -918,12 +1070,12 @@ async fn connect(endpoint: &str) -> Result> { /// or `Ok(None)` when the sandbox was created without a policy (the sandbox /// should discover one from disk or use the restrictive default). pub async fn fetch_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_name: &str, ) -> Result> { - debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to OpenShell server"); + debug!(endpoint = %gateway.url(), sandbox_name = %sandbox_name, "Connecting to OpenShell server"); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; debug!("Connected, fetching sandbox policy"); @@ -942,12 +1094,12 @@ pub async fn fetch_policy( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn fetch_settings_snapshot( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_name: &str, ) -> Result { let status = ClientSpanStatus::current(); - debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot"); - let mut client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot"); + let mut client = connect(gateway).await?; status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox_name, None).await) } @@ -1014,18 +1166,18 @@ async fn sync_policy_with_client( /// Performs the full discovery flow (fetch → sync → re-fetch) over one /// channel instead of establishing three separate connections. pub async fn discover_and_sync_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, discovered_policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result { debug!( - endpoint = %endpoint, + endpoint = %gateway.url(), sandbox = %sandbox, "Syncing discovered policy and re-fetching canonical version" ); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; // Sync the discovered policy to the gateway. sync_policy_with_client(&mut client, sandbox, discovered_policy, workspace).await?; @@ -1044,13 +1196,13 @@ pub async fn discover_and_sync_policy( /// Used by the supervisor to push baseline-path-enriched policies so the /// gateway stores the effective policy users see via `openshell sandbox get`. pub async fn sync_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result<()> { - debug!(endpoint = %endpoint, sandbox = %sandbox, "Syncing enriched policy to gateway"); - let mut client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), sandbox = %sandbox, "Syncing enriched policy to gateway"); + let mut client = connect(gateway).await?; sync_policy_with_client(&mut client, sandbox, policy, workspace).await } @@ -1061,13 +1213,13 @@ pub async fn sync_policy( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn sync_policy_and_fetch_snapshot( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result { let status = ClientSpanStatus::current(); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; sync_policy_with_client(&mut client, sandbox, policy, workspace).await?; status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await) } @@ -1080,7 +1232,7 @@ pub async fn sync_policy_and_fetch_snapshot( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn report_sandbox_configuration( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, instance_id: &str, snapshot: Option<&SettingsPollResult>, @@ -1088,7 +1240,7 @@ pub async fn report_sandbox_configuration( error: &str, ) -> Result<()> { let status = ClientSpanStatus::current(); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; client .report_sandbox_configuration(crate::proto::ReportSandboxConfigurationRequest { sandbox_id: sandbox_id.to_string(), @@ -1118,10 +1270,10 @@ pub async fn report_sandbox_configuration( /// environment represents a sandbox without provider credentials. Transport /// failure returns an error so callers can revoke credentials and retry. pub async fn fetch_provider_environment( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, ) -> Result { - fetch_provider_environment_with_stable_placeholders(endpoint, sandbox_id, false).await + fetch_provider_environment_with_stable_placeholders(gateway, sandbox_id, false).await } /// Fetch a snapshot without downgrading previously activated external stable credentials. @@ -1135,14 +1287,14 @@ pub async fn fetch_provider_environment( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn fetch_provider_environment_with_stable_placeholders( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { let status = ClientSpanStatus::current(); - debug!(endpoint = %endpoint, sandbox_id = %sandbox_id, "Fetching provider environment"); + debug!(endpoint = %gateway.url(), sandbox_id = %sandbox_id, "Fetching provider environment"); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; let response = client .get_sandbox_provider_environment(GetSandboxProviderEnvironmentRequest { @@ -1366,7 +1518,7 @@ mod provider_environment_tests { fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn exchange_provider_subject_token( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, provider: &str, credential_key: &str, @@ -1374,7 +1526,7 @@ pub async fn exchange_provider_subject_token( request_id: Option<&str>, ) -> std::result::Result { let status = ClientSpanStatus::current(); - let endpoint = token_grant_gateway_endpoint(endpoint)?; + let endpoint = token_grant_gateway_endpoint(gateway.url())?; if sandbox_id.is_empty() || provider.is_empty() || credential_key.is_empty() { return Err(gateway_grant_failure(TokenGrantFailureKind::Configuration)); } @@ -1388,7 +1540,7 @@ pub async fn exchange_provider_subject_token( // Discard those sources at the grant boundary; never diagnose from prose. // The shared channel bounds the connection attempt; the gateway bounds its // token-service calls, and a gateway deadline arrives as DeadlineExceeded. - let mut client = connect(endpoint.as_str()) + let mut client = connect(&gateway.with_url(endpoint.as_str())) .await .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Unclassified))?; let request = provider_subject_token_exchange_request( @@ -1559,6 +1711,8 @@ mod token_grant_exchange_tests { #[derive(Clone)] pub struct CachedOpenShellClient { client: OpenShellClient, + /// Session credentials rotated alongside extension credentials. + credentials: GatewayCredentials, workspace: Arc>, /// Extension credentials for this supervisor. Cloning the client shares /// the store, so the middleware registry and the polling loop that rotates @@ -1887,8 +2041,8 @@ pub struct ProviderSubjectTokenExchangeResult { } impl CachedOpenShellClient { - pub async fn connect(endpoint: &str) -> Result { - Self::connect_with_credentials(endpoint, ExtensionCredentialStore::new()).await + pub async fn connect(gateway: &GatewayEndpoint) -> Result { + Self::connect_with_credentials(gateway, ExtensionCredentialStore::new()).await } /// Connect while sharing an existing credential store. @@ -1898,13 +2052,14 @@ impl CachedOpenShellClient { /// credentials handed to the middleware registry are not the ones the loop /// rotates. pub async fn connect_with_credentials( - endpoint: &str, + gateway: &GatewayEndpoint, extension_credentials: ExtensionCredentialStore, ) -> Result { - debug!(endpoint = %endpoint, "Connecting openshell gRPC client for policy polling"); - let client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), "Connecting openshell gRPC client for policy polling"); + let client = connect(gateway).await?; Ok(Self { client, + credentials: gateway.credentials.clone(), workspace: Arc::new(tokio::sync::OnceCell::new()), extension_credentials, }) @@ -1972,6 +2127,7 @@ impl CachedOpenShellClient { let mut client = self.client.clone(); refresh_extension_credentials_with_client( &mut client, + &self.credentials, &self.extension_credentials, services, ) diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index c7a966a32b..2772f1a69c 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -870,7 +870,7 @@ async fn run_lifecycle( binary_path: host_proxy_binary_path(&sandbox_config), sandbox_id: Some(sandbox_id.clone()), sandbox_name: Some(sandbox_name.clone()), - openshell_endpoint: None, + gateway: None, provider_credentials: None, agent_proposals: openshell_core::proposals::AgentProposals::default(), denial_tx: None, diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 30f404752a..b191655d1a 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -14,3 +14,15 @@ allocator implementation belongs to the VM crate. The VM dependency disables default features, so this instrumentation does not require the compute runtime or Unix-only sandbox library. It is a development dependency only. + +## Session inputs + +`run_networking` takes a `NetworkSession` with the sandbox identity, the +session's `GatewayEndpoint`, the SPIFFE Workload API socket for provider token +grants, the `policy.local` denial source, and proxy CA files. None of these +are read from the process environment, so one process can run several +sessions. Each session owns its provider token cache and HTTP client; a token +acquired for one sandbox is never served to another. Without a Workload API +socket, token grants fail closed. A process hosting several sessions serves +`/v1/denials` from a per-session `SessionDenialLog` instead of the shared +shorthand log files. diff --git a/crates/openshell-supervisor-network/src/host.rs b/crates/openshell-supervisor-network/src/host.rs index b07bc0f537..5f111acb21 100644 --- a/crates/openshell-supervisor-network/src/host.rs +++ b/crates/openshell-supervisor-network/src/host.rs @@ -48,7 +48,8 @@ pub struct HostProxyConfig { pub binary_path: PathBuf, pub sandbox_id: Option, pub sandbox_name: Option, - pub openshell_endpoint: Option, + /// Gateway access with this sandbox's credentials. + pub gateway: Option, pub provider_credentials: Option, /// Shared feature state for the policy.local agent proposal surface. pub agent_proposals: AgentProposals, @@ -99,7 +100,7 @@ pub async fn start_host_proxy(config: HostProxyConfig) -> Result Pin> + Send + 'a>>; } -/// Resolves provider tokens using the supervisor's authenticated grant flow. -#[derive(Default)] -pub struct SpiffeTokenGrantResolver; +/// Resolves provider tokens through one session's authenticated grant flow. +pub struct SpiffeTokenGrantResolver { + grants: crate::token_grant::SessionTokenGrants, +} + +impl SpiffeTokenGrantResolver { + #[must_use] + pub fn new(grants: crate::token_grant::SessionTokenGrants) -> Self { + Self { grants } + } +} impl TokenGrantResolver for SpiffeTokenGrantResolver { fn obtain<'a>( @@ -63,8 +71,8 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { request: TokenGrantRequest<'a>, ) -> Pin> + Send + 'a>> { Box::pin(async move { - crate::token_grant::obtain_provider_token( - crate::token_grant::ObtainProviderTokenRequest { + self.grants + .obtain_provider_token(crate::token_grant::ObtainProviderTokenRequest { provider_name: request.provider_key, token_endpoint: request.token_endpoint, jwt_svid_audience: request.jwt_svid_audience, @@ -75,18 +83,12 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { grant_type: request.grant_type, requested_token_type: request.requested_token_type, request_id: request.request_id, - }, - ) - .await + }) + .await }) } } -/// Construct the production SPIFFE-backed resolver. -pub fn default_resolver() -> Arc { - Arc::new(SpiffeTokenGrantResolver) -} - /// A refused request: the grant failure and the credential whose grant failed. /// /// The credential is the `provider:credential` name pair of the selected diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index c3bb88b4ed..a581800c15 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -3,7 +3,9 @@ //! Sandbox-local policy advisor HTTP API. +use crate::l7::token_grant_injection::TokenGrantResolver; use miette::{IntoDiagnostic, Result}; +use openshell_core::grpc_client::GatewayEndpoint; use openshell_core::proposals::AgentProposals; use openshell_core::proto::{ L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, PolicyChunk, @@ -82,50 +84,153 @@ const SHORTHAND_LOG_PREFIX: &str = "openshell."; /// log entry (very long URL path, etc.) cannot blow up the response. const MAX_DENIAL_LINE_BYTES: usize = 4096; -#[derive(Debug)] +/// Most recent denial lines retained for one hosted session. +const MAX_SESSION_DENIAL_LINES: usize = 256; + +/// Where `/v1/denials` reads this sandbox's recent denials. +#[derive(Clone, Debug)] +pub enum DenialLogSource { + /// Shorthand log files written by a supervisor process that serves one + /// sandbox. + ShorthandFiles(PathBuf), + /// Denials recorded for this session alone. A process hosting several + /// sandboxes cannot expose its shared log files to any one of them. + Session(SessionDenialLog), +} + +impl Default for DenialLogSource { + fn default() -> Self { + Self::ShorthandFiles(PathBuf::from(LOG_DIR)) + } +} + +/// Bounded recent OCSF denial lines recorded for one hosted session. +#[derive(Clone, Default)] +pub struct SessionDenialLog { + lines: Arc>>, +} + +impl std::fmt::Debug for SessionDenialLog { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let retained = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len(); + formatter + .debug_struct("SessionDenialLog") + .field("retained", &retained) + .finish() + } +} + +impl SessionDenialLog { + /// Retain `line` when it is a rendered OCSF denial; ignore anything else. + pub fn record(&self, line: &str) { + let line = line.trim_end(); + if !is_ocsf_denial_line(line) { + return; + } + let mut lines = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if lines.len() == MAX_SESSION_DENIAL_LINES { + lines.pop_front(); + } + lines.push_back(line.to_string()); + } + + fn recent(&self, limit: usize) -> Vec { + self.lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .rev() + .take(limit) + .map(|line| surfaced_denial_line(line)) + .collect() + } +} + pub struct PolicyLocalContext { current_policy: Arc>>>, agent_proposals: AgentProposals, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, - shorthand_log_dir: PathBuf, + denials: DenialLogSource, + token_grants: Option>, workspace_rx: tokio::sync::watch::Receiver, } +impl std::fmt::Debug for PolicyLocalContext { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PolicyLocalContext") + .field("gateway", &self.gateway) + .field("sandbox_name", &self.sandbox_name) + .field("denials", &self.denials) + .field("token_grants", &self.token_grants.is_some()) + .finish_non_exhaustive() + } +} + impl PolicyLocalContext { pub fn new( current_policy: Option, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, ) -> Self { - Self::with_log_dir( - current_policy, - gateway_endpoint, - sandbox_name, + Self { + current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))), agent_proposals, - PathBuf::from(LOG_DIR), + gateway, + sandbox_name, + denials: DenialLogSource::default(), + token_grants: None, workspace_rx, - ) + } } + #[cfg(test)] fn with_log_dir( current_policy: Option, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, agent_proposals: AgentProposals, shorthand_log_dir: PathBuf, workspace_rx: tokio::sync::watch::Receiver, ) -> Self { - Self { - current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))), - agent_proposals, - gateway_endpoint, + Self::new( + current_policy, + gateway, sandbox_name, - shorthand_log_dir, + agent_proposals, workspace_rx, - } + ) + .with_denial_log(DenialLogSource::ShorthandFiles(shorthand_log_dir)) + } + + /// Serve `/v1/denials` from `denials` instead of the process log files. + #[must_use] + pub fn with_denial_log(mut self, denials: DenialLogSource) -> Self { + self.denials = denials; + self + } + + /// Acquire provider token grants for this sandbox's HTTP relays. + #[must_use] + pub fn with_token_grants(mut self, resolver: Arc) -> Self { + self.token_grants = Some(resolver); + self + } + + /// The session's provider token grant resolver, absent when grants are + /// unavailable and must fail closed. + pub(crate) fn token_grant_resolver(&self) -> Option> { + self.token_grants.clone() } pub async fn set_current_policy(&self, policy: ProtoSandboxPolicy) { @@ -318,7 +423,18 @@ async fn recent_denials_response( query: &str, ) -> (u16, serde_json::Value) { let limit = parse_last_query(query).unwrap_or(DEFAULT_DENIALS_LIMIT); - let log_dir = ctx.shorthand_log_dir.clone(); + let log_dir = match &ctx.denials { + DenialLogSource::ShorthandFiles(log_dir) => log_dir.clone(), + DenialLogSource::Session(log) => { + return ( + 200, + serde_json::json!({ + "denials": log.recent(limit), + "log_available": true, + }), + ); + } + }; // Distinguish "shorthand log exists and no denials happened" from "no log // file yet, so we have nothing to read." Without this flag the agent sees @@ -398,9 +514,7 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec { // proxy.rs FORWARD path so the on-disk shorthand log itself is // clean — tracked separately). Redact first so truncation cannot // slice mid-secret. - let redacted = redact_query_strings(line); - let surfaced = truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES); - lines.push(surfaced); + lines.push(surfaced_denial_line(line)); if lines.len() >= limit { return lines; } @@ -409,6 +523,11 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec { lines } +fn surfaced_denial_line(line: &str) -> String { + let redacted = redact_query_strings(line); + truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES) +} + /// Replace any `?` substring with `?[redacted]` to keep query-string /// secrets out of the agent's view. Walks per Unicode scalar value so multi-byte /// content is safe. A query is everything from `?` until the next whitespace or @@ -486,7 +605,7 @@ fn collect_shorthand_log_files(log_dir: &Path, max_files: usize) -> std::io::Res } async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_json::Value) { - let Some(endpoint) = ctx.gateway_endpoint.as_deref() else { + let Some(gateway) = ctx.gateway.as_ref() else { return ( 503, serde_json::json!({ @@ -526,7 +645,7 @@ async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_j Err(error) => return (400, error_payload("invalid_proposal", error)), }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await { Ok(client) => { client.set_workspace(workspace); client @@ -956,7 +1075,7 @@ struct LookupSession<'a> { async fn open_lookup_session( ctx: &PolicyLocalContext, ) -> std::result::Result, (u16, serde_json::Value)> { - let endpoint = ctx.gateway_endpoint.as_deref().ok_or_else(|| { + let gateway = ctx.gateway.as_ref().ok_or_else(|| { ( 503, error_payload( @@ -989,7 +1108,7 @@ async fn open_lookup_session( ), )); } - let client = openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint) + let client = openshell_core::grpc_client::CachedOpenShellClient::connect(gateway) .await .map_err(|e| (502, error_payload("gateway_connect_failed", e.to_string())))?; client.set_workspace(workspace); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index ffdb4674d1..bd93c78c57 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1227,11 +1227,11 @@ async fn handle_transparent_tcp_connection( &decision, None, None, - agent_proposals, // The transparent TCP path carries no PolicyLocalContext, so no // workspace is available here; matches the CONNECT path default when // policy-local context is absent. - String::new(), + None, + agent_proposals, relay::RelaySignals { activity: activity_tx.clone(), endpoint_observation: None, @@ -3019,16 +3019,12 @@ async fn handle_mediated_connection( // gate needs it) and drives the raw-tunnel branch below. // Build request-processing context shared by CONNECT and forward HTTP. - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); let mut ctx = relay::http_context( &decision, provider_credentials, secret_resolver.clone(), + policy_local_ctx.as_deref(), agent_proposals, - workspace, relay::RelaySignals { activity: activity_tx.clone(), endpoint_observation: endpoint_observation_tx, @@ -5673,16 +5669,12 @@ async fn handle_forward_proxy( .map_or_else(std::collections::HashMap::new, |query| { crate::l7::rest::parse_query_params(query).unwrap_or_default() }); - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); let mut l7_ctx = relay::http_context( &decision, provider_credentials, secret_resolver.clone(), + policy_local_ctx.as_deref(), agent_proposals, - workspace, relay::RelaySignals { activity: activity_tx.cloned(), endpoint_observation: endpoint_observation_tx, diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index dad4b12633..05f685e122 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -47,8 +47,8 @@ pub(super) fn http_context( decision: &EgressDecision, provider_credentials: Option, secret_resolver: Option>, + policy_local: Option<&crate::policy_local::PolicyLocalContext>, agent_proposals: openshell_core::proposals::AgentProposals, - workspace: String, signals: RelaySignals, ) -> L7EvalContext { // Provider-backed credentials must be acquired from the live state for @@ -58,9 +58,14 @@ pub(super) fn http_context( .is_none() .then_some(secret_resolver) .flatten(); + // Grants use the owning session's resolver; without one they fail closed. let token_grant_resolver = provider_credentials .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()); + .and(policy_local) + .and_then(crate::policy_local::PolicyLocalContext::token_grant_resolver); + let workspace = policy_local + .map(crate::policy_local::PolicyLocalContext::workspace) + .unwrap_or_default(); let policy_name = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone().unwrap_or_default(), NetworkAction::Deny { .. } => String::new(), diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index a7647d39ad..34f6e116d2 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -35,10 +35,13 @@ use crate::l7::tls::{ CertCache, ProxyTlsState, SandboxCa, build_upstream_client_config, read_system_ca_bundle, write_ca_files, }; +use crate::l7::token_grant_injection::SpiffeTokenGrantResolver; use crate::opa::OpaEngine; -use crate::policy_local::PolicyLocalContext; +use crate::policy_local::{DenialLogSource, PolicyLocalContext}; use crate::proxy::ProxyHandle; +use crate::token_grant::SessionTokenGrants; use openshell_core::endpoint_status::EndpointObservationSender; +use openshell_core::grpc_client::GatewayEndpoint; /// Concrete listener options supplied by trusted supervisor composition. /// This is not an isolation-backend capability or a serialized launch contract. @@ -189,6 +192,52 @@ fn append_additional_ca_bundle( Ok(()) } +/// Identity and authority of the one sandbox a [`run_networking`] call serves. +/// +/// A process hosting several sandboxes builds one per session; no value is +/// read from the process environment, so sessions cannot share gateway +/// credentials, provider token grants, denial history, or proxy CA files. +#[derive(Debug, Default)] +pub struct NetworkSession { + pub sandbox_id: Option, + pub sandbox_name: Option, + /// Gateway access with this session's credentials. + pub gateway: Option, + /// SPIFFE Workload API socket for provider token grants. Without one, + /// grants fail closed. + pub provider_workload_api_socket: Option, + /// Source served by `policy.local` `/v1/denials`. + pub denial_log: DenialLogSource, + /// Proxy interception CA and trust-file directory. + pub proxy_tls: ProxyTlsFiles, +} + +/// Provisioned proxy CA files and the directory that receives trust files. +/// +/// Without a provisioned CA the proxy generates an ephemeral one. Without a +/// directory it writes to the default container TLS root. +#[derive(Clone, Debug, Default)] +pub struct ProxyTlsFiles { + pub ca_certificate: Option, + pub ca_private_key: Option, + pub directory: Option, +} + +impl ProxyTlsFiles { + /// Files configured for a single-sandbox supervisor through its environment. + #[must_use] + pub fn from_environment() -> Self { + Self { + ca_certificate: std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT) + .map(std::path::PathBuf::from), + ca_private_key: std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY) + .map(std::path::PathBuf::from), + directory: std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR) + .map(std::path::PathBuf::from), + } + } +} + /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. /// @@ -235,9 +284,7 @@ pub async fn run_networking( entrypoint_pid: Arc, process_enabled: bool, provider_credentials: &ProviderCredentialState, - sandbox_id: Option<&str>, - sandbox_name: Option<&str>, - openshell_endpoint: Option<&str>, + session: NetworkSession, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -245,7 +292,6 @@ pub async fn run_networking( workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, additional_ca_bundle: Option<&str>, - proxy_tls_dir: Option<&std::path::Path>, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, network_mediation_source: Option>, @@ -254,15 +300,30 @@ pub async fn run_networking( // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. - let policy_local_ctx = Arc::new(PolicyLocalContext::new( - retained_proto.cloned(), - openshell_endpoint.map(str::to_string), - sandbox_name - .map(str::to_string) - .or_else(|| sandbox_id.map(str::to_string)), - agent_proposals.clone(), - workspace_rx, + let NetworkSession { + sandbox_id, + sandbox_name, + gateway, + provider_workload_api_socket, + denial_log, + proxy_tls, + } = session; + let token_grants = SpiffeTokenGrantResolver::new(SessionTokenGrants::new( + gateway.clone(), + sandbox_id.clone(), + provider_workload_api_socket, )); + let policy_local_ctx = Arc::new( + PolicyLocalContext::new( + retained_proto.cloned(), + gateway, + sandbox_name.or(sandbox_id), + agent_proposals.clone(), + workspace_rx, + ) + .with_denial_log(denial_log) + .with_token_grants(Arc::new(token_grants)), + ); // Readiness signal for the proxy accept loop: the proxy binds the TCP // listener immediately (so the OS backlog queues early SYN packets) but @@ -370,14 +431,10 @@ pub async fn run_networking( // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - let configured_ca = match ( - std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), - std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), - ) { - (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( - std::path::Path::new(&certificate), - std::path::Path::new(&private_key), - )?), + let configured_ca = match (&proxy_tls.ca_certificate, &proxy_tls.ca_private_key) { + (Some(certificate), Some(private_key)) => { + Some(SandboxCa::load_from_paths(certificate, private_key)?) + } (None, None) => None, _ => { return Err(miette::miette!( @@ -390,14 +447,9 @@ pub async fn run_networking( let durable_ca = configured_ca.is_some(); match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { - let configured_tls_dir = - std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR) - .map(std::path::PathBuf::from); - let tls_dir = proxy_tls_dir - .or(configured_tls_dir.as_deref()) - .unwrap_or_else(|| { - std::path::Path::new(openshell_core::container_paths::TLS_ROOT) - }); + let tls_dir = proxy_tls.directory.as_deref().unwrap_or_else(|| { + std::path::Path::new(openshell_core::container_paths::TLS_ROOT) + }); let mut system_ca_bundle = read_system_ca_bundle(); // A TLS-intercepting corporate proxy (issue #1792) re-signs // tunneled server certificates with the corporate CA, so the diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index e165754105..05331e128d 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -27,32 +27,30 @@ //! - `scopes` — `OAuth2` scopes to request (optional) //! - `cache_ttl` — Optional exact cache override; zero disables caching //! -//! ## Environment +//! ## Session scope //! -//! Requires `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to be set (path to -//! the SPIFFE Workload API socket). +//! Each supervised sandbox owns a [`SessionTokenGrants`]: its token cache, +//! HTTP client, gateway credentials, sandbox ID, and SPIFFE Workload API +//! socket. A token is never served to another sandbox. Without a configured +//! Workload API socket, grants fail closed with a configuration failure. use std::collections::HashMap; use std::future::Future; -use std::sync::{Arc, LazyLock, RwLock}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use openshell_core::grpc_client::GatewayEndpoint; use openshell_core::oauth::{ self, ACCESS_TOKEN_TYPE, OAuthTokenResponse, TokenExchangeParams, TokenGrantClient, TokenGrantParams, effective_client_assertion_type, effective_token_type, }; use openshell_core::proto::ProviderCredentialTokenGrantType; -use openshell_core::sandbox_env; use openshell_core::token_grant_failure::{ TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, }; use spiffe::{WorkloadApiClient, WorkloadApiError}; -/// Token cache shared across all provider token grants. -static TOKEN_CACHE: LazyLock = LazyLock::new(TokenCache::new); -// Initialization failures stay bounded and typed; credential acquisition must not panic. -static TOKEN_GRANT_HTTP_CLIENT: LazyLock> = - LazyLock::new(TokenGrantClient::new); const DEFAULT_TOKEN_CACHE_TTL_SECONDS: i64 = 300; const TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; const MAX_TOKEN_EXPIRES_IN_SECONDS: i64 = 3600; @@ -102,33 +100,154 @@ impl TokenCache { } } -/// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. +/// Provider token grants owned by one supervised sandbox. /// -/// This function fetches the sandbox's SPIFFE JWT-SVID from the local SPIRE -/// agent, then exchanges it for an access token with a POST request to the provider's -/// token endpoint with the JWT client assertion grant flow (RFC 7523). -/// -/// Tokens are cached per provider name with TTL. Subsequent calls return the -/// cached token if it has not expired. -/// -/// # Arguments -/// -/// * `provider_name` — Unique provider identifier (used as cache key) -/// * `token_endpoint` — `OAuth2` token service URL -/// * `jwt_svid_audience` — Optional audience to request when fetching the JWT-SVID -/// * `client_assertion_type` — Optional `OAuth2` client assertion type -/// * `audience` — Resource audience to request in the token request -/// * `scopes` — `OAuth2` scopes to request (may be empty) -/// * `cache_ttl_override` — Exact cache TTL; absence uses `expires_in` and zero disables caching -/// -/// # Errors -/// -/// Returns an error if: -/// - SPIFFE Workload API socket is not configured -/// - SPIRE agent is unreachable -/// - JWT-SVID fetch fails -/// - Token service request fails -/// - Token response is invalid +/// The cache, HTTP connection pool, gateway credentials, sandbox ID, and +/// workload identity socket all belong to this session, so a token acquired +/// for one sandbox is never served to another. +pub struct SessionTokenGrants { + cache: TokenCache, + // Initialization failures stay bounded and typed; acquisition must not panic. + http_client: Result, + gateway: Option, + sandbox_id: Option, + workload_api_socket: Option, +} + +impl SessionTokenGrants { + /// Grants authenticated as `sandbox_id` through `gateway`. A missing + /// Workload API socket makes every grant fail closed. + #[must_use] + pub fn new( + gateway: Option, + sandbox_id: Option, + workload_api_socket: Option, + ) -> Self { + Self { + cache: TokenCache::new(), + http_client: TokenGrantClient::new(), + gateway, + sandbox_id: sandbox_id.filter(|value| !value.trim().is_empty()), + workload_api_socket: workload_api_socket.filter(|path| !path.as_os_str().is_empty()), + } + } + + /// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. + /// + /// This fetches the sandbox's SPIFFE JWT-SVID from the session's Workload + /// API socket, then exchanges it for an access token with a POST request to + /// the provider's token endpoint with the JWT client assertion grant flow + /// (RFC 7523). + /// + /// Tokens are cached per provider name with TTL in this session's cache. + /// Subsequent calls return the cached token if it has not expired. + /// + /// # Errors + /// + /// Returns an error if: + /// - SPIFFE Workload API socket is not configured + /// - SPIRE agent is unreachable + /// - JWT-SVID fetch fails + /// - Token service request fails + /// - Token response is invalid + pub async fn obtain_provider_token( + &self, + request: ObtainProviderTokenRequest<'_>, + ) -> Result { + let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) + .map_err(|_| configuration_failure())?; + obtain_provider_token_with_grant( + ObtainProviderTokenInput { + cache: &self.cache, + provider_name: request.provider_name, + token_endpoint: request.token_endpoint, + jwt_svid_audience: request.jwt_svid_audience, + client_assertion_type: request.client_assertion_type, + audience: request.audience, + scopes: request.scopes, + cache_ttl_override: request.cache_ttl_override, + grant_type, + requested_token_type: request.requested_token_type, + }, + |jwt_audience| async move { + // Resolve local exchange prerequisites before fetching any workload credential. + let exchange_context = match grant_type { + ProviderCredentialTokenGrantType::ClientCredentials + | ProviderCredentialTokenGrantType::Unspecified => None, + ProviderCredentialTokenGrantType::TokenExchange => { + let (provider, credential_key) = + parse_provider_credential_key(request.provider_name)?; + let gateway = self.gateway.as_ref().ok_or_else(configuration_failure)?; + openshell_core::grpc_client::token_grant_gateway_endpoint(gateway.url())?; + let sandbox_id = self + .sandbox_id + .as_deref() + .ok_or_else(configuration_failure)?; + Some((provider, credential_key, gateway, sandbox_id)) + } + }; + let http_client = self.http_client.as_ref().map_err(|failure| *failure)?; + let socket_path = self + .workload_api_socket + .as_deref() + .ok_or_else(configuration_failure)?; + // Fetch JWT-SVID with authorization server as audience + // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm + let jwt_svid = fetch_jwt_svid_from_socket(socket_path, &jwt_audience).await?; + + match exchange_context { + None => { + // Perform OAuth2 JWT client assertion grant. The audience + // parameter in the token request specifies the resource server. + perform_token_grant( + http_client, + request.token_endpoint, + &jwt_svid, + request.client_assertion_type, + request.audience, + request.scopes, + request.request_id, + ) + .await + } + Some((provider, credential_key, gateway, sandbox_id)) => { + let intermediate = + openshell_core::grpc_client::exchange_provider_subject_token( + gateway, + sandbox_id, + provider, + credential_key, + &jwt_svid, + Some(request.request_id), + ) + .await?; + validate_access_token(&intermediate.access_token).map_err(|failure| { + failure.at_stage(TokenGrantStage::IntermediateExchange) + })?; + perform_token_exchange( + http_client, + request.token_endpoint, + &TokenExchangeParams { + client_assertion: &jwt_svid, + client_assertion_type: request.client_assertion_type, + subject_token: &intermediate.access_token, + subject_token_type: ACCESS_TOKEN_TYPE, + audience: request.audience, + scopes: request.scopes, + requested_token_type: request.requested_token_type, + }, + request.request_id, + ) + .await + } + } + }, + ) + .await + } +} + +/// One provider token request; see [`SessionTokenGrants::obtain_provider_token`]. pub struct ObtainProviderTokenRequest<'a> { pub provider_name: &'a str, pub token_endpoint: &'a str, @@ -144,96 +263,6 @@ pub struct ObtainProviderTokenRequest<'a> { pub request_id: &'a str, } -/// Obtain and cache a provider token, retaining only bounded grant failures. -pub async fn obtain_provider_token( - request: ObtainProviderTokenRequest<'_>, -) -> Result { - let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) - .map_err(|_| configuration_failure())?; - obtain_provider_token_with_grant( - ObtainProviderTokenInput { - cache: &TOKEN_CACHE, - provider_name: request.provider_name, - token_endpoint: request.token_endpoint, - jwt_svid_audience: request.jwt_svid_audience, - client_assertion_type: request.client_assertion_type, - audience: request.audience, - scopes: request.scopes, - cache_ttl_override: request.cache_ttl_override, - grant_type, - requested_token_type: request.requested_token_type, - }, - |jwt_audience| async move { - // Resolve local exchange prerequisites before fetching any workload credential. - let exchange_context = match grant_type { - ProviderCredentialTokenGrantType::ClientCredentials - | ProviderCredentialTokenGrantType::Unspecified => None, - ProviderCredentialTokenGrantType::TokenExchange => { - let (provider, credential_key) = - parse_provider_credential_key(request.provider_name)?; - let endpoint = supervisor_gateway_endpoint_from_env()?; - openshell_core::grpc_client::token_grant_gateway_endpoint(&endpoint)?; - Some(( - provider, - credential_key, - endpoint, - supervisor_sandbox_id_from_env()?, - )) - } - }; - // Fetch JWT-SVID with authorization server as audience - // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm - let jwt_svid = fetch_jwt_svid_for_token_grant(&jwt_audience).await?; - - match exchange_context { - None => { - // Perform OAuth2 JWT client assertion grant. The audience - // parameter in the token request specifies the resource server. - perform_token_grant( - request.token_endpoint, - &jwt_svid, - request.client_assertion_type, - request.audience, - request.scopes, - request.request_id, - ) - .await - } - Some((provider, credential_key, endpoint, sandbox_id)) => { - let intermediate = - openshell_core::grpc_client::exchange_provider_subject_token( - &endpoint, - &sandbox_id, - provider, - credential_key, - &jwt_svid, - Some(request.request_id), - ) - .await?; - validate_access_token(&intermediate.access_token).map_err(|failure| { - failure.at_stage(TokenGrantStage::IntermediateExchange) - })?; - perform_token_exchange( - request.token_endpoint, - &TokenExchangeParams { - client_assertion: &jwt_svid, - client_assertion_type: request.client_assertion_type, - subject_token: &intermediate.access_token, - subject_token_type: ACCESS_TOKEN_TYPE, - audience: request.audience, - scopes: request.scopes, - requested_token_type: request.requested_token_type, - }, - request.request_id, - ) - .await - } - } - }, - ) - .await -} - struct ObtainProviderTokenInput<'a> { cache: &'a TokenCache, provider_name: &'a str, @@ -297,21 +326,14 @@ where Ok(token_response.access_token) } -/// Fetch JWT-SVID from SPIRE agent for token grant authentication. +/// Fetch a JWT-SVID from the SPIRE agent for token grant authentication. /// -/// This function connects to the local SPIRE agent via the Workload API and -/// requests a JWT-SVID with the specified audience. The JWT-SVID is used as -/// the client assertion in the `OAuth2` grant request. -async fn fetch_jwt_svid_for_token_grant(audience: &str) -> Result { - let socket_path = provider_spiffe_workload_api_socket_from_env()?; - fetch_jwt_svid_from_socket(&socket_path, audience).await -} - +/// The JWT-SVID is used as the client assertion in the `OAuth2` grant request. async fn fetch_jwt_svid_from_socket( - socket_path: &str, + socket_path: &std::path::Path, audience: &str, ) -> Result { - let endpoint = openshell_core::spiffe::workload_api_endpoint(std::path::Path::new(socket_path)); + let endpoint = openshell_core::spiffe::workload_api_endpoint(socket_path); // Connect to SPIRE agent let client = WorkloadApiClient::connect_to(&endpoint) @@ -357,14 +379,8 @@ fn configuration_failure() -> TokenGrantFailure { ) } -fn provider_spiffe_workload_api_socket_from_env() -> Result { - std::env::var(sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - async fn perform_token_grant( + http_client: &TokenGrantClient, token_endpoint: &str, jwt_svid: &str, client_assertion_type: &str, @@ -373,9 +389,7 @@ async fn perform_token_grant( request_id: &str, ) -> Result { oauth::post_oauth_token_grant( - TOKEN_GRANT_HTTP_CLIENT - .as_ref() - .map_err(|failure| *failure)?, + http_client, token_endpoint, &TokenGrantParams { client_assertion: jwt_svid, @@ -389,19 +403,12 @@ async fn perform_token_grant( } async fn perform_token_exchange( + http_client: &TokenGrantClient, token_endpoint: &str, params: &TokenExchangeParams<'_>, request_id: &str, ) -> Result { - oauth::post_oauth_token_exchange( - TOKEN_GRANT_HTTP_CLIENT - .as_ref() - .map_err(|failure| *failure)?, - token_endpoint, - params, - Some(request_id), - ) - .await + oauth::post_oauth_token_exchange(http_client, token_endpoint, params, Some(request_id)).await } pub use oauth::validate_access_token; @@ -457,20 +464,6 @@ fn effective_jwt_svid_audience(token_endpoint: &str, jwt_svid_audience: &str) -> } } -fn supervisor_gateway_endpoint_from_env() -> Result { - std::env::var(sandbox_env::ENDPOINT) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - -fn supervisor_sandbox_id_from_env() -> Result { - std::env::var(sandbox_env::SANDBOX_ID) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - fn parse_provider_credential_key(key: &str) -> Result<(&str, &str), TokenGrantFailure> { openshell_core::dynamic_credential_key::credential_identity(key) .and_then(|identity| identity.split_once(':')) @@ -670,6 +663,10 @@ mod tests { const TEST_REQUEST_ID: &str = "4f1c2b3a-9d8e-4c7b-a6f5-e4d3c2b1a098"; + fn test_http_client() -> TokenGrantClient { + TokenGrantClient::new().expect("token grant HTTP client") + } + #[derive(Debug)] struct CapturedTokenRequest { request_line: String, @@ -1415,6 +1412,7 @@ mod tests { let scopes = vec!["read".to_string(), "write".to_string()]; let response = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "", @@ -1475,6 +1473,7 @@ mod tests { token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; let response = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "urn:ietf:params:oauth:client-assertion-type:jwt-spiffe", @@ -1504,9 +1503,17 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("malformed access token should fail closed"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("malformed access token should fail closed"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(200)); @@ -1516,9 +1523,17 @@ mod tests { async fn perform_token_grant_does_not_follow_redirects() { let (endpoint, request) = token_endpoint_redirect_once("http://127.0.0.1:1/stolen").await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("redirect response should fail closed"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("redirect response should fail closed"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(302)); @@ -1529,10 +1544,17 @@ mod tests { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; - let response = - perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect("token grant should succeed without audience or scopes"); + let response = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect("token grant should succeed without audience or scopes"); let request = request.await.expect("token endpoint task should finish"); assert_eq!(response.access_token, "access-123"); @@ -1553,6 +1575,7 @@ mod tests { .await; let err = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "", @@ -1580,9 +1603,17 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("token grant should fail on server error"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("token grant should fail on server error"); let _request = request.await.expect("token endpoint task should finish"); let message = err.to_string(); @@ -1597,9 +1628,17 @@ mod tests { async fn perform_token_grant_reports_malformed_success_json() { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":42"#).await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("malformed token response should fail"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("malformed token response should fail"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(200)); diff --git a/crates/openshell-supervisor-process/src/debug_rpc.rs b/crates/openshell-supervisor-process/src/debug_rpc.rs index 04cd1e9ba7..b4370c8954 100644 --- a/crates/openshell-supervisor-process/src/debug_rpc.rs +++ b/crates/openshell-supervisor-process/src/debug_rpc.rs @@ -24,7 +24,9 @@ use openshell_core::proto::{ }; use sha2::{Digest, Sha256}; -use openshell_core::grpc_client::{AuthedChannel, connect_channel_pub}; +use openshell_core::grpc_client::{ + AuthedChannel, GatewayCredentials, GatewayEndpoint, connect_channel_pub, +}; /// Entry point for the `debug-rpc` subcommand. Returns the process exit /// code; `main` propagates it. @@ -66,7 +68,8 @@ async fn open_client() -> Result> { let endpoint = std::env::var(openshell_core::sandbox_env::ENDPOINT) .into_diagnostic() .wrap_err("OPENSHELL_ENDPOINT must be set")?; - let channel = connect_channel_pub(&endpoint).await?; + let credentials = GatewayCredentials::from_environment(&endpoint).await?; + let channel = connect_channel_pub(&GatewayEndpoint::new(endpoint, credentials)).await?; Ok(OpenShellClient::new(channel)) } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 91e1fa5337..32e3fd4433 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -82,7 +82,7 @@ impl Drop for BoundaryAccess { #[allow(clippy::too_many_arguments)] pub async fn start_boundary_access( sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, + gateway: Option<&openshell_core::grpc_client::GatewayEndpoint>, ssh_socket_path: Option<&str>, shared_ssh_socket: bool, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -116,10 +116,10 @@ pub async fn start_boundary_access( // Gateway authentication, forwarding, and readiness do not depend on SSH // or the host OS. Acceptance remains false until the gateway authenticates // this supervisor, and reconnects retain main's retry behavior. - let (session_task, session_readiness) = match (openshell_endpoint, sandbox_id) { - (Some(endpoint), Some(id)) => { + let (session_task, session_readiness) = match (gateway, sandbox_id) { + (Some(gateway), Some(id)) => { let (task, accepted) = crate::supervisor_session::spawn_with_readiness( - endpoint.to_string(), + gateway.clone(), id.to_string(), ssh_socket_path.unwrap_or_default(), port_forward, @@ -223,7 +223,7 @@ async fn start_optional_ssh( /// Report the canonical process exit until the gateway acknowledges it. pub async fn report_main_process_exit( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, exit_code: i32, @@ -231,7 +231,7 @@ pub async fn report_main_process_exit( let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::report_main_process_exit( - endpoint, + gateway, sandbox_id, instance_id, exit_code, @@ -249,11 +249,15 @@ pub async fn report_main_process_exit( } /// Finalize canonical process terminal delivery until acknowledged. -pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { +pub async fn finalize_main_process_exit( + gateway: &openshell_core::grpc_client::GatewayEndpoint, + sandbox_id: &str, + instance_id: &str, +) { let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::finalize_main_process_exit( - endpoint, + gateway, sandbox_id, instance_id, ) @@ -352,9 +356,14 @@ mod tests { #[tokio::test] async fn no_ssh_access_retains_main_attachment_and_session_readiness() { let boundary = Arc::new(AccessBoundary); + let gateway = openshell_core::grpc_client::GatewayEndpoint::new( + "http://127.0.0.1:1", + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .expect("test credentials"), + ); let access = start_boundary_access( Some("sandbox"), - Some("http://127.0.0.1:1"), + Some(&gateway), None, false, None, diff --git a/crates/openshell-supervisor-process/src/log_push.rs b/crates/openshell-supervisor-process/src/log_push.rs index d12061e82e..930ef87d5d 100644 --- a/crates/openshell-supervisor-process/src/log_push.rs +++ b/crates/openshell-supervisor-process/src/log_push.rs @@ -7,32 +7,78 @@ //! channel to a background task. The task batches lines and streams them to //! the server using the `PushSandboxLogs` client-streaming RPC. -use openshell_core::grpc_client::CachedOpenShellClient; +use openshell_core::grpc_client::{CachedOpenShellClient, GatewayEndpoint}; use openshell_core::proto::{PushSandboxLogsRequest, SandboxLogLine}; +use std::cell::RefCell; +use std::sync::Arc; use tokio::sync::mpsc; use tracing::{Event, Subscriber}; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; +/// One sandbox's push queue. Lines are labelled with this sandbox and sent +/// only with the credentials of the task that drains the queue. +#[derive(Clone)] +pub struct LogSink { + sandbox_id: Arc, + tx: mpsc::Sender, +} + +impl LogSink { + #[must_use] + pub fn new(sandbox_id: impl Into>, tx: mpsc::Sender) -> Self { + Self { + sandbox_id: sandbox_id.into(), + tx, + } + } +} + +std::thread_local! { + /// Sink of the hosted session whose runtime owns this thread. + static THREAD_SINK: RefCell> = const { RefCell::new(None) }; +} + +/// Route events emitted on the calling thread to a hosted session's sink. +/// +/// Hosted sessions bind every thread of their runtime. Unbound threads push +/// nothing, so an event cannot reach another sandbox's log stream. +pub fn bind_thread_sink(sink: Option) { + THREAD_SINK.with(|current| *current.borrow_mut() = sink); +} + +#[derive(Clone)] +enum Route { + /// The process supervises one sandbox; every event belongs to it. + Fixed(LogSink), + /// Events belong to the session bound to the emitting thread, if any. + ThreadBound, +} + /// Tracing layer that pushes log events to the `OpenShell` server. /// /// Events are sent best-effort via `try_send` — if the channel is full the /// event is dropped. Logging must never block the sandbox. #[derive(Clone)] pub struct LogPushLayer { - sandbox_id: String, - tx: mpsc::Sender, + route: Route, max_level: tracing::Level, } impl LogPushLayer { pub fn new(sandbox_id: String, tx: mpsc::Sender) -> Self { + Self::with_route(Route::Fixed(LogSink::new(sandbox_id, tx))) + } + + /// Push each event to the sink bound to its emitting thread. + #[must_use] + pub fn thread_bound() -> Self { + Self::with_route(Route::ThreadBound) + } + + fn with_route(route: Route) -> Self { let max_level = parse_max_level(std::env::var("OPENSHELL_LOG_PUSH_LEVEL").ok().as_deref()); - Self { - sandbox_id, - tx, - max_level, - } + Self { route, max_level } } } @@ -50,62 +96,83 @@ impl Layer for LogPushLayer { if *meta.level() > self.max_level { return; } + match &self.route { + Route::Fixed(sink) => push_event(sink, event), + Route::ThreadBound => THREAD_SINK.with(|sink| { + if let Some(sink) = sink.borrow().as_ref() { + push_event(sink, event); + } + }), + } + } +} - // OCSF events carry their payload in a thread-local; extract the - // shorthand representation for the push message. Non-OCSF events - // use the original visitor-based extraction. - let (msg, fields) = if meta.target() == openshell_ocsf::OCSF_TARGET { - if let Some(ocsf_event) = openshell_ocsf::clone_current_event() { - ( - ocsf_event.format_shorthand(), - std::collections::HashMap::new(), - ) - } else { - return; - } +fn push_event(sink: &LogSink, event: &Event<'_>) { + let meta = event.metadata(); + // OCSF events carry their payload in a thread-local; extract the + // shorthand representation for the push message. Non-OCSF events + // use the original visitor-based extraction. + let (msg, fields) = if meta.target() == openshell_ocsf::OCSF_TARGET { + if let Some(ocsf_event) = openshell_ocsf::clone_current_event() { + ( + ocsf_event.format_shorthand(), + std::collections::HashMap::new(), + ) } else { - let mut visitor = LogVisitor::default(); - event.record(&mut visitor); - visitor.into_parts(meta.name()) - }; + return; + } + } else { + let mut visitor = LogVisitor::default(); + event.record(&mut visitor); + visitor.into_parts(meta.name()) + }; - let ts = openshell_core::time::now_ms(); + let ts = openshell_core::time::now_ms(); - let is_ocsf = meta.target() == openshell_ocsf::OCSF_TARGET; + let is_ocsf = meta.target() == openshell_ocsf::OCSF_TARGET; - let log = SandboxLogLine { - sandbox_id: self.sandbox_id.clone(), - event_time: openshell_core::time::timestamp_from_millis(ts).ok(), - level: if is_ocsf { - "OCSF".to_string() - } else { - meta.level().to_string() - }, - target: meta.target().to_string(), - message: msg, - source: "sandbox".to_string(), - fields, - }; + let log = SandboxLogLine { + sandbox_id: sink.sandbox_id.to_string(), + event_time: openshell_core::time::timestamp_from_millis(ts).ok(), + level: if is_ocsf { + "OCSF".to_string() + } else { + meta.level().to_string() + }, + target: meta.target().to_string(), + message: msg, + source: "sandbox".to_string(), + fields, + }; - // Best-effort: drop if the channel is full (don't block tracing). - let _ = self.tx.try_send(log); - } + // Best-effort: drop if the channel is full (don't block tracing). + let _ = sink.tx.try_send(log); } +/// Lines a sandbox's push queue holds before the layer starts dropping them. +pub const LOG_PUSH_QUEUE_LINES: usize = 1024; + /// Spawn a background task that batches and pushes log lines to the server. /// /// Returns the sender half of the channel (for the [`LogPushLayer`]) and the /// task handle. The task runs until the sender is dropped or the gRPC stream /// breaks. pub fn spawn_log_push_task( - endpoint: String, + gateway: GatewayEndpoint, sandbox_id: String, ) -> (mpsc::Sender, tokio::task::JoinHandle<()>) { - let (tx, rx) = mpsc::channel::(1024); - - let handle = tokio::spawn(run_push_loop(endpoint, sandbox_id, rx)); + let (tx, rx) = mpsc::channel::(LOG_PUSH_QUEUE_LINES); + (tx, start_log_push(gateway, sandbox_id, rx)) +} - (tx, handle) +/// Push lines from a queue created before the caller's runtime, with the +/// owning sandbox's gateway credentials. +pub fn start_log_push( + gateway: GatewayEndpoint, + sandbox_id: String, + lines: mpsc::Receiver, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(run_push_loop(gateway, sandbox_id, lines)) } /// Maximum backoff delay between reconnection attempts. @@ -114,7 +181,7 @@ const MAX_BACKOFF: tokio::time::Duration = tokio::time::Duration::from_secs(30); const INITIAL_BACKOFF: tokio::time::Duration = tokio::time::Duration::from_secs(1); async fn run_push_loop( - endpoint: String, + gateway: GatewayEndpoint, sandbox_id: String, mut rx: mpsc::Receiver, ) { @@ -127,7 +194,7 @@ async fn run_push_loop( attempt += 1; // --- Connect --- - let client = match CachedOpenShellClient::connect(&endpoint).await { + let client = match CachedOpenShellClient::connect(&gateway).await { Ok(c) => { if attempt > 1 { eprintln!("openshell: log push reconnected (attempt {attempt})"); @@ -339,8 +406,7 @@ mod tests { fn capture(capacity: usize, f: impl FnOnce()) -> Vec { let (tx, mut rx) = mpsc::channel::(capacity); let layer = LogPushLayer { - sandbox_id: "sb-test".to_string(), - tx, + route: Route::Fixed(LogSink::new("sb-test", tx)), max_level: tracing::Level::INFO, }; let subscriber = tracing_subscriber::registry().with(layer); diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 77620a94d1..71313e19c0 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -381,7 +381,7 @@ fn map_session_stream_message( /// The task runs for the lifetime of the sandbox process, reconnecting with /// exponential backoff on failures. pub fn spawn( - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -390,7 +390,7 @@ pub fn spawn( runtime: SessionRuntimeContext, ) -> tokio::task::JoinHandle<()> { spawn_with_readiness( - endpoint, + gateway, sandbox_id, ssh_socket_path, port_forward, @@ -403,7 +403,7 @@ pub fn spawn( /// Spawn the supervisor session and expose when the gateway has accepted it. pub fn spawn_with_readiness( - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -413,7 +413,7 @@ pub fn spawn_with_readiness( ) -> (tokio::task::JoinHandle<()>, watch::Receiver) { let (ready_tx, ready_rx) = watch::channel(false); let config = SessionConfig { - endpoint, + gateway, sandbox_id, ssh_socket_path, port_forward, @@ -429,7 +429,7 @@ pub fn spawn_with_readiness( } struct SessionConfig { - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -453,7 +453,7 @@ async fn run_session_loop(config: SessionConfig) { // dial `target` for the next attempt and mark it as redirected so that // replica serves us rather than redirecting again. Any failure sends us // back to the configured address, which load-balances across replicas. - let mut target = config.endpoint.clone(); + let mut target = config.gateway.clone(); let mut redirected = false; loop { @@ -466,8 +466,11 @@ async fn run_session_loop(config: SessionConfig) { match result { Ok(SessionOutcome::Closed) => { config.ready_tx.send_replace(false); - let event = - session_closed_event(&openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); + let event = session_closed_event( + &openshell_ocsf::ctx::ctx(), + target.url(), + &config.sandbox_id, + ); ocsf_emit!(event); break; } @@ -482,7 +485,7 @@ async fn run_session_loop(config: SessionConfig) { peer_endpoint = %peer_endpoint, "supervisor session: following gateway redirect" ); - target = peer_endpoint; + target = config.gateway.with_url(peer_endpoint); redirected = true; // No backoff: this is an expected handoff, not a failure. } @@ -490,15 +493,15 @@ async fn run_session_loop(config: SessionConfig) { let accepted = config.ready_tx.send_replace(false); let event = session_failed_event( &openshell_ocsf::ctx::ctx(), - &target, + target.url(), attempt, &e.to_string(), ); ocsf_emit!(event); // Fall back to the configured gateway address so a redirect // to a replica that is going away cannot strand the sandbox. - let failed_redirect_target = target != config.endpoint && !accepted; - target.clone_from(&config.endpoint); + let failed_redirect_target = target.url() != config.gateway.url() && !accepted; + target.clone_from(&config.gateway); redirected = redirect_survives_failure(redirected, accepted); if let Some(delay) = backoff.on_failure(accepted, failed_redirect_target) { tokio::time::sleep(delay).await; @@ -580,7 +583,7 @@ enum SessionOutcome { async fn run_single_session( config: &SessionConfig, - target: &str, + target: &grpc_client::GatewayEndpoint, redirected: bool, connection_epoch: u64, ) -> Result> { @@ -668,7 +671,7 @@ async fn run_single_session( } let event = session_established_event( &openshell_ocsf::ctx::ctx(), - target, + target.url(), &accepted.session_id, u32::try_from(heartbeat_secs).unwrap_or(u32::MAX), ); @@ -807,12 +810,12 @@ async fn apply_update( /// Report the canonical process result and wait for durable handling. pub async fn report_main_process_exit( - endpoint: &str, + gateway: &grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, exit_code: i32, ) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) + let channel = grpc_client::connect_channel_pub(gateway) .await .map_err(|error| format!("connect failed: {error}"))?; let mut client = OpenShellClient::new(channel); @@ -844,11 +847,11 @@ pub(crate) async fn test_bridge_ssh_relay( /// Confirm terminal delivery and permit ephemeral cleanup. pub async fn finalize_main_process_exit( - endpoint: &str, + gateway: &grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, ) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) + let channel = grpc_client::connect_channel_pub(gateway) .await .map_err(|error| format!("connect failed: {error}"))?; let mut client = OpenShellClient::new(channel); diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 8108c6ea03..864a24a9d9 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -22,6 +22,24 @@ only after boundary attachment and confirmation; the supervisor owns policy evaluation, live credentials, and listener lifetime. These settings do not add a proxy hook to the isolation interface or the shared Sandbox Protocol descriptor. +## Session ownership + +`run_session` supervises one sandbox generation from a `SessionConfig` and +`SessionControls`. Everything that belongs to that generation is an input or +is created by the session: gateway credentials and their renewal, the Sandbox +Protocol credential, policy, provider credentials and token grants, middleware +credentials, OCSF JSON settings, proxy trust files, the denial history served +by `policy.local`, the backend connection, access plane, readiness, and stop +signal. Gateway clients receive a `GatewayEndpoint`, which pairs the URL with +that session's `GatewayCredentials`; nothing is installed process-wide. + +`run_sandbox_with_backend` is the dedicated wrapper. It keeps the existing +process contract: it sets the process-wide OCSF context, reads launch inputs +from the environment, binds the readiness socket and port to session +readiness, and stops the session on SIGTERM or SIGINT. The binary shares one +`GatewayCredentials` between log push and supervision, so renewal by either +keeps both current. + ## Console logging Diagnostics and OCSF shorthand share a bounded, nonblocking stderr writer. With `ocsf_json_enabled=true`, the same writer also receives timestamped `OCSF-JSON` records. Each formatter submits a complete line in one write so concurrent producers cannot interleave records in the queue. The 1,024-line queue drops new lines when full rather than waiting for stderr. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index c291b42ef0..b9b5c1cdc9 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -702,6 +702,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { additional_ca_bundle: None, backend_descriptor: TestSetup::descriptor(), auth_bundle: setup.auth.clone(), + gateway_credentials: None, admitted_isolation_backend: Some(TEST_BACKEND.into()), main_exit_marker: Some(marker.clone()), }, diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 62e7ffb6d4..1a83fd0887 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -738,7 +738,7 @@ impl ConfigRuntime { ); let provider = match client .fetch_provider_environment( - &ctx.endpoint, + &ctx.gateway, &ctx.sandbox_id, ctx.provider_credentials.requires_stable_placeholders(), ) @@ -1159,7 +1159,7 @@ impl ConfigRuntime { pub async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - &ctx.endpoint, + &ctx.gateway, ctx.extension_credentials.clone(), ) .await?; @@ -1198,7 +1198,7 @@ async fn report_runtime_configuration( ); } openshell_core::grpc_client::report_sandbox_configuration( - &ctx.endpoint, + &ctx.gateway, &ctx.sandbox_id, instance_id, Some(snapshot), @@ -1618,6 +1618,7 @@ mod tests { use super::*; use crate::tests::{ proto_policy_fixture, proto_tcp_policy_fixture, settings_poll_result, startup_provider, + test_gateway, }; use crate::*; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1779,7 +1780,7 @@ mod tests { async fn fetch_provider_environment( &self, - _endpoint: &str, + _gateway: &openshell_core::grpc_client::GatewayEndpoint, _sandbox_id: &str, _requires_stable_placeholders: bool, ) -> Result { @@ -1826,12 +1827,12 @@ mod tests { async fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { self.inner - .fetch_provider_environment(endpoint, sandbox_id, requires_stable_placeholders) + .fetch_provider_environment(gateway, sandbox_id, requires_stable_placeholders) .await } @@ -2074,7 +2075,7 @@ mod tests { async fn fetch_provider_environment( &self, - _endpoint: &str, + _gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { @@ -2212,12 +2213,12 @@ mod tests { async fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { self.inner - .fetch_provider_environment(endpoint, sandbox_id, requires_stable_placeholders) + .fetch_provider_environment(gateway, sandbox_id, requires_stable_placeholders) .await } @@ -2709,7 +2710,7 @@ mod tests { ); } PolicyPollLoopContext { - endpoint: String::new(), + gateway: test_gateway(), sandbox_id: "sandbox-test".to_string(), sandbox: "sandbox-test-name".to_string(), opa_engine, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index d78c4c04b9..e27bd23455 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -303,6 +303,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl payload: Vec::new(), }, auth_bundle, + gateway_credentials: None, admitted_isolation_backend: Some("test-held".into()), main_exit_marker: None, }, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 9628d811a0..ab05a5e1fb 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -601,6 +601,8 @@ pub async fn run_network_proxy( ); let (_, workspace_rx) = tokio::sync::watch::channel(String::new()); let tls_dir = prepare_network_proxy_tls_dir(tls_dir)?; + let mut proxy_tls = openshell_supervisor_network::run::ProxyTlsFiles::from_environment(); + proxy_tls.directory = Some(tls_dir.path.clone()); let mut networking = openshell_supervisor_network::run::run_networking( &policy, None, @@ -609,9 +611,11 @@ pub async fn run_network_proxy( Arc::new(AtomicU32::new(0)), false, &provider_credentials, - None, - Some("network-proxy"), - None, + openshell_supervisor_network::run::NetworkSession { + sandbox_name: Some("network-proxy".to_string()), + proxy_tls, + ..Default::default() + }, None, None, None, @@ -619,7 +623,6 @@ pub async fn run_network_proxy( workspace_rx, &upstream_proxy_args, additional_ca_bundle.as_deref(), - Some(&tls_dir.path), None, #[cfg(target_os = "linux")] None, @@ -714,6 +717,7 @@ pub async fn run_sandbox( additional_ca_bundle, backend_descriptor, auth_bundle, + gateway_credentials: None, admitted_isolation_backend, main_exit_marker, }, @@ -742,10 +746,19 @@ pub struct SandboxRunConfig { pub additional_ca_bundle: Option, pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, + /// Credentials issued from `auth_bundle` and shared with other gateway + /// clients of this process, such as log push. Created when absent. + pub gateway_credentials: Option, pub admitted_isolation_backend: Option, pub main_exit_marker: Option, } +/// Supervise the one sandbox generation this process was launched for. +/// +/// The process-wide OCSF context, SIGTERM/SIGINT handling, readiness +/// endpoints, and the environment-provided launch inputs belong to the +/// dedicated process; [`run_session`] owns everything else. +/// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. #[allow(clippy::similar_names)] @@ -773,9 +786,260 @@ pub async fn run_sandbox_with_backend( additional_ca_bundle, backend_descriptor, auth_bundle, + gateway_credentials, admitted_isolation_backend, main_exit_marker, } = config; + + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; the boundary mediation source carries + // workload-side connection metadata. + if !openshell_ocsf::ctx::set_ctx(session_event_context( + sandbox_id.as_deref(), + sandbox.as_deref(), + std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + )) { + debug!("OCSF context already initialized, keeping existing"); + } + + let (readiness, session_readiness) = tokio::sync::watch::channel(false); + let control_readiness = health_socket_path + .map(ReadinessEndpoint::Unix) + .into_iter() + .chain(health_port.map(ReadinessEndpoint::Tcp)) + .map(|endpoint| ControlReadiness::start(endpoint, Some(session_readiness.clone()))) + .collect::>>()?; + let result = Box::pin(run_session( + backend_setup, + SessionConfig { + command, + workdir, + timeout_secs, + interactive, + await_main_process_attachment, + sandbox_id, + sandbox, + openshell_endpoint, + policy_rules, + policy_data, + ssh_socket_path, + shared_ssh_socket: shared_ssh_socket_from_env(), + ocsf_json: openshell_ocsf::OcsfJsonSettings { + enabled: ocsf_enabled, + target_version: ocsf_schema_version, + }, + upstream_proxy_args, + additional_ca_bundle, + backend_descriptor, + auth_bundle, + gateway_credentials, + admitted_isolation_backend, + main_exit_marker, + trace_context: TraceContext::from_environment(), + provider_workload_api_socket: std::env::var_os( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ) + .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from), + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + }, + SessionControls { + shutdown: Box::pin(wait_for_control_shutdown_signal()), + readiness, + }, + )) + .await; + drop(control_readiness); + result +} + +/// OCSF identity of one supervised sandbox. +#[must_use] +pub fn session_event_context( + sandbox_id: Option<&str>, + sandbox: Option<&str>, + container_image: String, +) -> EventContext { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + EventContext { + sandbox_id: sandbox_id.unwrap_or_default().to_string(), + sandbox_name: sandbox.unwrap_or_default().to_string(), + container_image, + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + origin: openshell_ocsf::EventOrigin::Supervisor, + } +} + +/// W3C trace context of the operation that launched a sandbox generation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TraceContext { + pub traceparent: String, + pub tracestate: Option, +} + +impl TraceContext { + /// The context a driver passed to a dedicated supervisor process. + #[must_use] + pub fn from_environment() -> Option { + let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) + .ok() + .filter(|value| !value.is_empty())?; + Some(Self { + traceparent, + tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) + .ok() + .filter(|value| !value.is_empty()), + }) + } +} + +/// Trusted inputs owned by one supervised sandbox generation. +/// +/// Nothing in a session reads process-wide sandbox state: identity, +/// credentials, policy inputs, logging settings, and file locations all come +/// from here, so one process can host several sessions. +pub struct SessionConfig { + pub command: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, + pub await_main_process_attachment: bool, + pub sandbox_id: Option, + pub sandbox: Option, + pub openshell_endpoint: Option, + pub policy_rules: Option, + pub policy_data: Option, + pub ssh_socket_path: Option, + /// Grant the SSH socket to the supervisor's group rather than its owner. + pub shared_ssh_socket: bool, + /// OCSF JSON output settings this session's gateway configuration controls. + pub ocsf_json: openshell_ocsf::OcsfJsonSettings, + pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + pub additional_ca_bundle: Option, + pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, + pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, + /// Credentials issued from `auth_bundle`; created from it when absent. + pub gateway_credentials: Option, + pub admitted_isolation_backend: Option, + pub main_exit_marker: Option, + /// Parent of the startup span. + pub trace_context: Option, + /// SPIFFE Workload API socket for provider token grants; absent fails closed. + pub provider_workload_api_socket: Option, + pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, + /// Source of `policy.local` recent denials. + pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, +} + +/// How the owner of a session stops it and observes its readiness. +pub struct SessionControls { + /// Completes when the owner asks the session to stop its workload. It is + /// first polled once the workload has started. + pub shutdown: Pin + Send>>, + /// True while the gateway has accepted this session's supervision and its + /// access plane is retained; false otherwise. + pub readiness: tokio::sync::watch::Sender, +} + +/// Mirrors gateway session acceptance into the owner's readiness until the +/// access plane is released. +struct SessionReadiness { + readiness: tokio::sync::watch::Sender, + forward: Option>, +} + +impl SessionReadiness { + fn start( + readiness: tokio::sync::watch::Sender, + accepted: Option>, + ) -> Self { + // Without a gateway session, an attached access plane is ready. + let Some(mut accepted) = accepted else { + readiness.send_replace(true); + return Self { + readiness, + forward: None, + }; + }; + let sender = readiness.clone(); + let forward = tokio::spawn(async move { + loop { + sender.send_replace(*accepted.borrow_and_update()); + if accepted.changed().await.is_err() { + sender.send_replace(false); + return; + } + } + }); + Self { + readiness, + forward: Some(forward), + } + } + + fn release(&mut self) { + if let Some(forward) = self.forward.take() { + forward.abort(); + } + self.readiness.send_replace(false); + } +} + +impl Drop for SessionReadiness { + fn drop(&mut self) { + self.release(); + } +} + +/// Supervise one sandbox generation until its workload exits or the owner +/// stops it. +/// +/// Trusted composition chooses the setup before shared admission, policy, and +/// lifecycle handling. Payload contents never select a backend implementation. +#[allow(clippy::similar_names)] +pub async fn run_session( + backend_setup: &dyn backend_setup::BackendSetup, + config: SessionConfig, + controls: SessionControls, +) -> Result { + let SessionConfig { + command, + workdir, + timeout_secs, + interactive, + await_main_process_attachment, + sandbox_id, + sandbox, + openshell_endpoint, + policy_rules, + policy_data, + ssh_socket_path, + shared_ssh_socket, + ocsf_json, + upstream_proxy_args, + additional_ca_bundle, + backend_descriptor, + auth_bundle, + gateway_credentials, + admitted_isolation_backend, + main_exit_marker, + trace_context, + provider_workload_api_socket, + proxy_tls, + denial_log, + } = config; + let SessionControls { + shutdown: shutdown_requested, + readiness, + } = controls; // The current client attachment adapter is SSH; retaining a process's I/O // internally does not make it reachable by a foreground client. if await_main_process_attachment && (!cfg!(unix) || ssh_socket_path.is_none()) { @@ -791,33 +1055,10 @@ pub async fn run_sandbox_with_backend( |(program, args)| (program.clone(), args.to_vec()), ); - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; the boundary mediation source carries - // workload-side connection metadata. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(EventContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - origin: openshell_ocsf::EventOrigin::Supervisor, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - // Extension credentials are owned by this supervisor and shared by every - // gateway connection it opens, so the middleware registry's bearer slots - // and the policy poll loop that rotates them stay the same objects. + // Extension credentials are owned by this supervisor session and shared + // by every gateway connection it opens, so the middleware registry's + // bearer slots and the policy poll loop that rotates them stay the same + // objects. let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); let selected_backend = backend_setup::SelectedBackend::select( @@ -827,7 +1068,17 @@ pub async fn run_sandbox_with_backend( sandbox_id.as_deref(), &auth_bundle, )?; - let sandbox_bearer = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth_bundle)?; + let gateway_credentials = match gateway_credentials { + Some(credentials) => credentials, + None => openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth_bundle)?, + }; + let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { + miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") + })?; + // Every gateway client of this session uses its own credentials. + let gateway = openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new(endpoint, gateway_credentials.clone()) + }); #[cfg(unix)] let ssh_host_key = if ssh_socket_path.is_some() { Some(openshell_supervisor_process::ssh::parse_host_key( @@ -852,7 +1103,13 @@ pub async fn run_sandbox_with_backend( sandbox.id = sandbox_id.as_deref().unwrap_or_default(), otel.status_code = tracing::field::Empty, ); - openshell_otel::set_parent_from_environment(&startup); + if let Some(trace_context) = &trace_context { + openshell_otel::set_parent_from_trace_context( + &startup, + Some(&trace_context.traceparent), + trace_context.tracestate.as_deref(), + ); + } let startup_status = startup.in_scope(openshell_otel::ErrorStatusGuard::current); let (image_yaml, invalid_image) = selected_backend .discover_policy(sandbox_bearer.clone()) @@ -872,7 +1129,6 @@ pub async fn run_sandbox_with_backend( let vm_policy_identity = selected_backend.vm_policy_identity(); // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); let ( policy, @@ -886,7 +1142,7 @@ pub async fn run_sandbox_with_backend( ) = load_policy_with_gateway( sandbox_id.clone(), sandbox.clone(), - openshell_endpoint.clone(), + gateway.clone(), policy_rules, policy_data, &extension_credentials, @@ -894,7 +1150,7 @@ pub async fn run_sandbox_with_backend( vm_policy_identity, Some(image_discovery), &RemoteStartupGateway { - endpoint: openshell_endpoint.clone().unwrap_or_default(), + gateway: gateway.clone(), }, ) .instrument(tracing::info_span!(parent: &startup, "supervisor.policy.load")) @@ -915,8 +1171,8 @@ pub async fn run_sandbox_with_backend( // Fetch provider environment variables from the server. // This is done after loading the policy so the sandbox can still start // even if provider env fetch fails (graceful degradation). - let environment = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id) + let environment = if let (Some(id), Some(gateway)) = (&sandbox_id, &gateway) { + match openshell_core::grpc_client::fetch_provider_environment(gateway, id) .instrument(startup.clone()) .await { @@ -1109,9 +1365,14 @@ pub async fn run_sandbox_with_backend( // unnecessarily delay DNS and network readiness. false, &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), + openshell_supervisor_network::run::NetworkSession { + sandbox_id: sandbox_id.clone(), + sandbox_name: sandbox_name_for_agg.clone(), + gateway: gateway.clone(), + provider_workload_api_socket, + denial_log, + proxy_tls, + }, denial_tx, activity_tx, endpoint_observation_tx.clone(), @@ -1119,7 +1380,6 @@ pub async fn run_sandbox_with_backend( workspace_rx.clone(), &upstream_proxy_args, additional_ca_bundle.as_deref(), - None, remote_host_gateway_ip, #[cfg(target_os = "linux")] None, @@ -1142,14 +1402,14 @@ pub async fn run_sandbox_with_backend( // Spawn the denial-aggregator flush task. The aggregator drains proxy // denial events, batches them, and ships summaries to the gateway via // `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + if let (Some(rx), Some(gateway)) = (denial_rx, gateway.as_ref()) { // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall // back to the ID when the name isn't set. let agg_name = sandbox_name_for_agg .clone() .or_else(|| sandbox_id.clone()) .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); + let agg_gateway = gateway.clone(); let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") .ok() .and_then(|v| v.parse().ok()) @@ -1163,12 +1423,12 @@ pub async fn run_sandbox_with_backend( aggregator .run( |summaries| { - let endpoint = agg_endpoint.clone(); + let gateway = agg_gateway.clone(); let sandbox_name = agg_name.clone(); let workspace = denial_workspace_rx.borrow().clone(); async move { if let Err(e) = flush_proposals_to_gateway( - &endpoint, + &gateway, &sandbox_name, &workspace, summaries, @@ -1188,12 +1448,12 @@ pub async fn run_sandbox_with_backend( // Spawn the activity-aggregator flush task. The aggregator drains // anonymous activity events from the proxy, sanitizes deny groups, // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + if let (Some(rx), Some(gateway)) = (activity_rx, gateway.as_ref()) { let agg_name = sandbox_name_for_agg .clone() .or_else(|| sandbox_id.clone()) .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); + let agg_gateway = gateway.clone(); let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") .ok() @@ -1208,12 +1468,12 @@ pub async fn run_sandbox_with_backend( aggregator .run( move |summary| { - let endpoint = agg_endpoint.clone(); + let gateway = agg_gateway.clone(); let sandbox_name = agg_name.clone(); let workspace = activity_workspace_rx.borrow().clone(); async move { if let Err(e) = flush_activity_to_gateway( - &endpoint, + &gateway, &sandbox_name, &workspace, summary, @@ -1231,18 +1491,18 @@ pub async fn run_sandbox_with_backend( } // Spawn background policy poll task (gRPC mode only). - if let (Some(id), Some(sandbox), Some(endpoint), Some(engine)) = ( + if let (Some(id), Some(sandbox), Some(gateway), Some(engine)) = ( sandbox_id.as_deref(), sandbox.as_deref(), - openshell_endpoint.as_deref(), + gateway.as_ref(), opa_engine.as_ref(), ) { let poll_id = id.to_string(); let poll_sandbox = sandbox.to_string(); - let poll_endpoint = endpoint.to_string(); + let poll_gateway = gateway.clone(); let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_ocsf_schema_version = ocsf_schema_version.clone(); + let poll_ocsf_enabled = ocsf_json.enabled.clone(); + let poll_ocsf_schema_version = ocsf_json.target_version.clone(); let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); @@ -1255,7 +1515,7 @@ pub async fn run_sandbox_with_backend( .and_then(|v| v.parse().ok()) .unwrap_or(10); let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, + gateway: poll_gateway, sandbox_id: poll_id, sandbox: poll_sandbox, opa_engine: poll_engine, @@ -1326,9 +1586,9 @@ pub async fn run_sandbox_with_backend( let agent = running.agent(); let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( sandbox_id.as_deref(), - openshell_endpoint.as_deref(), + gateway.as_ref(), ssh_socket_path.as_deref(), - shared_ssh_socket_from_env(), + shared_ssh_socket, networking .as_ref() .and_then(|runtime| runtime.ca_file_paths.clone()), @@ -1345,29 +1605,23 @@ pub async fn run_sandbox_with_backend( info!(backend = %backend_name, "Control-mode access plane started"); startup_status.finish(Ok::<_, ()>(())).ok(); drop(startup); - let _provider_reporter = - sandbox_id - .as_ref() - .zip(openshell_endpoint.as_ref()) - .map(|(id, endpoint)| { - provider_readiness.start_reporter( - endpoint.clone(), - id.clone(), - provider_credentials.clone(), - supervisor_session_id.clone(), - running.exec(), - ) - }); - let mut control_readiness = health_socket_path - .map(ReadinessEndpoint::Unix) - .into_iter() - .chain(health_port.map(ReadinessEndpoint::Tcp)) - .map(|endpoint| ControlReadiness::start(endpoint, boundary_access.session_readiness())) - .collect::>>()?; + let _provider_reporter = sandbox_id + .as_ref() + .zip(gateway.as_ref()) + .map(|(id, gateway)| { + provider_readiness.start_reporter( + gateway.clone(), + id.clone(), + provider_credentials.clone(), + supervisor_session_id.clone(), + running.exec(), + ) + }); + let mut session_readiness = + SessionReadiness::start(readiness, boundary_access.session_readiness()); let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); - let shutdown_requested = wait_for_control_shutdown_signal(); - tokio::pin!(shutdown_requested); + let mut shutdown_requested = shutdown_requested; let wait = async move { wait_agent .wait() @@ -1417,7 +1671,7 @@ pub async fn run_sandbox_with_backend( } }; if !retain_access { - control_readiness.clear(); + session_readiness.release(); } boundary_access .publish_main_exit(exit_code, await_main_process_attachment) @@ -1431,11 +1685,10 @@ pub async fn run_sandbox_with_backend( .wrap_err("persist canonical-process completion marker")?; } if !completion_cancelled - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { let report = openshell_supervisor_process::delegated::report_main_process_exit( - endpoint, + gateway, id, &instance_id, exit_code, @@ -1449,11 +1702,10 @@ pub async fn run_sandbox_with_backend( completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; } if !completion_cancelled - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( - endpoint, + gateway, id, &instance_id, ); @@ -1462,13 +1714,13 @@ pub async fn run_sandbox_with_backend( } if completion_cancelled { retain_access = false; - control_readiness.clear(); + session_readiness.release(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } - drop(control_readiness); + drop(session_readiness); drop(running); drop(boundary_access); exit_code @@ -1521,7 +1773,7 @@ fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io:: /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_name: &str, workspace: &str, summaries: Vec, @@ -1529,7 +1781,7 @@ async fn flush_proposals_to_gateway( use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::{DenialSummary, L7RequestSample}; - let client = CachedOpenShellClient::connect(endpoint).await?; + let client = CachedOpenShellClient::connect(gateway).await?; client.set_workspace(workspace.to_string()); let proto_summaries: Vec = summaries @@ -1591,7 +1843,7 @@ async fn flush_proposals_to_gateway( /// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. async fn flush_activity_to_gateway( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_name: &str, workspace: &str, summary: activity_aggregator::FlushableActivitySummary, @@ -1599,7 +1851,7 @@ async fn flush_activity_to_gateway( use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - let client = CachedOpenShellClient::connect(endpoint).await?; + let client = CachedOpenShellClient::connect(gateway).await?; client.set_workspace(workspace.to_string()); let proto_summary = NetworkActivitySummary { @@ -2153,7 +2405,15 @@ trait StartupGateway: Send + Sync { } struct RemoteStartupGateway { - endpoint: String, + gateway: Option, +} + +impl RemoteStartupGateway { + fn gateway(&self) -> Result<&openshell_core::grpc_client::GatewayEndpoint> { + self.gateway + .as_ref() + .ok_or_else(|| miette::miette!("supervisor has no gateway endpoint")) + } } impl StartupGateway for RemoteStartupGateway { @@ -2161,13 +2421,13 @@ impl StartupGateway for RemoteStartupGateway { &self, sandbox: &str, ) -> Result { - openshell_core::grpc_client::fetch_settings_snapshot(&self.endpoint, sandbox).await + openshell_core::grpc_client::fetch_settings_snapshot(self.gateway()?, sandbox).await } async fn provider( &self, id: &str, ) -> Result { - openshell_core::grpc_client::fetch_provider_environment(&self.endpoint, id).await + openshell_core::grpc_client::fetch_provider_environment(self.gateway()?, id).await } async fn sync( &self, @@ -2176,7 +2436,7 @@ impl StartupGateway for RemoteStartupGateway { workspace: &str, ) -> Result { openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - &self.endpoint, + self.gateway()?, sandbox, policy, workspace, @@ -2192,7 +2452,7 @@ impl StartupGateway for RemoteStartupGateway { error: &str, ) -> Result<()> { openshell_core::grpc_client::report_sandbox_configuration( - &self.endpoint, + self.gateway()?, id, instance_id, snapshot, @@ -2291,7 +2551,7 @@ impl CapturedProviderEnvironment { async fn load_policy( sandbox_id: Option, sandbox: Option, - openshell_endpoint: Option, + openshell_gateway: Option, policy_rules: Option, policy_data: Option, extension_credentials: &openshell_extension_core::ExtensionCredentialStore, @@ -2309,7 +2569,7 @@ async fn load_policy( load_policy_with_gateway( sandbox_id, sandbox, - openshell_endpoint.clone(), + openshell_gateway.clone(), policy_rules, policy_data, extension_credentials, @@ -2317,7 +2577,7 @@ async fn load_policy( None, None, &RemoteStartupGateway { - endpoint: openshell_endpoint.unwrap_or_default(), + gateway: openshell_gateway, }, ) .await @@ -2330,7 +2590,7 @@ async fn load_policy( async fn load_policy_with_gateway( sandbox_id: Option, sandbox: Option, - openshell_endpoint: Option, + openshell_gateway: Option, policy_rules: Option, policy_data: Option, extension_credentials: &openshell_extension_core::ExtensionCredentialStore, @@ -2351,7 +2611,7 @@ async fn load_policy_with_gateway( use openshell_core::proto::ConfigurationAdmissionState; // File mode: load OPA engine from rego rules + YAML data (dev override) if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - if sandbox_id.is_some() && openshell_endpoint.is_some() { + if sandbox_id.is_some() && openshell_gateway.is_some() { return Err(miette::miette!( "Local policy overrides cannot be combined with gateway-managed activation; replace the sandbox policy through the gateway" )); @@ -2417,12 +2677,11 @@ async fn load_policy_with_gateway( } // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(sandbox), Some(endpoint)) = (&sandbox_id, &sandbox, &openshell_endpoint) - { + if let (Some(id), Some(sandbox), Some(endpoint)) = (&sandbox_id, &sandbox, &openshell_gateway) { info!( sandbox_id = %id, sandbox = %sandbox, - endpoint = %endpoint, + endpoint = %endpoint.url(), "Fetching sandbox policy via gRPC" ); let instance_id = uuid::Uuid::new_v4().to_string(); @@ -3511,14 +3770,14 @@ trait PolicyGatewayClient: Clone + Send + Sync + 'static { /// Refresh credentials while preserving the activated delivery requirement. fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> impl Future> + Send { async move { openshell_core::grpc_client::fetch_provider_environment_with_stable_placeholders( - endpoint, + gateway, sandbox_id, requires_stable_placeholders, ) @@ -3706,7 +3965,7 @@ fn enqueue_policy_status(sender: &UnboundedSender, update: P /// and preserves the original construction error as the reported message. A /// delivery failure here is swallowed so it can never mask that error. async fn report_initial_policy_failure( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, revision: Option<&LoadedPolicyRevision>, error: &miette::Report, @@ -3717,7 +3976,7 @@ async fn report_initial_policy_failure( }) else { return; }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await { Ok(client) => client, Err(e) => { warn!(error = %e, "Failed to connect to report initial policy failure"); @@ -3749,7 +4008,7 @@ async fn report_initial_policy_failure( /// When the entrypoint PID is available, policy reloads include symlink /// resolution for binary paths via the container filesystem. struct PolicyPollLoopContext { - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, /// Canonical sandbox reference used by name-scoped configuration APIs. sandbox: String, @@ -4141,6 +4400,14 @@ fn emit_policy_validation_failure( )] mod tests { use super::*; + + pub fn test_gateway() -> openshell_core::grpc_client::GatewayEndpoint { + openshell_core::grpc_client::GatewayEndpoint::new( + "http://unused.invalid", + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .expect("test gateway credentials"), + ) + } use std::sync::atomic::{AtomicUsize, Ordering}; /// Admission must reject this input before any backend decodes its payload @@ -4745,7 +5012,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".into()), Some("sandbox".into()), - Some("http://unused.invalid".into()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -4884,7 +5151,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -4961,7 +5228,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5007,7 +5274,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5340,7 +5607,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5489,7 +5756,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5815,7 +6082,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5910,7 +6177,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 0442fd10a2..daafbc364b 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -413,11 +413,10 @@ fn main() -> Result<()> { let setup = backend_setup(&args, &admitted_backend)?; let descriptor = backend_descriptor(&args, &admitted_backend)?; let auth = auth_bundle(&args)?; - // Install the driver-provisioned session before starting log push or - // any other gateway client. `run_sandbox` obtains the same Sandbox - // Protocol bearer slot after validating the descriptor binding. - let _ = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth)?; - Some((setup, descriptor, auth, admitted_backend)) + // Log push and supervision share these credentials, so either one's + // renewal keeps the other's gateway token current. + let credentials = openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth)?; + Some((setup, descriptor, auth, admitted_backend, credentials)) } else { None }; @@ -443,7 +442,7 @@ fn main() -> Result<()> { }); let ocsf_headers = isolation_inputs .as_ref() - .and_then(|(_, _, auth, _)| auth.ocsf_otlp_headers.as_ref()) + .and_then(|(_, _, auth, _, _)| auth.ocsf_otlp_headers.as_ref()) .map(openshell_core::jwt::SecretOtlpHeaders::expose_secret); let (ocsf_export, ocsf_setup_error) = runtime.block_on(async { start_ocsf_export( @@ -457,11 +456,18 @@ fn main() -> Result<()> { let exit_code = runtime.block_on(async move { openshell_crypto::tls::ensure_default_provider(); - let log_push_state = if args.role == SupervisorRole::IsolationBackend - && let (Some(sandbox_id), Some(endpoint)) = (&args.sandbox_id, &args.openshell_endpoint) - { + let log_push_state = if let (Some(sandbox_id), Some(endpoint), Some(credentials)) = ( + &args.sandbox_id, + &args.openshell_endpoint, + isolation_inputs + .as_ref() + .map(|(_, _, _, _, credentials)| credentials), + ) { let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), + openshell_core::grpc_client::GatewayEndpoint::new( + endpoint.as_str(), + credentials.clone(), + ), sandbox_id.clone(), ); let layer = @@ -538,8 +544,13 @@ fn main() -> Result<()> { match args.role { SupervisorRole::IsolationBackend => { info!(command = ?command, "Starting sandbox supervision"); - let Some((backend_setup, backend_descriptor, auth_bundle, admitted_backend)) = - isolation_inputs + let Some(( + backend_setup, + backend_descriptor, + auth_bundle, + admitted_backend, + gateway_credentials, + )) = isolation_inputs else { return Err(miette::miette!( "isolation-backend role started without validated runtime inputs" @@ -567,6 +578,7 @@ fn main() -> Result<()> { additional_ca_bundle: args.additional_ca_bundle, backend_descriptor, auth_bundle, + gateway_credentials: Some(gateway_credentials), admitted_isolation_backend: Some(admitted_backend), main_exit_marker: args.main_exit_marker, }, diff --git a/crates/openshell-supervisor/src/provider_readiness.rs b/crates/openshell-supervisor/src/provider_readiness.rs index af883dc265..347f7ca3e9 100644 --- a/crates/openshell-supervisor/src/provider_readiness.rs +++ b/crates/openshell-supervisor/src/provider_readiness.rs @@ -283,7 +283,7 @@ impl Tracker { /// Report while the owner holds this handle; reconnects use the accepted session. pub(crate) fn start_reporter( &self, - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, credentials: ProviderCredentialState, sessions: watch::Receiver>, @@ -297,7 +297,7 @@ impl Tracker { sessions, boundary, GatewayReporter { - endpoint, + gateway, sandbox_id, }, ) @@ -399,7 +399,7 @@ trait ReportClient: Send + Sync { } struct GatewayReporter { - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, } @@ -409,7 +409,7 @@ impl ReportClient for GatewayReporter { observation: ProviderReadinessObservation, ) -> miette::Result { openshell_core::grpc_client::report_provider_readiness( - &self.endpoint, + &self.gateway, &self.sandbox_id, observation, ) From bce3e09f0773f579b15e8c3cc52a7b3316fb75db Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 17:06:21 -0700 Subject: [PATCH 3/6] feat(supervisor): host concurrent sessions on session-bound runtimes SessionHost runs several supervisor sessions in one process. Each session gets its own Tokio runtime, and every thread that runtime starts, plus the thread driving it, is bound to the session's OCSF context, OCSF JSON settings, log sink, and denial log. Tasks, blocking work, reconnects, and credential renewal spawned by a session therefore keep its identity, while threads outside any session carry none and push no logs. Gateway credentials stay explicit per session. Removing a session stops its workload gracefully, or cancels it at once while it is still starting, then shuts down only its runtime and revokes its gateway credentials. The key stays registered until the session thread has finished, so the same generation cannot gain a second owner while one is stopping. A panicking session reports a failure, clears its readiness, and is revoked the same way. Hosted sessions get a private proxy trust directory and serve policy.local denials from their own log. A focused test runs two sessions concurrently and checks that nested and blocking tasks see their own identity, gateway requests carry their own bearer, log lines reach only their own stream, and removing one leaves the other supervising. Closes #4699 Signed-off-by: Drew Newberry --- Cargo.lock | 5 + crates/openshell-supervisor/Cargo.toml | 5 + crates/openshell-supervisor/README.md | 18 + crates/openshell-supervisor/src/lib.rs | 1 + .../openshell-supervisor/src/session_host.rs | 938 ++++++++++++++++++ 5 files changed, 967 insertions(+) create mode 100644 crates/openshell-supervisor/src/session_host.rs diff --git a/Cargo.lock b/Cargo.lock index 0a2fcb8c92..025ba026d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5093,8 +5093,13 @@ dependencies = [ name = "openshell-supervisor" version = "0.0.0" dependencies = [ + "bytes", "clap", "futures", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", "miette", "nix 0.29.0", "openshell-core", diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index 9e2e0ea8ad..a42c370404 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -58,6 +58,11 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } tonic-prost = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } tokio = { workspace = true, features = ["test-util"] } futures = { workspace = true } temp-env = "0.3" diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 864a24a9d9..12c08cebb0 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -40,6 +40,24 @@ readiness, and stops the session on SIGTERM or SIGINT. The binary shares one `GatewayCredentials` between log push and supervision, so renewal by either keeps both current. +`session_host::SessionHost` runs several sessions in one process. Each session +gets its own Tokio runtime; every thread of that runtime, and the thread +driving it, is bound to the session's OCSF context, OCSF JSON settings, log +sink, and denial log. Spawned tasks, blocking work, callbacks, reconnects, and +renewal therefore inherit the session's identity, and threads outside a +session carry none and push no logs. The OCSF context also supplies the +sandbox identity in middleware requests. + +Removing a session stops its workload, shuts down only its runtime, and +revokes its gateway credentials; the key stays registered until the session's +thread has finished. A session that panics reports a failure and is revoked +the same way. A session that is still starting is cancelled at once; the +sandbox runtime then loses supervision and fails closed, as it does when a +dedicated supervisor exits. A hosted session without an explicit proxy trust +directory receives a private one for its lifetime. Hosted processes must +install `LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to +`SessionDenialWriter`. + ## Console logging Diagnostics and OCSF shorthand share a bounded, nonblocking stderr writer. With `ocsf_json_enabled=true`, the same writer also receives timestamped `OCSF-JSON` records. Each formatter submits a complete line in one write so concurrent producers cannot interleave records in the queue. The 1,024-line queue drops new lines when full rather than waiting for stderr. diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index ab05a5e1fb..bd402b5d38 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -24,6 +24,7 @@ mod denial_aggregator; mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; +pub mod session_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; diff --git a/crates/openshell-supervisor/src/session_host.rs b/crates/openshell-supervisor/src/session_host.rs new file mode 100644 index 0000000000..fcb4e989ff --- /dev/null +++ b/crates/openshell-supervisor/src/session_host.rs @@ -0,0 +1,938 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Several supervisor sessions in one process. +//! +//! Each session runs on its own Tokio runtime. Every thread that runtime +//! starts, and the thread driving it, is bound to the session's OCSF context, +//! OCSF JSON settings, log sink, and denial log, so tasks, callbacks, +//! reconnects, and credential renewal that the session spawns inherit its +//! identity. Unbound threads carry no sandbox identity and push no logs. +//! Gateway credentials are never bound to threads; each session passes its +//! own explicitly. Stopping a session shuts down only its runtime and revokes +//! its gateway credentials. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use miette::Result; +use openshell_core::grpc_client::GatewayCredentials; +use openshell_ocsf::{EventContext, OcsfJsonSettings}; +use openshell_supervisor_network::policy_local::SessionDenialLog; +use openshell_supervisor_process::log_push::LogSink; +use tokio::sync::watch; + +use crate::SessionControls; + +/// Worker threads in each session runtime. +const SESSION_WORKER_THREADS: usize = 2; +/// Bound on joining a session thread after its runtime is told to stop. +const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Per-session values bound to every thread of the session's runtime. +#[derive(Clone)] +pub struct SessionBinding { + pub event_context: Arc, + pub ocsf_json: OcsfJsonSettings, + pub log_sink: Option, + pub denial_log: SessionDenialLog, +} + +impl SessionBinding { + fn bind(&self) { + openshell_ocsf::ctx::bind_thread_ctx(Some(self.event_context.clone())); + openshell_ocsf::bind_thread_json_settings(Some(self.ocsf_json.clone())); + openshell_supervisor_process::log_push::bind_thread_sink(self.log_sink.clone()); + bind_thread_denial_log(Some(self.denial_log.clone())); + } + + fn unbind() { + openshell_ocsf::ctx::bind_thread_ctx(None); + openshell_ocsf::bind_thread_json_settings(None); + openshell_supervisor_process::log_push::bind_thread_sink(None); + bind_thread_denial_log(None); + } +} + +std::thread_local! { + static THREAD_DENIAL_LOG: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +fn bind_thread_denial_log(log: Option) { + THREAD_DENIAL_LOG.with(|current| *current.borrow_mut() = log); +} + +/// Shorthand-log destination for hosted sessions' recent denials. +/// +/// Each line goes to the denial log of the session bound to the emitting +/// thread. Lines from unbound threads are dropped, so no session can read +/// another's denials through `policy.local`. +#[derive(Debug, Default, Clone, Copy)] +pub struct SessionDenialWriter; + +impl std::io::Write for SessionDenialWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + THREAD_DENIAL_LOG.with(|log| { + if let Some(log) = log.borrow().as_ref() { + for line in String::from_utf8_lossy(bytes).lines() { + log.record(line); + } + } + }); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Observable lifecycle of one hosted session. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SessionPhase { + /// Startup has not yet produced an accepted gateway session. + Starting, + /// The workload started and the gateway accepted supervision at least once. + Supervising, + /// The owner asked the session to stop. + Stopping, + /// Supervision ended with the workload's exit status. + Exited(i32), + /// Startup or supervision failed; the workload is no longer supervised. + Failed(String), +} + +impl SessionPhase { + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited(_) | Self::Failed(_)) + } +} + +/// Current state of one hosted session. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SessionStatus { + pub phase: SessionPhase, + /// Gateway supervision is currently accepted for this session. + pub ready: bool, +} + +/// Body of a hosted session, constructed on its own runtime. +pub type SessionBody = + Box Pin>>> + Send + 'static>; + +/// The hosted-session registration was rejected. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum SessionHostError { + #[error("a session with this key is already hosted")] + Duplicate, + #[error("start session thread: {0}")] + Thread(String), +} + +struct HostedSession { + /// Distinguishes this session from a later one that reuses its key. + serial: u64, + stop: watch::Sender, + abort: watch::Sender, + phase: watch::Receiver, + ready: watch::Receiver, + thread: Option>, + /// Final status of the removal in progress, shared by every remover. + removal: Option>>, +} + +impl HostedSession { + fn status(&self) -> SessionStatus { + let phase = self.phase.borrow().clone(); + // The runner always reports a terminal phase, even while unwinding; + // a closed channel without one still means supervision has ended. + if !phase.is_terminal() && self.phase.has_changed().is_err() { + return SessionStatus { + phase: SessionPhase::Failed("session thread ended unexpectedly".to_string()), + ready: false, + }; + } + SessionStatus { + phase, + ready: *self.ready.borrow(), + } + } +} + +type Sessions = Arc>>; + +fn lock(sessions: &Sessions) -> MutexGuard<'_, HashMap> { + sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Runs and stops supervisor sessions, each on its own bound runtime. +#[derive(Default)] +pub struct SessionHost { + sessions: Sessions, + next_serial: AtomicU64, +} + +impl SessionHost { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Start `body` on a new runtime bound to `binding`. + /// + /// `credentials`, when present, are revoked once the session's runtime has + /// stopped, so nothing can reuse them after the session ends. + /// + /// # Errors + /// + /// Returns [`SessionHostError::Duplicate`] when `key` is already hosted, + /// including a session that ended but has not been removed. + pub fn start( + &self, + key: &str, + binding: SessionBinding, + credentials: Option, + body: SessionBody, + ) -> std::result::Result<(), SessionHostError> { + let mut sessions = lock(&self.sessions); + if sessions.contains_key(key) { + return Err(SessionHostError::Duplicate); + } + let (stop, stop_requested) = watch::channel(false); + let (abort, abort_requested) = watch::channel(false); + let (phase_tx, phase) = watch::channel(SessionPhase::Starting); + let (ready_tx, ready) = watch::channel(false); + let thread = std::thread::Builder::new() + .name("openshell-session".to_string()) + .spawn(move || { + run_hosted_session( + &binding, + credentials, + body, + SessionSignals { + stop_requested, + abort_requested, + phase: phase_tx, + ready: ready_tx, + }, + ); + }) + .map_err(|error| SessionHostError::Thread(error.to_string()))?; + sessions.insert( + key.to_string(), + HostedSession { + serial: self.next_serial.fetch_add(1, Ordering::Relaxed), + stop, + abort, + phase, + ready, + thread: Some(thread), + removal: None, + }, + ); + Ok(()) + } + + /// Status of a hosted session, including one that ended but has not been + /// removed. + #[must_use] + pub fn status(&self, key: &str) -> Option { + lock(&self.sessions).get(key).map(HostedSession::status) + } + + /// Keys of every hosted session. + #[must_use] + pub fn keys(&self) -> Vec { + lock(&self.sessions).keys().cloned().collect() + } + + /// Stop one session and forget it. Other sessions keep running. + /// + /// A supervising session terminates its workload gracefully within + /// `grace`; one still starting is cancelled at once, because startup has + /// no safe intermediate state to wind down. Removal runs on its own task, + /// so it completes even when the caller stops waiting, and concurrent + /// callers share it and its result. The key stays registered until the + /// session's thread has finished, so the same key cannot start a second + /// owner while this one is stopping. Returns the final status, or `None` + /// when `key` is not hosted. + pub async fn remove(&self, key: &str, grace: Duration) -> Option { + let (mut removed, phase) = self.begin_removal(key, grace)?; + // The removal task ends without a report only with its runtime. + removed + .wait_for(Option::is_some) + .await + .map_or_else(|_| Some(terminal_status(&phase)), |status| status.clone()) + } + + fn begin_removal( + &self, + key: &str, + grace: Duration, + ) -> Option<( + watch::Receiver>, + watch::Receiver, + )> { + let mut sessions = lock(&self.sessions); + let session = sessions.get_mut(key)?; + if let Some(removed) = &session.removal { + return Some((removed.clone(), session.phase.clone())); + } + session.stop.send_replace(true); + if *session.phase.borrow() == SessionPhase::Starting { + session.abort.send_replace(true); + } + let (report, removed) = watch::channel(None); + session.removal = Some(removed.clone()); + tokio::spawn( + Removal { + sessions: self.sessions.clone(), + key: key.to_string(), + serial: session.serial, + phase: session.phase.clone(), + thread: session.thread.take(), + grace, + report, + } + .run(), + ); + Some((removed, session.phase.clone())) + } +} + +fn terminal_status(phase: &watch::Receiver) -> SessionStatus { + SessionStatus { + phase: phase.borrow().clone(), + ready: false, + } +} + +/// The single owner of one session's removal. +struct Removal { + sessions: Sessions, + key: String, + serial: u64, + phase: watch::Receiver, + thread: Option>, + grace: Duration, + report: watch::Sender>, +} + +impl Removal { + async fn run(mut self) { + let finished = + tokio::time::timeout(self.grace, self.phase.wait_for(SessionPhase::is_terminal)) + .await + .is_ok(); + if !finished + && let Some(session) = lock(&self.sessions) + .get(&self.key) + .filter(|session| session.serial == self.serial) + { + session.abort.send_replace(true); + } + if let Some(thread) = self.thread.take() { + let _ = tokio::task::spawn_blocking(move || thread.join()).await; + } + let removed = { + let mut sessions = lock(&self.sessions); + match sessions.get(&self.key) { + Some(session) if session.serial == self.serial => sessions.remove(&self.key), + _ => None, + } + }; + let status = + removed.map_or_else(|| terminal_status(&self.phase), |session| session.status()); + self.report.send_replace(Some(status)); + } +} + +struct SessionSignals { + stop_requested: watch::Receiver, + abort_requested: watch::Receiver, + phase: watch::Sender, + ready: watch::Sender, +} + +/// Reports a hosted session's end and revokes its credentials, including when +/// the session thread unwinds. +struct SessionExit { + credentials: Option, + phase: watch::Sender, + ready: watch::Sender, + outcome: Option, +} + +impl Drop for SessionExit { + fn drop(&mut self) { + if let Some(credentials) = self.credentials.take() { + credentials.revoke(); + } + self.ready.send_replace(false); + let outcome = self.outcome.take().unwrap_or_else(|| { + SessionPhase::Failed("session thread ended without reporting a result".to_string()) + }); + self.phase.send_replace(outcome); + SessionBinding::unbind(); + } +} + +fn run_hosted_session( + binding: &SessionBinding, + credentials: Option, + body: SessionBody, + signals: SessionSignals, +) { + let SessionSignals { + stop_requested, + mut abort_requested, + phase, + ready, + } = signals; + let mut exit = SessionExit { + credentials, + phase: phase.clone(), + ready: ready.clone(), + outcome: None, + }; + binding.bind(); + let thread_binding = binding.clone(); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(SESSION_WORKER_THREADS) + .thread_name("openshell-session-worker") + .enable_all() + .on_thread_start(move || thread_binding.bind()) + .on_thread_stop(SessionBinding::unbind) + .build(); + let outcome = match runtime { + Ok(runtime) => { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runtime.block_on(async { + let (readiness, mut accepted) = watch::channel(false); + let mirror_phase = phase.clone(); + let mirror_ready = ready.clone(); + let mirror = tokio::spawn(async move { + loop { + let is_ready = *accepted.borrow_and_update(); + mirror_ready.send_replace(is_ready); + if is_ready { + mirror_phase.send_if_modified(|current| { + let promote = *current == SessionPhase::Starting; + if promote { + *current = SessionPhase::Supervising; + } + promote + }); + } + if accepted.changed().await.is_err() { + mirror_ready.send_replace(false); + return; + } + } + }); + let mut stop = stop_requested.clone(); + let stop_phase = phase.clone(); + let shutdown = Box::pin(async move { + let _ = stop.wait_for(|requested| *requested).await; + stop_phase.send_replace(SessionPhase::Stopping); + }); + let session = body(SessionControls { + shutdown, + readiness, + }); + let outcome = tokio::select! { + result = session => result, + _ = abort_requested.wait_for(|requested| *requested) => { + Err(miette::miette!("session was cancelled before it finished")) + } + }; + mirror.abort(); + outcome + }) + })); + runtime.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); + outcome.unwrap_or_else(|_| Err(miette::miette!("session panicked"))) + } + Err(error) => Err(miette::miette!("build session runtime: {error}")), + }; + exit.outcome = Some(match outcome { + Ok(code) => SessionPhase::Exited(code), + Err(error) => SessionPhase::Failed(error.to_string()), + }); +} + +/// Trusted inputs for one supervisor session hosted with others. +pub struct HostedSessionSpec { + /// Unique key for this generation within the host. + pub key: String, + pub backend_setup: Box, + pub config: crate::SessionConfig, + /// Workload image recorded in this session's OCSF events. + pub container_image: String, +} + +impl SessionHost { + /// Start a supervisor session with its own OCSF identity, OCSF JSON + /// settings, log push stream, denial log, and gateway credentials. + /// + /// # Errors + /// + /// Fails when the launch credentials are invalid or the key is hosted. + pub fn start_supervisor_session(&self, spec: HostedSessionSpec) -> Result<()> { + let HostedSessionSpec { + key, + backend_setup, + mut config, + container_image, + } = spec; + let credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; + // Sessions must never share proxy trust files; give one a private + // directory that lives exactly as long as the session. + let private_tls_dir = if config.proxy_tls.directory.is_none() { + let prepared = crate::prepare_network_proxy_tls_dir(None)?; + config.proxy_tls.directory = Some(prepared.path.clone()); + Some(prepared) + } else { + None + }; + let gateway = config.openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new(endpoint, credentials.clone()) + }); + let (log_tx, log_rx) = tokio::sync::mpsc::channel( + openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, + ); + let log_push = config.sandbox_id.clone().zip(gateway); + let binding = SessionBinding { + event_context: Arc::new(crate::session_event_context( + config.sandbox_id.as_deref(), + config.sandbox.as_deref(), + container_image, + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: log_push + .as_ref() + .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), + denial_log: SessionDenialLog::default(), + }; + config.gateway_credentials = Some(credentials.clone()); + config.ocsf_json = binding.ocsf_json.clone(); + config.denial_log = openshell_supervisor_network::policy_local::DenialLogSource::Session( + binding.denial_log.clone(), + ); + let body: SessionBody = Box::new(move |controls| { + Box::pin(async move { + let _private_tls_dir = private_tls_dir; + let _log_push = log_push.map(|(sandbox_id, gateway)| { + openshell_supervisor_process::log_push::start_log_push( + gateway, sandbox_id, log_rx, + ) + }); + Box::pin(crate::run_session(backend_setup.as_ref(), config, controls)).await + }) + }); + self.start(&key, binding, Some(credentials), body) + .map_err(|error| miette::miette!("{error}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use http_body_util::BodyExt as _; + use openshell_core::grpc_client::GatewayEndpoint; + use openshell_core::proto::SandboxLogLine; + use tokio::sync::mpsc; + use tracing_subscriber::layer::SubscriberExt as _; + + type Captured = Arc)>>>; + + /// Records each gRPC request's bearer and body, then answers `UNIMPLEMENTED`. + async fn capture_gateway() -> (String, Captured) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let captured = Captured::default(); + let sink = captured.clone(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let sink = sink.clone(); + tokio::spawn(async move { + let service = hyper::service::service_fn( + move |request: http::Request| { + let sink = sink.clone(); + async move { + let authorization = request + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let body = request + .into_body() + .collect() + .await + .map(|body| body.to_bytes().to_vec()) + .unwrap_or_default(); + sink.lock().unwrap().push((authorization, body)); + http::Response::builder() + .header("content-type", "application/grpc") + .header("grpc-status", "12") + .body(http_body_util::Empty::::new()) + } + }, + ); + let _ = hyper::server::conn::http2::Builder::new( + hyper_util::rt::TokioExecutor::new(), + ) + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + }); + } + }); + (format!("http://{address}"), captured) + } + + fn bearers_for(captured: &Captured, sandbox: &str) -> Vec { + captured + .lock() + .unwrap() + .iter() + .filter(|(_, body)| { + body.windows(sandbox.len()) + .any(|bytes| bytes == sandbox.as_bytes()) + }) + .map(|(authorization, _)| authorization.clone()) + .collect() + } + + struct Session { + identity: tokio::sync::oneshot::Receiver, + logs: mpsc::Receiver, + call_again: mpsc::Sender<()>, + alive: mpsc::Receiver<()>, + credentials: GatewayCredentials, + } + + fn start_session(host: &SessionHost, gateway_url: &str, sandbox: &'static str) -> Session { + let credentials = + GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); + let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); + let (log_tx, logs) = mpsc::channel(16); + let binding = SessionBinding { + event_context: Arc::new(crate::session_event_context( + Some(sandbox), + Some(sandbox), + String::new(), + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: Some(LogSink::new(sandbox, log_tx)), + denial_log: SessionDenialLog::default(), + }; + let (identity_tx, identity) = tokio::sync::oneshot::channel(); + let (call_again, mut calls) = mpsc::channel(1); + let (alive_tx, alive) = mpsc::channel::<()>(1); + let body: SessionBody = Box::new(move |controls| { + Box::pin(async move { + let SessionControls { + shutdown, + readiness, + } = controls; + // Nested tasks and the blocking pool inherit the session identity. + let observed = tokio::spawn(async { + tokio::task::spawn_blocking(|| openshell_ocsf::ctx::ctx().sandbox_id.clone()) + .await + .unwrap() + }) + .await + .unwrap(); + let _ = identity_tx.send(observed); + let _ = + openshell_core::grpc_client::fetch_settings_snapshot(&gateway, sandbox).await; + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(openshell_supervisor_process::log_push::LogPushLayer::thread_bound()), + ); + tokio::spawn(async move { + tracing::dispatcher::with_default(&dispatch, || { + tracing::info!(target: "session_test", "emitted by {sandbox}"); + }); + }) + .await + .unwrap(); + // A long-lived task the session never stops itself. + tokio::spawn(async move { + let _alive = alive_tx; + std::future::pending::<()>().await; + }); + readiness.send_replace(true); + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => return Ok(0), + Some(()) = calls.recv() => { + let _ = openshell_core::grpc_client::fetch_settings_snapshot( + &gateway, sandbox, + ) + .await; + } + } + } + }) + }); + host.start(sandbox, binding, Some(credentials.clone()), body) + .unwrap(); + Session { + identity, + logs, + call_again, + alive, + credentials, + } + } + + async fn wait_supervising(host: &SessionHost, key: &str) { + tokio::time::timeout(Duration::from_secs(10), async { + while host.status(key).map(|status| status.phase) != Some(SessionPhase::Supervising) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("session reaches supervision"); + } + + fn unbound() -> SessionBinding { + SessionBinding { + event_context: Arc::new(crate::session_event_context(None, None, String::new())), + ocsf_json: OcsfJsonSettings::default(), + log_sink: None, + denial_log: SessionDenialLog::default(), + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn stopping_and_panicking_sessions_keep_their_key_and_fail_closed() { + let host = Arc::new(SessionHost::new()); + let body: SessionBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + // Winding down takes time; the key must stay taken meanwhile. + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(0) + }) + }); + host.start("generation", unbound(), None, body).unwrap(); + wait_supervising(&host, "generation").await; + let removal = tokio::spawn({ + let host = host.clone(); + async move { host.remove("generation", Duration::from_secs(10)).await } + }); + tokio::time::timeout(Duration::from_secs(10), async { + while host.status("generation").map(|status| status.phase) + != Some(SessionPhase::Stopping) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("session observes the stop request"); + let replacement: SessionBody = Box::new(|_| Box::pin(async { Ok(0) })); + assert_eq!( + host.start("generation", unbound(), None, replacement), + Err(SessionHostError::Duplicate) + ); + let stopped = removal.await.unwrap().expect("session was hosted"); + assert_eq!(stopped.phase, SessionPhase::Exited(0)); + assert!(host.status("generation").is_none()); + + let credentials = GatewayCredentials::from_gateway_token("panicking-token").unwrap(); + let panicking: SessionBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + tokio::task::yield_now().await; + panic!("hosted session failure"); + }) + }); + host.start("panicking", unbound(), Some(credentials.clone()), panicking) + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(status) = host.status("panicking") + && status.phase.is_terminal() + { + return status; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("panicking session reports an end"); + assert_eq!( + status, + SessionStatus { + phase: SessionPhase::Failed("session panicked".to_string()), + ready: false, + } + ); + let revoked = GatewayEndpoint::new("http://127.0.0.1:9", credentials); + let error = openshell_core::grpc_client::fetch_settings_snapshot(&revoked, "panicking") + .await + .unwrap_err(); + assert!(error.to_string().contains("revoked"), "{error}"); + assert!( + host.remove("panicking", Duration::from_secs(1)) + .await + .is_some() + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn removal_outlives_its_caller_and_reports_to_every_remover() { + let host = Arc::new(SessionHost::new()); + let winding_down = |code| -> SessionBody { + Box::new(move |controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(code) + }) + }) + }; + host.start("generation", unbound(), None, winding_down(3)) + .unwrap(); + wait_supervising(&host, "generation").await; + // A caller that gives up, such as an expired RPC, strands nothing. + assert!( + tokio::time::timeout( + Duration::from_millis(20), + host.remove("generation", Duration::from_secs(10)) + ) + .await + .is_err() + ); + let removers = [(); 2].map(|()| { + let host = host.clone(); + tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) + }); + for remover in removers { + let status = remover.await.unwrap().expect("session was hosted"); + assert_eq!(status.phase, SessionPhase::Exited(3)); + } + assert!(host.status("generation").is_none()); + + // The key is free again, and the earlier removal cannot touch its new owner. + host.start("generation", unbound(), None, winding_down(4)) + .unwrap(); + wait_supervising(&host, "generation").await; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + host.status("generation").map(|status| status.phase), + Some(SessionPhase::Supervising) + ); + let status = host.remove("generation", Duration::from_secs(10)).await; + assert_eq!( + status.map(|status| status.phase), + Some(SessionPhase::Exited(4)) + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn hosted_sessions_keep_identity_credentials_and_lifecycle_separate() { + let (gateway_url, captured) = capture_gateway().await; + let host = SessionHost::new(); + let mut alpha = start_session(&host, &gateway_url, "sandbox-alpha"); + let mut beta = start_session(&host, &gateway_url, "sandbox-beta"); + wait_supervising(&host, "sandbox-alpha").await; + wait_supervising(&host, "sandbox-beta").await; + assert!(matches!( + host.start( + "sandbox-alpha", + SessionBinding { + event_context: Arc::new(crate::session_event_context( + None, + None, + String::new() + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: None, + denial_log: SessionDenialLog::default(), + }, + None, + Box::new(|_| Box::pin(async { Ok(0) })), + ), + Err(SessionHostError::Duplicate) + )); + + assert_eq!((&mut alpha.identity).await.unwrap(), "sandbox-alpha"); + assert_eq!((&mut beta.identity).await.unwrap(), "sandbox-beta"); + // Other tests in this process may set the process-wide context; a + // thread outside every session must still carry no session identity. + let outside = openshell_ocsf::ctx::ctx().sandbox_id.clone(); + assert!(!["sandbox-alpha", "sandbox-beta"].contains(&outside.as_str())); + assert_eq!( + bearers_for(&captured, "sandbox-alpha"), + ["Bearer token-sandbox-alpha"] + ); + assert_eq!( + bearers_for(&captured, "sandbox-beta"), + ["Bearer token-sandbox-beta"] + ); + for (session, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { + let line = session.logs.try_recv().expect("session log line"); + assert_eq!(line.sandbox_id, sandbox); + assert_eq!(line.message, format!("emitted by {sandbox}")); + assert!(session.logs.try_recv().is_err()); + } + + let stopped = host + .remove("sandbox-alpha", Duration::from_secs(10)) + .await + .expect("alpha was hosted"); + assert_eq!(stopped.phase, SessionPhase::Exited(0)); + assert!(!stopped.ready); + assert!(host.status("sandbox-alpha").is_none()); + assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); + let revoked = GatewayEndpoint::new(gateway_url.as_str(), alpha.credentials.clone()); + let error = openshell_core::grpc_client::fetch_settings_snapshot(&revoked, "sandbox-alpha") + .await + .unwrap_err(); + assert!(error.to_string().contains("revoked"), "{error}"); + + beta.call_again.send(()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while bearers_for(&captured, "sandbox-beta").len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("beta keeps calling the gateway"); + assert_eq!( + bearers_for(&captured, "sandbox-beta"), + ["Bearer token-sandbox-beta", "Bearer token-sandbox-beta"] + ); + assert_eq!( + host.status("sandbox-beta").map(|status| status.phase), + Some(SessionPhase::Supervising) + ); + assert!( + beta.alive + .try_recv() + .is_err_and(|error| error == mpsc::error::TryRecvError::Empty) + ); + + let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; + assert_eq!( + stopped.map(|status| status.phase), + Some(SessionPhase::Exited(0)) + ); + } +} From ccfdfbbb66726b0f9c8f3f053f93040a85319043 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 19:59:01 -0700 Subject: [PATCH 4/6] refactor(supervisor): name hosted lifetimes sandbox instances A sandbox instance is one running generation of a stable Sandbox resource. Gateway transport connections and the gateway supervisor sessions they carry can reconnect within one instance, so the hosted object, its configuration, controls, host, and per-instance network, denial, and token-grant state now use the sandbox instance name. Rename session_host to sandbox_instance_host, run_session to run_sandbox_instance, and SessionConfig, SessionControls, SessionHost, and related hosted types to their SandboxInstance equivalents. Gateway supervisor sessions, SSH sessions, and authentication session types keep their names. No wire, configuration, or lifecycle behavior changes. Signed-off-by: Drew Newberry --- crates/openshell-core/src/grpc_client.rs | 53 +- crates/openshell-ocsf/src/builders/mod.rs | 7 +- crates/openshell-ocsf/src/ctx.rs | 15 +- .../src/tracing_layers/jsonl_layer.rs | 15 +- crates/openshell-otel/src/propagation.rs | 4 +- crates/openshell-supervisor-network/README.md | 20 +- .../src/l7/token_grant_injection.rs | 6 +- .../src/policy_local.rs | 24 +- .../src/proxy/relay.rs | 2 +- .../openshell-supervisor-network/src/run.rs | 18 +- .../src/token_grant.rs | 16 +- .../src/log_push.rs | 9 +- crates/openshell-supervisor/README.md | 62 +-- crates/openshell-supervisor/src/lib.rs | 76 +-- ...ssion_host.rs => sandbox_instance_host.rs} | 462 ++++++++++-------- 15 files changed, 422 insertions(+), 367 deletions(-) rename crates/openshell-supervisor/src/{session_host.rs => sandbox_instance_host.rs} (65%) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 80b20b0264..75672d0263 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -5,7 +5,7 @@ //! `OpenShell` server. //! //! Every request carries a sandbox bearer credential in the `Authorization` -//! header. Each supervisor session owns its [`GatewayCredentials`], usually +//! header. Each sandbox instance owns its [`GatewayCredentials`], usually //! built from the trusted launch bundle with //! [`GatewayCredentials::from_auth_bundle`]. Diagnostic tools can instead //! resolve a token with [`GatewayCredentials::from_environment`] from one of: @@ -16,8 +16,8 @@ //! caller exchanges it for a gateway JWT via `IssueSandboxToken`. //! //! Callers reach the gateway through a [`GatewayEndpoint`], which pairs the -//! URL with one session's credentials. No credential is installed -//! process-wide, so sessions hosted by one supervisor process cannot use or +//! URL with one sandbox instance's credentials. No credential is installed +//! process-wide, so instances hosted by one supervisor process cannot use or //! rotate each other's identity. use std::collections::HashMap; @@ -65,7 +65,7 @@ pub const MAX_SUPERVISOR_STREAM_MESSAGE_BYTES: usize = 9 * 1024 * 1024; /// Shared, refreshable Bearer header. All [`AuthInterceptor`] clones read /// the same slot, so the renewal task can replace the token in place without -/// rebuilding the channel. `None` means the owning session revoked it. +/// rebuilding the channel. `None` means the owning sandbox instance revoked it. type TokenSlot = Arc>>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -75,7 +75,7 @@ enum TokenSource { K8sServiceAccount, } -/// Gateway credentials owned by one supervisor session. +/// Gateway credentials owned by one sandbox instance. /// /// Clients built from a [`GatewayEndpoint`] read only these slots, and the /// renewal task started by the first connection updates only them. Dropping @@ -200,7 +200,7 @@ impl GatewayCredentials { } /// Start the renewal task once for these credentials, on the caller's - /// runtime, so a hosted session's renewal stops with that session. + /// runtime, so a hosted sandbox instance's renewal stops with that instance. fn start_renewal(&self, channel: &AuthedChannel, plain_channel: &Channel, endpoint: &str) { let mut renewal = self .inner @@ -239,14 +239,16 @@ impl GatewayCredentials { .bearer .write() .unwrap_or_else(std::sync::PoisonError::into_inner); - // A revoked session must not be revived by an in-flight response. + // Revoked instance credentials must not be revived by an in-flight + // response. if slot.is_some() { *slot = Some(bearer); } } } -/// The gateway URL a session uses, paired with that session's credentials. +/// The gateway URL a sandbox instance uses, paired with that instance's +/// credentials. #[derive(Clone)] pub struct GatewayEndpoint { url: Arc, @@ -271,7 +273,7 @@ impl GatewayEndpoint { &self.credentials } - /// Reach another gateway address with the same session credentials. + /// Reach another gateway address with the same sandbox instance credentials. #[must_use] pub fn with_url(&self, url: impl Into>) -> Self { Self::new(url, self.credentials.clone()) @@ -357,7 +359,9 @@ impl tonic::service::Interceptor for AuthInterceptor { .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .clone() - .ok_or_else(|| Status::unauthenticated("gateway session credentials were revoked"))?; + .ok_or_else(|| { + Status::unauthenticated("sandbox instance gateway credentials were revoked") + })?; req.metadata_mut().insert("authorization", bearer); #[cfg(feature = "trace-context")] let req = @@ -458,11 +462,14 @@ impl Drop for ClientSpanStatus { } } -/// Build a Bearer-authenticated channel to the gateway with one session's -/// credentials. The first connection starts that session's renewal task. +/// Build a Bearer-authenticated channel to the gateway with one sandbox +/// instance's credentials. The first connection starts that instance's +/// renewal task. async fn connect_channel(gateway: &GatewayEndpoint) -> Result { if gateway.credentials.is_revoked() { - return Err(miette::miette!("gateway session credentials were revoked")); + return Err(miette::miette!( + "sandbox instance gateway credentials were revoked" + )); } let channel = build_plain_channel(gateway.url()).await?; let intercepted = InterceptedService::new( @@ -577,8 +584,9 @@ pub async fn report_provider_readiness( .into_diagnostic() } -/// Slots and transport retained by one session's renewal task. It holds no -/// reference to [`GatewayCredentials`], so dropping them still stops it. +/// Slots and transport retained by one sandbox instance's renewal task. It +/// holds no reference to [`GatewayCredentials`], so dropping them still stops +/// it. struct RenewalTarget { channel: AuthedChannel, bearer: TokenSlot, @@ -590,10 +598,10 @@ struct RenewalTarget { } /// Background task that renews the sandbox JWT at ~80% of its remaining -/// lifetime. The new token replaces the value in the session's slot, so all -/// in-flight and future clients of that session pick it up on their next -/// request. The loop never panics: every failure is logged and re-attempted -/// after a bounded backoff. +/// lifetime. The new token replaces the value in the sandbox instance's slot, +/// so all in-flight and future clients of that instance pick it up on their +/// next request. The loop never panics: every failure is logged and +/// re-attempted after a bounded backoff. async fn refresh_token_loop(target: RenewalTarget) { let RenewalTarget { channel, @@ -832,7 +840,7 @@ fn compute_refresh_delay(slot: &TokenSlot) -> Option { let current = slot .read() .unwrap_or_else(std::sync::PoisonError::into_inner); - // A revoked session has nothing to renew. + // Revoked instance credentials have nothing to renew. current.as_ref()?.to_str().ok().map(str::to_string) } .unwrap_or_default(); @@ -1058,7 +1066,7 @@ mod workspace_tests { } } -/// Connect to the `OpenShell` server with one session's credentials. +/// Connect to the `OpenShell` server with one sandbox instance's credentials. async fn connect(gateway: &GatewayEndpoint) -> Result> { let channel = connect_channel(gateway).await?; Ok(OpenShellClient::new(channel)) @@ -1711,7 +1719,8 @@ mod token_grant_exchange_tests { #[derive(Clone)] pub struct CachedOpenShellClient { client: OpenShellClient, - /// Session credentials rotated alongside extension credentials. + /// Sandbox instance gateway credentials rotated alongside extension + /// credentials. credentials: GatewayCredentials, workspace: Arc>, /// Extension credentials for this supervisor. Cloning the client shares diff --git a/crates/openshell-ocsf/src/builders/mod.rs b/crates/openshell-ocsf/src/builders/mod.rs index 5c2219cc6d..7026979860 100644 --- a/crates/openshell-ocsf/src/builders/mod.rs +++ b/crates/openshell-ocsf/src/builders/mod.rs @@ -200,10 +200,11 @@ pub struct EventContext { pub origin: EventOrigin, } -/// Event context borrowed from the caller or shared by a hosted session. +/// Event context borrowed from the caller or shared by a hosted sandbox +/// instance. /// -/// Builders accept either form, so events attributed to a hosted supervisor -/// session need not leak a `'static` context for each session. +/// Builders accept either form, so events attributed to a hosted sandbox +/// instance need not leak a `'static` context for each instance. #[derive(Debug, Clone)] pub enum ContextRef<'a> { Borrowed(&'a EventContext), diff --git a/crates/openshell-ocsf/src/ctx.rs b/crates/openshell-ocsf/src/ctx.rs index 9dd542a9aa..726d4ad0a9 100644 --- a/crates/openshell-ocsf/src/ctx.rs +++ b/crates/openshell-ocsf/src/ctx.rs @@ -1,11 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process-wide and session-bound [`EventContext`]. +//! Process-wide and sandbox-instance-bound [`EventContext`]. //! //! A dedicated supervisor initialises the process-wide context once via -//! [`set_ctx`]. A process hosting several supervisor sessions instead binds -//! each session's context to the threads of that session's runtime with +//! [`set_ctx`]. A process hosting several sandbox instances instead binds +//! each instance's context to the threads of that instance's runtime with //! [`bind_thread_ctx`]; a bound context takes precedence. [`ctx`] falls back //! to a default context when neither is set (e.g. unit tests that exercise //! builders without booting the sandbox). @@ -28,7 +28,7 @@ static OCSF_CTX_FALLBACK: LazyLock = LazyLock::new(|| EventContext }); std::thread_local! { - /// Context of the hosted session whose runtime owns this thread. + /// Context of the hosted sandbox instance whose runtime owns this thread. static THREAD_CTX: RefCell>> = const { RefCell::new(None) }; } @@ -40,16 +40,17 @@ pub fn set_ctx(ctx: EventContext) -> bool { OCSF_CTX.set(ctx).is_ok() } -/// Attribute events emitted on the calling thread to one hosted session. +/// Attribute events emitted on the calling thread to one hosted sandbox +/// instance. /// -/// Session runtimes bind every thread they start; `None` unbinds it. +/// Instance runtimes bind every thread they start; `None` unbinds it. pub fn bind_thread_ctx(ctx: Option>) { THREAD_CTX.with(|current| *current.borrow_mut() = ctx); } /// Return the context for events emitted on the calling thread. /// -/// A session-bound context wins over the process-wide one. Falls back to a +/// An instance-bound context wins over the process-wide one. Falls back to a /// default context if neither is set. #[must_use] pub fn ctx() -> ContextRef<'static> { diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index 3f0936e841..40484ce43b 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -16,10 +16,10 @@ use tracing_subscriber::layer::Context; use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; -/// OCSF JSON output settings owned by one hosted supervisor session. +/// OCSF JSON output settings owned by one hosted sandbox instance. /// -/// Sessions apply their own `ocsf_json_enabled` and schema-version settings -/// to these values. Binding them to the session's runtime threads makes every +/// Instances apply their own `ocsf_json_enabled` and schema-version settings +/// to these values. Binding them to the instance's runtime threads makes every /// [`OcsfJsonlLayer`] use them for events those threads emit. #[derive(Clone, Debug, Default)] pub struct OcsfJsonSettings { @@ -32,7 +32,8 @@ std::thread_local! { const { std::cell::RefCell::new(None) }; } -/// Apply one hosted session's JSON settings to events emitted on this thread. +/// Apply one hosted sandbox instance's JSON settings to events emitted on this +/// thread. pub fn bind_thread_json_settings(settings: Option) { THREAD_JSON_SETTINGS.with(|current| *current.borrow_mut() = settings); } @@ -111,9 +112,9 @@ where return; } - // A hosted session's settings replace the process-wide ones. - let session = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); - let (enabled, target_version) = session.as_ref().map_or( + // A hosted sandbox instance's settings replace the process-wide ones. + let instance = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); + let (enabled, target_version) = instance.as_ref().map_or( (self.enabled.as_ref(), self.target_version.as_ref()), |settings| (Some(&settings.enabled), Some(&settings.target_version)), ); diff --git a/crates/openshell-otel/src/propagation.rs b/crates/openshell-otel/src/propagation.rs index 82e5fa784d..02f2bdb120 100644 --- a/crates/openshell-otel/src/propagation.rs +++ b/crates/openshell-otel/src/propagation.rs @@ -124,8 +124,8 @@ pub fn set_parent_from_environment(span: &tracing::Span) { /// Parent `span` under an explicit W3C `traceparent` and `tracestate`. /// /// A process supervising several sandboxes receives each launch's context -/// with its session instead of through process environment. Leaves `span` -/// unchanged when the values carry no valid context. +/// with its sandbox instance instead of through process environment. Leaves +/// `span` unchanged when the values carry no valid context. pub fn set_parent_from_trace_context( span: &tracing::Span, traceparent: Option<&str>, diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index b191655d1a..7ce403a15e 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -15,14 +15,14 @@ default features, so this instrumentation does not require the compute runtime or Unix-only sandbox library. It is a development dependency only. -## Session inputs +## Sandbox instance inputs -`run_networking` takes a `NetworkSession` with the sandbox identity, the -session's `GatewayEndpoint`, the SPIFFE Workload API socket for provider token -grants, the `policy.local` denial source, and proxy CA files. None of these -are read from the process environment, so one process can run several -sessions. Each session owns its provider token cache and HTTP client; a token -acquired for one sandbox is never served to another. Without a Workload API -socket, token grants fail closed. A process hosting several sessions serves -`/v1/denials` from a per-session `SessionDenialLog` instead of the shared -shorthand log files. +`run_networking` takes a `SandboxInstanceNetwork` with the sandbox identity, +the sandbox instance's `GatewayEndpoint`, the SPIFFE Workload API socket for +provider token grants, the `policy.local` denial source, and proxy CA files. +None of these are read from the process environment, so one process can run +several sandbox instances. Each instance owns its provider token cache and +HTTP client; a token acquired for one sandbox is never served to another. +Without a Workload API socket, token grants fail closed. A process hosting +several instances serves `/v1/denials` from a per-instance +`SandboxInstanceDenialLog` instead of the shared shorthand log files. diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 456cf1ab51..65d7b70d08 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -53,14 +53,14 @@ pub trait TokenGrantResolver: Send + Sync { ) -> Pin> + Send + 'a>>; } -/// Resolves provider tokens through one session's authenticated grant flow. +/// Resolves provider tokens through one sandbox instance's authenticated grant flow. pub struct SpiffeTokenGrantResolver { - grants: crate::token_grant::SessionTokenGrants, + grants: crate::token_grant::SandboxInstanceTokenGrants, } impl SpiffeTokenGrantResolver { #[must_use] - pub fn new(grants: crate::token_grant::SessionTokenGrants) -> Self { + pub fn new(grants: crate::token_grant::SandboxInstanceTokenGrants) -> Self { Self { grants } } } diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index a581800c15..033f492aa7 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -84,8 +84,8 @@ const SHORTHAND_LOG_PREFIX: &str = "openshell."; /// log entry (very long URL path, etc.) cannot blow up the response. const MAX_DENIAL_LINE_BYTES: usize = 4096; -/// Most recent denial lines retained for one hosted session. -const MAX_SESSION_DENIAL_LINES: usize = 256; +/// Most recent denial lines retained for one hosted sandbox instance. +const MAX_INSTANCE_DENIAL_LINES: usize = 256; /// Where `/v1/denials` reads this sandbox's recent denials. #[derive(Clone, Debug)] @@ -93,9 +93,9 @@ pub enum DenialLogSource { /// Shorthand log files written by a supervisor process that serves one /// sandbox. ShorthandFiles(PathBuf), - /// Denials recorded for this session alone. A process hosting several + /// Denials recorded for this sandbox instance alone. A process hosting several /// sandboxes cannot expose its shared log files to any one of them. - Session(SessionDenialLog), + SandboxInstance(SandboxInstanceDenialLog), } impl Default for DenialLogSource { @@ -104,13 +104,13 @@ impl Default for DenialLogSource { } } -/// Bounded recent OCSF denial lines recorded for one hosted session. +/// Bounded recent OCSF denial lines recorded for one hosted sandbox instance. #[derive(Clone, Default)] -pub struct SessionDenialLog { +pub struct SandboxInstanceDenialLog { lines: Arc>>, } -impl std::fmt::Debug for SessionDenialLog { +impl std::fmt::Debug for SandboxInstanceDenialLog { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let retained = self .lines @@ -118,13 +118,13 @@ impl std::fmt::Debug for SessionDenialLog { .unwrap_or_else(std::sync::PoisonError::into_inner) .len(); formatter - .debug_struct("SessionDenialLog") + .debug_struct("SandboxInstanceDenialLog") .field("retained", &retained) .finish() } } -impl SessionDenialLog { +impl SandboxInstanceDenialLog { /// Retain `line` when it is a rendered OCSF denial; ignore anything else. pub fn record(&self, line: &str) { let line = line.trim_end(); @@ -135,7 +135,7 @@ impl SessionDenialLog { .lines .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if lines.len() == MAX_SESSION_DENIAL_LINES { + if lines.len() == MAX_INSTANCE_DENIAL_LINES { lines.pop_front(); } lines.push_back(line.to_string()); @@ -227,7 +227,7 @@ impl PolicyLocalContext { self } - /// The session's provider token grant resolver, absent when grants are + /// The sandbox instance's provider token grant resolver, absent when grants are /// unavailable and must fail closed. pub(crate) fn token_grant_resolver(&self) -> Option> { self.token_grants.clone() @@ -425,7 +425,7 @@ async fn recent_denials_response( let limit = parse_last_query(query).unwrap_or(DEFAULT_DENIALS_LIMIT); let log_dir = match &ctx.denials { DenialLogSource::ShorthandFiles(log_dir) => log_dir.clone(), - DenialLogSource::Session(log) => { + DenialLogSource::SandboxInstance(log) => { return ( 200, serde_json::json!({ diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 05f685e122..3e722e3b50 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -58,7 +58,7 @@ pub(super) fn http_context( .is_none() .then_some(secret_resolver) .flatten(); - // Grants use the owning session's resolver; without one they fail closed. + // Grants use the owning sandbox instance's resolver; without one they fail closed. let token_grant_resolver = provider_credentials .as_ref() .and(policy_local) diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 34f6e116d2..340b3984c5 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -39,7 +39,7 @@ use crate::l7::token_grant_injection::SpiffeTokenGrantResolver; use crate::opa::OpaEngine; use crate::policy_local::{DenialLogSource, PolicyLocalContext}; use crate::proxy::ProxyHandle; -use crate::token_grant::SessionTokenGrants; +use crate::token_grant::SandboxInstanceTokenGrants; use openshell_core::endpoint_status::EndpointObservationSender; use openshell_core::grpc_client::GatewayEndpoint; @@ -194,14 +194,14 @@ fn append_additional_ca_bundle( /// Identity and authority of the one sandbox a [`run_networking`] call serves. /// -/// A process hosting several sandboxes builds one per session; no value is -/// read from the process environment, so sessions cannot share gateway +/// A process hosting several sandboxes builds one per sandbox instance; no value +/// is read from the process environment, so instances cannot share gateway /// credentials, provider token grants, denial history, or proxy CA files. #[derive(Debug, Default)] -pub struct NetworkSession { +pub struct SandboxInstanceNetwork { pub sandbox_id: Option, pub sandbox_name: Option, - /// Gateway access with this session's credentials. + /// Gateway access with this instance's credentials. pub gateway: Option, /// SPIFFE Workload API socket for provider token grants. Without one, /// grants fail closed. @@ -284,7 +284,7 @@ pub async fn run_networking( entrypoint_pid: Arc, process_enabled: bool, provider_credentials: &ProviderCredentialState, - session: NetworkSession, + instance: SandboxInstanceNetwork, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -300,15 +300,15 @@ pub async fn run_networking( // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. - let NetworkSession { + let SandboxInstanceNetwork { sandbox_id, sandbox_name, gateway, provider_workload_api_socket, denial_log, proxy_tls, - } = session; - let token_grants = SpiffeTokenGrantResolver::new(SessionTokenGrants::new( + } = instance; + let token_grants = SpiffeTokenGrantResolver::new(SandboxInstanceTokenGrants::new( gateway.clone(), sandbox_id.clone(), provider_workload_api_socket, diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index 05331e128d..83d36d36b3 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -27,9 +27,9 @@ //! - `scopes` — `OAuth2` scopes to request (optional) //! - `cache_ttl` — Optional exact cache override; zero disables caching //! -//! ## Session scope +//! ## Sandbox instance scope //! -//! Each supervised sandbox owns a [`SessionTokenGrants`]: its token cache, +//! Each sandbox instance owns a [`SandboxInstanceTokenGrants`]: its token cache, //! HTTP client, gateway credentials, sandbox ID, and SPIFFE Workload API //! socket. A token is never served to another sandbox. Without a configured //! Workload API socket, grants fail closed with a configuration failure. @@ -103,9 +103,9 @@ impl TokenCache { /// Provider token grants owned by one supervised sandbox. /// /// The cache, HTTP connection pool, gateway credentials, sandbox ID, and -/// workload identity socket all belong to this session, so a token acquired +/// workload identity socket all belong to this sandbox instance, so a token acquired /// for one sandbox is never served to another. -pub struct SessionTokenGrants { +pub struct SandboxInstanceTokenGrants { cache: TokenCache, // Initialization failures stay bounded and typed; acquisition must not panic. http_client: Result, @@ -114,7 +114,7 @@ pub struct SessionTokenGrants { workload_api_socket: Option, } -impl SessionTokenGrants { +impl SandboxInstanceTokenGrants { /// Grants authenticated as `sandbox_id` through `gateway`. A missing /// Workload API socket makes every grant fail closed. #[must_use] @@ -134,12 +134,12 @@ impl SessionTokenGrants { /// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. /// - /// This fetches the sandbox's SPIFFE JWT-SVID from the session's Workload + /// This fetches the sandbox's SPIFFE JWT-SVID from the instance's Workload /// API socket, then exchanges it for an access token with a POST request to /// the provider's token endpoint with the JWT client assertion grant flow /// (RFC 7523). /// - /// Tokens are cached per provider name with TTL in this session's cache. + /// Tokens are cached per provider name with TTL in this instance's cache. /// Subsequent calls return the cached token if it has not expired. /// /// # Errors @@ -247,7 +247,7 @@ impl SessionTokenGrants { } } -/// One provider token request; see [`SessionTokenGrants::obtain_provider_token`]. +/// One provider token request; see [`SandboxInstanceTokenGrants::obtain_provider_token`]. pub struct ObtainProviderTokenRequest<'a> { pub provider_name: &'a str, pub token_endpoint: &'a str, diff --git a/crates/openshell-supervisor-process/src/log_push.rs b/crates/openshell-supervisor-process/src/log_push.rs index 930ef87d5d..c2ee97760e 100644 --- a/crates/openshell-supervisor-process/src/log_push.rs +++ b/crates/openshell-supervisor-process/src/log_push.rs @@ -35,13 +35,14 @@ impl LogSink { } std::thread_local! { - /// Sink of the hosted session whose runtime owns this thread. + /// Sink of the hosted sandbox instance whose runtime owns this thread. static THREAD_SINK: RefCell> = const { RefCell::new(None) }; } -/// Route events emitted on the calling thread to a hosted session's sink. +/// Route events emitted on the calling thread to a hosted sandbox instance's +/// sink. /// -/// Hosted sessions bind every thread of their runtime. Unbound threads push +/// Hosted instances bind every thread of their runtime. Unbound threads push /// nothing, so an event cannot reach another sandbox's log stream. pub fn bind_thread_sink(sink: Option) { THREAD_SINK.with(|current| *current.borrow_mut() = sink); @@ -51,7 +52,7 @@ pub fn bind_thread_sink(sink: Option) { enum Route { /// The process supervises one sandbox; every event belongs to it. Fixed(LogSink), - /// Events belong to the session bound to the emitting thread, if any. + /// Events belong to the sandbox instance bound to the emitting thread, if any. ThreadBound, } diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 12c08cebb0..d603af388f 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -22,41 +22,47 @@ only after boundary attachment and confirmation; the supervisor owns policy evaluation, live credentials, and listener lifetime. These settings do not add a proxy hook to the isolation interface or the shared Sandbox Protocol descriptor. -## Session ownership - -`run_session` supervises one sandbox generation from a `SessionConfig` and -`SessionControls`. Everything that belongs to that generation is an input or -is created by the session: gateway credentials and their renewal, the Sandbox -Protocol credential, policy, provider credentials and token grants, middleware -credentials, OCSF JSON settings, proxy trust files, the denial history served -by `policy.local`, the backend connection, access plane, readiness, and stop -signal. Gateway clients receive a `GatewayEndpoint`, which pairs the URL with -that session's `GatewayCredentials`; nothing is installed process-wide. +## Sandbox instance ownership + +A sandbox instance is one running generation of a stable Sandbox resource. +Gateway connections, and the gateway supervisor sessions carried over them, +can reconnect within an instance without creating another one; the generation +remains the fencing identifier. + +`run_sandbox_instance` supervises one sandbox instance from a +`SandboxInstanceConfig` and `SandboxInstanceControls`. Everything that belongs +to that instance is an input or is created by the instance: gateway +credentials and their renewal, the Sandbox Protocol credential, policy, +provider credentials and token grants, middleware credentials, OCSF JSON +settings, proxy trust files, the denial history served by `policy.local`, the +backend connection, access plane, readiness, and stop signal. Gateway clients +receive a `GatewayEndpoint`, which pairs the URL with that instance's +`GatewayCredentials`; nothing is installed process-wide. `run_sandbox_with_backend` is the dedicated wrapper. It keeps the existing process contract: it sets the process-wide OCSF context, reads launch inputs -from the environment, binds the readiness socket and port to session -readiness, and stops the session on SIGTERM or SIGINT. The binary shares one +from the environment, binds the readiness socket and port to instance +readiness, and stops the instance on SIGTERM or SIGINT. The binary shares one `GatewayCredentials` between log push and supervision, so renewal by either keeps both current. -`session_host::SessionHost` runs several sessions in one process. Each session -gets its own Tokio runtime; every thread of that runtime, and the thread -driving it, is bound to the session's OCSF context, OCSF JSON settings, log -sink, and denial log. Spawned tasks, blocking work, callbacks, reconnects, and -renewal therefore inherit the session's identity, and threads outside a -session carry none and push no logs. The OCSF context also supplies the -sandbox identity in middleware requests. - -Removing a session stops its workload, shuts down only its runtime, and -revokes its gateway credentials; the key stays registered until the session's -thread has finished. A session that panics reports a failure and is revoked -the same way. A session that is still starting is cancelled at once; the -sandbox runtime then loses supervision and fails closed, as it does when a -dedicated supervisor exits. A hosted session without an explicit proxy trust -directory receives a private one for its lifetime. Hosted processes must +`sandbox_instance_host::SandboxInstanceHost` runs several sandbox instances in +one process. Each instance gets its own Tokio runtime; every thread of that +runtime, and the thread driving it, is bound to the instance's OCSF context, +OCSF JSON settings, log sink, and denial log. Spawned tasks, blocking work, +callbacks, reconnects, and renewal therefore inherit the instance's identity, +and threads outside an instance carry none and push no logs. The OCSF context +also supplies the sandbox identity in middleware requests. + +Removing an instance stops its workload, shuts down only its runtime, and +revokes its gateway credentials; the key stays registered until the +instance's thread has finished. An instance that panics reports a failure and +is revoked the same way. An instance that is still starting is cancelled at +once; the sandbox runtime then loses supervision and fails closed, as it does +when a dedicated supervisor exits. A hosted instance without an explicit proxy +trust directory receives a private one for its lifetime. Hosted processes must install `LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to -`SessionDenialWriter`. +`SandboxInstanceDenialWriter`. ## Console logging diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index bd402b5d38..69ee814a34 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -24,7 +24,7 @@ mod denial_aggregator; mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; -pub mod session_host; +pub mod sandbox_instance_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -612,7 +612,7 @@ pub async fn run_network_proxy( Arc::new(AtomicU32::new(0)), false, &provider_credentials, - openshell_supervisor_network::run::NetworkSession { + openshell_supervisor_network::run::SandboxInstanceNetwork { sandbox_name: Some("network-proxy".to_string()), proxy_tls, ..Default::default() @@ -754,11 +754,11 @@ pub struct SandboxRunConfig { pub main_exit_marker: Option, } -/// Supervise the one sandbox generation this process was launched for. +/// Supervise the one sandbox instance this process was launched for. /// /// The process-wide OCSF context, SIGTERM/SIGINT handling, readiness /// endpoints, and the environment-provided launch inputs belong to the -/// dedicated process; [`run_session`] owns everything else. +/// dedicated process; [`run_sandbox_instance`] owns everything else. /// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. @@ -796,7 +796,7 @@ pub async fn run_sandbox_with_backend( // during policy loading (filesystem config, validation) have a context. // Proxy IP/port use defaults here; the boundary mediation source carries // workload-side connection metadata. - if !openshell_ocsf::ctx::set_ctx(session_event_context( + if !openshell_ocsf::ctx::set_ctx(sandbox_instance_event_context( sandbox_id.as_deref(), sandbox.as_deref(), std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), @@ -804,16 +804,16 @@ pub async fn run_sandbox_with_backend( debug!("OCSF context already initialized, keeping existing"); } - let (readiness, session_readiness) = tokio::sync::watch::channel(false); + let (readiness, instance_readiness) = tokio::sync::watch::channel(false); let control_readiness = health_socket_path .map(ReadinessEndpoint::Unix) .into_iter() .chain(health_port.map(ReadinessEndpoint::Tcp)) - .map(|endpoint| ControlReadiness::start(endpoint, Some(session_readiness.clone()))) + .map(|endpoint| ControlReadiness::start(endpoint, Some(instance_readiness.clone()))) .collect::>>()?; - let result = Box::pin(run_session( + let result = Box::pin(run_sandbox_instance( backend_setup, - SessionConfig { + SandboxInstanceConfig { command, workdir, timeout_secs, @@ -846,7 +846,7 @@ pub async fn run_sandbox_with_backend( proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), }, - SessionControls { + SandboxInstanceControls { shutdown: Box::pin(wait_for_control_shutdown_signal()), readiness, }, @@ -858,7 +858,7 @@ pub async fn run_sandbox_with_backend( /// OCSF identity of one supervised sandbox. #[must_use] -pub fn session_event_context( +pub fn sandbox_instance_event_context( sandbox_id: Option<&str>, sandbox: Option<&str>, container_image: String, @@ -902,12 +902,14 @@ impl TraceContext { } } -/// Trusted inputs owned by one supervised sandbox generation. +/// Trusted inputs owned by one supervised sandbox instance. /// -/// Nothing in a session reads process-wide sandbox state: identity, +/// A sandbox instance is one running generation of a stable Sandbox resource; +/// gateway connections may reconnect within it without creating another. +/// Nothing in an instance reads process-wide sandbox state: identity, /// credentials, policy inputs, logging settings, and file locations all come -/// from here, so one process can host several sessions. -pub struct SessionConfig { +/// from here, so one process can host several instances. +pub struct SandboxInstanceConfig { pub command: Vec, pub workdir: Option, pub timeout_secs: u64, @@ -921,7 +923,7 @@ pub struct SessionConfig { pub ssh_socket_path: Option, /// Grant the SSH socket to the supervisor's group rather than its owner. pub shared_ssh_socket: bool, - /// OCSF JSON output settings this session's gateway configuration controls. + /// OCSF JSON output settings this instance's gateway configuration controls. pub ocsf_json: openshell_ocsf::OcsfJsonSettings, pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, pub additional_ca_bundle: Option, @@ -940,24 +942,24 @@ pub struct SessionConfig { pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, } -/// How the owner of a session stops it and observes its readiness. -pub struct SessionControls { - /// Completes when the owner asks the session to stop its workload. It is +/// How the owner of a sandbox instance stops it and observes its readiness. +pub struct SandboxInstanceControls { + /// Completes when the owner asks the instance to stop its workload. It is /// first polled once the workload has started. pub shutdown: Pin + Send>>, - /// True while the gateway has accepted this session's supervision and its + /// True while the gateway has accepted this instance's supervision and its /// access plane is retained; false otherwise. pub readiness: tokio::sync::watch::Sender, } /// Mirrors gateway session acceptance into the owner's readiness until the /// access plane is released. -struct SessionReadiness { +struct SandboxInstanceReadiness { readiness: tokio::sync::watch::Sender, forward: Option>, } -impl SessionReadiness { +impl SandboxInstanceReadiness { fn start( readiness: tokio::sync::watch::Sender, accepted: Option>, @@ -994,24 +996,24 @@ impl SessionReadiness { } } -impl Drop for SessionReadiness { +impl Drop for SandboxInstanceReadiness { fn drop(&mut self) { self.release(); } } -/// Supervise one sandbox generation until its workload exits or the owner +/// Supervise one sandbox instance until its workload exits or the owner /// stops it. /// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. #[allow(clippy::similar_names)] -pub async fn run_session( +pub async fn run_sandbox_instance( backend_setup: &dyn backend_setup::BackendSetup, - config: SessionConfig, - controls: SessionControls, + config: SandboxInstanceConfig, + controls: SandboxInstanceControls, ) -> Result { - let SessionConfig { + let SandboxInstanceConfig { command, workdir, timeout_secs, @@ -1037,7 +1039,7 @@ pub async fn run_session( proxy_tls, denial_log, } = config; - let SessionControls { + let SandboxInstanceControls { shutdown: shutdown_requested, readiness, } = controls; @@ -1056,7 +1058,7 @@ pub async fn run_session( |(program, args)| (program.clone(), args.to_vec()), ); - // Extension credentials are owned by this supervisor session and shared + // Extension credentials are owned by this sandbox instance and shared // by every gateway connection it opens, so the middleware registry's // bearer slots and the policy poll loop that rotates them stay the same // objects. @@ -1076,7 +1078,7 @@ pub async fn run_session( let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") })?; - // Every gateway client of this session uses its own credentials. + // Every gateway client of this instance uses its own credentials. let gateway = openshell_endpoint.as_deref().map(|endpoint| { openshell_core::grpc_client::GatewayEndpoint::new(endpoint, gateway_credentials.clone()) }); @@ -1366,7 +1368,7 @@ pub async fn run_session( // unnecessarily delay DNS and network readiness. false, &provider_credentials, - openshell_supervisor_network::run::NetworkSession { + openshell_supervisor_network::run::SandboxInstanceNetwork { sandbox_id: sandbox_id.clone(), sandbox_name: sandbox_name_for_agg.clone(), gateway: gateway.clone(), @@ -1618,8 +1620,8 @@ pub async fn run_session( running.exec(), ) }); - let mut session_readiness = - SessionReadiness::start(readiness, boundary_access.session_readiness()); + let mut instance_readiness = + SandboxInstanceReadiness::start(readiness, boundary_access.session_readiness()); let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); let mut shutdown_requested = shutdown_requested; @@ -1672,7 +1674,7 @@ pub async fn run_session( } }; if !retain_access { - session_readiness.release(); + instance_readiness.release(); } boundary_access .publish_main_exit(exit_code, await_main_process_attachment) @@ -1715,13 +1717,13 @@ pub async fn run_session( } if completion_cancelled { retain_access = false; - session_readiness.release(); + instance_readiness.release(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } - drop(session_readiness); + drop(instance_readiness); drop(running); drop(boundary_access); exit_code diff --git a/crates/openshell-supervisor/src/session_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs similarity index 65% rename from crates/openshell-supervisor/src/session_host.rs rename to crates/openshell-supervisor/src/sandbox_instance_host.rs index fcb4e989ff..4511b40176 100644 --- a/crates/openshell-supervisor/src/session_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -1,16 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Several supervisor sessions in one process. +//! Several sandbox instances in one supervisor process. //! -//! Each session runs on its own Tokio runtime. Every thread that runtime -//! starts, and the thread driving it, is bound to the session's OCSF context, +//! A sandbox instance is one running generation of a stable Sandbox +//! resource. Gateway transport connections, and the gateway supervisor +//! sessions carried over them, may reconnect within one instance without +//! creating another. +//! +//! Each instance runs on its own Tokio runtime. Every thread that runtime +//! starts, and the thread driving it, is bound to the instance's OCSF context, //! OCSF JSON settings, log sink, and denial log, so tasks, callbacks, -//! reconnects, and credential renewal that the session spawns inherit its +//! reconnects, and credential renewal that the instance spawns inherit its //! identity. Unbound threads carry no sandbox identity and push no logs. -//! Gateway credentials are never bound to threads; each session passes its -//! own explicitly. Stopping a session shuts down only its runtime and revokes -//! its gateway credentials. +//! Gateway credentials are never bound to threads; each instance passes its +//! own explicitly. Stopping an instance shuts down only its runtime and +//! revokes its gateway credentials. use std::collections::HashMap; use std::future::Future; @@ -22,27 +27,27 @@ use std::time::Duration; use miette::Result; use openshell_core::grpc_client::GatewayCredentials; use openshell_ocsf::{EventContext, OcsfJsonSettings}; -use openshell_supervisor_network::policy_local::SessionDenialLog; +use openshell_supervisor_network::policy_local::SandboxInstanceDenialLog; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; -use crate::SessionControls; +use crate::SandboxInstanceControls; -/// Worker threads in each session runtime. -const SESSION_WORKER_THREADS: usize = 2; -/// Bound on joining a session thread after its runtime is told to stop. +/// Worker threads in each sandbox instance runtime. +const INSTANCE_WORKER_THREADS: usize = 2; +/// Bound on joining an instance thread after its runtime is told to stop. const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); -/// Per-session values bound to every thread of the session's runtime. +/// Per-instance values bound to every thread of the instance's runtime. #[derive(Clone)] -pub struct SessionBinding { +pub struct SandboxInstanceBinding { pub event_context: Arc, pub ocsf_json: OcsfJsonSettings, pub log_sink: Option, - pub denial_log: SessionDenialLog, + pub denial_log: SandboxInstanceDenialLog, } -impl SessionBinding { +impl SandboxInstanceBinding { fn bind(&self) { openshell_ocsf::ctx::bind_thread_ctx(Some(self.event_context.clone())); openshell_ocsf::bind_thread_json_settings(Some(self.ocsf_json.clone())); @@ -59,23 +64,23 @@ impl SessionBinding { } std::thread_local! { - static THREAD_DENIAL_LOG: std::cell::RefCell> = + static THREAD_DENIAL_LOG: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; } -fn bind_thread_denial_log(log: Option) { +fn bind_thread_denial_log(log: Option) { THREAD_DENIAL_LOG.with(|current| *current.borrow_mut() = log); } -/// Shorthand-log destination for hosted sessions' recent denials. +/// Shorthand-log destination for hosted sandbox instances' recent denials. /// -/// Each line goes to the denial log of the session bound to the emitting -/// thread. Lines from unbound threads are dropped, so no session can read +/// Each line goes to the denial log of the instance bound to the emitting +/// thread. Lines from unbound threads are dropped, so no instance can read /// another's denials through `policy.local`. #[derive(Debug, Default, Clone, Copy)] -pub struct SessionDenialWriter; +pub struct SandboxInstanceDenialWriter; -impl std::io::Write for SessionDenialWriter { +impl std::io::Write for SandboxInstanceDenialWriter { fn write(&mut self, bytes: &[u8]) -> std::io::Result { THREAD_DENIAL_LOG.with(|log| { if let Some(log) = log.borrow().as_ref() { @@ -92,14 +97,14 @@ impl std::io::Write for SessionDenialWriter { } } -/// Observable lifecycle of one hosted session. +/// Observable lifecycle of one hosted sandbox instance. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum SessionPhase { +pub enum SandboxInstancePhase { /// Startup has not yet produced an accepted gateway session. Starting, /// The workload started and the gateway accepted supervision at least once. Supervising, - /// The owner asked the session to stop. + /// The owner asked the instance to stop. Stopping, /// Supervision ended with the workload's exit status. Exited(i32), @@ -107,80 +112,85 @@ pub enum SessionPhase { Failed(String), } -impl SessionPhase { +impl SandboxInstancePhase { #[must_use] pub fn is_terminal(&self) -> bool { matches!(self, Self::Exited(_) | Self::Failed(_)) } } -/// Current state of one hosted session. +/// Current state of one hosted sandbox instance. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct SessionStatus { - pub phase: SessionPhase, - /// Gateway supervision is currently accepted for this session. +pub struct SandboxInstanceStatus { + pub phase: SandboxInstancePhase, + /// Gateway supervision is currently accepted for this instance. pub ready: bool, } -/// Body of a hosted session, constructed on its own runtime. -pub type SessionBody = - Box Pin>>> + Send + 'static>; +/// Body of a hosted sandbox instance, constructed on its own runtime. +pub type SandboxInstanceBody = Box< + dyn FnOnce(SandboxInstanceControls) -> Pin>>> + + Send + + 'static, +>; -/// The hosted-session registration was rejected. +/// The hosted sandbox instance registration was rejected. #[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum SessionHostError { - #[error("a session with this key is already hosted")] +pub enum SandboxInstanceHostError { + #[error("a sandbox instance with this key is already hosted")] Duplicate, - #[error("start session thread: {0}")] + #[error("start sandbox instance thread: {0}")] Thread(String), } -struct HostedSession { - /// Distinguishes this session from a later one that reuses its key. +struct HostedSandboxInstance { + /// Distinguishes this instance from a later one that reuses its key. serial: u64, stop: watch::Sender, abort: watch::Sender, - phase: watch::Receiver, + phase: watch::Receiver, ready: watch::Receiver, thread: Option>, /// Final status of the removal in progress, shared by every remover. - removal: Option>>, + removal: Option>>, } -impl HostedSession { - fn status(&self) -> SessionStatus { +impl HostedSandboxInstance { + fn status(&self) -> SandboxInstanceStatus { let phase = self.phase.borrow().clone(); // The runner always reports a terminal phase, even while unwinding; // a closed channel without one still means supervision has ended. if !phase.is_terminal() && self.phase.has_changed().is_err() { - return SessionStatus { - phase: SessionPhase::Failed("session thread ended unexpectedly".to_string()), + return SandboxInstanceStatus { + phase: SandboxInstancePhase::Failed( + "sandbox instance thread ended unexpectedly".to_string(), + ), ready: false, }; } - SessionStatus { + SandboxInstanceStatus { phase, ready: *self.ready.borrow(), } } } -type Sessions = Arc>>; +type Instances = Arc>>; -fn lock(sessions: &Sessions) -> MutexGuard<'_, HashMap> { - sessions +fn lock(instances: &Instances) -> MutexGuard<'_, HashMap> { + instances .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } -/// Runs and stops supervisor sessions, each on its own bound runtime. +/// Runs and stops sandbox instances, each on its own bound runtime. #[derive(Default)] -pub struct SessionHost { - sessions: Sessions, +pub struct SandboxInstanceHost { + instances: Instances, next_serial: AtomicU64, } -impl SessionHost { +impl SandboxInstanceHost { #[must_use] pub fn new() -> Self { Self::default() @@ -188,36 +198,36 @@ impl SessionHost { /// Start `body` on a new runtime bound to `binding`. /// - /// `credentials`, when present, are revoked once the session's runtime has - /// stopped, so nothing can reuse them after the session ends. + /// `credentials`, when present, are revoked once the instance's runtime + /// has stopped, so nothing can reuse them after the instance ends. /// /// # Errors /// - /// Returns [`SessionHostError::Duplicate`] when `key` is already hosted, - /// including a session that ended but has not been removed. + /// Returns [`SandboxInstanceHostError::Duplicate`] when `key` is already hosted, + /// including an instance that ended but has not been removed. pub fn start( &self, key: &str, - binding: SessionBinding, + binding: SandboxInstanceBinding, credentials: Option, - body: SessionBody, - ) -> std::result::Result<(), SessionHostError> { - let mut sessions = lock(&self.sessions); - if sessions.contains_key(key) { - return Err(SessionHostError::Duplicate); + body: SandboxInstanceBody, + ) -> std::result::Result<(), SandboxInstanceHostError> { + let mut instances = lock(&self.instances); + if instances.contains_key(key) { + return Err(SandboxInstanceHostError::Duplicate); } let (stop, stop_requested) = watch::channel(false); let (abort, abort_requested) = watch::channel(false); - let (phase_tx, phase) = watch::channel(SessionPhase::Starting); + let (phase_tx, phase) = watch::channel(SandboxInstancePhase::Starting); let (ready_tx, ready) = watch::channel(false); let thread = std::thread::Builder::new() - .name("openshell-session".to_string()) + .name("openshell-instance".to_string()) .spawn(move || { - run_hosted_session( + run_hosted_sandbox_instance( &binding, credentials, body, - SessionSignals { + SandboxInstanceSignals { stop_requested, abort_requested, phase: phase_tx, @@ -225,10 +235,10 @@ impl SessionHost { }, ); }) - .map_err(|error| SessionHostError::Thread(error.to_string()))?; - sessions.insert( + .map_err(|error| SandboxInstanceHostError::Thread(error.to_string()))?; + instances.insert( key.to_string(), - HostedSession { + HostedSandboxInstance { serial: self.next_serial.fetch_add(1, Ordering::Relaxed), stop, abort, @@ -241,30 +251,32 @@ impl SessionHost { Ok(()) } - /// Status of a hosted session, including one that ended but has not been - /// removed. + /// Status of a hosted sandbox instance, including one that ended but has + /// not been removed. #[must_use] - pub fn status(&self, key: &str) -> Option { - lock(&self.sessions).get(key).map(HostedSession::status) + pub fn status(&self, key: &str) -> Option { + lock(&self.instances) + .get(key) + .map(HostedSandboxInstance::status) } - /// Keys of every hosted session. + /// Keys of every hosted sandbox instance. #[must_use] pub fn keys(&self) -> Vec { - lock(&self.sessions).keys().cloned().collect() + lock(&self.instances).keys().cloned().collect() } - /// Stop one session and forget it. Other sessions keep running. + /// Stop one sandbox instance and forget it. Other instances keep running. /// - /// A supervising session terminates its workload gracefully within + /// A supervising instance terminates its workload gracefully within /// `grace`; one still starting is cancelled at once, because startup has /// no safe intermediate state to wind down. Removal runs on its own task, /// so it completes even when the caller stops waiting, and concurrent /// callers share it and its result. The key stays registered until the - /// session's thread has finished, so the same key cannot start a second + /// instance's thread has finished, so the same key cannot start a second /// owner while this one is stopping. Returns the final status, or `None` /// when `key` is not hosted. - pub async fn remove(&self, key: &str, grace: Duration) -> Option { + pub async fn remove(&self, key: &str, grace: Duration) -> Option { let (mut removed, phase) = self.begin_removal(key, grace)?; // The removal task ends without a report only with its runtime. removed @@ -278,126 +290,132 @@ impl SessionHost { key: &str, grace: Duration, ) -> Option<( - watch::Receiver>, - watch::Receiver, + watch::Receiver>, + watch::Receiver, )> { - let mut sessions = lock(&self.sessions); - let session = sessions.get_mut(key)?; - if let Some(removed) = &session.removal { - return Some((removed.clone(), session.phase.clone())); + let mut instances = lock(&self.instances); + let instance = instances.get_mut(key)?; + if let Some(removed) = &instance.removal { + return Some((removed.clone(), instance.phase.clone())); } - session.stop.send_replace(true); - if *session.phase.borrow() == SessionPhase::Starting { - session.abort.send_replace(true); + instance.stop.send_replace(true); + if *instance.phase.borrow() == SandboxInstancePhase::Starting { + instance.abort.send_replace(true); } let (report, removed) = watch::channel(None); - session.removal = Some(removed.clone()); + instance.removal = Some(removed.clone()); tokio::spawn( Removal { - sessions: self.sessions.clone(), + instances: self.instances.clone(), key: key.to_string(), - serial: session.serial, - phase: session.phase.clone(), - thread: session.thread.take(), + serial: instance.serial, + phase: instance.phase.clone(), + thread: instance.thread.take(), grace, report, } .run(), ); - Some((removed, session.phase.clone())) + Some((removed, instance.phase.clone())) } } -fn terminal_status(phase: &watch::Receiver) -> SessionStatus { - SessionStatus { +fn terminal_status(phase: &watch::Receiver) -> SandboxInstanceStatus { + SandboxInstanceStatus { phase: phase.borrow().clone(), ready: false, } } -/// The single owner of one session's removal. +/// The single owner of one sandbox instance's removal. struct Removal { - sessions: Sessions, + instances: Instances, key: String, serial: u64, - phase: watch::Receiver, + phase: watch::Receiver, thread: Option>, grace: Duration, - report: watch::Sender>, + report: watch::Sender>, } impl Removal { async fn run(mut self) { - let finished = - tokio::time::timeout(self.grace, self.phase.wait_for(SessionPhase::is_terminal)) - .await - .is_ok(); + let finished = tokio::time::timeout( + self.grace, + self.phase.wait_for(SandboxInstancePhase::is_terminal), + ) + .await + .is_ok(); if !finished - && let Some(session) = lock(&self.sessions) + && let Some(instance) = lock(&self.instances) .get(&self.key) - .filter(|session| session.serial == self.serial) + .filter(|instance| instance.serial == self.serial) { - session.abort.send_replace(true); + instance.abort.send_replace(true); } if let Some(thread) = self.thread.take() { let _ = tokio::task::spawn_blocking(move || thread.join()).await; } let removed = { - let mut sessions = lock(&self.sessions); - match sessions.get(&self.key) { - Some(session) if session.serial == self.serial => sessions.remove(&self.key), + let mut instances = lock(&self.instances); + match instances.get(&self.key) { + Some(instance) if instance.serial == self.serial => instances.remove(&self.key), _ => None, } }; - let status = - removed.map_or_else(|| terminal_status(&self.phase), |session| session.status()); + let status = removed.map_or_else( + || terminal_status(&self.phase), + |instance| instance.status(), + ); self.report.send_replace(Some(status)); } } -struct SessionSignals { +struct SandboxInstanceSignals { stop_requested: watch::Receiver, abort_requested: watch::Receiver, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, } -/// Reports a hosted session's end and revokes its credentials, including when -/// the session thread unwinds. -struct SessionExit { +/// Reports a hosted sandbox instance's end and revokes its credentials, +/// including when the instance thread unwinds. +struct SandboxInstanceExit { credentials: Option, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, - outcome: Option, + outcome: Option, } -impl Drop for SessionExit { +impl Drop for SandboxInstanceExit { fn drop(&mut self) { if let Some(credentials) = self.credentials.take() { credentials.revoke(); } self.ready.send_replace(false); let outcome = self.outcome.take().unwrap_or_else(|| { - SessionPhase::Failed("session thread ended without reporting a result".to_string()) + SandboxInstancePhase::Failed( + "sandbox instance thread ended without reporting a result".to_string(), + ) }); self.phase.send_replace(outcome); - SessionBinding::unbind(); + SandboxInstanceBinding::unbind(); } } -fn run_hosted_session( - binding: &SessionBinding, +fn run_hosted_sandbox_instance( + binding: &SandboxInstanceBinding, credentials: Option, - body: SessionBody, - signals: SessionSignals, + body: SandboxInstanceBody, + signals: SandboxInstanceSignals, ) { - let SessionSignals { + let SandboxInstanceSignals { stop_requested, mut abort_requested, phase, ready, } = signals; - let mut exit = SessionExit { + let mut exit = SandboxInstanceExit { credentials, phase: phase.clone(), ready: ready.clone(), @@ -406,11 +424,11 @@ fn run_hosted_session( binding.bind(); let thread_binding = binding.clone(); let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(SESSION_WORKER_THREADS) - .thread_name("openshell-session-worker") + .worker_threads(INSTANCE_WORKER_THREADS) + .thread_name("openshell-instance-worker") .enable_all() .on_thread_start(move || thread_binding.bind()) - .on_thread_stop(SessionBinding::unbind) + .on_thread_stop(SandboxInstanceBinding::unbind) .build(); let outcome = match runtime { Ok(runtime) => { @@ -425,9 +443,9 @@ fn run_hosted_session( mirror_ready.send_replace(is_ready); if is_ready { mirror_phase.send_if_modified(|current| { - let promote = *current == SessionPhase::Starting; + let promote = *current == SandboxInstancePhase::Starting; if promote { - *current = SessionPhase::Supervising; + *current = SandboxInstancePhase::Supervising; } promote }); @@ -442,16 +460,16 @@ fn run_hosted_session( let stop_phase = phase.clone(); let shutdown = Box::pin(async move { let _ = stop.wait_for(|requested| *requested).await; - stop_phase.send_replace(SessionPhase::Stopping); + stop_phase.send_replace(SandboxInstancePhase::Stopping); }); - let session = body(SessionControls { + let instance = body(SandboxInstanceControls { shutdown, readiness, }); let outcome = tokio::select! { - result = session => result, + result = instance => result, _ = abort_requested.wait_for(|requested| *requested) => { - Err(miette::miette!("session was cancelled before it finished")) + Err(miette::miette!("sandbox instance was cancelled before it finished")) } }; mirror.abort(); @@ -459,43 +477,43 @@ fn run_hosted_session( }) })); runtime.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); - outcome.unwrap_or_else(|_| Err(miette::miette!("session panicked"))) + outcome.unwrap_or_else(|_| Err(miette::miette!("sandbox instance panicked"))) } - Err(error) => Err(miette::miette!("build session runtime: {error}")), + Err(error) => Err(miette::miette!("build sandbox instance runtime: {error}")), }; exit.outcome = Some(match outcome { - Ok(code) => SessionPhase::Exited(code), - Err(error) => SessionPhase::Failed(error.to_string()), + Ok(code) => SandboxInstancePhase::Exited(code), + Err(error) => SandboxInstancePhase::Failed(error.to_string()), }); } -/// Trusted inputs for one supervisor session hosted with others. -pub struct HostedSessionSpec { +/// Trusted inputs for one sandbox instance hosted with others. +pub struct SandboxInstanceSpec { /// Unique key for this generation within the host. pub key: String, pub backend_setup: Box, - pub config: crate::SessionConfig, - /// Workload image recorded in this session's OCSF events. + pub config: crate::SandboxInstanceConfig, + /// Workload image recorded in this instance's OCSF events. pub container_image: String, } -impl SessionHost { - /// Start a supervisor session with its own OCSF identity, OCSF JSON +impl SandboxInstanceHost { + /// Start a sandbox instance with its own OCSF identity, OCSF JSON /// settings, log push stream, denial log, and gateway credentials. /// /// # Errors /// /// Fails when the launch credentials are invalid or the key is hosted. - pub fn start_supervisor_session(&self, spec: HostedSessionSpec) -> Result<()> { - let HostedSessionSpec { + pub fn start_sandbox_instance(&self, spec: SandboxInstanceSpec) -> Result<()> { + let SandboxInstanceSpec { key, backend_setup, mut config, container_image, } = spec; let credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; - // Sessions must never share proxy trust files; give one a private - // directory that lives exactly as long as the session. + // Instances must never share proxy trust files; give one a private + // directory that lives exactly as long as the instance. let private_tls_dir = if config.proxy_tls.directory.is_none() { let prepared = crate::prepare_network_proxy_tls_dir(None)?; config.proxy_tls.directory = Some(prepared.path.clone()); @@ -510,8 +528,8 @@ impl SessionHost { openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, ); let log_push = config.sandbox_id.clone().zip(gateway); - let binding = SessionBinding { - event_context: Arc::new(crate::session_event_context( + let binding = SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( config.sandbox_id.as_deref(), config.sandbox.as_deref(), container_image, @@ -520,14 +538,15 @@ impl SessionHost { log_sink: log_push .as_ref() .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), }; config.gateway_credentials = Some(credentials.clone()); config.ocsf_json = binding.ocsf_json.clone(); - config.denial_log = openshell_supervisor_network::policy_local::DenialLogSource::Session( - binding.denial_log.clone(), - ); - let body: SessionBody = Box::new(move |controls| { + config.denial_log = + openshell_supervisor_network::policy_local::DenialLogSource::SandboxInstance( + binding.denial_log.clone(), + ); + let body: SandboxInstanceBody = Box::new(move |controls| { Box::pin(async move { let _private_tls_dir = private_tls_dir; let _log_push = log_push.map(|(sandbox_id, gateway)| { @@ -535,7 +554,12 @@ impl SessionHost { gateway, sandbox_id, log_rx, ) }); - Box::pin(crate::run_session(backend_setup.as_ref(), config, controls)).await + Box::pin(crate::run_sandbox_instance( + backend_setup.as_ref(), + config, + controls, + )) + .await }) }); self.start(&key, binding, Some(credentials), body) @@ -613,7 +637,7 @@ mod tests { .collect() } - struct Session { + struct Instance { identity: tokio::sync::oneshot::Receiver, logs: mpsc::Receiver, call_again: mpsc::Sender<()>, @@ -621,31 +645,35 @@ mod tests { credentials: GatewayCredentials, } - fn start_session(host: &SessionHost, gateway_url: &str, sandbox: &'static str) -> Session { + fn start_instance( + host: &SandboxInstanceHost, + gateway_url: &str, + sandbox: &'static str, + ) -> Instance { let credentials = GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); let (log_tx, logs) = mpsc::channel(16); - let binding = SessionBinding { - event_context: Arc::new(crate::session_event_context( + let binding = SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( Some(sandbox), Some(sandbox), String::new(), )), ocsf_json: OcsfJsonSettings::default(), log_sink: Some(LogSink::new(sandbox, log_tx)), - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), }; let (identity_tx, identity) = tokio::sync::oneshot::channel(); let (call_again, mut calls) = mpsc::channel(1); let (alive_tx, alive) = mpsc::channel::<()>(1); - let body: SessionBody = Box::new(move |controls| { + let body: SandboxInstanceBody = Box::new(move |controls| { Box::pin(async move { - let SessionControls { + let SandboxInstanceControls { shutdown, readiness, } = controls; - // Nested tasks and the blocking pool inherit the session identity. + // Nested tasks and the blocking pool inherit the instance identity. let observed = tokio::spawn(async { tokio::task::spawn_blocking(|| openshell_ocsf::ctx::ctx().sandbox_id.clone()) .await @@ -662,12 +690,12 @@ mod tests { ); tokio::spawn(async move { tracing::dispatcher::with_default(&dispatch, || { - tracing::info!(target: "session_test", "emitted by {sandbox}"); + tracing::info!(target: "instance_test", "emitted by {sandbox}"); }); }) .await .unwrap(); - // A long-lived task the session never stops itself. + // A long-lived task the instance never stops itself. tokio::spawn(async move { let _alive = alive_tx; std::future::pending::<()>().await; @@ -689,7 +717,7 @@ mod tests { }); host.start(sandbox, binding, Some(credentials.clone()), body) .unwrap(); - Session { + Instance { identity, logs, call_again, @@ -698,29 +726,35 @@ mod tests { } } - async fn wait_supervising(host: &SessionHost, key: &str) { + async fn wait_supervising(host: &SandboxInstanceHost, key: &str) { tokio::time::timeout(Duration::from_secs(10), async { - while host.status(key).map(|status| status.phase) != Some(SessionPhase::Supervising) { + while host.status(key).map(|status| status.phase) + != Some(SandboxInstancePhase::Supervising) + { tokio::time::sleep(Duration::from_millis(10)).await; } }) .await - .expect("session reaches supervision"); + .expect("instance reaches supervision"); } - fn unbound() -> SessionBinding { - SessionBinding { - event_context: Arc::new(crate::session_event_context(None, None, String::new())), + fn unbound() -> SandboxInstanceBinding { + SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( + None, + None, + String::new(), + )), ocsf_json: OcsfJsonSettings::default(), log_sink: None, - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), } } #[tokio::test(flavor = "multi_thread")] - async fn stopping_and_panicking_sessions_keep_their_key_and_fail_closed() { - let host = Arc::new(SessionHost::new()); - let body: SessionBody = Box::new(|controls| { + async fn stopping_and_panicking_instances_keep_their_key_and_fail_closed() { + let host = Arc::new(SandboxInstanceHost::new()); + let body: SandboxInstanceBody = Box::new(|controls| { Box::pin(async move { controls.readiness.send_replace(true); controls.shutdown.await; @@ -737,28 +771,28 @@ mod tests { }); tokio::time::timeout(Duration::from_secs(10), async { while host.status("generation").map(|status| status.phase) - != Some(SessionPhase::Stopping) + != Some(SandboxInstancePhase::Stopping) { tokio::time::sleep(Duration::from_millis(5)).await; } }) .await - .expect("session observes the stop request"); - let replacement: SessionBody = Box::new(|_| Box::pin(async { Ok(0) })); + .expect("instance observes the stop request"); + let replacement: SandboxInstanceBody = Box::new(|_| Box::pin(async { Ok(0) })); assert_eq!( host.start("generation", unbound(), None, replacement), - Err(SessionHostError::Duplicate) + Err(SandboxInstanceHostError::Duplicate) ); - let stopped = removal.await.unwrap().expect("session was hosted"); - assert_eq!(stopped.phase, SessionPhase::Exited(0)); + let stopped = removal.await.unwrap().expect("instance was hosted"); + assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); assert!(host.status("generation").is_none()); let credentials = GatewayCredentials::from_gateway_token("panicking-token").unwrap(); - let panicking: SessionBody = Box::new(|controls| { + let panicking: SandboxInstanceBody = Box::new(|controls| { Box::pin(async move { controls.readiness.send_replace(true); tokio::task::yield_now().await; - panic!("hosted session failure"); + panic!("hosted sandbox instance failure"); }) }); host.start("panicking", unbound(), Some(credentials.clone()), panicking) @@ -774,11 +808,11 @@ mod tests { } }) .await - .expect("panicking session reports an end"); + .expect("panicking instance reports an end"); assert_eq!( status, - SessionStatus { - phase: SessionPhase::Failed("session panicked".to_string()), + SandboxInstanceStatus { + phase: SandboxInstancePhase::Failed("sandbox instance panicked".to_string()), ready: false, } ); @@ -796,8 +830,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn removal_outlives_its_caller_and_reports_to_every_remover() { - let host = Arc::new(SessionHost::new()); - let winding_down = |code| -> SessionBody { + let host = Arc::new(SandboxInstanceHost::new()); + let winding_down = |code| -> SandboxInstanceBody { Box::new(move |controls| { Box::pin(async move { controls.readiness.send_replace(true); @@ -824,8 +858,8 @@ mod tests { tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) }); for remover in removers { - let status = remover.await.unwrap().expect("session was hosted"); - assert_eq!(status.phase, SessionPhase::Exited(3)); + let status = remover.await.unwrap().expect("instance was hosted"); + assert_eq!(status.phase, SandboxInstancePhase::Exited(3)); } assert!(host.status("generation").is_none()); @@ -836,46 +870,46 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!( host.status("generation").map(|status| status.phase), - Some(SessionPhase::Supervising) + Some(SandboxInstancePhase::Supervising) ); let status = host.remove("generation", Duration::from_secs(10)).await; assert_eq!( status.map(|status| status.phase), - Some(SessionPhase::Exited(4)) + Some(SandboxInstancePhase::Exited(4)) ); } #[tokio::test(flavor = "multi_thread")] - async fn hosted_sessions_keep_identity_credentials_and_lifecycle_separate() { + async fn hosted_instances_keep_identity_credentials_and_lifecycle_separate() { let (gateway_url, captured) = capture_gateway().await; - let host = SessionHost::new(); - let mut alpha = start_session(&host, &gateway_url, "sandbox-alpha"); - let mut beta = start_session(&host, &gateway_url, "sandbox-beta"); + let host = SandboxInstanceHost::new(); + let mut alpha = start_instance(&host, &gateway_url, "sandbox-alpha"); + let mut beta = start_instance(&host, &gateway_url, "sandbox-beta"); wait_supervising(&host, "sandbox-alpha").await; wait_supervising(&host, "sandbox-beta").await; assert!(matches!( host.start( "sandbox-alpha", - SessionBinding { - event_context: Arc::new(crate::session_event_context( + SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( None, None, String::new() )), ocsf_json: OcsfJsonSettings::default(), log_sink: None, - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), }, None, Box::new(|_| Box::pin(async { Ok(0) })), ), - Err(SessionHostError::Duplicate) + Err(SandboxInstanceHostError::Duplicate) )); assert_eq!((&mut alpha.identity).await.unwrap(), "sandbox-alpha"); assert_eq!((&mut beta.identity).await.unwrap(), "sandbox-beta"); // Other tests in this process may set the process-wide context; a - // thread outside every session must still carry no session identity. + // thread outside every instance must still carry no instance identity. let outside = openshell_ocsf::ctx::ctx().sandbox_id.clone(); assert!(!["sandbox-alpha", "sandbox-beta"].contains(&outside.as_str())); assert_eq!( @@ -886,18 +920,18 @@ mod tests { bearers_for(&captured, "sandbox-beta"), ["Bearer token-sandbox-beta"] ); - for (session, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { - let line = session.logs.try_recv().expect("session log line"); + for (instance, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { + let line = instance.logs.try_recv().expect("instance log line"); assert_eq!(line.sandbox_id, sandbox); assert_eq!(line.message, format!("emitted by {sandbox}")); - assert!(session.logs.try_recv().is_err()); + assert!(instance.logs.try_recv().is_err()); } let stopped = host .remove("sandbox-alpha", Duration::from_secs(10)) .await .expect("alpha was hosted"); - assert_eq!(stopped.phase, SessionPhase::Exited(0)); + assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); assert!(!stopped.ready); assert!(host.status("sandbox-alpha").is_none()); assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); @@ -921,7 +955,7 @@ mod tests { ); assert_eq!( host.status("sandbox-beta").map(|status| status.phase), - Some(SessionPhase::Supervising) + Some(SandboxInstancePhase::Supervising) ); assert!( beta.alive @@ -932,7 +966,7 @@ mod tests { let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; assert_eq!( stopped.map(|status| status.phase), - Some(SessionPhase::Exited(0)) + Some(SandboxInstancePhase::Exited(0)) ); } } From e1be7e9c35fe4dac942eef38520736f013aa7895 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 20:21:32 -0700 Subject: [PATCH 5/6] refactor(supervisor): launch sandbox instances from one config and prepared bindings Replace SandboxRunConfig and its field-by-field copy into SandboxInstanceConfig with a single launch configuration. Live per-instance state (gateway credentials, OCSF context and JSON settings, log sink, and denial log) is now an explicit SandboxInstanceBindings value that the owner prepares once, instead of placeholders the runner or host overwrote. The dedicated binary loads launch inputs from its arguments and environment and shares one GatewayCredentials with log push; run_sandbox_with_backend keeps the process-wide OCSF context, health endpoints, and signal handling. The instance host binds the same SandboxInstanceBindings to its runtime threads and revokes their credentials when the instance ends. Signed-off-by: Drew Newberry --- crates/openshell-supervisor/README.md | 45 +-- .../src/backend_setup/tests.rs | 58 ++-- .../src/legacy_startup_regression_tests.rs | 64 ++-- crates/openshell-supervisor/src/lib.rs | 321 +++++++----------- crates/openshell-supervisor/src/main.rs | 66 ++-- .../src/sandbox_instance_host.rs | 174 +++++----- 6 files changed, 337 insertions(+), 391 deletions(-) diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index d603af388f..ac2f3ed888 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -4,7 +4,7 @@ The supervisor loads and reconciles policy, maintains provider credentials, appl ## Backend startup -The public `run_sandbox` entry point selects the OpenShell Sandbox Protocol backend and collects its startup inputs into a public `SandboxRunConfig`. Shared startup receives that config and the trusted backend setup separately. The `backend_setup` module owns that backend's launch-data decoder, workload policy discovery, and client construction. Descriptor contents cannot select an implementation. +The supervisor binary selects a backend setup and collects its launch inputs into a public `SandboxInstanceConfig`. Shared startup receives that config and the trusted backend setup separately. The `backend_setup` module owns that backend's launch-data decoder, workload policy discovery, and client construction. Descriptor contents cannot select an implementation. Shared startup checks the admitted backend name before passing the opaque payload to its decoder. It then compares the decoded sandbox, session, and runtime generation with the trusted launch inputs before installing credentials or discovering workload policy. A mismatch stops startup. @@ -14,7 +14,7 @@ The supervisor admits policy and prepares credentials before constructing and at The client receives the supervisor's live provider state, bearer-token slot, and CA-path slot. Provider refresh, token rotation, and later CA publication must remain visible through those shared handles. Startup does not create independent copies of their current values. -`backend_setup`, `SandboxRunConfig`, and `run_sandbox_with_backend` are public so the supervisor binary can select a registered backend. External backends should implement the `IsolationBackend` and `SandboxProtocol` services rather than link this crate. +`backend_setup`, `SandboxInstanceConfig`, `SandboxInstanceBindings`, and `run_sandbox_with_backend` are public so the supervisor binary can select a registered backend. External backends should implement the `IsolationBackend` and `SandboxProtocol` services rather than link this crate. The startup result separates the backend transport payload from optional authenticated CONNECT listener settings. Shared networking starts the listener @@ -29,27 +29,32 @@ Gateway connections, and the gateway supervisor sessions carried over them, can reconnect within an instance without creating another one; the generation remains the fencing identifier. -`run_sandbox_instance` supervises one sandbox instance from a -`SandboxInstanceConfig` and `SandboxInstanceControls`. Everything that belongs -to that instance is an input or is created by the instance: gateway -credentials and their renewal, the Sandbox Protocol credential, policy, -provider credentials and token grants, middleware credentials, OCSF JSON -settings, proxy trust files, the denial history served by `policy.local`, the -backend connection, access plane, readiness, and stop signal. Gateway clients -receive a `GatewayEndpoint`, which pairs the URL with that instance's -`GatewayCredentials`; nothing is installed process-wide. - -`run_sandbox_with_backend` is the dedicated wrapper. It keeps the existing -process contract: it sets the process-wide OCSF context, reads launch inputs -from the environment, binds the readiness socket and port to instance -readiness, and stops the instance on SIGTERM or SIGINT. The binary shares one -`GatewayCredentials` between log push and supervision, so renewal by either -keeps both current. +`run_sandbox_instance` supervises one sandbox instance from three inputs. +`SandboxInstanceConfig` is the single launch configuration: identity, command, +policy inputs, auth bundle, backend descriptor, and file locations. +`SandboxInstanceBindings` is the live state its owner prepares once: the +`GatewayCredentials` issued from the auth bundle, the OCSF context and JSON +settings, the log sink, and the denial history served by `policy.local`. +`SandboxInstanceControls` carries the owner's stop signal and readiness. +Everything else, including credential renewal, policy, provider credentials and +token grants, middleware credentials, the backend connection, and the access +plane, is created by the instance. Gateway clients receive a `GatewayEndpoint`, +which pairs the URL with that instance's `GatewayCredentials`; nothing is +installed process-wide. + +The dedicated process wrapper keeps the existing process contract. The binary +loads the launch configuration from its arguments and environment and prepares +`SandboxInstanceBindings::dedicated`, sharing one `GatewayCredentials` with log +push and the OCSF JSON settings with its logging layers, so renewal by either +gateway client keeps both current. `run_sandbox_with_backend` sets the +process-wide OCSF context, binds the readiness socket and port to instance +readiness, and stops the instance on SIGTERM or SIGINT. `sandbox_instance_host::SandboxInstanceHost` runs several sandbox instances in one process. Each instance gets its own Tokio runtime; every thread of that -runtime, and the thread driving it, is bound to the instance's OCSF context, -OCSF JSON settings, log sink, and denial log. Spawned tasks, blocking work, +runtime, and the thread driving it, is bound to the instance's +`SandboxInstanceBindings`: OCSF context, OCSF JSON settings, log sink, and +denial log. Spawned tasks, blocking work, callbacks, reconnects, and renewal therefore inherit the instance's identity, and threads outside an instance carry none and push no logs. The OCSF context also supplies the sandbox identity in middleware requests. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index b9b5c1cdc9..b4704bf623 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -680,32 +680,42 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .require_networking .store(true, Ordering::SeqCst); let startup = |await_main_process_attachment| { + let config = crate::SandboxInstanceConfig { + command: vec!["test-agent".into(), "test-argument".into()], + workdir: Some("/test-workspace".into()), + timeout_secs: 15, + interactive: false, + await_main_process_attachment, + sandbox_id: Some(setup.sandbox_id.clone()), + sandbox: None, + container_image: String::new(), + openshell_endpoint: None, + policy_rules: Some(rules.to_string_lossy().into_owned()), + policy_data: Some(data.to_string_lossy().into_owned()), + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: TestSetup::descriptor(), + auth_bundle: setup.auth.clone(), + admitted_isolation_backend: Some(TEST_BACKEND.into()), + main_exit_marker: Some(marker.clone()), + trace_context: None, + provider_workload_api_socket: None, + // The binary loads the proxy trust directory from its environment. + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + }; + let bindings = crate::SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), + openshell_ocsf::OcsfJsonSettings::default(), + ); Box::pin(crate::run_sandbox_with_backend( &setup, - crate::SandboxRunConfig { - command: vec!["test-agent".into(), "test-argument".into()], - workdir: Some("/test-workspace".into()), - timeout_secs: 15, - interactive: false, - await_main_process_attachment, - sandbox_id: Some(setup.sandbox_id.clone()), - sandbox: None, - openshell_endpoint: None, - policy_rules: Some(rules.to_string_lossy().into_owned()), - policy_data: Some(data.to_string_lossy().into_owned()), - ssh_socket_path: None, - health_socket_path: Some(readiness.clone()), - health_port: Some(health_port), - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(Mutex::new(String::new())), - upstream_proxy_args: UpstreamProxyArgs::default(), - additional_ca_bundle: None, - backend_descriptor: TestSetup::descriptor(), - auth_bundle: setup.auth.clone(), - gateway_credentials: None, - admitted_isolation_backend: Some(TEST_BACKEND.into()), - main_exit_marker: Some(marker.clone()), - }, + config, + bindings, + Some(readiness.clone()), + Some(health_port), )) }; diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index e27bd23455..699880f1ad 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -11,7 +11,7 @@ use openshell_core::proto::{ UpdateConfigResponse, gateway_message, supervisor_message, }; use openshell_isolation_interface::contract::{BackendError, ResolvedWorkloadIdentity}; -use std::sync::{Mutex, atomic::Ordering}; +use std::sync::atomic::Ordering; use tonic::codegen::tokio_stream::{StreamExt as _, wrappers::TcpListenerStream}; use tonic::codegen::{Body, BoxFuture, BoxStream, Bytes, Service, StdError, http}; @@ -277,37 +277,41 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl session_id, probe: probe.clone(), }; - let startup = run_sandbox_with_backend( - &setup, - SandboxRunConfig { - command: vec!["true".into()], - workdir: None, - timeout_secs: 0, - interactive: false, - await_main_process_attachment: false, - sandbox_id: Some("sb-legacy-restart".into()), - sandbox: Some("legacy-restart".into()), - openshell_endpoint: Some(endpoint), - policy_rules: None, - policy_data: None, - ssh_socket_path: None, - health_socket_path: None, - health_port: None, - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(Mutex::new(String::new())), - upstream_proxy_args: - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), - additional_ca_bundle: None, - backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { - backend_name: "test-held".into(), - payload: Vec::new(), - }, - auth_bundle, - gateway_credentials: None, - admitted_isolation_backend: Some("test-held".into()), - main_exit_marker: None, + let config = SandboxInstanceConfig { + command: vec!["true".into()], + workdir: None, + timeout_secs: 0, + interactive: false, + await_main_process_attachment: false, + sandbox_id: Some("sb-legacy-restart".into()), + sandbox: Some("legacy-restart".into()), + container_image: String::new(), + openshell_endpoint: Some(endpoint), + policy_rules: None, + policy_data: None, + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { + backend_name: "test-held".into(), + payload: Vec::new(), }, + auth_bundle, + admitted_isolation_backend: Some("test-held".into()), + main_exit_marker: None, + trace_context: None, + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + }; + let bindings = SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) + .unwrap(), + openshell_ocsf::OcsfJsonSettings::default(), ); + let startup = run_sandbox_with_backend(&setup, config, bindings, None, None); tokio::pin!(startup); let result = tokio::select! { () = probe.ready_event.notified() => None, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 69ee814a34..39699751e2 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -67,7 +67,7 @@ use openshell_ocsf::{ /// /// The singleton lives in `openshell-ocsf` so both supervisor leaves can /// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +/// `run_sandbox_with_backend()` startup via `openshell_ocsf::ctx::set_ctx`. pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; /// The sandbox rejected the image working directory while starting the agent. @@ -388,7 +388,10 @@ use provider_readiness::{EnvironmentIdentity, Tracker as ProviderReadinessTracke use tokio::sync::mpsc::UnboundedSender; use tokio::time::timeout; -fn shared_ssh_socket_from_env() -> bool { +/// Whether a dedicated supervisor's environment grants the SSH socket to its +/// group rather than its owner. +#[must_use] +pub fn shared_ssh_socket_from_env() -> bool { std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) .is_ok_and(|value| shared_ssh_socket_value(&value)) } @@ -661,146 +664,35 @@ pub async fn run_network_proxy( } } -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::implicit_hasher, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - health_socket_path: Option, - health_port: Option, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, - additional_ca_bundle: Option, - backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, - auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - admitted_isolation_backend: Option, - main_exit_marker: Option, -) -> Result { - // Shared startup retains policy and networking state; box it to keep callers' futures small. - Box::pin(run_sandbox_with_backend( - &backend_setup::OpenShellBackendSetup, - SandboxRunConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - health_socket_path, - health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials: None, - admitted_isolation_backend, - main_exit_marker, - }, - )) - .await -} - -/// Trusted inputs for one supervisor launch. -pub struct SandboxRunConfig { - pub command: Vec, - pub workdir: Option, - pub timeout_secs: u64, - pub interactive: bool, - pub await_main_process_attachment: bool, - pub sandbox_id: Option, - pub sandbox: Option, - pub openshell_endpoint: Option, - pub policy_rules: Option, - pub policy_data: Option, - pub ssh_socket_path: Option, - pub health_socket_path: Option, - pub health_port: Option, - pub ocsf_enabled: Arc, - pub ocsf_schema_version: Arc>, - pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, - pub additional_ca_bundle: Option, - pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, - pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - /// Credentials issued from `auth_bundle` and shared with other gateway - /// clients of this process, such as log push. Created when absent. - pub gateway_credentials: Option, - pub admitted_isolation_backend: Option, - pub main_exit_marker: Option, -} - /// Supervise the one sandbox instance this process was launched for. /// -/// The process-wide OCSF context, SIGTERM/SIGINT handling, readiness -/// endpoints, and the environment-provided launch inputs belong to the -/// dedicated process; [`run_sandbox_instance`] owns everything else. +/// This is the dedicated process wrapper. It sets the process-wide OCSF +/// context from `bindings`, publishes instance readiness on the health socket +/// and port, and stops the instance on SIGTERM or SIGINT; +/// [`run_sandbox_instance`] owns everything else. The caller loads `config` +/// from the process's arguments and environment and prepares `bindings` once, +/// sharing their gateway credentials and OCSF JSON settings with the +/// process's log push and logging layers. /// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. -#[allow(clippy::similar_names)] +/// +/// # Errors +/// +/// Returns an error if a readiness endpoint cannot be prepared or the +/// instance fails to start or supervise its workload. pub async fn run_sandbox_with_backend( backend_setup: &dyn backend_setup::BackendSetup, - config: SandboxRunConfig, + config: SandboxInstanceConfig, + bindings: SandboxInstanceBindings, + health_socket_path: Option, + health_port: Option, ) -> Result { - let SandboxRunConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - health_socket_path, - health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials, - admitted_isolation_backend, - main_exit_marker, - } = config; - // Initialize the process-wide OCSF context early so that events emitted // during policy loading (filesystem config, validation) have a context. // Proxy IP/port use defaults here; the boundary mediation source carries // workload-side connection metadata. - if !openshell_ocsf::ctx::set_ctx(sandbox_instance_event_context( - sandbox_id.as_deref(), - sandbox.as_deref(), - std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - )) { + if !openshell_ocsf::ctx::set_ctx((*bindings.event_context).clone()) { debug!("OCSF context already initialized, keeping existing"); } @@ -813,39 +705,8 @@ pub async fn run_sandbox_with_backend( .collect::>>()?; let result = Box::pin(run_sandbox_instance( backend_setup, - SandboxInstanceConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - shared_ssh_socket: shared_ssh_socket_from_env(), - ocsf_json: openshell_ocsf::OcsfJsonSettings { - enabled: ocsf_enabled, - target_version: ocsf_schema_version, - }, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials, - admitted_isolation_backend, - main_exit_marker, - trace_context: TraceContext::from_environment(), - provider_workload_api_socket: std::env::var_os( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - ) - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from), - proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), - denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), - }, + config, + bindings, SandboxInstanceControls { shutdown: Box::pin(wait_for_control_shutdown_signal()), readiness, @@ -902,13 +763,14 @@ impl TraceContext { } } -/// Trusted inputs owned by one supervised sandbox instance. +/// Trusted launch inputs of one supervised sandbox instance. /// /// A sandbox instance is one running generation of a stable Sandbox resource; /// gateway connections may reconnect within it without creating another. -/// Nothing in an instance reads process-wide sandbox state: identity, -/// credentials, policy inputs, logging settings, and file locations all come -/// from here, so one process can host several instances. +/// Nothing in an instance reads process-wide sandbox state: identity, policy +/// inputs, and file locations come from here, and live per-instance state +/// from its [`SandboxInstanceBindings`], so one process can host several +/// instances. pub struct SandboxInstanceConfig { pub command: Vec, pub workdir: Option, @@ -917,20 +779,18 @@ pub struct SandboxInstanceConfig { pub await_main_process_attachment: bool, pub sandbox_id: Option, pub sandbox: Option, + /// Workload image recorded in this instance's OCSF events. + pub container_image: String, pub openshell_endpoint: Option, pub policy_rules: Option, pub policy_data: Option, pub ssh_socket_path: Option, /// Grant the SSH socket to the supervisor's group rather than its owner. pub shared_ssh_socket: bool, - /// OCSF JSON output settings this instance's gateway configuration controls. - pub ocsf_json: openshell_ocsf::OcsfJsonSettings, pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, pub additional_ca_bundle: Option, pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - /// Credentials issued from `auth_bundle`; created from it when absent. - pub gateway_credentials: Option, pub admitted_isolation_backend: Option, pub main_exit_marker: Option, /// Parent of the startup span. @@ -938,10 +798,59 @@ pub struct SandboxInstanceConfig { /// SPIFFE Workload API socket for provider token grants; absent fails closed. pub provider_workload_api_socket: Option, pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, +} + +impl SandboxInstanceConfig { + /// OCSF identity of this sandbox instance. + #[must_use] + pub fn event_context(&self) -> EventContext { + sandbox_instance_event_context( + self.sandbox_id.as_deref(), + self.sandbox.as_deref(), + self.container_image.clone(), + ) + } +} + +/// Live state bound to one sandbox instance, prepared once by its owner. +/// +/// A dedicated supervisor shares these with its process-wide logging and log +/// push; a hosted instance binds them to every thread of its runtime. Every +/// gateway client of the instance, including log push and credential renewal, +/// uses the same `gateway_credentials`, so renewal by one keeps all current. +#[derive(Clone)] +pub struct SandboxInstanceBindings { + /// Gateway credentials issued from the instance's auth bundle. + pub gateway_credentials: openshell_core::grpc_client::GatewayCredentials, + pub event_context: Arc, + /// OCSF JSON output settings this instance's gateway configuration controls. + pub ocsf_json: openshell_ocsf::OcsfJsonSettings, + /// Log push destination of a hosted instance's threads. A dedicated + /// process pushes through its process-wide layer instead. + pub log_sink: Option, /// Source of `policy.local` recent denials. pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, } +impl SandboxInstanceBindings { + /// Bindings of a process dedicated to the instance `config` launches. Its + /// process-wide log push and shorthand log files serve the instance. + #[must_use] + pub fn dedicated( + config: &SandboxInstanceConfig, + gateway_credentials: openshell_core::grpc_client::GatewayCredentials, + ocsf_json: openshell_ocsf::OcsfJsonSettings, + ) -> Self { + Self { + gateway_credentials, + event_context: Arc::new(config.event_context()), + ocsf_json, + log_sink: None, + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + } + } +} + /// How the owner of a sandbox instance stops it and observes its readiness. pub struct SandboxInstanceControls { /// Completes when the owner asks the instance to stop its workload. It is @@ -1011,6 +920,7 @@ impl Drop for SandboxInstanceReadiness { pub async fn run_sandbox_instance( backend_setup: &dyn backend_setup::BackendSetup, config: SandboxInstanceConfig, + bindings: SandboxInstanceBindings, controls: SandboxInstanceControls, ) -> Result { let SandboxInstanceConfig { @@ -1021,24 +931,30 @@ pub async fn run_sandbox_instance( await_main_process_attachment, sandbox_id, sandbox, + container_image: _, openshell_endpoint, policy_rules, policy_data, ssh_socket_path, shared_ssh_socket, - ocsf_json, upstream_proxy_args, additional_ca_bundle, backend_descriptor, auth_bundle, - gateway_credentials, admitted_isolation_backend, main_exit_marker, trace_context, provider_workload_api_socket, proxy_tls, - denial_log, } = config; + // The owner binds the event context and log sink to the instance's + // threads; the instance itself uses the rest. + let SandboxInstanceBindings { + gateway_credentials, + ocsf_json, + denial_log, + .. + } = bindings; let SandboxInstanceControls { shutdown: shutdown_requested, readiness, @@ -1071,10 +987,6 @@ pub async fn run_sandbox_instance( sandbox_id.as_deref(), &auth_bundle, )?; - let gateway_credentials = match gateway_credentials { - Some(credentials) => credentials, - None => openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth_bundle)?, - }; let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") })?; @@ -4435,29 +4347,44 @@ mod tests { ssh_host_private_key: None, ocsf_otlp_headers: None, }; - let error = run_sandbox( - vec!["true".to_string()], - None, - 0, - false, - false, - Some("sandbox-1".to_string()), - None, - None, - None, - None, - None, - None, - None, - Arc::new(AtomicBool::new(false)), - Arc::new(std::sync::Mutex::new(String::new())), - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), - None, - openshell_isolation_interface::contract::BackendDescriptor { + let config = SandboxInstanceConfig { + command: vec!["true".to_string()], + workdir: None, + timeout_secs: 0, + interactive: false, + await_main_process_attachment: false, + sandbox_id: Some("sandbox-1".to_string()), + sandbox: None, + container_image: String::new(), + openshell_endpoint: None, + policy_rules: None, + policy_data: None, + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { backend_name: "in-process-test".to_string(), payload: b"in-process-only-launch-data".to_vec(), }, auth_bundle, + admitted_isolation_backend: None, + main_exit_marker: None, + trace_context: None, + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + }; + let bindings = SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) + .expect("credentials"), + openshell_ocsf::OcsfJsonSettings::default(), + ); + let error = run_sandbox_with_backend( + &backend_setup::OpenShellBackendSetup, + config, + bindings, None, None, ) diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index daafbc364b..b222c1c196 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -556,32 +556,50 @@ fn main() -> Result<()> { "isolation-backend role started without validated runtime inputs" )); }; + let config = openshell_supervisor::SandboxInstanceConfig { + command, + workdir, + timeout_secs: args.timeout, + interactive, + await_main_process_attachment, + sandbox_id: args.sandbox_id, + sandbox: args.sandbox, + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + openshell_endpoint: args.openshell_endpoint, + policy_rules: args.policy_rules, + policy_data: args.policy_data, + ssh_socket_path: args.ssh_socket_path, + shared_ssh_socket: openshell_supervisor::shared_ssh_socket_from_env(), + upstream_proxy_args, + additional_ca_bundle: args.additional_ca_bundle, + backend_descriptor, + auth_bundle, + admitted_isolation_backend: Some(admitted_backend), + main_exit_marker: args.main_exit_marker, + trace_context: openshell_supervisor::TraceContext::from_environment(), + provider_workload_api_socket: std::env::var_os( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ) + .filter(|path| !path.is_empty()) + .map(PathBuf::from), + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + }; + // Log push already shares these credentials and the logging + // layers these OCSF JSON settings. + let bindings = openshell_supervisor::SandboxInstanceBindings::dedicated( + &config, + gateway_credentials, + openshell_ocsf::OcsfJsonSettings { + enabled: ocsf_enabled, + target_version: ocsf_schema_version, + }, + ); let result = Box::pin(openshell_supervisor::run_sandbox_with_backend( backend_setup.as_ref(), - openshell_supervisor::SandboxRunConfig { - command, - workdir, - timeout_secs: args.timeout, - interactive, - await_main_process_attachment, - sandbox_id: args.sandbox_id, - sandbox: args.sandbox, - openshell_endpoint: args.openshell_endpoint, - policy_rules: args.policy_rules, - policy_data: args.policy_data, - ssh_socket_path: args.ssh_socket_path, - health_socket_path: args.health_socket_path, - health_port: args.health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle: args.additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials: Some(gateway_credentials), - admitted_isolation_backend: Some(admitted_backend), - main_exit_marker: args.main_exit_marker, - }, + config, + bindings, + args.health_socket_path, + args.health_port, )) .await; supervisor_exit(result) diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 4511b40176..65a8b27534 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -9,13 +9,13 @@ //! creating another. //! //! Each instance runs on its own Tokio runtime. Every thread that runtime -//! starts, and the thread driving it, is bound to the instance's OCSF context, -//! OCSF JSON settings, log sink, and denial log, so tasks, callbacks, -//! reconnects, and credential renewal that the instance spawns inherit its -//! identity. Unbound threads carry no sandbox identity and push no logs. -//! Gateway credentials are never bound to threads; each instance passes its -//! own explicitly. Stopping an instance shuts down only its runtime and -//! revokes its gateway credentials. +//! starts, and the thread driving it, is bound to the instance's +//! [`SandboxInstanceBindings`]: OCSF context, OCSF JSON settings, log sink, and +//! denial log, so tasks, callbacks, reconnects, and credential renewal that the +//! instance spawns inherit its identity. Unbound threads carry no sandbox +//! identity and push no logs. Gateway credentials are never bound to threads; +//! each instance passes its own explicitly. Stopping an instance shuts down +//! only its runtime and revokes its gateway credentials. use std::collections::HashMap; use std::future::Future; @@ -26,41 +26,34 @@ use std::time::Duration; use miette::Result; use openshell_core::grpc_client::GatewayCredentials; -use openshell_ocsf::{EventContext, OcsfJsonSettings}; -use openshell_supervisor_network::policy_local::SandboxInstanceDenialLog; +use openshell_ocsf::OcsfJsonSettings; +use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; -use crate::SandboxInstanceControls; +use crate::{SandboxInstanceBindings, SandboxInstanceControls}; /// Worker threads in each sandbox instance runtime. const INSTANCE_WORKER_THREADS: usize = 2; /// Bound on joining an instance thread after its runtime is told to stop. const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); -/// Per-instance values bound to every thread of the instance's runtime. -#[derive(Clone)] -pub struct SandboxInstanceBinding { - pub event_context: Arc, - pub ocsf_json: OcsfJsonSettings, - pub log_sink: Option, - pub denial_log: SandboxInstanceDenialLog, +/// Bind the calling thread to one instance's bindings. +fn bind_thread(bindings: &SandboxInstanceBindings) { + openshell_ocsf::ctx::bind_thread_ctx(Some(bindings.event_context.clone())); + openshell_ocsf::bind_thread_json_settings(Some(bindings.ocsf_json.clone())); + openshell_supervisor_process::log_push::bind_thread_sink(bindings.log_sink.clone()); + bind_thread_denial_log(match &bindings.denial_log { + DenialLogSource::SandboxInstance(log) => Some(log.clone()), + DenialLogSource::ShorthandFiles(_) => None, + }); } -impl SandboxInstanceBinding { - fn bind(&self) { - openshell_ocsf::ctx::bind_thread_ctx(Some(self.event_context.clone())); - openshell_ocsf::bind_thread_json_settings(Some(self.ocsf_json.clone())); - openshell_supervisor_process::log_push::bind_thread_sink(self.log_sink.clone()); - bind_thread_denial_log(Some(self.denial_log.clone())); - } - - fn unbind() { - openshell_ocsf::ctx::bind_thread_ctx(None); - openshell_ocsf::bind_thread_json_settings(None); - openshell_supervisor_process::log_push::bind_thread_sink(None); - bind_thread_denial_log(None); - } +fn unbind_thread() { + openshell_ocsf::ctx::bind_thread_ctx(None); + openshell_ocsf::bind_thread_json_settings(None); + openshell_supervisor_process::log_push::bind_thread_sink(None); + bind_thread_denial_log(None); } std::thread_local! { @@ -196,10 +189,10 @@ impl SandboxInstanceHost { Self::default() } - /// Start `body` on a new runtime bound to `binding`. + /// Start `body` on a new runtime bound to `bindings`. /// - /// `credentials`, when present, are revoked once the instance's runtime - /// has stopped, so nothing can reuse them after the instance ends. + /// The bindings' gateway credentials are revoked once the instance's + /// runtime has stopped, so nothing can reuse them after the instance ends. /// /// # Errors /// @@ -208,8 +201,7 @@ impl SandboxInstanceHost { pub fn start( &self, key: &str, - binding: SandboxInstanceBinding, - credentials: Option, + bindings: SandboxInstanceBindings, body: SandboxInstanceBody, ) -> std::result::Result<(), SandboxInstanceHostError> { let mut instances = lock(&self.instances); @@ -224,8 +216,7 @@ impl SandboxInstanceHost { .name("openshell-instance".to_string()) .spawn(move || { run_hosted_sandbox_instance( - &binding, - credentials, + &bindings, body, SandboxInstanceSignals { stop_requested, @@ -381,7 +372,7 @@ struct SandboxInstanceSignals { /// Reports a hosted sandbox instance's end and revokes its credentials, /// including when the instance thread unwinds. struct SandboxInstanceExit { - credentials: Option, + credentials: GatewayCredentials, phase: watch::Sender, ready: watch::Sender, outcome: Option, @@ -389,9 +380,7 @@ struct SandboxInstanceExit { impl Drop for SandboxInstanceExit { fn drop(&mut self) { - if let Some(credentials) = self.credentials.take() { - credentials.revoke(); - } + self.credentials.revoke(); self.ready.send_replace(false); let outcome = self.outcome.take().unwrap_or_else(|| { SandboxInstancePhase::Failed( @@ -399,13 +388,12 @@ impl Drop for SandboxInstanceExit { ) }); self.phase.send_replace(outcome); - SandboxInstanceBinding::unbind(); + unbind_thread(); } } fn run_hosted_sandbox_instance( - binding: &SandboxInstanceBinding, - credentials: Option, + bindings: &SandboxInstanceBindings, body: SandboxInstanceBody, signals: SandboxInstanceSignals, ) { @@ -416,19 +404,19 @@ fn run_hosted_sandbox_instance( ready, } = signals; let mut exit = SandboxInstanceExit { - credentials, + credentials: bindings.gateway_credentials.clone(), phase: phase.clone(), ready: ready.clone(), outcome: None, }; - binding.bind(); - let thread_binding = binding.clone(); + bind_thread(bindings); + let thread_bindings = bindings.clone(); let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(INSTANCE_WORKER_THREADS) .thread_name("openshell-instance-worker") .enable_all() - .on_thread_start(move || thread_binding.bind()) - .on_thread_stop(SandboxInstanceBinding::unbind) + .on_thread_start(move || bind_thread(&thread_bindings)) + .on_thread_stop(unbind_thread) .build(); let outcome = match runtime { Ok(runtime) => { @@ -493,8 +481,6 @@ pub struct SandboxInstanceSpec { pub key: String, pub backend_setup: Box, pub config: crate::SandboxInstanceConfig, - /// Workload image recorded in this instance's OCSF events. - pub container_image: String, } impl SandboxInstanceHost { @@ -509,9 +495,7 @@ impl SandboxInstanceHost { key, backend_setup, mut config, - container_image, } = spec; - let credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; // Instances must never share proxy trust files; give one a private // directory that lives exactly as long as the instance. let private_tls_dir = if config.proxy_tls.directory.is_none() { @@ -521,31 +505,30 @@ impl SandboxInstanceHost { } else { None }; - let gateway = config.openshell_endpoint.as_deref().map(|endpoint| { - openshell_core::grpc_client::GatewayEndpoint::new(endpoint, credentials.clone()) - }); + // Log push, supervision, and renewal share these credentials. + let gateway_credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; + let log_push = config + .sandbox_id + .clone() + .zip(config.openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new( + endpoint, + gateway_credentials.clone(), + ) + })); let (log_tx, log_rx) = tokio::sync::mpsc::channel( openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, ); - let log_push = config.sandbox_id.clone().zip(gateway); - let binding = SandboxInstanceBinding { - event_context: Arc::new(crate::sandbox_instance_event_context( - config.sandbox_id.as_deref(), - config.sandbox.as_deref(), - container_image, - )), + let bindings = SandboxInstanceBindings { + gateway_credentials, + event_context: Arc::new(config.event_context()), ocsf_json: OcsfJsonSettings::default(), log_sink: log_push .as_ref() .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), - denial_log: SandboxInstanceDenialLog::default(), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), }; - config.gateway_credentials = Some(credentials.clone()); - config.ocsf_json = binding.ocsf_json.clone(); - config.denial_log = - openshell_supervisor_network::policy_local::DenialLogSource::SandboxInstance( - binding.denial_log.clone(), - ); + let instance_bindings = bindings.clone(); let body: SandboxInstanceBody = Box::new(move |controls| { Box::pin(async move { let _private_tls_dir = private_tls_dir; @@ -557,12 +540,13 @@ impl SandboxInstanceHost { Box::pin(crate::run_sandbox_instance( backend_setup.as_ref(), config, + instance_bindings, controls, )) .await }) }); - self.start(&key, binding, Some(credentials), body) + self.start(&key, bindings, body) .map_err(|error| miette::miette!("{error}")) } } @@ -654,7 +638,8 @@ mod tests { GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); let (log_tx, logs) = mpsc::channel(16); - let binding = SandboxInstanceBinding { + let bindings = SandboxInstanceBindings { + gateway_credentials: credentials.clone(), event_context: Arc::new(crate::sandbox_instance_event_context( Some(sandbox), Some(sandbox), @@ -662,7 +647,7 @@ mod tests { )), ocsf_json: OcsfJsonSettings::default(), log_sink: Some(LogSink::new(sandbox, log_tx)), - denial_log: SandboxInstanceDenialLog::default(), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), }; let (identity_tx, identity) = tokio::sync::oneshot::channel(); let (call_again, mut calls) = mpsc::channel(1); @@ -715,8 +700,7 @@ mod tests { } }) }); - host.start(sandbox, binding, Some(credentials.clone()), body) - .unwrap(); + host.start(sandbox, bindings, body).unwrap(); Instance { identity, logs, @@ -738,8 +722,9 @@ mod tests { .expect("instance reaches supervision"); } - fn unbound() -> SandboxInstanceBinding { - SandboxInstanceBinding { + fn unbound() -> SandboxInstanceBindings { + SandboxInstanceBindings { + gateway_credentials: GatewayCredentials::from_gateway_token("unbound-token").unwrap(), event_context: Arc::new(crate::sandbox_instance_event_context( None, None, @@ -747,7 +732,7 @@ mod tests { )), ocsf_json: OcsfJsonSettings::default(), log_sink: None, - denial_log: SandboxInstanceDenialLog::default(), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), } } @@ -763,7 +748,7 @@ mod tests { Ok(0) }) }); - host.start("generation", unbound(), None, body).unwrap(); + host.start("generation", unbound(), body).unwrap(); wait_supervising(&host, "generation").await; let removal = tokio::spawn({ let host = host.clone(); @@ -780,7 +765,7 @@ mod tests { .expect("instance observes the stop request"); let replacement: SandboxInstanceBody = Box::new(|_| Box::pin(async { Ok(0) })); assert_eq!( - host.start("generation", unbound(), None, replacement), + host.start("generation", unbound(), replacement), Err(SandboxInstanceHostError::Duplicate) ); let stopped = removal.await.unwrap().expect("instance was hosted"); @@ -795,8 +780,15 @@ mod tests { panic!("hosted sandbox instance failure"); }) }); - host.start("panicking", unbound(), Some(credentials.clone()), panicking) - .unwrap(); + host.start( + "panicking", + SandboxInstanceBindings { + gateway_credentials: credentials.clone(), + ..unbound() + }, + panicking, + ) + .unwrap(); let status = tokio::time::timeout(Duration::from_secs(10), async { loop { if let Some(status) = host.status("panicking") @@ -841,7 +833,7 @@ mod tests { }) }) }; - host.start("generation", unbound(), None, winding_down(3)) + host.start("generation", unbound(), winding_down(3)) .unwrap(); wait_supervising(&host, "generation").await; // A caller that gives up, such as an expired RPC, strands nothing. @@ -864,7 +856,7 @@ mod tests { assert!(host.status("generation").is_none()); // The key is free again, and the earlier removal cannot touch its new owner. - host.start("generation", unbound(), None, winding_down(4)) + host.start("generation", unbound(), winding_down(4)) .unwrap(); wait_supervising(&host, "generation").await; tokio::time::sleep(Duration::from_millis(50)).await; @@ -890,17 +882,7 @@ mod tests { assert!(matches!( host.start( "sandbox-alpha", - SandboxInstanceBinding { - event_context: Arc::new(crate::sandbox_instance_event_context( - None, - None, - String::new() - )), - ocsf_json: OcsfJsonSettings::default(), - log_sink: None, - denial_log: SandboxInstanceDenialLog::default(), - }, - None, + unbound(), Box::new(|_| Box::pin(async { Ok(0) })), ), Err(SandboxInstanceHostError::Duplicate) From 34d2ee006b19798cf4bfc1f2a5b44ee2fc2764e7 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 21 Oct 2026 18:07:16 -0700 Subject: [PATCH 6/6] fix(supervisor): update Windows host proxy test config Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/src/host.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-network/src/host.rs b/crates/openshell-supervisor-network/src/host.rs index 5f111acb21..45b21127c4 100644 --- a/crates/openshell-supervisor-network/src/host.rs +++ b/crates/openshell-supervisor-network/src/host.rs @@ -258,7 +258,7 @@ mod tests { binary_path, sandbox_id: Some("sandbox-123".to_string()), sandbox_name: Some("agent-box".to_string()), - openshell_endpoint: None, + gateway: None, provider_credentials: None, agent_proposals: AgentProposals::new(true), denial_tx: None,