diff --git a/crates/navigator-bootstrap/src/docker.rs b/crates/navigator-bootstrap/src/docker.rs index 4ced403d63..08470b6ebd 100644 --- a/crates/navigator-bootstrap/src/docker.rs +++ b/crates/navigator-bootstrap/src/docker.rs @@ -355,6 +355,26 @@ pub async fn ensure_container( env_vars.push(format!("REGISTRY_USERNAME={username}")); env_vars.push(format!("REGISTRY_PASSWORD={password}")); } + + // When the primary registry is NOT the distribution registry (e.g. a + // local registry in push-mode), we still need containerd credentials for + // the distribution registry so that community sandbox images + // (d1i0nduu2f6qxk.cloudfront.net/nemoclaw-community/sandboxes/*) can be + // pulled at runtime. Pass the distribution registry credentials as a + // separate set of env vars so the entrypoint can add a second block to + // registries.yaml. + if registry_host != pull_registry() { + env_vars.push(format!("DIST_REGISTRY_HOST={}", pull_registry())); + env_vars.push(format!( + "DIST_REGISTRY_USERNAME={}", + pull_registry_username() + )); + env_vars.push(format!( + "DIST_REGISTRY_PASSWORD={}", + pull_registry_password() + )); + } + if !extra_sans.is_empty() { env_vars.push(format!("EXTRA_SANS={}", extra_sans.join(","))); } diff --git a/crates/navigator-cli/src/run.rs b/crates/navigator-cli/src/run.rs index fa2bbd7397..6993299997 100644 --- a/crates/navigator-cli/src/run.rs +++ b/crates/navigator-cli/src/run.rs @@ -1090,10 +1090,13 @@ pub async fn sandbox_create( // When a custom image is specified, clear the default run_as_user/group // to prevent failures on images that lack the "sandbox" user/group. + // If no explicit policy was provided, start from the restrictive default + // so the server stores a policy with cleared identity — otherwise the + // sandbox falls back to disk discovery which may re-introduce + // run_as_user: sandbox for images that don't have that user. if image.is_some() { - if let Some(ref mut p) = policy { - navigator_policy::clear_process_identity(p); - } + let p = policy.get_or_insert_with(navigator_policy::restrictive_default_policy); + navigator_policy::clear_process_identity(p); } let template = image.map(|img| SandboxTemplate { diff --git a/crates/navigator-server/src/grpc.rs b/crates/navigator-server/src/grpc.rs index 86094b9c14..0a733c47c0 100644 --- a/crates/navigator-server/src/grpc.rs +++ b/crates/navigator-server/src/grpc.rs @@ -186,6 +186,11 @@ impl Navigator for NavigatorService { ..Default::default() }; + // Persist to the store FIRST so the sandbox watcher always finds + // the record with `spec` populated. If we created the k8s + // resource first, the watcher could race us and write a fallback + // record with `spec: None`, causing the supervisor to fail with + // "sandbox has no spec". self.state.sandbox_index.update_from_sandbox(&sandbox); self.state @@ -194,49 +199,47 @@ impl Navigator for NavigatorService { .await .map_err(|e| Status::internal(format!("persist sandbox failed: {e}")))?; - self.state.sandbox_watch_bus.notify(&id); - + // Now create the Kubernetes resource. If this fails, clean up + // the store entry to avoid orphans. match self.state.sandbox_client.create(&sandbox).await { - Ok(_) => { - info!( - sandbox_id = %id, - sandbox_name = %name, - "CreateSandbox request completed successfully" - ); - Ok(Response::new(SandboxResponse { - sandbox: Some(sandbox), - })) - } + Ok(_) => {} Err(kube::Error::Api(err)) if err.code == 409 => { + // Clean up the store entry we just wrote. + let _ = self.state.store.delete("sandbox", &id).await; + self.state.sandbox_index.remove_sandbox(&id); warn!( sandbox_id = %id, sandbox_name = %name, "Sandbox already exists in Kubernetes" ); - if let Err(e) = self.state.store.delete(Sandbox::object_type(), &id).await { - warn!(sandbox_id = %id, error = %e, "Failed to clean up store after conflict"); - } - self.state.sandbox_index.remove_sandbox(&id); - self.state.sandbox_watch_bus.notify(&id); - Err(Status::already_exists("sandbox already exists")) + return Err(Status::already_exists("sandbox already exists")); } Err(err) => { + // Clean up the store entry we just wrote. + let _ = self.state.store.delete("sandbox", &id).await; + self.state.sandbox_index.remove_sandbox(&id); warn!( sandbox_id = %id, sandbox_name = %name, error = %err, "CreateSandbox request failed" ); - if let Err(e) = self.state.store.delete(Sandbox::object_type(), &id).await { - warn!(sandbox_id = %id, error = %e, "Failed to clean up store after creation failure"); - } - self.state.sandbox_index.remove_sandbox(&id); - self.state.sandbox_watch_bus.notify(&id); - Err(Status::internal(format!( + return Err(Status::internal(format!( "create sandbox in kubernetes failed: {err}" - ))) + ))); } } + + self.state.sandbox_watch_bus.notify(&id); + + info!( + sandbox_id = %id, + sandbox_name = %name, + "CreateSandbox request completed successfully" + ); + Ok(Response::new(SandboxResponse { + sandbox: Some(sandbox), + })) } type WatchSandboxStream = ReceiverStream>; diff --git a/crates/navigator-server/src/lib.rs b/crates/navigator-server/src/lib.rs index 398acd7dc1..7c7a4683ba 100644 --- a/crates/navigator-server/src/lib.rs +++ b/crates/navigator-server/src/lib.rs @@ -30,7 +30,7 @@ pub use grpc::NavigatorService; pub use http::{health_router, http_router}; pub use multiplex::{MultiplexService, MultiplexedService}; use persistence::Store; -use sandbox::{SandboxClient, spawn_sandbox_watcher}; +use sandbox::{SandboxClient, spawn_sandbox_watcher, spawn_store_reconciler}; use sandbox_index::SandboxIndex; use sandbox_watch::{SandboxWatchBus, spawn_kube_event_tailer}; pub use tls::TlsAcceptor; @@ -130,6 +130,13 @@ pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Resul state.sandbox_watch_bus.clone(), state.tracing_log_bus.clone(), ); + spawn_store_reconciler( + store.clone(), + state.sandbox_client.clone(), + state.sandbox_index.clone(), + state.sandbox_watch_bus.clone(), + state.tracing_log_bus.clone(), + ); spawn_kube_event_tailer(state.clone()); // Create the multiplexed service diff --git a/deploy/docker/cluster-entrypoint.sh b/deploy/docker/cluster-entrypoint.sh index 9cca0ff0e0..73498b22bf 100644 --- a/deploy/docker/cluster-entrypoint.sh +++ b/deploy/docker/cluster-entrypoint.sh @@ -113,6 +113,18 @@ mirrors: REGEOF + # If the distribution registry is a separate host (e.g. we're using a + # local registry for component images), add it as an additional mirror + # so community sandbox images can be pulled at runtime. + if [ -n "${DIST_REGISTRY_HOST:-}" ] && [ "${DIST_REGISTRY_HOST}" != "${REGISTRY_HOST}" ]; then + echo "Adding distribution registry mirror for ${DIST_REGISTRY_HOST}" + cat >> "$REGISTRIES_YAML" <> "$REGISTRIES_YAML" <> "$REGISTRIES_YAML" <> "$REGISTRIES_YAML" <