From 23f52031a652db1885fe315f893d0f89ddb2bf1a Mon Sep 17 00:00:00 2001 From: Yuedong Wu Date: Tue, 13 Oct 2026 10:46:19 +0800 Subject: [PATCH] feat(kubernetes): support emptyDir sandbox workspaces Signed-off-by: Yuedong Wu --- crates/openshell-driver-kubernetes/README.md | 12 ++- .../openshell-driver-kubernetes/src/config.rs | 44 +++++++++ .../openshell-driver-kubernetes/src/driver.rs | 94 +++++++++++++++---- crates/openshell-driver-kubernetes/src/lib.rs | 2 +- .../openshell-driver-kubernetes/src/main.rs | 10 +- docs/how-it-works/gateways/configuration.mdx | 4 + docs/how-it-works/sandboxes/runtimes.mdx | 7 +- docs/kubernetes/setup.mdx | 2 +- skills/debug-openshell-cluster/SKILL.md | 2 +- 9 files changed, 151 insertions(+), 26 deletions(-) diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index bc51f3c24b..d87d036ac4 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -134,9 +134,9 @@ errors, avoiding a gateway-side watch restart and its associated watch gap. ## Workspace Persistence -Sandbox pods use a PVC-backed `/sandbox` workspace. An init container seeds the -PVC from the image's original `/sandbox` contents on first start and writes a -sentinel so subsequent starts skip the copy. +By default, sandbox pods use a PVC-backed `/sandbox` workspace. An init +container seeds the PVC from the image's original `/sandbox` contents on first +start and writes a sentinel so subsequent starts skip the copy. This is a stopgap persistence model. It preserves user files across pod rescheduling but duplicates the base workspace and does not automatically apply @@ -161,6 +161,12 @@ runtime via `OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE` and that OpenShell provisions automatically; they have no effect when a `driver_config` mount attaches an existing PVC under `/sandbox`, which skips the default PVC. +Set `workspace_volume = "empty_dir"` (standalone driver: +`OPENSHELL_K8S_WORKSPACE_VOLUME`) to back `/sandbox` with an `emptyDir` capped +at `workspace_default_storage_size` instead of a PVC, for clusters without +dynamic provisioning. The init container then seeds `/sandbox` on every pod +start, so stop and start or any other pod replacement resets it. + ## Credentials, TLS, and Relay Both Pods set `automountServiceAccountToken: false`. The supervisor receives an diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index acbd0d006f..fda01569d2 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -141,6 +141,32 @@ impl FromStr for WorkspaceMode { } } +/// Volume that backs each sandbox's `/sandbox` workspace. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceVolume { + /// A per-sandbox PVC that survives Pod replacement. + #[default] + PersistentVolumeClaim, + /// An `emptyDir` capped at `workspace_default_storage_size` that lasts as + /// long as the workload Pod. + EmptyDir, +} + +impl FromStr for WorkspaceVolume { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "persistent_volume_claim" => Ok(Self::PersistentVolumeClaim), + "empty_dir" => Ok(Self::EmptyDir), + other => Err(format!( + "invalid workspace volume '{other}'; expected persistent_volume_claim or empty_dir" + )), + } + } +} + fn deserialize_provider_spiffe_workload_api_socket_path<'de, D>( deserializer: D, ) -> Result @@ -252,6 +278,9 @@ pub struct KubernetesComputeConfig { pub client_tls_secret_name: String, pub host_gateway_ip: String, pub enable_user_namespaces: bool, + /// Volume that backs `/sandbox`. Default `persistent_volume_claim`; + /// `empty_dir` needs no `StorageClass`. + pub workspace_volume: WorkspaceVolume, pub workspace_default_storage_size: String, /// Kubernetes `StorageClass` name for the default workspace PVC. /// Empty string (default) = omit `storageClassName`, using the cluster's @@ -374,6 +403,7 @@ impl Default for KubernetesComputeConfig { client_tls_secret_name: String::new(), host_gateway_ip: String::new(), enable_user_namespaces: false, + workspace_volume: WorkspaceVolume::default(), workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(), workspace_storage_class: String::new(), default_runtime_class_name: String::new(), @@ -1061,6 +1091,20 @@ mod tests { assert_eq!(cfg.workspace_storage_class, "fast-ssd"); } + #[test] + fn serde_override_workspace_volume() { + let json = serde_json::json!({ "workspace_volume": "empty_dir" }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.workspace_volume, WorkspaceVolume::EmptyDir); + } + + #[test] + fn serde_rejects_invalid_workspace_volume() { + let json = serde_json::json!({ "workspace_volume": "ephemeral" }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + #[test] fn serde_override_service_account_name() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f6546e7b8f..8053b27f23 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -5,8 +5,9 @@ use crate::config::{ DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, OperatorNamespaceAllowlist, WorkspaceMode, is_dns_1123_label, - managed_namespace, managed_namespace_prefix, validate_managed_namespace_name, + KubernetesComputeConfig, OperatorNamespaceAllowlist, WorkspaceMode, WorkspaceVolume, + is_dns_1123_label, managed_namespace, managed_namespace_prefix, + validate_managed_namespace_name, }; use crate::isolation::{ BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeBoundarySpec, @@ -651,11 +652,13 @@ fn kubernetes_driver_volume_mount_to_k8s( // --------------------------------------------------------------------------- // Default workspace persistence (temporary — will be replaced by snapshotting) // --------------------------------------------------------------------------- -// Every sandbox pod gets a PVC-backed `/sandbox` directory so that user data -// (installed packages, files, dotfiles) survives pod rescheduling across -// gateway stop/start cycles. An init container seeds the PVC with the +// By default, every sandbox pod gets a PVC-backed `/sandbox` directory so that +// user data (installed packages, files, dotfiles) survives pod rescheduling +// across gateway stop/start cycles. An init container seeds the PVC with the // image's original `/sandbox` contents on first use so that the Python venv, -// skills, and shell config are not lost when the empty PVC is mounted. +// skills, and shell config are not lost when the empty PVC is mounted. With +// `workspace_volume = "empty_dir"`, a size-capped `emptyDir` replaces the PVC +// and the init container seeds it on every pod start. // // NOTE: This PVC + init-container approach is a stopgap. It has known // limitations: image upgrades don't propagate into existing PVCs, the init @@ -1970,6 +1973,7 @@ impl KubernetesComputeDriver { service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, enable_user_namespaces: self.config.enable_user_namespaces, + workspace_volume: self.config.workspace_volume, workspace_default_storage_size: &self.config.workspace_default_storage_size, workspace_storage_class: &self.config.workspace_storage_class, default_runtime_class_name: &self.config.default_runtime_class_name, @@ -6232,16 +6236,11 @@ fn default_workspace_volume_claim_templates( storage_size: &str, storage_class: &str, ) -> serde_json::Value { - let size = if storage_size.is_empty() { - DEFAULT_WORKSPACE_STORAGE_SIZE - } else { - storage_size - }; let mut spec = serde_json::json!({ "accessModes": ["ReadWriteOnce"], "resources": { "requests": { - "storage": size + "storage": workspace_storage_size(storage_size) } } }); @@ -6256,6 +6255,15 @@ fn default_workspace_volume_claim_templates( }]) } +/// Size of the workspace volume: the PVC request, or the `emptyDir` limit. +fn workspace_storage_size(storage_size: &str) -> &str { + if storage_size.is_empty() { + DEFAULT_WORKSPACE_STORAGE_SIZE + } else { + storage_size + } +} + /// Parameters shared by `sandbox_to_k8s_spec` and `sandbox_template_to_k8s`. #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { @@ -6267,6 +6275,7 @@ struct SandboxPodParams<'a> { service_account_name: &'a str, sandbox_id: &'a str, enable_user_namespaces: bool, + workspace_volume: WorkspaceVolume, workspace_default_storage_size: &'a str, workspace_storage_class: &'a str, default_runtime_class_name: &'a str, @@ -6291,6 +6300,7 @@ impl Default for SandboxPodParams<'_> { service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", enable_user_namespaces: false, + workspace_volume: WorkspaceVolume::default(), workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE, workspace_storage_class: "", default_runtime_class_name: "", @@ -6336,7 +6346,7 @@ fn sandbox_to_k8s_spec( let mut root = serde_json::Map::new(); // Determine early whether OpenShell should inject its default workspace - // PVC. Explicit Kubernetes driver-config mounts under /sandbox/ take + // volume. Explicit Kubernetes driver-config mounts under /sandbox/ take // ownership of workspace persistence. // We need this flag before building the podTemplate because the workspace // persistence transforms are applied inside sandbox_template_to_k8s. @@ -6366,7 +6376,7 @@ fn sandbox_to_k8s_spec( } } - if inject_workspace { + if inject_workspace && params.workspace_volume == WorkspaceVolume::PersistentVolumeClaim { root.insert( "volumeClaimTemplates".to_string(), default_workspace_volume_claim_templates( @@ -6605,9 +6615,9 @@ fn sandbox_template_to_k8s_with_validated_config( apply_supervisor_sandbox_runtime_boundary(&mut result, params); - // Inject workspace persistence (init container + PVC volume mount) so - // that /sandbox data survives pod rescheduling. Skipped when the user - // provides custom storage through driver_config. + // Inject the workspace (init container + `/sandbox` mount, plus the + // `emptyDir` when it replaces the PVC). Skipped when the user provides + // custom storage through driver_config. if inject_workspace { apply_workspace_persistence( &mut result, @@ -6616,6 +6626,17 @@ fn sandbox_template_to_k8s_with_validated_config( None, Some((params.sandbox_uid, params.sandbox_gid)), ); + if params.workspace_volume == WorkspaceVolume::EmptyDir { + result["spec"]["volumes"] + .as_array_mut() + .expect("pod volumes must be an array") + .push(serde_json::json!({ + "name": WORKSPACE_VOLUME_NAME, + "emptyDir": { + "sizeLimit": workspace_storage_size(params.workspace_default_storage_size) + } + })); + } } result @@ -11270,6 +11291,45 @@ mod tests { .get("storageClassName") .is_none() ); + assert!( + !cr["spec"]["podTemplate"]["spec"]["volumes"] + .as_array() + .unwrap() + .iter() + .any(|volume| volume["name"] == WORKSPACE_VOLUME_NAME) + ); + } + + #[test] + fn empty_dir_workspace_volume_replaces_claim_template() { + let params = SandboxPodParams { + workspace_volume: WorkspaceVolume::EmptyDir, + workspace_default_storage_size: "5Gi", + ..SandboxPodParams::default() + }; + let cr = sandbox_to_k8s_spec_for_test(Some(&SandboxSpec::default()), ¶ms); + + assert!(cr["spec"].get("volumeClaimTemplates").is_none()); + let pod_spec = &cr["spec"]["podTemplate"]["spec"]; + let workspace = pod_spec["volumes"] + .as_array() + .unwrap() + .iter() + .find(|volume| volume["name"] == WORKSPACE_VOLUME_NAME) + .unwrap(); + assert_eq!( + workspace["emptyDir"], + serde_json::json!({"sizeLimit": "5Gi"}) + ); + assert!( + pod_spec["containers"][0]["volumeMounts"] + .as_array() + .unwrap() + .contains(&serde_json::json!({ + "name": WORKSPACE_VOLUME_NAME, + "mountPath": WORKSPACE_MOUNT_PATH + })) + ); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index aa980db44f..9dc0b5e0fd 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -13,7 +13,7 @@ mod sandbox_runtime; pub use config::{ DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesImagePullPolicy, KubernetesSandboxRuntimeConfig, - ManagedSshIngressConfig, WorkspaceMode, managed_namespace_prefix, + ManagedSshIngressConfig, WorkspaceMode, WorkspaceVolume, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 448147af6c..4e5e503697 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -13,7 +13,7 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_kubernetes::{ ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, KubernetesImagePullPolicy, - KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode, + KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode, WorkspaceVolume, }; #[derive(Parser, Debug)] @@ -179,6 +179,13 @@ struct Args { #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, + #[arg( + long, + env = "OPENSHELL_K8S_WORKSPACE_VOLUME", + default_value = "persistent_volume_claim" + )] + workspace_volume: WorkspaceVolume, + /// Lifetime (seconds) of the projected `ServiceAccount` token /// kubelet writes into each sandbox pod for the `IssueSandboxToken` /// bootstrap exchange. Kubelet enforces a minimum of 600s; the @@ -288,6 +295,7 @@ async fn main() -> Result<()> { client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), enable_user_namespaces: args.enable_user_namespaces, + workspace_volume: args.workspace_volume, workspace_default_storage_size: std::env::var( "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", ) diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 30a75f7f6a..b5c6179ddb 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -882,6 +882,10 @@ ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" enable_user_namespaces = false +# Volume backing /sandbox: "persistent_volume_claim" (default) or "empty_dir". +# An emptyDir needs no StorageClass, is capped at workspace_default_storage_size, +# and lasts only as long as the sandbox pod. +# workspace_volume = "empty_dir" workspace_default_storage_size = "10Gi" # Kubernetes StorageClass for the workspace PVC. Empty (default) omits the # field, using the cluster's default StorageClass. Set this on clusters with no diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index cc76cf98d2..c100a240d5 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -251,7 +251,8 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Project or stage only `ca.crt` into supervisor Pods to authenticate the gateway. | | `sandbox_runtime_image` | `sandboxRuntime.image.*` | Override the sandbox runtime image. | | `supervisor_image` | `supervisor.image.*` | Override the supervisor image. | -| `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size. | +| `workspace_volume` | `gatewayConfig.openshell.drivers.kubernetes.workspace_volume` | `persistent_volume_claim` (default) or `empty_dir`, which needs no `StorageClass`. | +| `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size, or the `empty_dir` size limit. | | `workspace_storage_class` | `server.workspaceStorageClass` | `StorageClass` for workspace PVCs. Set this if the cluster has no default `StorageClass`. | | `allowed_pod_annotations` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. Keys under `openshell.ai` and `agents.x-k8s.io` are reserved and can't be allowlisted. | | `allowed_pod_labels` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_labels` | Pod label keys that sandbox templates may set, in the same format as `allowed_pod_annotations`. Empty (default) rejects all template labels. Labels select pods for NetworkPolicy rules and Services, so only allow keys that no other rule in the sandbox namespace selects on. | @@ -261,6 +262,8 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `proxy_auth_secret_name` / `proxy_auth_secret_key` | `upstreamProxy.authSecret.name` / `.key` | Secret holding the proxy `user:pass` credential. | | `proxy_ca_bundle` | `upstreamProxy.caBundle.configMapName` / `.key` | ConfigMap with the proxy's CA certificate. | +An `empty_dir` workspace lasts only as long as the sandbox pod, so stopping and starting the sandbox resets `/sandbox` to the image's contents, and exceeding the size limit evicts the pod. + For the full list, see the [Gateway Configuration File](/how-it-works/gateways/configuration) reference. On OpenShift, copy the cluster proxy's CA into the gateway namespace and set `upstreamProxy.caBundle.configMapName` to `corporate-proxy-ca`: @@ -279,7 +282,7 @@ Workspace storage settings cannot change after a sandbox is created. Delete and ### Kubernetes PVC Mounts -Mount existing PersistentVolumeClaims into the agent container through driver config. Any mount under `/sandbox` replaces the default workspace PVC for that sandbox. +Mount existing PersistentVolumeClaims into the agent container through driver config. Any mount under `/sandbox` replaces the default workspace volume for that sandbox. ```shell openshell sandbox create \ diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 9b924c9d06..3267d4f5e0 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -230,7 +230,7 @@ openshell status ## Upgrade OpenShell -To upgrade, rerun `helm upgrade --install` with the new `--version` and the values you installed with. Running sandboxes keep their supervisor and sandbox runtime images until they stop. When a stopped sandbox starts, it takes both images from the upgraded gateway and keeps its workload image and workspace. +To upgrade, rerun `helm upgrade --install` with the new `--version` and the values you installed with. Running sandboxes keep their supervisor and sandbox runtime images until they stop. When a stopped sandbox starts, it takes both images from the upgraded gateway and keeps its workload image and workspace PVC. If you override images, set `supervisor.image` and `sandboxRuntime.image` to the same release, or set only `global.image.tag`. An incompatible pair fails when a sandbox starts, and the supervisor log shows the cause while the start is in progress: diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ffedd4504a..de0f4f8418 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1160,7 +1160,7 @@ credential failures. | CPU-only VM startup reports that read-only `/proc` cannot be removed | Older supervisors can infer GPU requirements from host devices | Check gateway, VM driver, and bundled supervisor versions; use matching updated artifacts. CPU-only VMs do not require read-write `/proc`. | | Docker GPU sandbox fails before startup | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` | | Kubernetes gateway pod pending | PVC unbound, taint, selector, or insufficient resources | `kubectl -n openshell describe pod ` | -| Kubernetes sandbox pod stuck pending, workspace PVC unbound | Cluster has no default `StorageClass` and OpenShell does not set `storageClassName` on the workspace PVC (clusters with a default `StorageClass` bind fine without it) | `kubectl -n openshell describe pvc`; set `server.workspaceStorageClass` (gateway config `workspace_storage_class`) to a valid `StorageClass` | +| Kubernetes sandbox pod stuck pending, workspace PVC unbound | Cluster has no default `StorageClass` and OpenShell does not set `storageClassName` on the workspace PVC (clusters with a default `StorageClass` bind fine without it) | `kubectl -n openshell describe pvc`; set `server.workspaceStorageClass` (gateway config `workspace_storage_class`) to a valid `StorageClass`, or set gateway config `workspace_volume = "empty_dir"` if `/sandbox` need not survive stop and start | | Kubernetes gateway pod crash loops | Missing secret, bad DB URL, bad TLS config | `kubectl -n openshell logs deployment/openshell -c openshell-gateway` or `kubectl -n openshell logs statefulset/openshell -c openshell-gateway` | | Mutating RPCs return `UNAVAILABLE` with "timed out waiting for a concurrent mutation" | Advisory-lock contention, a full lock pool on one replica (4 connections unless `database_lock_max_connections` is set), a slow PostgreSQL, or older gateways still running during an upgrade | `openshell_server_mutation_lock_timeouts_total`, `openshell_server_mutation_lock_connections_in_use` against its capacity, the `detail` field of `mutation lock acquisition timed out` in gateway logs, `mutation lock held longer than the lock wait timeout` warnings and `openshell_server_mutation_lock_hold_seconds`, the `pg_locks` query in Step 6 | | Mutating RPCs return `INTERNAL` with "could not open a mutation lock connection" | PostgreSQL out of connection slots, restarting, or unreachable | `openshell_server_mutation_lock_errors_total`, PostgreSQL logs, `SELECT count(*) FROM pg_stat_activity` against `max_connections`, `database_max_connections` and `database_lock_max_connections`, the connection sizing in the High Availability guide |