From 4a0738dacedd92e0eeaf89407edab9a61d6a7ea1 Mon Sep 17 00:00:00 2001 From: politerealism Date: Tue, 6 Oct 2026 10:05:20 -0400 Subject: [PATCH 1/3] test(conformance): migrate workspace_lifecycle to the conformance suite Port the workspace and workspace-scoped provider CRUD/isolation/deletion-guard coverage from e2e/rust/tests/workspace_lifecycle.rs into two driver-agnostic scenarios (workspace-lifecycle, workspace-terminating) under tests/suites/conformance/, following the OpenShellRunner/Scenario pattern established for sandbox_lifecycle. Remove the old per-driver test file and its Cargo.toml and tests/artifacts.nix follow-up-exclusion entries. Normalizes miette's line-wrapped diagnostic text before substring-matching error messages; the original test's exact-match assertions were fragile to the terminal width the CLI happened to detect when rendering errors. Verified against an ephemeral Podman gateway. Signed-off-by: politerealism (cherry picked from commit b3b16f0977e67cbb22546d30ef9d7da1cfb2d236) --- crates/openshell-conformance/src/lib.rs | 38 +- .../src/scenarios/mod.rs | 2 + .../src/scenarios/workspace_lifecycle.rs | 518 ++++++++++++++++++ e2e/rust/Cargo.toml | 5 - e2e/rust/tests/workspace_lifecycle.rs | 278 ---------- tests/artifacts.nix | 1 - .../cli/tests/workspace_lifecycle.rs | 33 ++ 7 files changed, 590 insertions(+), 285 deletions(-) create mode 100644 crates/openshell-conformance/src/scenarios/workspace_lifecycle.rs delete mode 100644 e2e/rust/tests/workspace_lifecycle.rs create mode 100644 tests/suites/conformance/cli/tests/workspace_lifecycle.rs diff --git a/crates/openshell-conformance/src/lib.rs b/crates/openshell-conformance/src/lib.rs index 4d1fcc1308..dd34fc8f10 100644 --- a/crates/openshell-conformance/src/lib.rs +++ b/crates/openshell-conformance/src/lib.rs @@ -27,7 +27,7 @@ pub use scenarios::{ FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO, - SMOKE_SCENARIO, + SMOKE_SCENARIO, WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO, }; /// A portable conformance scenario exercised by the Cargo test suite. @@ -94,6 +94,28 @@ impl CommandResult { self.elapsed } + /// Check combined stdout+stderr for `needle`, normalizing miette's + /// line-wrapped, `│`-continued diagnostic text first so phrase matches + /// don't depend on the terminal width the CLI detected when rendering. + pub fn output_contains(&self, needle: &str) -> bool { + self.normalized_output().contains(needle) + } + + /// Case-insensitive variant of [`Self::output_contains`]. + pub fn output_contains_ignore_case(&self, needle: &str) -> bool { + self.normalized_output() + .to_lowercase() + .contains(&needle.to_lowercase()) + } + + fn normalized_output(&self) -> String { + format!("{}{}", self.stdout, self.stderr) + .replace(['\n', '│'], " ") + .split_whitespace() + .collect::>() + .join(" ") + } + pub fn json(&self) -> Result { serde_json::from_str(&self.stdout).map_err(|source| RunnerError::InvalidJson { context: self.context(), @@ -110,6 +132,20 @@ impl CommandResult { Err(self.failure_diagnostic(&self.expectation)) } + /// Assert the command's success matches `expect_success` (`false` asserts + /// the command fails, e.g. a rejected duplicate-name or blocked-while-managed case). + pub fn require_outcome(&self, expect_success: bool) -> Result<(), String> { + if self.success() == expect_success { + Ok(()) + } else { + Err(self.failure_diagnostic(if expect_success { + "command succeeds" + } else { + "command fails" + })) + } + } + pub fn failure_diagnostic(&self, expectation: &str) -> String { format!( "{}\nexpected: {expectation}\nactual: exit {} after {:.1?}\ncommand: {}\nstdout:\n{}\nstderr:\n{}", diff --git a/crates/openshell-conformance/src/scenarios/mod.rs b/crates/openshell-conformance/src/scenarios/mod.rs index 72a90925f0..9cafd6023a 100644 --- a/crates/openshell-conformance/src/scenarios/mod.rs +++ b/crates/openshell-conformance/src/scenarios/mod.rs @@ -7,6 +7,7 @@ mod file_transfer; mod policy_behavior; mod sandbox_lifecycle; mod smoke; +mod workspace_lifecycle; pub use file_transfer::{ FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, @@ -17,3 +18,4 @@ pub use policy_behavior::{ }; pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO; pub use smoke::SMOKE_SCENARIO; +pub use workspace_lifecycle::{WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO}; diff --git a/crates/openshell-conformance/src/scenarios/workspace_lifecycle.rs b/crates/openshell-conformance/src/scenarios/workspace_lifecycle.rs new file mode 100644 index 0000000000..7b6e16f697 --- /dev/null +++ b/crates/openshell-conformance/src/scenarios/workspace_lifecycle.rs @@ -0,0 +1,518 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Portable workspace lifecycle conformance scenarios. + +use std::time::Duration; + +use serde::Deserialize; + +use crate::{ + CommandResult, OpenShellRunner, Scenario, ScenarioFuture, combine_results, + output_reports_not_found, +}; + +const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); +const PROVIDER_TYPE: &str = "openai"; +const PROVIDER_CREDENTIAL: &str = "OPENAI_API_KEY=test-value"; + +/// Certify workspace and workspace-scoped provider CRUD, isolation from the +/// default workspace, and the deletion guard that blocks removing a +/// workspace while resources still exist in it. +/// +/// - **Behavior:** a created workspace is retrievable and listed as `Active`. A provider created in it appears in its own workspace listing and in the all-workspaces listing (owned by that workspace), and not in the default workspace. Deleting the workspace is refused while the provider exists and succeeds once it is removed, after which the workspace can no longer be retrieved. +/// - **Preconditions:** a reachable gateway that can create workspaces and providers. The provider uses the `openai` type with a placeholder credential and is never contacted. +/// - **Permissions:** an identity allowed to create and delete workspaces and providers. +/// - **Side effects:** creates one workspace and one provider named from the run id. Both are deleted on success and on failure; a cleanup failure fails the scenario. +/// - **Known limitation:** `workspace get` has no structured output, so only its exit status is checked, and the deletion-guard error is matched by phrase because the CLI exposes no structured error. +pub const WORKSPACE_LIFECYCLE_SCENARIO: Scenario = Scenario { + name: "workspace-lifecycle", + run: run_workspace_lifecycle, +}; + +/// Certify that a terminating workspace rejects new resource creation while +/// still allowing deletion of its existing resources. +/// +/// - **Behavior:** after deleting a workspace that still holds a resource, the workspace is listed as `Terminating`. While it is terminating, creating another resource in it is rejected, deleting its existing resource succeeds, and deleting the workspace then succeeds. +/// - **Preconditions:** a reachable gateway that can create workspaces and providers. The provider uses the `openai` type with a placeholder credential and is never contacted. +/// - **Permissions:** an identity allowed to create and delete workspaces and providers. +/// - **Side effects:** creates one workspace and up to two providers named from the run id. All are deleted on success and on failure; a cleanup failure fails the scenario. +/// - **Known limitation:** the rejected create is matched by phrase because the CLI exposes no structured error. +pub const WORKSPACE_TERMINATING_SCENARIO: Scenario = Scenario { + name: "workspace-terminating", + run: run_workspace_terminating, +}; + +fn run_workspace_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + let workspace = format!("ws-{}-cr", runner.id()); + let provider = format!("pv-{}-cr", runner.id()); + + let result = workspace_full_crud_lifecycle(runner, &workspace, &provider).await; + let cleanup = cleanup_workspace(runner, &workspace, &[&provider]).await; + combine_results(result, cleanup) + }) +} + +fn run_workspace_terminating(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + let workspace = format!("ws-{}-tm", runner.id()); + let provider = format!("pv-{}-tm", runner.id()); + let blocked_provider = format!("pv-{}-bl", runner.id()); + + let result = + workspace_terminating_rejects_creates(runner, &workspace, &provider, &blocked_provider) + .await; + let cleanup = cleanup_workspace(runner, &workspace, &[&provider, &blocked_provider]).await; + combine_results(result, cleanup) + }) +} + +async fn workspace_full_crud_lifecycle( + runner: &OpenShellRunner, + workspace: &str, + provider: &str, +) -> Result<(), String> { + run_cli( + runner, + "create", + format!("workspace '{workspace}' is created"), + &["workspace", "create", "--name", workspace], + true, + ) + .await?; + + run_cli( + runner, + "get", + format!("workspace '{workspace}' can be retrieved"), + &["workspace", "get", workspace], + true, + ) + .await?; + require_workspace_status(runner, "list-created", workspace, "Active").await?; + + run_cli( + runner, + "provider-create", + format!("provider '{provider}' is created in workspace '{workspace}'"), + &[ + "provider", + "create", + "--name", + provider, + "--type", + PROVIDER_TYPE, + "--credential", + PROVIDER_CREDENTIAL, + "--workspace", + workspace, + ], + true, + ) + .await?; + + require_provider_listed( + runner, + "provider-list-scoped", + &["--workspace", workspace], + provider, + workspace, + ) + .await?; + require_provider_not_listed(runner, "provider-list-default", &[], provider).await?; + require_provider_listed( + runner, + "provider-list-all", + &["--all-workspaces"], + provider, + workspace, + ) + .await?; + + let blocked_delete = run_cli( + runner, + "delete-blocked", + format!("workspace '{workspace}' delete is blocked while resources exist"), + &["workspace", "delete", workspace], + false, + ) + .await?; + require_contains( + &blocked_delete, + "still contains resources", + "blocked deletion error names the blocking resources", + )?; + + run_cli( + runner, + "provider-delete", + format!("provider '{provider}' is deleted"), + &["provider", "delete", provider, "--workspace", workspace], + true, + ) + .await?; + + run_cli( + runner, + "delete", + format!("workspace '{workspace}' delete succeeds once empty"), + &["workspace", "delete", workspace], + true, + ) + .await?; + + run_cli( + runner, + "get-after-delete", + format!("workspace '{workspace}' get fails after deletion"), + &["workspace", "get", workspace], + false, + ) + .await + .map(|_| ()) +} + +async fn workspace_terminating_rejects_creates( + runner: &OpenShellRunner, + workspace: &str, + provider: &str, + blocked_provider: &str, +) -> Result<(), String> { + run_cli( + runner, + "create", + format!("workspace '{workspace}' is created"), + &["workspace", "create", "--name", workspace], + true, + ) + .await?; + + run_cli( + runner, + "provider-create", + format!("provider '{provider}' is created"), + &[ + "provider", + "create", + "--name", + provider, + "--type", + PROVIDER_TYPE, + "--credential", + PROVIDER_CREDENTIAL, + "--workspace", + workspace, + ], + true, + ) + .await?; + + run_cli( + runner, + "delete-blocked", + format!("workspace '{workspace}' delete is blocked by '{provider}'"), + &["workspace", "delete", workspace], + false, + ) + .await?; + + require_workspace_status(runner, "list-terminating", workspace, "Terminating").await?; + + let blocked_create = run_cli( + runner, + "create-blocked", + format!("provider create in terminating workspace '{workspace}' fails"), + &[ + "provider", + "create", + "--name", + blocked_provider, + "--type", + PROVIDER_TYPE, + "--credential", + PROVIDER_CREDENTIAL, + "--workspace", + workspace, + ], + false, + ) + .await?; + require_contains( + &blocked_create, + "being deleted", + "blocked create error names the terminating workspace", + )?; + + run_cli( + runner, + "provider-delete", + format!("provider '{provider}' is deleted"), + &["provider", "delete", provider, "--workspace", workspace], + true, + ) + .await?; + + run_cli( + runner, + "delete-retry", + format!("workspace '{workspace}' delete succeeds once empty"), + &["workspace", "delete", workspace], + true, + ) + .await + .map(|_| ()) +} + +/// Delete the scenario's providers and workspace. A successful run has already +/// removed them, so "not found" is fine; any other failure is returned so the +/// scenario fails instead of leaking resources silently. +async fn cleanup_workspace( + runner: &OpenShellRunner, + workspace: &str, + providers: &[&str], +) -> Result<(), String> { + let mut failures = Vec::new(); + for provider in providers { + cleanup_delete( + runner, + "cleanup/provider-delete", + format!("provider '{provider}' is deleted or already absent"), + &["provider", "delete", provider, "--workspace", workspace], + &mut failures, + ) + .await; + } + cleanup_delete( + runner, + "cleanup/workspace-delete", + format!("workspace '{workspace}' is deleted or already absent"), + &["workspace", "delete", workspace], + &mut failures, + ) + .await; + + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("\n\n")) + } +} + +async fn cleanup_delete( + runner: &OpenShellRunner, + step: &str, + expectation: String, + args: &[&str], + failures: &mut Vec, +) { + match runner + .step(step) + .description(expectation.clone()) + .with_timeout(COMMAND_TIMEOUT) + .run(args) + .await + { + Ok(result) if result.success() || output_reports_not_found(&result) => {} + Ok(result) => failures.push(result.failure_diagnostic(&expectation)), + Err(error) => failures.push(error.to_string()), + } +} + +#[derive(Debug, Deserialize)] +struct WorkspaceListPage { + workspaces: Vec, + next_page_token: String, +} + +#[derive(Debug, Deserialize)] +struct WorkspaceEntry { + name: String, + status: String, +} + +#[derive(Debug, Deserialize)] +struct ProviderListPage { + providers: Vec, + next_page_token: String, +} + +#[derive(Debug, Deserialize)] +struct ProviderEntry { + name: String, + #[serde(default)] + workspace: String, +} + +/// Find `workspace` in the paginated `workspace list --output json` listing. +async fn find_workspace( + runner: &OpenShellRunner, + step: &str, + workspace: &str, +) -> Result, String> { + let mut page_token = String::new(); + let mut page = 0u32; + loop { + let result = run_cli( + runner, + &format!("{step}/{page}"), + format!("workspace list page {page} succeeds"), + &[ + "workspace", + "list", + "--page-token", + &page_token, + "--output", + "json", + ], + true, + ) + .await?; + let listing = result + .json::() + .map_err(|error| error.to_string())?; + if let Some(entry) = listing + .workspaces + .into_iter() + .find(|entry| entry.name == workspace) + { + return Ok(Some(entry)); + } + if listing.next_page_token.is_empty() { + return Ok(None); + } + page_token = listing.next_page_token; + page = page + .checked_add(1) + .ok_or_else(|| "workspace list page counter overflowed".to_string())?; + } +} + +/// Require that the workspace listing shows `workspace` itself with `status`, +/// not merely that some workspace has that status. +async fn require_workspace_status( + runner: &OpenShellRunner, + step: &str, + workspace: &str, + status: &str, +) -> Result<(), String> { + match find_workspace(runner, step, workspace).await? { + Some(entry) if entry.status == status => Ok(()), + Some(entry) => Err(format!( + "{step}: workspace '{workspace}' has status '{}', expected '{status}'", + entry.status + )), + None => Err(format!( + "{step}: workspace '{workspace}' is not in the workspace list" + )), + } +} + +/// Find `provider` in the paginated `provider list --output json` listing +/// selected by `scope` (`--workspace `, `--all-workspaces`, or none for +/// the default workspace). +async fn find_provider( + runner: &OpenShellRunner, + step: &str, + scope: &[&str], + provider: &str, +) -> Result, String> { + let mut page_token = String::new(); + let mut page = 0u32; + loop { + let mut args = vec![ + "provider", + "list", + "--page-token", + page_token.as_str(), + "--output", + "json", + ]; + args.extend_from_slice(scope); + let result = run_cli( + runner, + &format!("{step}/{page}"), + format!("provider list page {page} succeeds"), + &args, + true, + ) + .await?; + let listing = result + .json::() + .map_err(|error| error.to_string())?; + if let Some(entry) = listing + .providers + .into_iter() + .find(|entry| entry.name == provider) + { + return Ok(Some(entry)); + } + if listing.next_page_token.is_empty() { + return Ok(None); + } + page_token = listing.next_page_token; + page = page + .checked_add(1) + .ok_or_else(|| "provider list page counter overflowed".to_string())?; + } +} + +/// Require that `provider` appears in the selected listing, owned by `workspace`. +async fn require_provider_listed( + runner: &OpenShellRunner, + step: &str, + scope: &[&str], + provider: &str, + workspace: &str, +) -> Result<(), String> { + match find_provider(runner, step, scope, provider).await? { + Some(entry) if entry.workspace == workspace => Ok(()), + Some(entry) => Err(format!( + "{step}: provider '{provider}' is listed in workspace '{}', expected '{workspace}'", + entry.workspace + )), + None => Err(format!("{step}: provider '{provider}' is not listed")), + } +} + +/// Require that `provider` does not appear in the selected listing. +async fn require_provider_not_listed( + runner: &OpenShellRunner, + step: &str, + scope: &[&str], + provider: &str, +) -> Result<(), String> { + match find_provider(runner, step, scope, provider).await? { + None => Ok(()), + Some(entry) => Err(format!( + "{step}: provider '{provider}' (workspace '{}') is listed but should not be", + entry.workspace + )), + } +} + +async fn run_cli( + runner: &OpenShellRunner, + step: &str, + description: impl Into, + args: &[&str], + expect_success: bool, +) -> Result { + let result = runner + .step(step) + .description(description) + .with_timeout(COMMAND_TIMEOUT) + .run(args) + .await + .map_err(|error| error.to_string())?; + result.require_outcome(expect_success)?; + Ok(result) +} + +fn require_contains(result: &CommandResult, needle: &str, expectation: &str) -> Result<(), String> { + if result.output_contains(needle) { + Ok(()) + } else { + Err(result.failure_diagnostic(expectation)) + } +} diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 01315baef5..da9076cafd 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -204,11 +204,6 @@ name = "forward_proxy_jsonrpc_l7" path = "tests/forward_proxy_jsonrpc_l7.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "workspace_lifecycle" -path = "tests/workspace_lifecycle.rs" -required-features = ["e2e"] - [[test]] name = "sandbox_templates" path = "tests/sandbox_templates.rs" diff --git a/e2e/rust/tests/workspace_lifecycle.rs b/e2e/rust/tests/workspace_lifecycle.rs deleted file mode 100644 index 2fe946389e..0000000000 --- a/e2e/rust/tests/workspace_lifecycle.rs +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for workspace resource lifecycle. -//! -//! Covers: -//! - Workspace CRUD -//! - Provider creation scoped to a workspace -//! - Workspace isolation (resources in one workspace are invisible in another) -//! - `--all-workspaces` listing -//! - Deletion guard (workspace cannot be deleted while resources exist) -//! - Successful deletion after resource cleanup - -use std::process::Stdio; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; - -const WORKSPACE: &str = "lifecycle-test"; -const WORKSPACE_TERM: &str = "terminating-test"; -const PROVIDER: &str = "lifecycle-prov"; -const PROVIDER_TERM: &str = "term-prov"; - -struct CliResult { - output: String, - success: bool, -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -struct WorkspaceCleanup; - -impl Drop for WorkspaceCleanup { - fn drop(&mut self) { - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["provider", "delete", PROVIDER, "--workspace", WORKSPACE]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - let _ = std::process::Command::new(&bin) - .args(["workspace", "delete", WORKSPACE]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -#[tokio::test] -async fn workspace_full_crud_lifecycle() { - let _cleanup = WorkspaceCleanup; - - // 1. Create workspace. - let res = run_cli(&["workspace", "create", "--name", WORKSPACE]).await; - assert!(res.success, "workspace create failed: {}", res.output); - - // 2. Verify workspace exists. - let res = run_cli(&["workspace", "get", WORKSPACE]).await; - assert!(res.success, "workspace get failed: {}", res.output); - assert!( - res.output.contains(WORKSPACE), - "workspace get output should contain workspace name: {}", - res.output - ); - - // 3. Create provider in the workspace. - let res = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE, - ]) - .await; - assert!( - res.success, - "provider create in workspace failed: {}", - res.output - ); - - // 4. List providers in workspace — should see our provider. - let res = run_cli(&["provider", "list", "--workspace", WORKSPACE]).await; - assert!(res.success, "provider list failed: {}", res.output); - assert!( - res.output.contains(PROVIDER), - "workspace-scoped list should show the provider: {}", - res.output - ); - - // 5. List providers in default workspace — should NOT see our provider. - let res = run_cli(&["provider", "list"]).await; - assert!( - res.success, - "provider list (default) failed: {}", - res.output - ); - assert!( - !res.output.contains(PROVIDER), - "default workspace should not contain the workspace-scoped provider: {}", - res.output - ); - - // 6. List providers with --all-workspaces — should see our provider. - let res = run_cli(&["provider", "list", "--all-workspaces"]).await; - assert!( - res.success, - "provider list --all-workspaces failed: {}", - res.output - ); - assert!( - res.output.contains(PROVIDER), - "--all-workspaces should include the workspace-scoped provider: {}", - res.output - ); - - // 7. Attempt workspace deletion — should fail because provider exists. - let res = run_cli(&["workspace", "delete", WORKSPACE]).await; - assert!( - !res.success, - "workspace delete should fail while resources exist: {}", - res.output - ); - assert!( - res.output.contains("still contains resources"), - "error should mention blocking resources: {}", - res.output - ); - - // 8. Delete the provider. - let res = run_cli(&["provider", "delete", PROVIDER, "--workspace", WORKSPACE]).await; - assert!(res.success, "provider delete failed: {}", res.output); - - // 9. Workspace deletion should now succeed. - let res = run_cli(&["workspace", "delete", WORKSPACE]).await; - assert!( - res.success, - "workspace delete should succeed after resources removed: {}", - res.output - ); - - // 10. Verify workspace is gone. - let res = run_cli(&["workspace", "get", WORKSPACE]).await; - assert!( - !res.success, - "workspace get should fail after deletion: {}", - res.output - ); -} - -struct TerminatingCleanup; - -impl Drop for TerminatingCleanup { - fn drop(&mut self) { - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args([ - "provider", - "delete", - PROVIDER_TERM, - "--workspace", - WORKSPACE_TERM, - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - let _ = std::process::Command::new(&bin) - .args(["workspace", "delete", WORKSPACE_TERM]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -#[tokio::test] -async fn workspace_terminating_rejects_creates() { - let _cleanup = TerminatingCleanup; - - // 1. Create workspace. - let res = run_cli(&["workspace", "create", "--name", WORKSPACE_TERM]).await; - assert!(res.success, "workspace create failed: {}", res.output); - - // 2. Create a provider to block deletion. - let res = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER_TERM, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!(res.success, "provider create failed: {}", res.output); - - // 3. Attempt deletion — fails, but workspace is now Terminating. - let res = run_cli(&["workspace", "delete", WORKSPACE_TERM]).await; - assert!( - !res.success, - "delete should fail with blocker: {}", - res.output - ); - - // 4. Workspace list should show Terminating status. - let res = run_cli(&["workspace", "list"]).await; - assert!(res.success, "workspace list failed: {}", res.output); - assert!( - res.output.contains("Terminating"), - "workspace list should show Terminating status: {}", - res.output - ); - - // 5. Creating a new provider should be rejected. - let res = run_cli(&[ - "provider", - "create", - "--name", - "should-fail", - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!( - !res.success, - "provider create should fail in terminating workspace: {}", - res.output - ); - assert!( - res.output.contains("being deleted"), - "error should mention workspace is being deleted: {}", - res.output - ); - - // 6. Delete the blocking provider (deletes still work in Terminating). - let res = run_cli(&[ - "provider", - "delete", - PROVIDER_TERM, - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!(res.success, "provider delete failed: {}", res.output); - - // 7. Retry deletion — should succeed (idempotent on already-Terminating). - let res = run_cli(&["workspace", "delete", WORKSPACE_TERM]).await; - assert!( - res.success, - "workspace delete should succeed after cleanup: {}", - res.output - ); -} diff --git a/tests/artifacts.nix b/tests/artifacts.nix index 50faab5f7b..2a3c12b90c 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -135,7 +135,6 @@ let "sandbox_lifecycle" # Needs a prebuilt musl DNS probe in guest artifact mode; tracked in #3009. "transparent_tcp" - "workspace_lifecycle" ]; podmanE2eArchiveFilter = let diff --git a/tests/suites/conformance/cli/tests/workspace_lifecycle.rs b/tests/suites/conformance/cli/tests/workspace_lifecycle.rs new file mode 100644 index 0000000000..7c5de2b07b --- /dev/null +++ b/tests/suites/conformance/cli/tests/workspace_lifecycle.rs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-agnostic workspace lifecycle conformance tests. + +use openshell_conformance::{ + OpenShellRunner, Scenario, WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO, +}; + +/// Exercise workspace and workspace-scoped provider CRUD through the candidate CLI. +#[tokio::test] +async fn full_crud_lifecycle() { + run(WORKSPACE_LIFECYCLE_SCENARIO).await; +} + +/// Exercise terminating-workspace create rejection through the candidate CLI. +#[tokio::test] +async fn terminating_rejects_creates() { + run(WORKSPACE_TERMINATING_SCENARIO).await; +} + +async fn run(scenario: Scenario) { + let mut runner = OpenShellRunner::from_env(scenario.name) + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + scenario.run(&mut runner).await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("{} conformance scenario failed:\n{error}", scenario.name); + } +} From ced747a3473605ab0652f99d1046d3a3dc55c4de Mon Sep 17 00:00:00 2001 From: politerealism Date: Tue, 6 Oct 2026 10:20:25 -0400 Subject: [PATCH 2/3] test(conformance): migrate upload_create into file-transfer conformance Port sandbox create --upload coverage from e2e/rust/tests/upload_create.rs into a new file-transfer/create-upload scenario in crates/openshell-conformance/src/scenarios/file_transfer.rs, grouped alongside the other file/template-transfer conformance scenarios rather than left scattered in the Podman-specific e2e suite. Wire it into the FILE_TRANSFER_SCENARIO chain and the conformance CLI test crate, and remove the old per-driver test file (picked up by Cargo's default test auto-discovery; it had no Cargo.toml or artifacts.nix entries to clean up). Verified against an ephemeral Podman gateway: the new create-upload scenario and the existing path-safety scenario pass. round-trip and git-filtering already fail against current main on code this change does not touch (the CLI doesn't warn/reject non-Git-tree uploads the way those scenarios assert) -- pre-existing, out of scope here. Signed-off-by: politerealism (cherry picked from commit 398dfe3491716722c57ad9696770aff824230d57) Signed-off-by: politerealism --- crates/openshell-conformance/src/lib.rs | 10 +- .../src/scenarios/file_transfer.rs | 206 ++++++++++++++++++ .../src/scenarios/mod.rs | 5 +- e2e/rust/tests/upload_create.rs | 113 ---------- .../conformance/cli/tests/file_transfer.rs | 25 ++- 5 files changed, 238 insertions(+), 121 deletions(-) delete mode 100644 e2e/rust/tests/upload_create.rs diff --git a/crates/openshell-conformance/src/lib.rs b/crates/openshell-conformance/src/lib.rs index dd34fc8f10..63ba8d8adb 100644 --- a/crates/openshell-conformance/src/lib.rs +++ b/crates/openshell-conformance/src/lib.rs @@ -24,10 +24,12 @@ use tokio::time::sleep; use self::executor::{CliExecutionError, CliExecutor, ProcessCli}; pub use scenarios::{ - FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, - FILE_TRANSFER_ROUND_TRIP_SCENARIO, MECHANISTIC_PROPOSAL_SCENARIO, - NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO, - SMOKE_SCENARIO, WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO, + FILE_TRANSFER_CREATE_UPLOAD_DIRECTORY_SCENARIO, FILE_TRANSFER_CREATE_UPLOAD_FILE_SCENARIO, + FILE_TRANSFER_CREATE_UPLOAD_MULTIPLE_SCENARIO, FILE_TRANSFER_GIT_FILTERING_SCENARIO, + FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, + MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, + SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO, WORKSPACE_LIFECYCLE_SCENARIO, + WORKSPACE_TERMINATING_SCENARIO, }; /// A portable conformance scenario exercised by the Cargo test suite. diff --git a/crates/openshell-conformance/src/scenarios/file_transfer.rs b/crates/openshell-conformance/src/scenarios/file_transfer.rs index 0af908008c..81611af774 100644 --- a/crates/openshell-conformance/src/scenarios/file_transfer.rs +++ b/crates/openshell-conformance/src/scenarios/file_transfer.rs @@ -35,6 +35,40 @@ pub const FILE_TRANSFER_PATH_SAFETY_SCENARIO: Scenario = Scenario { run: run_path_safety, }; +/// Certify that `sandbox create --upload :` preserves the source +/// directory's basename under the destination. +/// +/// - **Behavior:** a directory passed to `sandbox create --upload :` is available in the sandbox under `/`, keeping the source directory's basename. Verified by reading a file from inside the sandbox. +/// - **Preconditions:** a reachable gateway whose compute driver can create sandboxes from the default image (supplied by target preparation). The test writes its upload sources to a local temporary directory. +/// - **Permissions:** an identity allowed to create, execute in, and delete sandboxes. +/// - **Side effects:** creates one sandbox named from the run id, which the runner removes on success and on failure (a cleanup failure fails the scenario), and a local temporary directory. +pub const FILE_TRANSFER_CREATE_UPLOAD_DIRECTORY_SCENARIO: Scenario = Scenario { + name: "file-transfer/create-upload-directory", + run: run_create_upload_directory, +}; + +/// Certify that `sandbox create` accepts several `--upload` flags at once. +/// +/// - **Behavior:** several `--upload` flags on one `sandbox create` are all pre-loaded to their own destinations. Verified by reading a file from each destination inside the sandbox. +/// - **Preconditions:** a reachable gateway whose compute driver can create sandboxes from the default image (supplied by target preparation). The test writes its upload sources to a local temporary directory. +/// - **Permissions:** an identity allowed to create, execute in, and delete sandboxes. +/// - **Side effects:** creates one sandbox named from the run id, which the runner removes on success and on failure (a cleanup failure fails the scenario), and a local temporary directory. +pub const FILE_TRANSFER_CREATE_UPLOAD_MULTIPLE_SCENARIO: Scenario = Scenario { + name: "file-transfer/create-upload-multiple", + run: run_create_upload_multiple, +}; + +/// Certify that `sandbox create --upload ` pre-loads a single file. +/// +/// - **Behavior:** a single file passed to `sandbox create --upload` is available in the sandbox at the requested destination. Verified by reading it from inside the sandbox. +/// - **Preconditions:** a reachable gateway whose compute driver can create sandboxes from the default image (supplied by target preparation). The test writes its upload sources to a local temporary directory. +/// - **Permissions:** an identity allowed to create, execute in, and delete sandboxes. +/// - **Side effects:** creates one sandbox named from the run id, which the runner removes on success and on failure (a cleanup failure fails the scenario), and a local temporary directory. +pub const FILE_TRANSFER_CREATE_UPLOAD_FILE_SCENARIO: Scenario = Scenario { + name: "file-transfer/create-upload-file", + run: run_create_upload_file, +}; + fn run_round_trip(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { Box::pin(async move { let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "round-trip").await?; @@ -66,6 +100,178 @@ fn run_path_safety(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { }) } +fn run_create_upload_directory(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { create_upload_directory_preserves_source_basename(runner).await }) +} + +fn run_create_upload_multiple(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { create_with_multiple_uploads(runner).await }) +} + +fn run_create_upload_file(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { create_upload_single_file(runner).await }) +} + +/// A sandbox created with `--upload :` preserves the source +/// directory's basename under ``. +async fn create_upload_directory_preserves_source_basename( + runner: &mut OpenShellRunner, +) -> Result<(), String> { + let local = + tempfile::tempdir().map_err(fs_error("create create-upload/directory temporary dir"))?; + let source = local.path().join("project"); + fs::create_dir_all(source.join("src")).map_err(fs_error("create project/src"))?; + fs::write(source.join("marker.txt"), "upload-create-marker") + .map_err(fs_error("write marker.txt"))?; + fs::write(source.join("src/main.py"), "print('hello')").map_err(fs_error("write main.py"))?; + + let sandbox_name = format!("ct-{}-cu1", runner.id()); + create_with_uploads( + runner, + "create-upload/directory", + &sandbox_name, + &[(&source, "/sandbox/data")], + ) + .await?; + + let output = exec_capture( + runner, + &sandbox_name, + "create-upload/directory/verify", + "cat /sandbox/data/project/marker.txt", + ) + .await?; + if !output.contains("upload-create-marker") { + return Err(format!( + "expected uploaded marker content in sandbox output:\n{output}" + )); + } + + delete_sandbox(runner, &sandbox_name).await +} + +/// Two `--upload` specs in a single `sandbox create` call both land in the +/// sandbox before the command runs. +async fn create_with_multiple_uploads(runner: &mut OpenShellRunner) -> Result<(), String> { + let local = tempfile::tempdir().map_err(fs_error("create multiple-uploads temporary dir"))?; + let dir_a = local.path().join("alpha"); + let dir_b = local.path().join("beta"); + fs::create_dir_all(&dir_a).map_err(fs_error("create alpha"))?; + fs::create_dir_all(&dir_b).map_err(fs_error("create beta"))?; + fs::write(dir_a.join("a.txt"), "content-alpha").map_err(fs_error("write a.txt"))?; + fs::write(dir_b.join("b.txt"), "content-beta").map_err(fs_error("write b.txt"))?; + + let sandbox_name = format!("ct-{}-cu2", runner.id()); + create_with_uploads( + runner, + "create-upload/multiple", + &sandbox_name, + &[(&dir_a, "/sandbox/alpha"), (&dir_b, "/sandbox/beta")], + ) + .await?; + + let output = exec_capture( + runner, + &sandbox_name, + "create-upload/multiple/verify", + "cat /sandbox/alpha/alpha/a.txt /sandbox/beta/beta/b.txt", + ) + .await?; + if !output.contains("content-alpha") || !output.contains("content-beta") { + return Err(format!( + "expected both uploaded contents in sandbox output:\n{output}" + )); + } + + delete_sandbox(runner, &sandbox_name).await +} + +/// `--upload` with a single file (not a directory) works. +async fn create_upload_single_file(runner: &mut OpenShellRunner) -> Result<(), String> { + let local = tempfile::tempdir().map_err(fs_error("create single-file temporary dir"))?; + let file_path = local.path().join("config.txt"); + fs::write(&file_path, "single-file-upload-test").map_err(fs_error("write config.txt"))?; + + let sandbox_name = format!("ct-{}-cu3", runner.id()); + create_with_uploads( + runner, + "create-upload/single-file", + &sandbox_name, + &[(&file_path, "/sandbox")], + ) + .await?; + + let output = exec_capture( + runner, + &sandbox_name, + "create-upload/single-file/verify", + "cat /sandbox/config.txt", + ) + .await?; + if !output.contains("single-file-upload-test") { + return Err(format!( + "expected single-file content in sandbox output:\n{output}" + )); + } + + delete_sandbox(runner, &sandbox_name).await +} + +/// Run `sandbox create --detach --upload :... --no-git-ignore`. +async fn create_with_uploads( + runner: &mut OpenShellRunner, + step: &str, + sandbox_name: &str, + uploads: &[(&Path, &str)], +) -> Result<(), String> { + runner.track_sandbox(sandbox_name); + let mut args = vec![ + "sandbox".to_string(), + "create".to_string(), + "--name".to_string(), + sandbox_name.to_string(), + "--detach".to_string(), + ]; + for (local, dest) in uploads { + let local = local + .to_str() + .ok_or_else(|| format!("local upload path is not UTF-8: {}", local.display()))?; + args.push("--upload".to_string()); + args.push(format!("{local}:{dest}")); + } + args.push("--no-git-ignore".to_string()); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + + let result = runner + .step(step) + .description(format!("sandbox '{sandbox_name}' is created with uploads")) + .with_timeout(CREATE_TIMEOUT) + .run(&args) + .await + .map_err(|error| error.to_string())?; + result.require_success() +} + +/// Exec a command in the sandbox and return its combined stdout+stderr. +async fn exec_capture( + runner: &OpenShellRunner, + sandbox: &str, + step: &str, + script: &str, +) -> Result { + let result = runner + .step(step) + .description(format!("sandbox '{sandbox}' exec for {step} succeeds")) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", "exec", "--name", sandbox, "--no-tty", "--", "sh", "-c", script, + ]) + .await + .map_err(|error| error.to_string())?; + result.require_success()?; + Ok(format!("{}{}", result.stdout(), result.stderr())) +} + async fn prepare_sandbox( runner: &mut OpenShellRunner, group: &str, diff --git a/crates/openshell-conformance/src/scenarios/mod.rs b/crates/openshell-conformance/src/scenarios/mod.rs index 9cafd6023a..0d1db283bb 100644 --- a/crates/openshell-conformance/src/scenarios/mod.rs +++ b/crates/openshell-conformance/src/scenarios/mod.rs @@ -10,8 +10,9 @@ mod smoke; mod workspace_lifecycle; pub use file_transfer::{ - FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, - FILE_TRANSFER_ROUND_TRIP_SCENARIO, + FILE_TRANSFER_CREATE_UPLOAD_DIRECTORY_SCENARIO, FILE_TRANSFER_CREATE_UPLOAD_FILE_SCENARIO, + FILE_TRANSFER_CREATE_UPLOAD_MULTIPLE_SCENARIO, FILE_TRANSFER_GIT_FILTERING_SCENARIO, + FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, }; pub use policy_behavior::{ MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, diff --git a/e2e/rust/tests/upload_create.rs b/e2e/rust/tests/upload_create.rs deleted file mode 100644 index 5cc3964841..0000000000 --- a/e2e/rust/tests/upload_create.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E test: `sandbox create --upload` pre-loads files before running a command. -//! -//! Validates that the `--upload :` flag on `sandbox create` -//! transfers files into the sandbox before the user command executes, -//! so the command can read the uploaded content. -//! -//! Prerequisites: -//! - A running openshell gateway (`mise run gateway:docker`) -//! - The `openshell` binary (built automatically from the workspace) - -use std::fs; - -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -/// Create a sandbox with `--upload dir:/sandbox/data` and verify directory -/// uploads preserve the source basename at `/sandbox/data//...`. -#[tokio::test] -async fn create_with_upload_directory_preserves_source_basename() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - - // Create a directory with files to upload. - let upload_dir = tmpdir.path().join("project"); - fs::create_dir_all(upload_dir.join("src")).expect("create project/src"); - fs::write(upload_dir.join("marker.txt"), "upload-create-marker").expect("write marker.txt"); - fs::write(upload_dir.join("src/main.py"), "print('hello')").expect("write main.py"); - - let upload_str = upload_dir.to_str().expect("upload path is UTF-8"); - let remote_marker = "/sandbox/data/project/marker.txt"; - - // The command reads the marker file — if upload worked, its content - // appears in the output. - let mut guard = - SandboxGuard::create_with_upload(upload_str, "/sandbox/data", &["cat", remote_marker]) - .await - .expect("sandbox create --upload"); - - let clean = strip_ansi(&guard.create_output); - assert!( - clean.contains("upload-create-marker"), - "expected uploaded marker content in sandbox output:\n{clean}" - ); - - guard.cleanup().await; -} - -/// Two `--upload` specs in a single `sandbox create` call should both land in -/// the sandbox before the command runs. -#[tokio::test] -async fn create_with_multiple_uploads() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - - let dir_a = tmpdir.path().join("alpha"); - let dir_b = tmpdir.path().join("beta"); - fs::create_dir_all(&dir_a).expect("create alpha"); - fs::create_dir_all(&dir_b).expect("create beta"); - fs::write(dir_a.join("a.txt"), "content-alpha").expect("write a.txt"); - fs::write(dir_b.join("b.txt"), "content-beta").expect("write b.txt"); - - let spec_a = dir_a.to_str().expect("alpha path is UTF-8"); - let spec_b = dir_b.to_str().expect("beta path is UTF-8"); - - let mut guard = SandboxGuard::create_with_uploads( - &[(spec_a, "/sandbox/alpha"), (spec_b, "/sandbox/beta")], - &[ - "sh", - "-c", - "cat /sandbox/alpha/alpha/a.txt /sandbox/beta/beta/b.txt", - ], - ) - .await - .expect("sandbox create with multiple --upload flags"); - - let clean = strip_ansi(&guard.create_output); - assert!( - clean.contains("content-alpha"), - "expected alpha content in output:\n{clean}" - ); - assert!( - clean.contains("content-beta"), - "expected beta content in output:\n{clean}" - ); - - guard.cleanup().await; -} - -/// `--upload` with a single file (not a directory) should work. -#[tokio::test] -async fn create_with_upload_single_file() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let file_path = tmpdir.path().join("config.txt"); - fs::write(&file_path, "single-file-upload-test").expect("write config.txt"); - - let file_str = file_path.to_str().expect("file path is UTF-8"); - - let mut guard = - SandboxGuard::create_with_upload(file_str, "/sandbox", &["cat", "/sandbox/config.txt"]) - .await - .expect("sandbox create --upload single file"); - - let clean = strip_ansi(&guard.create_output); - assert!( - clean.contains("single-file-upload-test"), - "expected single-file content in sandbox output:\n{clean}" - ); - - guard.cleanup().await; -} diff --git a/tests/suites/conformance/cli/tests/file_transfer.rs b/tests/suites/conformance/cli/tests/file_transfer.rs index 167506b87b..0130b9ab76 100644 --- a/tests/suites/conformance/cli/tests/file_transfer.rs +++ b/tests/suites/conformance/cli/tests/file_transfer.rs @@ -4,8 +4,11 @@ //! Driver-agnostic sandbox file-transfer conformance tests. use openshell_conformance::{ - FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, - FILE_TRANSFER_ROUND_TRIP_SCENARIO, OpenShellRunner, Scenario, + FILE_TRANSFER_CREATE_UPLOAD_DIRECTORY_SCENARIO, FILE_TRANSFER_CREATE_UPLOAD_FILE_SCENARIO, + FILE_TRANSFER_CREATE_UPLOAD_MULTIPLE_SCENARIO, + FILE_TRANSFER_GIT_FILTERING_SCENARIO, + FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, OpenShellRunner, + Scenario, }; /// Exercise file and directory round trips through the candidate CLI. @@ -26,6 +29,24 @@ async fn path_safety() { run(FILE_TRANSFER_PATH_SAFETY_SCENARIO).await; } +/// Exercise directory-basename preservation for `sandbox create --upload`. +#[tokio::test] +async fn create_upload_directory() { + run(FILE_TRANSFER_CREATE_UPLOAD_DIRECTORY_SCENARIO).await; +} + +/// Exercise several `--upload` flags on one `sandbox create`. +#[tokio::test] +async fn create_upload_multiple() { + run(FILE_TRANSFER_CREATE_UPLOAD_MULTIPLE_SCENARIO).await; +} + +/// Exercise single-file `sandbox create --upload` pre-loading. +#[tokio::test] +async fn create_upload_file() { + run(FILE_TRANSFER_CREATE_UPLOAD_FILE_SCENARIO).await; +} + async fn run(scenario: Scenario) { let mut runner = OpenShellRunner::from_env(scenario.name) .expect("candidate openshell CLI is available"); From dc4ac8df4d8ac6e5d5dac5f2d1ff8329ad1310d1 Mon Sep 17 00:00:00 2001 From: politerealism Date: Tue, 6 Oct 2026 11:18:33 -0400 Subject: [PATCH 3/3] test(conformance): migrate sandbox_templates to the conformance suite Port reusable sandbox workload template CRUD and template-backed sandbox creation coverage from e2e/rust/tests/sandbox_templates.rs into four driver-agnostic scenarios (sandbox-templates/lifecycle, get-after-delete, duplicate-name, missing-template) under tests/suites/conformance/, following the OpenShellRunner/Scenario pattern. Remove the old per-driver test file and its Cargo.toml entry. Verified against an ephemeral Podman gateway. Signed-off-by: politerealism (cherry picked from commit 617b8bc990c956abbf857eaae44b57b2f150601f) --- crates/openshell-conformance/src/lib.rs | 6 +- .../src/scenarios/mod.rs | 6 + .../src/scenarios/sandbox_templates.rs | 369 ++++++++++++++++++ e2e/rust/Cargo.toml | 5 - e2e/rust/tests/sandbox_templates.rs | 275 ------------- .../cli/tests/sandbox_templates.rs | 47 +++ 6 files changed, 426 insertions(+), 282 deletions(-) create mode 100644 crates/openshell-conformance/src/scenarios/sandbox_templates.rs delete mode 100644 e2e/rust/tests/sandbox_templates.rs create mode 100644 tests/suites/conformance/cli/tests/sandbox_templates.rs diff --git a/crates/openshell-conformance/src/lib.rs b/crates/openshell-conformance/src/lib.rs index 63ba8d8adb..e1460c57f2 100644 --- a/crates/openshell-conformance/src/lib.rs +++ b/crates/openshell-conformance/src/lib.rs @@ -28,8 +28,10 @@ pub use scenarios::{ FILE_TRANSFER_CREATE_UPLOAD_MULTIPLE_SCENARIO, FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, - SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO, WORKSPACE_LIFECYCLE_SCENARIO, - WORKSPACE_TERMINATING_SCENARIO, + SANDBOX_LIFECYCLE_SCENARIO, SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO, + SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO, SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO, + SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO, SANDBOX_TEMPLATES_SCENARIO, SMOKE_SCENARIO, + WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO, }; /// A portable conformance scenario exercised by the Cargo test suite. diff --git a/crates/openshell-conformance/src/scenarios/mod.rs b/crates/openshell-conformance/src/scenarios/mod.rs index 0d1db283bb..064a7932ed 100644 --- a/crates/openshell-conformance/src/scenarios/mod.rs +++ b/crates/openshell-conformance/src/scenarios/mod.rs @@ -6,6 +6,7 @@ mod file_transfer; mod policy_behavior; mod sandbox_lifecycle; +mod sandbox_templates; mod smoke; mod workspace_lifecycle; @@ -18,5 +19,10 @@ pub use policy_behavior::{ MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, }; pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO; +pub use sandbox_templates::{ + SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO, SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO, + SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO, SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO, + SANDBOX_TEMPLATES_SCENARIO, +}; pub use smoke::SMOKE_SCENARIO; pub use workspace_lifecycle::{WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO}; diff --git a/crates/openshell-conformance/src/scenarios/sandbox_templates.rs b/crates/openshell-conformance/src/scenarios/sandbox_templates.rs new file mode 100644 index 0000000000..9ccf9c2546 --- /dev/null +++ b/crates/openshell-conformance/src/scenarios/sandbox_templates.rs @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Portable sandbox workload template conformance scenarios. + +use std::time::Duration; + +use serde_json::Value; + +use crate::{ + CommandResult, OpenShellRunner, Scenario, ScenarioFuture, combine_results, + output_reports_not_found, +}; + +const CREATE_TIMEOUT: Duration = Duration::from_mins(10); +const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); + +/// Certify reusable sandbox workload template behavior through the public CLI. +/// +/// - **Behavior:** runs the four template scenarios below in sequence. +/// - **Preconditions:** a reachable gateway whose compute driver can create sandboxes from the default image (supplied by target preparation, not pulled from a public service by the test). +/// - **Permissions:** an identity allowed to create and delete sandbox templates and sandboxes. +/// - **Side effects:** each scenario creates the resources it names from the run id and removes them on success and on failure; a cleanup failure fails the scenario. +pub const SANDBOX_TEMPLATES_SCENARIO: Scenario = Scenario { + name: "sandbox-templates", + run: run_sandbox_templates, +}; + +/// Certify that a sandbox created from a template inherits its resources, +/// labels, and environment. +/// +/// - **Behavior:** a template created with resource limits, a label, and an environment variable can be retrieved with those values and appears in the template list. A sandbox created from it carries the template's resources, labels, and environment. +/// - **Preconditions:** a reachable gateway whose compute driver can create sandboxes from the default image (supplied by target preparation, not pulled from a public service by the test). +/// - **Permissions:** an identity allowed to create and delete sandbox templates and sandboxes. +/// - **Side effects:** creates one template and one sandbox named from the run id. The sandbox is removed by the runner's tracked cleanup and the template by the scenario; a cleanup failure fails the scenario. +pub const SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO: Scenario = Scenario { + name: "sandbox-templates/lifecycle", + run: run_lifecycle, +}; + +/// Certify that a deleted template is no longer fetchable. +/// +/// - **Behavior:** after a template is deleted, retrieving it fails with a template-not-found error. +/// - **Preconditions:** a reachable gateway. No sandbox is created. +/// - **Permissions:** an identity allowed to create and delete sandbox templates and sandboxes. +/// - **Side effects:** creates one template named from the run id and removes it; a cleanup failure fails the scenario. +pub const SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO: Scenario = Scenario { + name: "sandbox-templates/get-after-delete", + run: run_get_after_delete, +}; + +/// Certify that creating a template with a name already in use fails. +/// +/// - **Behavior:** creating a second template with a name already in use is rejected. +/// - **Preconditions:** a reachable gateway. No sandbox is created. +/// - **Permissions:** an identity allowed to create and delete sandbox templates and sandboxes. +/// - **Side effects:** creates one template named from the run id and removes it; a cleanup failure fails the scenario. +pub const SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO: Scenario = Scenario { + name: "sandbox-templates/duplicate-name", + run: run_duplicate_name, +}; + +/// Certify that creating a sandbox from a nonexistent template fails. +/// +/// - **Behavior:** creating a sandbox from a template that does not exist fails with a template-not-found error. +/// - **Preconditions:** a reachable gateway. +/// - **Permissions:** an identity allowed to create sandboxes. +/// - **Side effects:** the sandbox name is tracked so a sandbox that was unexpectedly created is removed; a cleanup failure fails the scenario. +pub const SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO: Scenario = Scenario { + name: "sandbox-templates/missing-template", + run: run_missing_template, +}; + +fn run_sandbox_templates(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO.run(runner).await?; + SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO + .run(runner) + .await?; + SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO.run(runner).await?; + SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO.run(runner).await + }) +} + +fn run_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + let template_name = format!("ct-{}-tl", runner.id()); + let sandbox_name = format!("ct-{}-stl", runner.id()); + + let result = lifecycle(runner, &template_name, &sandbox_name).await; + let cleanup = cleanup_template(runner, &template_name).await; + combine_results(result, cleanup) + }) +} + +async fn lifecycle( + runner: &mut OpenShellRunner, + template_name: &str, + sandbox_name: &str, +) -> Result<(), String> { + runner + .step("lifecycle/create") + .description(format!("template '{template_name}' is created")) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "template", + "create", + template_name, + "--cpu", + "500m", + "--memory", + "512Mi", + "--label", + "e2e=sandbox-template", + "--env", + "FEATURE_FLAG=on", + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let get = runner + .step("lifecycle/get") + .description(format!("template '{template_name}' can be retrieved")) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "template", + "get", + template_name, + "--output", + "json", + ]) + .await + .map_err(|error| error.to_string())?; + get.require_success()?; + let template_json: Value = get.json().map_err(|error| error.to_string())?; + require_eq(&template_json["name"], template_name, "template name")?; + require_eq( + &template_json["labels"]["e2e"], + "sandbox-template", + "template label e2e", + )?; + require_eq( + &template_json["environment"]["FEATURE_FLAG"], + "on", + "template environment FEATURE_FLAG", + )?; + require_eq(&template_json["resources"]["cpu"], "500m", "template cpu")?; + require_eq( + &template_json["resources"]["memory"], + "512Mi", + "template memory", + )?; + + let list = runner + .step("lifecycle/list") + .description("template list includes the created template") + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "list", "--names"]) + .await + .map_err(|error| error.to_string())?; + list.require_success()?; + if !list + .stdout() + .lines() + .any(|line| line.trim() == template_name) + { + return Err(list.failure_diagnostic(&format!("template list includes '{template_name}'"))); + } + + runner.track_sandbox(sandbox_name); + let create_sandbox = runner + .step("lifecycle/sandbox-create") + .description(format!( + "sandbox '{sandbox_name}' is created from template '{template_name}'" + )) + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + sandbox_name, + "--template", + template_name, + "--", + "sh", + "-lc", + "test \"$FEATURE_FLAG\" = on && echo template-env-ok", + ]) + .await + .map_err(|error| error.to_string())?; + create_sandbox.require_success()?; + if !create_sandbox.output_contains_ignore_case("template-env-ok") { + return Err(create_sandbox.failure_diagnostic("sandbox inherits template environment")); + } + + runner + .step("lifecycle/sandbox-delete") + .description(format!("sandbox '{sandbox_name}' is deleted")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "delete", sandbox_name]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + runner.forget_sandbox(sandbox_name); + Ok(()) +} + +fn run_get_after_delete(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + let template_name = format!("ct-{}-tgd", runner.id()); + let result = get_after_delete(runner, &template_name).await; + let cleanup = cleanup_template(runner, &template_name).await; + combine_results(result, cleanup) + }) +} + +async fn get_after_delete(runner: &OpenShellRunner, template_name: &str) -> Result<(), String> { + runner + .step("get-after-delete/create") + .description(format!("template '{template_name}' is created")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "create", template_name]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + runner + .step("get-after-delete/delete") + .description(format!("template '{template_name}' is deleted")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "delete", template_name]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let get_deleted = runner + .step("get-after-delete/get") + .description(format!( + "deleted template '{template_name}' is not fetchable" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "get", template_name]) + .await + .map_err(|error| error.to_string())?; + if get_deleted.success() { + return Err(get_deleted.failure_diagnostic("deleted template is not fetchable")); + } + if !output_mentions_template_not_found(&get_deleted) { + return Err(get_deleted.failure_diagnostic("deleted template returns not-found")); + } + Ok(()) +} + +fn run_duplicate_name(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + let template_name = format!("ct-{}-tdn", runner.id()); + let result = duplicate_name(runner, &template_name).await; + let cleanup = cleanup_template(runner, &template_name).await; + combine_results(result, cleanup) + }) +} + +async fn duplicate_name(runner: &OpenShellRunner, template_name: &str) -> Result<(), String> { + runner + .step("duplicate-name/create") + .description(format!("template '{template_name}' is created")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "create", template_name]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let duplicate = runner + .step("duplicate-name/duplicate") + .description(format!( + "duplicate template '{template_name}' creation is rejected" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "create", template_name]) + .await + .map_err(|error| error.to_string())?; + if duplicate.success() { + return Err(duplicate.failure_diagnostic("duplicate template creation is rejected")); + } + if !duplicate.output_contains_ignore_case("already exists") { + return Err(duplicate.failure_diagnostic("duplicate template error reports already-exists")); + } + Ok(()) +} + +fn run_missing_template(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { + let template_name = format!("ct-{}-tmt", runner.id()); + let sandbox_name = format!("ct-{}-smt", runner.id()); + missing_template(runner, &template_name, &sandbox_name).await + }) +} + +async fn missing_template( + runner: &mut OpenShellRunner, + template_name: &str, + sandbox_name: &str, +) -> Result<(), String> { + runner.track_sandbox(sandbox_name); + let create_sandbox = runner + .step("missing-template/create") + .description(format!( + "sandbox create from missing template '{template_name}' fails" + )) + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + sandbox_name, + "--template", + template_name, + "--", + "true", + ]) + .await + .map_err(|error| error.to_string())?; + + if create_sandbox.success() { + return Err(create_sandbox.failure_diagnostic("sandbox create from missing template fails")); + } + runner.forget_sandbox(sandbox_name); + if !output_mentions_template_not_found(&create_sandbox) { + return Err(create_sandbox + .failure_diagnostic("sandbox create from missing template reports not-found")); + } + Ok(()) +} + +/// Delete the scenario's template. A successful run has already removed it, so +/// "not found" is fine; any other failure is returned so the scenario fails +/// instead of leaking the template silently. +async fn cleanup_template(runner: &OpenShellRunner, template_name: &str) -> Result<(), String> { + let expectation = format!("template '{template_name}' is deleted or already absent"); + match runner + .step("cleanup/template-delete") + .description(expectation.clone()) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "template", "delete", template_name]) + .await + { + Ok(result) if result.success() || output_reports_not_found(&result) => Ok(()), + Ok(result) => Err(result.failure_diagnostic(&expectation)), + Err(error) => Err(error.to_string()), + } +} + +fn output_mentions_template_not_found(result: &CommandResult) -> bool { + result.output_contains_ignore_case("sandbox template") + && result.output_contains_ignore_case("not found") +} + +fn require_eq(actual: &Value, expected: &str, label: &str) -> Result<(), String> { + if actual.as_str() == Some(expected) { + Ok(()) + } else { + Err(format!( + "{label} mismatch: expected {expected:?}, got {actual:?}" + )) + } +} diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index da9076cafd..0298474e61 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -204,11 +204,6 @@ name = "forward_proxy_jsonrpc_l7" path = "tests/forward_proxy_jsonrpc_l7.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "sandbox_templates" -path = "tests/sandbox_templates.rs" -required-features = ["e2e"] - [[test]] name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" diff --git a/e2e/rust/tests/sandbox_templates.rs b/e2e/rust/tests/sandbox_templates.rs deleted file mode 100644 index 21c9233daf..0000000000 --- a/e2e/rust/tests/sandbox_templates.rs +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for reusable sandbox workload templates. - -use std::process::Stdio; -use std::time::{SystemTime, UNIX_EPOCH}; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; - -struct CliResult { - output: String, - success: bool, -} - -struct TemplateGuard { - name: String, -} - -impl TemplateGuard { - fn new(name: String) -> Self { - Self { name } - } - - async fn cleanup(mut self) { - delete_template(&self.name).await; - self.name.clear(); - } -} - -impl Drop for TemplateGuard { - fn drop(&mut self) { - if self.name.is_empty() { - return; - } - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["sandbox", "template", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -async fn delete_template(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "template", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -fn unique_name(prefix: &str) -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before epoch") - .as_millis(); - let suffix = millis % 1_000_000; - format!("{prefix}-{suffix:06}") -} - -fn output_contains(output: &str, needle: &str) -> bool { - output.to_lowercase().contains(&needle.to_lowercase()) -} - -fn output_mentions_template_not_found(output: &str) -> bool { - output_contains(output, "sandbox template") && output_contains(output, "not found") -} - -#[tokio::test] -async fn sandbox_create_from_template_uses_reusable_workload() { - let template_name = unique_name("tmpl"); - let sandbox_name = unique_name("sb-tmpl"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&[ - "sandbox", - "template", - "create", - &template_name, - "--cpu", - "500m", - "--memory", - "512Mi", - "--label", - "e2e=sandbox-template", - "--env", - "FEATURE_FLAG=on", - ]) - .await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let get_template = run_cli(&[ - "sandbox", - "template", - "get", - &template_name, - "--output", - "json", - ]) - .await; - assert!( - get_template.success, - "sandbox template get failed:\n{}", - get_template.output - ); - let template_json: Value = - serde_json::from_str(&get_template.output).expect("template JSON output"); - assert_eq!(template_json["name"].as_str(), Some(template_name.as_str())); - assert_eq!( - template_json["labels"]["e2e"].as_str(), - Some("sandbox-template") - ); - assert_eq!( - template_json["environment"]["FEATURE_FLAG"].as_str(), - Some("on") - ); - assert_eq!(template_json["resources"]["cpu"].as_str(), Some("500m")); - assert_eq!(template_json["resources"]["memory"].as_str(), Some("512Mi")); - - let list_templates = run_cli(&["sandbox", "template", "list", "--names"]).await; - assert!( - list_templates.success, - "sandbox template list failed:\n{}", - list_templates.output - ); - assert!( - list_templates - .output - .lines() - .any(|line| line.trim() == template_name), - "template list should include {template_name}:\n{}", - list_templates.output - ); - - let mut sandbox = SandboxGuard::create(&[ - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "sh", - "-lc", - "test \"$FEATURE_FLAG\" = on && echo template-env-ok", - ]) - .await - .expect("sandbox create from template should succeed"); - - assert!( - sandbox.create_output.contains("template-env-ok"), - "sandbox should inherit template environment:\n{}", - sandbox.create_output - ); - - sandbox.cleanup().await; - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_get_after_delete_returns_not_found() { - let template_name = unique_name("tmpl-del"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let delete_template = run_cli(&["sandbox", "template", "delete", &template_name]).await; - assert!( - delete_template.success, - "sandbox template delete failed:\n{}", - delete_template.output - ); - - let get_deleted = run_cli(&["sandbox", "template", "get", &template_name]).await; - assert!( - !get_deleted.success, - "deleted template should not be fetchable:\n{}", - get_deleted.output - ); - assert!( - output_mentions_template_not_found(&get_deleted.output), - "deleted template should return not-found:\n{}", - get_deleted.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_create_rejects_duplicate_name() { - let template_name = unique_name("tmpl-dupe"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let duplicate = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - !duplicate.success, - "duplicate sandbox template create should fail:\n{}", - duplicate.output - ); - assert!( - output_contains(&duplicate.output, "already exists"), - "duplicate sandbox template create should report already-exists:\n{}", - duplicate.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_create_from_missing_template_returns_not_found() { - let template_name = unique_name("tmpl-missing"); - let sandbox_name = unique_name("sb-missing"); - - let create_sandbox = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "true", - ]) - .await; - - if create_sandbox.success { - let _ = run_cli(&["sandbox", "delete", &sandbox_name]).await; - } - - assert!( - !create_sandbox.success, - "sandbox create from missing template should fail:\n{}", - create_sandbox.output - ); - assert!( - output_mentions_template_not_found(&create_sandbox.output), - "sandbox create from missing template should report not-found:\n{}", - create_sandbox.output - ); -} diff --git a/tests/suites/conformance/cli/tests/sandbox_templates.rs b/tests/suites/conformance/cli/tests/sandbox_templates.rs new file mode 100644 index 0000000000..575dff6144 --- /dev/null +++ b/tests/suites/conformance/cli/tests/sandbox_templates.rs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-agnostic sandbox workload template conformance tests. + +use openshell_conformance::{ + OpenShellRunner, SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO, + SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO, SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO, + SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO, Scenario, +}; + +/// Exercise template create/get/list and template-backed sandbox creation. +#[tokio::test] +async fn lifecycle() { + run(SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO).await; +} + +/// Exercise that a deleted template returns not-found. +#[tokio::test] +async fn get_after_delete() { + run(SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO).await; +} + +/// Exercise that duplicate template creation is rejected. +#[tokio::test] +async fn duplicate_name() { + run(SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO).await; +} + +/// Exercise that sandbox creation from a missing template returns not-found. +#[tokio::test] +async fn missing_template() { + run(SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO).await; +} + +async fn run(scenario: Scenario) { + let mut runner = OpenShellRunner::from_env(scenario.name) + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + scenario.run(&mut runner).await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("{} conformance scenario failed:\n{error}", scenario.name); + } +}