diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 80bbd58f7e..3870d8c407 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -214,11 +214,6 @@ name = "workspace_lifecycle" path = "tests/workspace_lifecycle.rs" required-features = ["e2e"] -[[test]] -name = "sandbox_templates" -path = "tests/sandbox_templates.rs" -required-features = ["e2e"] - [[test]] name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" diff --git a/e2e/rust/tests/sandbox_templates.rs b/e2e/rust/tests/sandbox_templates.rs deleted file mode 100644 index 21c9233daf..0000000000 --- a/e2e/rust/tests/sandbox_templates.rs +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for reusable sandbox workload templates. - -use std::process::Stdio; -use std::time::{SystemTime, UNIX_EPOCH}; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; - -struct CliResult { - output: String, - success: bool, -} - -struct TemplateGuard { - name: String, -} - -impl TemplateGuard { - fn new(name: String) -> Self { - Self { name } - } - - async fn cleanup(mut self) { - delete_template(&self.name).await; - self.name.clear(); - } -} - -impl Drop for TemplateGuard { - fn drop(&mut self) { - if self.name.is_empty() { - return; - } - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["sandbox", "template", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -async fn delete_template(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "template", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -fn unique_name(prefix: &str) -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before epoch") - .as_millis(); - let suffix = millis % 1_000_000; - format!("{prefix}-{suffix:06}") -} - -fn output_contains(output: &str, needle: &str) -> bool { - output.to_lowercase().contains(&needle.to_lowercase()) -} - -fn output_mentions_template_not_found(output: &str) -> bool { - output_contains(output, "sandbox template") && output_contains(output, "not found") -} - -#[tokio::test] -async fn sandbox_create_from_template_uses_reusable_workload() { - let template_name = unique_name("tmpl"); - let sandbox_name = unique_name("sb-tmpl"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&[ - "sandbox", - "template", - "create", - &template_name, - "--cpu", - "500m", - "--memory", - "512Mi", - "--label", - "e2e=sandbox-template", - "--env", - "FEATURE_FLAG=on", - ]) - .await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let get_template = run_cli(&[ - "sandbox", - "template", - "get", - &template_name, - "--output", - "json", - ]) - .await; - assert!( - get_template.success, - "sandbox template get failed:\n{}", - get_template.output - ); - let template_json: Value = - serde_json::from_str(&get_template.output).expect("template JSON output"); - assert_eq!(template_json["name"].as_str(), Some(template_name.as_str())); - assert_eq!( - template_json["labels"]["e2e"].as_str(), - Some("sandbox-template") - ); - assert_eq!( - template_json["environment"]["FEATURE_FLAG"].as_str(), - Some("on") - ); - assert_eq!(template_json["resources"]["cpu"].as_str(), Some("500m")); - assert_eq!(template_json["resources"]["memory"].as_str(), Some("512Mi")); - - let list_templates = run_cli(&["sandbox", "template", "list", "--names"]).await; - assert!( - list_templates.success, - "sandbox template list failed:\n{}", - list_templates.output - ); - assert!( - list_templates - .output - .lines() - .any(|line| line.trim() == template_name), - "template list should include {template_name}:\n{}", - list_templates.output - ); - - let mut sandbox = SandboxGuard::create(&[ - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "sh", - "-lc", - "test \"$FEATURE_FLAG\" = on && echo template-env-ok", - ]) - .await - .expect("sandbox create from template should succeed"); - - assert!( - sandbox.create_output.contains("template-env-ok"), - "sandbox should inherit template environment:\n{}", - sandbox.create_output - ); - - sandbox.cleanup().await; - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_get_after_delete_returns_not_found() { - let template_name = unique_name("tmpl-del"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let delete_template = run_cli(&["sandbox", "template", "delete", &template_name]).await; - assert!( - delete_template.success, - "sandbox template delete failed:\n{}", - delete_template.output - ); - - let get_deleted = run_cli(&["sandbox", "template", "get", &template_name]).await; - assert!( - !get_deleted.success, - "deleted template should not be fetchable:\n{}", - get_deleted.output - ); - assert!( - output_mentions_template_not_found(&get_deleted.output), - "deleted template should return not-found:\n{}", - get_deleted.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_create_rejects_duplicate_name() { - let template_name = unique_name("tmpl-dupe"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let duplicate = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - !duplicate.success, - "duplicate sandbox template create should fail:\n{}", - duplicate.output - ); - assert!( - output_contains(&duplicate.output, "already exists"), - "duplicate sandbox template create should report already-exists:\n{}", - duplicate.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_create_from_missing_template_returns_not_found() { - let template_name = unique_name("tmpl-missing"); - let sandbox_name = unique_name("sb-missing"); - - let create_sandbox = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "true", - ]) - .await; - - if create_sandbox.success { - let _ = run_cli(&["sandbox", "delete", &sandbox_name]).await; - } - - assert!( - !create_sandbox.success, - "sandbox create from missing template should fail:\n{}", - create_sandbox.output - ); - assert!( - output_mentions_template_not_found(&create_sandbox.output), - "sandbox create from missing template should report not-found:\n{}", - create_sandbox.output - ); -} diff --git a/e2e/suites/conformance/README.md b/e2e/suites/conformance/README.md index e0ecece555..8a421f648a 100644 --- a/e2e/suites/conformance/README.md +++ b/e2e/suites/conformance/README.md @@ -10,7 +10,8 @@ conformance area and user story. The CLI is the current client interface; lifecy - `cli/tests//.rs`: test functions, steps, and assertions for a story. - `cli/tests//helpers.rs`: setup and assertions shared by that area's stories. -The current areas are `file_transfer`, `lifecycle`, `policy_advisor`, and `smoke`. +The current areas are `file_transfer`, `lifecycle`, `policy_advisor`, `sandbox_templates`, +and `smoke`. Each area compiles into one test binary; individual stories remain selectable by their module prefix. `main.rs` contains only module declarations. @@ -66,7 +67,8 @@ must supply the workload image and tools used by the selected scenarios. Missing CLI or gateway prerequisites fail the run rather than silently passing. Run one group with `--test file_transfer`, `--test lifecycle`, -`--test policy_advisor`, or `--test smoke`, before the `--` separator. A group or +`--test policy_advisor`, `--test sandbox_templates`, or `--test smoke`, before the +`--` separator. A group or individual-test filter exercises only part of the suite. The tmachine conformance testsuite runs these same entry points from the diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/duplicate_name.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/duplicate_name.rs new file mode 100644 index 0000000000..4f7a23aa70 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/duplicate_name.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{fails, finish, succeeds, template_json}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify creating a template with an existing name is rejected and leaves the +/// original definition unchanged. +#[tokio::test] +async fn sandbox_template_create_rejects_duplicate_name() { + let mut runner = OpenShellRunner::from_env("sandbox-templates/duplicate-name") + .expect("candidate openshell CLI is available"); + let template = format!("ct-{}-dn", runner.id()); + let result = async { + runner.check_gateway_status().await?; + let r = &runner; + let first = [ + "sandbox", + "template", + "create", + &template, + "--label", + "e2e=first", + ]; + succeeds(r, "create", "template is created", &first).await?; + let second = [ + "sandbox", + "template", + "create", + &template, + "--label", + "e2e=second", + ]; + fails( + r, + "duplicate", + "a second template with the same name is rejected", + &second, + ) + .await?; + + let json = template_json(r, "get", &template).await?; + if json["labels"]["e2e"].as_str() != Some("first") { + return Err(format!( + "duplicate create changed the template: {}", + json["labels"] + )); + } + succeeds( + r, + "delete", + "template is deleted", + &["sandbox", "template", "delete", &template], + ) + .await + } + .await; + if let Err(error) = finish(runner, result, &[&template]).await { + panic!("sandbox-templates/duplicate-name conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/from_template.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/from_template.rs new file mode 100644 index 0000000000..6307d19ed1 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/from_template.rs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{CREATE_TIMEOUT, finish, run, succeeds}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a sandbox created from a template gets the template's environment. +/// +/// Side effects: creates one sandbox and one template, and deletes both. +#[tokio::test] +async fn sandbox_create_from_template_uses_reusable_workload() { + let mut runner = OpenShellRunner::from_env("sandbox-templates/from-template") + .expect("candidate openshell CLI is available"); + let template = format!("ct-{}-ft", runner.id()); + let sandbox = format!("ct-{}-fs", runner.id()); + let result = async { + runner.check_gateway_status().await?; + let r = &mut runner; + let create = [ + "sandbox", + "template", + "create", + &template, + "--env", + "FEATURE_FLAG=on", + ]; + succeeds(r, "template-create", "template is created", &create).await?; + + r.track_sandbox(&sandbox); + let create = [ + "sandbox", + "create", + "--name", + &sandbox, + "--detach", + "--template", + &template, + ]; + r.step("sandbox-create") + .description("sandbox is created from the template") + .with_timeout(CREATE_TIMEOUT) + .run(&create) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let exec = ["sandbox", "exec", "--name", &sandbox, "--no-tty", "--"]; + let read = [ + exec.as_slice(), + &["sh", "-c", "printf %s \"$FEATURE_FLAG\""], + ] + .concat(); + let flag = run( + r, + "read-env", + "sandbox sees the template environment", + &read, + ) + .await?; + flag.require_success()?; + if flag.stdout() != "on" { + return Err(flag.failure_diagnostic("FEATURE_FLAG is on in the sandbox")); + } + succeeds( + r, + "sandbox-delete", + "sandbox is deleted", + &["sandbox", "delete", &sandbox], + ) + .await?; + r.forget_sandbox(&sandbox); + succeeds( + r, + "template-delete", + "template is deleted", + &["sandbox", "template", "delete", &template], + ) + .await + } + .await; + if let Err(error) = finish(runner, result, &[&template]).await { + panic!("sandbox-templates/from-template conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/get_after_delete.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/get_after_delete.rs new file mode 100644 index 0000000000..3927613f0a --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/get_after_delete.rs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{fails, finish, require_listed, succeeds}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a deleted template cannot be retrieved and is no longer listed. +#[tokio::test] +async fn sandbox_template_get_after_delete_returns_not_found() { + let mut runner = OpenShellRunner::from_env("sandbox-templates/get-after-delete") + .expect("candidate openshell CLI is available"); + let template = format!("ct-{}-gd", runner.id()); + let result = async { + runner.check_gateway_status().await?; + let r = &runner; + succeeds( + r, + "create", + "template is created", + &["sandbox", "template", "create", &template], + ) + .await?; + succeeds( + r, + "delete", + "template is deleted", + &["sandbox", "template", "delete", &template], + ) + .await?; + fails( + r, + "get", + "deleted template is gone", + &["sandbox", "template", "get", &template], + ) + .await?; + require_listed(r, "list", &template, false).await + } + .await; + if let Err(error) = finish(runner, result, &[&template]).await { + panic!("sandbox-templates/get-after-delete conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/helpers.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/helpers.rs new file mode 100644 index 0000000000..b148078f98 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/helpers.rs @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_e2e_support::{CommandResult, OpenShellRunner}; +use serde_json::Value; +use std::time::Duration; + +const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); +pub const CREATE_TIMEOUT: Duration = Duration::from_mins(10); + +/// Run a CLI command and return its result without judging the outcome. +pub async fn run( + runner: &OpenShellRunner, + step: &str, + expectation: &str, + args: &[&str], +) -> Result { + runner + .step(step) + .description(expectation) + .with_timeout(COMMAND_TIMEOUT) + .run(args) + .await + .map_err(|error| error.to_string()) +} + +/// Run a CLI command that must succeed. +pub async fn succeeds( + runner: &OpenShellRunner, + step: &str, + expectation: &str, + args: &[&str], +) -> Result<(), String> { + run(runner, step, expectation, args) + .await? + .require_success() +} + +/// Run a CLI command that must fail. +pub async fn fails( + runner: &OpenShellRunner, + step: &str, + expectation: &str, + args: &[&str], +) -> Result<(), String> { + let result = run(runner, step, expectation, args).await?; + if result.success() { + return Err(result.failure_diagnostic(expectation)); + } + Ok(()) +} + +/// Return the template's definition from `sandbox template get --output json`. +pub async fn template_json( + runner: &OpenShellRunner, + step: &str, + name: &str, +) -> Result { + let args = ["sandbox", "template", "get", name, "--output", "json"]; + let result = run(runner, step, "template can be retrieved", &args).await?; + result.require_success()?; + result.json().map_err(|error| error.to_string()) +} + +/// Require whether `name` appears in the template listing. +pub async fn require_listed( + runner: &OpenShellRunner, + step: &str, + name: &str, + listed: bool, +) -> Result<(), String> { + let args = ["sandbox", "template", "list", "--names"]; + let result = run(runner, step, "template list succeeds", &args).await?; + result.require_success()?; + if result.stdout().lines().any(|line| line.trim() == name) == listed { + return Ok(()); + } + Err(result.failure_diagnostic(&format!("template '{name}' listed: {listed}"))) +} + +/// Finish a story. A passing story already removed what it created, so +/// templates are deleted only after a failure; both failures are kept. The +/// runner then removes the sandboxes it tracks. +pub async fn finish( + runner: OpenShellRunner, + result: Result<(), String>, + templates: &[&str], +) -> Result<(), String> { + let mut cleanup = Vec::new(); + if result.is_err() { + for template in templates { + let args = ["sandbox", "template", "delete", template]; + match run( + &runner, + "cleanup", + "template is deleted or already absent", + &args, + ) + .await + { + Ok(deleted) if deleted.success() || reports_not_found(&deleted) => {} + Ok(deleted) => cleanup.push(deleted.failure_diagnostic("cleanup delete succeeds")), + Err(error) => cleanup.push(error), + } + } + } + let result = if cleanup.is_empty() { + result + } else { + let error = result.err().unwrap_or_default(); + Err(format!( + "{error}\n\ncleanup also failed:\n{}", + cleanup.join("\n\n") + )) + }; + runner.finish(result).await +} + +fn reports_not_found(result: &CommandResult) -> bool { + let output = format!("{}{}", result.stdout(), result.stderr()); + output + .to_ascii_lowercase() + .contains("sandbox template not found") +} diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/main.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/main.rs new file mode 100644 index 0000000000..79ab5d354e --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/main.rs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox template conformance stories. +//! +//! Preconditions: a gateway that lets the test identity create and delete +//! sandbox templates and sandboxes, and a target that supplies the default +//! sandbox image. +//! +//! Side effects: each story creates templates and sandboxes named from the run +//! id and removes them even when it fails. +//! +//! Known limitation: the CLI exposes no structured error, so rejections are +//! checked by exit status and by the resulting state, not by message. + +mod duplicate_name; +mod from_template; +mod get_after_delete; +mod helpers; +mod missing_template; +mod round_trip; diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/missing_template.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/missing_template.rs new file mode 100644 index 0000000000..d76b5f4f11 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/missing_template.rs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{CREATE_TIMEOUT, fails, finish}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify creating a sandbox from a template that does not exist is rejected +/// and leaves no sandbox behind. The same flags succeed for an existing +/// template in the from-template story, so the rejection is about the template. +#[tokio::test] +async fn sandbox_create_from_missing_template_returns_not_found() { + let mut runner = OpenShellRunner::from_env("sandbox-templates/missing-template") + .expect("candidate openshell CLI is available"); + let template = format!("ct-{}-mt", runner.id()); + let sandbox = format!("ct-{}-ms", runner.id()); + let result = async { + runner.check_gateway_status().await?; + let r = &mut runner; + r.track_sandbox(&sandbox); + let create = [ + "sandbox", + "create", + "--name", + &sandbox, + "--detach", + "--template", + &template, + ]; + let rejected = r + .step("create") + .description("sandbox create from a missing template is rejected") + .with_timeout(CREATE_TIMEOUT) + .run(&create) + .await + .map_err(|error| error.to_string())?; + if rejected.success() { + return Err(rejected.failure_diagnostic("create from a missing template fails")); + } + let get = ["sandbox", "get", &sandbox]; + fails(r, "get", "no sandbox was created", &get).await + } + .await; + if let Err(error) = finish(runner, result, &[]).await { + panic!("sandbox-templates/missing-template conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/sandbox_templates/round_trip.rs b/e2e/suites/conformance/cli/tests/sandbox_templates/round_trip.rs new file mode 100644 index 0000000000..e047e7247b --- /dev/null +++ b/e2e/suites/conformance/cli/tests/sandbox_templates/round_trip.rs @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{finish, require_listed, succeeds, template_json}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a template's name, labels, environment and resources survive a +/// create and get, and that it is listed. +/// +/// Resources are metadata here. Enforcing them is driver-specific. +#[tokio::test] +async fn sandbox_template_round_trips_its_definition() { + let mut runner = OpenShellRunner::from_env("sandbox-templates/round-trip") + .expect("candidate openshell CLI is available"); + let template = format!("ct-{}-tr", runner.id()); + let result = async { + runner.check_gateway_status().await?; + let r = &runner; + let create = [ + "sandbox", + "template", + "create", + &template, + "--cpu", + "500m", + "--memory", + "512Mi", + "--label", + "e2e=sandbox-template", + "--env", + "FEATURE_FLAG=on", + ]; + succeeds(r, "create", "template is created", &create).await?; + + let json = template_json(r, "get", &template).await?; + let checks = [ + ("name", &json["name"], template.as_str()), + ("label e2e", &json["labels"]["e2e"], "sandbox-template"), + ( + "env FEATURE_FLAG", + &json["environment"]["FEATURE_FLAG"], + "on", + ), + ("cpu", &json["resources"]["cpu"], "500m"), + ("memory", &json["resources"]["memory"], "512Mi"), + ]; + for (label, actual, expected) in checks { + if actual.as_str() != Some(expected) { + return Err(format!("template {label} is {actual}, expected {expected}")); + } + } + require_listed(r, "list", &template, true).await?; + succeeds( + r, + "delete", + "template is deleted", + &["sandbox", "template", "delete", &template], + ) + .await + } + .await; + if let Err(error) = finish(runner, result, &[&template]).await { + panic!("sandbox-templates/round-trip conformance story failed:\n{error}"); + } +}