diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index b299a5d1d7..87c71f3724 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -779,56 +779,6 @@ async fn canonical_main_nonzero_exit_preserves_status() { ); } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_exit_zero_reaches_completed() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-success-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 0"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached successful canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached successful canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Completed", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached successful main did not complete:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Completed"), - "detached successful main should retain Completed status:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_nonzero_exit_reaches_error() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-failure-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 11"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached failing canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached failing canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached failing main did not reach Error:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "detached failing main should retain its terminal result:\n{details}" - ); - - sandbox.cleanup().await; -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn canonical_main_and_exec_receive_declared_environment() { @@ -868,40 +818,6 @@ async fn canonical_main_and_exec_receive_declared_environment() { } } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() { - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", "exit 11"]) - .await - .expect("fast detached main exit should not be reported as a provisioning failure"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("fast detached main did not reach Error:\n{err}")); - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox.name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - get_output.status.success(), - "sandbox get failed:\n{details}" - ); - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "fast detached main should retain its workload result:\n{details}" - ); - - sandbox.cleanup().await; -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn canonical_tty_main_uses_sandbox_environment() { diff --git a/e2e/suites/conformance/cli/tests/lifecycle/detached_exit_nonzero.rs b/e2e/suites/conformance/cli/tests/lifecycle/detached_exit_nonzero.rs new file mode 100644 index 0000000000..8e93b36d30 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/detached_exit_nonzero.rs @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{expect_terminal_result, run_gated_main}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a detached main process that exits nonzero leaves the sandbox in Error +/// with the code recorded. +#[tokio::test] +async fn detached_canonical_main_nonzero_exit_reaches_error() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/detached-exit-nonzero") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-dn", runner.id()); + run_gated_main(runner, &sandbox_name, 11, "detached-exit-nonzero").await?; + expect_terminal_result( + runner, + &sandbox_name, + "Error", + 11, + "detached-exit-nonzero/error", + ) + .await?; + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/detached-exit-nonzero conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/lifecycle/detached_exit_zero.rs b/e2e/suites/conformance/cli/tests/lifecycle/detached_exit_zero.rs new file mode 100644 index 0000000000..ebe3f3a925 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/detached_exit_zero.rs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{expect_terminal_result, run_gated_main}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a detached main process that exits 0 leaves the sandbox in Completed. +#[tokio::test] +async fn detached_canonical_main_exit_zero_reaches_completed() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/detached-exit-zero") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-dz", runner.id()); + run_gated_main(runner, &sandbox_name, 0, "detached-exit-zero").await?; + expect_terminal_result( + runner, + &sandbox_name, + "Completed", + 0, + "detached-exit-zero/completed", + ) + .await?; + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/detached-exit-zero conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/lifecycle/detached_fast_exit.rs b/e2e/suites/conformance/cli/tests/lifecycle/detached_fast_exit.rs new file mode 100644 index 0000000000..6d58e8dcc9 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/detached_fast_exit.rs @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{create_with_main, expect_terminal_result}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a detached main process that exits while the sandbox is still +/// provisioning is reported as a workload result, not a provisioning failure. +#[tokio::test] +async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/detached-fast-exit") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-fx", runner.id()); + create_with_main( + runner, + &sandbox_name, + &["--detach"], + &["sh", "-c", "exit 11"], + "detached-fast-exit/create", + ) + .await? + .require_success()?; + let state = expect_terminal_result( + runner, + &sandbox_name, + "Error", + 11, + "detached-fast-exit/error", + ) + .await?; + // Error alone does not separate a workload result from a provisioning + // failure; the documented Ready reason does. + let ready = state + .conditions + .iter() + .find(|condition| condition.kind == "Ready"); + if ready.map(|condition| condition.reason.as_str()) != Some("MainProcessFailed") { + return Err(format!( + "sandbox '{sandbox_name}' Ready condition is {ready:?}; expected reason MainProcessFailed" + )); + } + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/detached-fast-exit conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/lifecycle/helpers.rs b/e2e/suites/conformance/cli/tests/lifecycle/helpers.rs index f972141d80..fabb3a9687 100644 --- a/e2e/suites/conformance/cli/tests/lifecycle/helpers.rs +++ b/e2e/suites/conformance/cli/tests/lifecycle/helpers.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use openshell_e2e_support::CommandResult; use openshell_e2e_support::OpenShellRunner; use openshell_e2e_support::Poll; use serde::Deserialize; @@ -18,6 +19,16 @@ pub struct SandboxState { pub id: String, pub name: String, pub phase: String, + pub exit_code: Option, + #[serde(default)] + pub conditions: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct SandboxCondition { + #[serde(rename = "type")] + pub kind: String, + pub reason: String, } #[derive(Debug, Deserialize)] @@ -32,29 +43,156 @@ pub async fn create_running_sandbox( main: &str, step: &str, ) -> Result<(), String> { + create_with_main( + runner, + sandbox_name, + &["--detach"], + &["sh", "-lc", main], + &format!("{step}/create"), + ) + .await? + .require_success()?; + wait_for_phase(runner, sandbox_name, "Ready", &format!("{step}/ready")) + .await + .map(|_| ()) +} + +/// Create a sandbox whose main process is `main` and return the create result +/// without judging it. `flags` are extra `sandbox create` flags such as +/// `--detach`. +pub async fn create_with_main( + runner: &mut OpenShellRunner, + sandbox_name: &str, + flags: &[&str], + main: &[&str], + step: &str, +) -> Result { runner.track_sandbox(sandbox_name); - let create = runner - .step(format!("{step}/create")) + let mut args = vec!["sandbox", "create", "--name", sandbox_name, "--no-tty"]; + args.extend_from_slice(flags); + args.push("--"); + args.extend_from_slice(main); + runner + .step(step) .description(format!("sandbox '{sandbox_name}' is created")) .with_timeout(CREATE_TIMEOUT) - .run(&[ - "sandbox", - "create", - "--name", - sandbox_name, - "--detach", - "--no-tty", - "--", - "sh", - "-lc", - main, - ]) + .run(&args) .await - .map_err(|error| error.to_string())?; - create.require_success()?; - wait_for_phase(runner, sandbox_name, "Ready", &format!("{step}/ready")) + .map_err(|error| error.to_string()) +} + +/// Wait until the sandbox is in the terminal `phase` with the main process's +/// `exit_code` recorded. The exit code can arrive after a driver-observed +/// phase change, so a missing code is polled, not failed; a different code +/// fails at once. +pub async fn expect_terminal_result( + runner: &mut OpenShellRunner, + sandbox_name: &str, + phase: &str, + exit_code: i32, + step: &str, +) -> Result { + let sandbox_name = sandbox_name.to_string(); + let phase = phase.to_string(); + let step = step.to_string(); + let poll_step = step.clone(); + runner + .poll_until( + &poll_step, + TRANSITION_TIMEOUT, + TRANSITION_INTERVAL, + async move |runner| { + let result = runner + .step(format!("{step}/get")) + .description(format!( + "sandbox '{sandbox_name}' is {phase} with exit code {exit_code}" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) + .await; + match result { + Ok(result) if !result.success() => Poll::Pending(result.failure_diagnostic( + &format!("sandbox '{sandbox_name}' can be retrieved"), + )), + Ok(result) => match result.json::() { + Ok(state) if state.exit_code.is_some_and(|code| code != exit_code) => { + Poll::Failed(format!( + "sandbox '{sandbox_name}' recorded exit code {:?}; expected {exit_code}", + state.exit_code + )) + } + Ok(state) if state.phase == phase && state.exit_code == Some(exit_code) => { + Poll::Ready(state) + } + Ok(state) => Poll::Pending(format!( + "sandbox '{sandbox_name}' is {:?} with exit code {:?}; expected {phase:?} with {exit_code}", + state.phase, state.exit_code + )), + Err(error) => Poll::Failed(error.to_string()), + }, + Err(error) => Poll::Pending(error.to_string()), + } + }, + ) .await - .map(|_| ()) + .map_err(|error| error.to_string()) +} + +/// Create a detached sandbox whose main process waits for a release file and +/// then exits with `exit_code`, and release it once the sandbox is created. The +/// gate makes the exit happen after creation, not during it. +pub async fn run_gated_main( + runner: &mut OpenShellRunner, + sandbox_name: &str, + exit_code: i32, + step: &str, +) -> Result<(), String> { + let release = "/sandbox/.openshell-release"; + let main = format!("while [ ! -e '{release}' ]; do sleep 0.05; done; exit {exit_code}"); + create_with_main( + runner, + sandbox_name, + &["--detach"], + &["sh", "-c", &main], + &format!("{step}/create"), + ) + .await? + .require_success()?; + exec_succeeds( + runner, + sandbox_name, + &["touch", release], + &format!("{step}/release"), + ) + .await +} + +/// Run `command` in the sandbox and require it to succeed. +pub async fn exec_succeeds( + runner: &OpenShellRunner, + sandbox_name: &str, + command: &[&str], + step: &str, +) -> Result<(), String> { + let mut args = vec!["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]; + args.extend_from_slice(command); + runner + .step(step) + .description(format!("sandbox '{sandbox_name}' exec succeeds")) + .with_timeout(COMMAND_TIMEOUT) + .run(&args) + .await + .map_err(|error| error.to_string())? + .require_success() +} + +/// Require the main process's `line` among the create command's output. +pub fn require_output_line(result: &CommandResult, line: &str) -> Result<(), String> { + if result.stdout().lines().any(|output| output.trim() == line) { + Ok(()) + } else { + Err(result.failure_diagnostic(&format!("main process output includes {line:?}"))) + } } pub async fn run_lifecycle_command( diff --git a/e2e/suites/conformance/cli/tests/lifecycle/main.rs b/e2e/suites/conformance/cli/tests/lifecycle/main.rs index a7958f54fb..3461aa8743 100644 --- a/e2e/suites/conformance/cli/tests/lifecycle/main.rs +++ b/e2e/suites/conformance/cli/tests/lifecycle/main.rs @@ -4,5 +4,10 @@ //! Lifecycle conformance stories. mod delete_stopped; +mod detached_exit_nonzero; +mod detached_exit_zero; +mod detached_fast_exit; mod helpers; +mod main_exit_nonzero; +mod main_exit_zero; mod stop_start; diff --git a/e2e/suites/conformance/cli/tests/lifecycle/main_exit_nonzero.rs b/e2e/suites/conformance/cli/tests/lifecycle/main_exit_nonzero.rs new file mode 100644 index 0000000000..d7dc2fd9d4 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/main_exit_nonzero.rs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{create_with_main, expect_terminal_result, require_output_line}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a main process that exits nonzero fails the create command with that +/// code and leaves the sandbox in Error with the code recorded. +#[tokio::test] +async fn canonical_main_nonzero_exit_preserves_status() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/main-exit-nonzero") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-mn", runner.id()); + let create = create_with_main( + runner, + &sandbox_name, + &[], + &["sh", "-c", "echo failed-main; exit 7"], + "main-exit-nonzero/create", + ) + .await?; + if create.exit_code() != Some(7) { + return Err(create.failure_diagnostic("create exits with the main process code 7")); + } + require_output_line(&create, "failed-main")?; + expect_terminal_result(runner, &sandbox_name, "Error", 7, "main-exit-nonzero/error") + .await?; + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/main-exit-nonzero conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/lifecycle/main_exit_zero.rs b/e2e/suites/conformance/cli/tests/lifecycle/main_exit_zero.rs new file mode 100644 index 0000000000..0b6fa25185 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/main_exit_zero.rs @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{create_with_main, expect_terminal_result, require_output_line}; +use openshell_e2e_support::OpenShellRunner; + +/// Verify a main process that exits 0 leaves a retained sandbox in Completed. +#[tokio::test] +async fn canonical_main_exit_zero_completes_persistent_sandbox() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/main-exit-zero") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-mz", runner.id()); + let create = create_with_main( + runner, + &sandbox_name, + &[], + &["echo", "OK"], + "main-exit-zero/create", + ) + .await?; + create.require_success()?; + require_output_line(&create, "OK")?; + expect_terminal_result( + runner, + &sandbox_name, + "Completed", + 0, + "main-exit-zero/completed", + ) + .await?; + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/main-exit-zero conformance story failed:\n{error}"); + } +}