feat(runner): add the test runner, plans and tmachine Session API

.github/workflows/branch-checks.yml

@@ -222,7 +222,11 @@ jobs:
           cargo fmt --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all -- --check
           cargo fmt --manifest-path examples/isolation-backend-v8/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/suites/sandbox/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/suites/providers/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/suites/podman/Cargo.toml --all -- --check
 
   cargo-lockfiles:
     name: Cargo lockfiles
@@ -240,7 +244,9 @@ jobs:
 
       - name: Verify Cargo lockfiles
         shell: nix develop -c bash -euo pipefail {0}
-        run: tasks/scripts/check-cargo-lockfiles.sh
+        run: |
+          tasks/scripts/check-cargo-lockfiles.sh
+          uv run tasks/scripts/check_cargo_lockfile_alignment.py
 
   rust-lint:
     name: Rust lint (${{ matrix.system }})
@@ -280,7 +286,11 @@ jobs:
           cargo clippy --locked --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path examples/isolation-backend-v8/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/suites/sandbox/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/providers/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/podman/Cargo.toml --all-targets -- -D warnings
 
   rust-test:
     name: Rust test (${{ matrix.system }})
@@ -330,6 +340,17 @@ jobs:
           cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml
+
+      # What runs where is generated from tests/plans, tests/environments and
+      # the tests themselves. A change to any of them must update the checked-in
+      # tests/expected-matrix.json, so it shows up as a diff in review.
+      - name: Check expected test matrix
+        shell: nix develop -c bash -euo pipefail {0}
+        run: |
+          cargo run --locked --quiet --manifest-path tests/runner/Cargo.toml --bin openshell-test -- check-matrix
+          uv run tasks/scripts/check_plan_workflow_matrix.py
 
   rust-postgres-test:
     name: Rust PostgreSQL tests

.github/workflows/branch-e2e.yml

@@ -240,7 +240,7 @@ jobs:
       deb-artifact-name: deb-linux-amd64
       rpm-artifact-name: rpm-linux-x86_64
 
-  # Run driver-independent conformance tests.
+  # Run the driver-independent sandbox suites, written on testkit (tests/testkit).
   conformance-integration:
     needs: prepare-integration
     permissions:
@@ -254,11 +254,11 @@ jobs:
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
       test-matrix: >-
         [
-          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
-          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
-          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
+          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"sandbox"},
+          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"sandbox"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"sandbox"},
+          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"sandbox"},
+          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"sandbox"}
         ]
 
   # Run feature-specific integration tests:

.github/workflows/integration-runner.yml

@@ -24,11 +24,11 @@ on:
         type: string
         default: >-
           [
-            {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
-            {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
-            {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
-            {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
-            {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
+            {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"sandbox"},
+            {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"sandbox"},
+            {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"sandbox"},
+            {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"sandbox"},
+            {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"sandbox"}
           ]
 
 permissions:
@@ -73,7 +73,13 @@ jobs:
         uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
         with:
           path: ~/.cache/tmachine
-          key: tmachine-${{ runner.os }}-${{ runner.arch }}
+          # tmachine keys its disk layers on a hash of the Ansible tree.
+          # actions/cache doesn't save after an exact key hit, so a fixed key
+          # would rebuild the changed layers on every run. Key on the playbooks
+          # and fall back to the newest entry for the same runner.
+          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**') }}
+          restore-keys: |
+            tmachine-${{ runner.os }}-${{ runner.arch }}-
 
       - name: Download integration inputs
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1

.github/workflows/integration-test.yml

@@ -27,11 +27,11 @@ on:
         type: string
         default: >-
           [
-            {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
-            {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
-            {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
-            {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
-            {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
+            {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"sandbox"},
+            {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"sandbox"},
+            {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"sandbox"},
+            {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"sandbox"},
+            {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"sandbox"}
           ]
 
 permissions:

.github/workflows/release-dev.yml

@@ -125,11 +125,11 @@ jobs:
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
       test-matrix: >-
         [
-          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
-          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
-          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
+          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"sandbox"},
+          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"sandbox"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"sandbox"},
+          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"sandbox"},
+          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"sandbox"}
         ]
 
   feature-specific-integration:

.github/workflows/release-qualification.yml

@@ -78,11 +78,11 @@ jobs:
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
       test-matrix: >-
         [
-          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
-          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
-          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"conformance"},
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
+          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"sandbox"},
+          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"sandbox"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha","testsuite":"sandbox"},
+          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"sandbox"},
+          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"sandbox"}
         ]
 
   feature-specific-integration:

AGENTS.md

@@ -22,7 +22,6 @@ Do not rely on this file for a full inventory. The detailed public and contribut
 | Path | Components | Purpose |
 |------|-----------|---------|
 | `crates/openshell-cli/` | CLI binary | User-facing command-line interface |
-| `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner |
 | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary |
 | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O |
 | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking |
@@ -62,7 +61,9 @@ Do not rely on this file for a full inventory. The detailed public and contribut
 | `python/openshell/` | Python SDK | Python bindings and CLI packaging |
 | `sdk/typescript/` | TypeScript SDK | Native Connect client, curated sandbox API, and generated protobuf types |
 | `proto/` | Protobuf definitions | gRPC service contracts |
-| `tests/testkit/` | E2E testkit | Support library and `#[testkit::test]` attribute for SDK-driven E2E suites |
+| `tests/testkit/` | E2E testkit | Support library, `#[testkit::test]` attribute, and `testkit-spec` parser for SDK-driven E2E suites |
+| `tests/tmachine/` | Test machines | QEMU machines provisioned with Ansible, and the `Session` API that drives them |
+| `tests/runner/` | Test runner | `openshell-test`: plans, environments, capabilities, waivers, JUnit result checks, CI matrix generation |
 | `deploy/` | Docker, Helm, K8s | Dockerfiles, Helm chart, manifests |
 | `docs/` | Published docs | MDX pages, navigation, and content assets |
 | `fern/` | Docs site config | Fern site config, components, and theme assets |

CONTRIBUTING.md

@@ -345,6 +345,9 @@ These are the primary `mise` tasks for day-to-day development:
 | `proto/`        | Protocol buffer definitions and [public API conventions](proto/README.md) |
 | `tasks/`        | `mise` task definitions and build scripts     |
 | `tests/testkit/` | Support library for SDK-driven end-to-end suites |
+| `tests/tmachine/` | QEMU test machines provisioned with Ansible, and the `Session` API that drives them |
+| `tests/runner/` | `openshell-test`: expands test plans into jobs, runs them in tmachine environments, and checks the results |
+| `tests/plans/`, `tests/environments/` | Test plans, and the environments they run in with the capabilities each provides |
 | `deploy/`       | Dockerfiles, Helm chart, Kubernetes manifests |
 | `docs/`         | Published Fern docs source, navigation, and content assets |
 | `fern/`         | Fern site config, components, and theme assets |