TaylorMutch · Oct 20, 2026 · Oct 16, 2026
diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md
@@ -125,6 +125,8 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
 
 ## View Job Logs
 
+For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
+
 `Pre-release` temporarily omits Codex Security from its `Security Scan` job.
 `Release` uses the passing prerelease qualification result. Other scanners and
 qualification gates still run. Use the standalone

diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml
@@ -6,8 +6,10 @@
   hosts: all
   gather_facts: false
   vars:
-    # Conformance is skipped when the gateway never connects.
-    conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}"
+    conformance_failed: >-
+      {{ gateway_status is failed or
+         (controller_readiness_failed | default(false)) or
+         conformance_result.rc != 0 }}
   tasks:
     - name: Wait for SSH
       ansible.builtin.wait_for_connection:
@@ -72,6 +74,21 @@
       delay: 5
       ignore_errors: true
 
+    - name: Check for K3s
+      ansible.builtin.stat:
+        path: /usr/local/bin/k3s
+      register: conformance_k3s
+
+    - name: Prepare the Kubernetes controller
+      when: gateway_status is succeeded and conformance_k3s.stat.exists
+      block:
+        - name: Verify Agent Sandbox reconciliation
+          ansible.builtin.include_tasks: k3s-ready.yaml
+      rescue:
+        - name: Record controller readiness failure
+          ansible.builtin.set_fact:
+            controller_readiness_failed: true
+
     - name: Run OpenShell conformance archive
       ansible.builtin.command:
         argv:
@@ -94,7 +111,7 @@
       register: conformance_result
       changed_when: false
       failed_when: false
-      when: gateway_status is succeeded
+      when: gateway_status is succeeded and not (controller_readiness_failed | default(false))
 
     # Preserve both streams for failures without adding passing-test output to
     # every tmachine run.
@@ -114,6 +131,12 @@
           - gateway_status is succeeded
         fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics
 
+    - name: Require a ready Kubernetes controller
+      ansible.builtin.assert:
+        that:
+          - not (controller_readiness_failed | default(false))
+        fail_msg: Agent Sandbox reconciliation did not become ready before conformance; see controller diagnostics
+
     - name: Require OpenShell conformance success
       ansible.builtin.assert:
         that:

diff --git a/tests/ansible/playbooks/conformance/k3s-ready.yaml b/tests/ansible/playbooks/conformance/k3s-ready.yaml
@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Check Agent Sandbox reconciliation after boot
+  become: true
+  block:
+    - name: Choose a unique controller probe name
+      ansible.builtin.set_fact:
+        controller_probe_name: "conformance-startup-{{ 9999999999999999 | random }}"
+
+    - name: Apply a scheduling-gated controller probe
+      ansible.builtin.command:
+        argv:
+          - timeout
+          - 15s
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=10s
+          - apply
+          - --filename=-
+          - --output=json
+        stdin: |
+          apiVersion: agents.x-k8s.io/v1beta1
+          kind: Sandbox
+          metadata:
+            name: "{{ controller_probe_name }}"
+            namespace: openshell
+          spec:
+            podTemplate:
+              spec:
+                automountServiceAccountToken: false
+                schedulingGates:
+                  - name: openshell.ai/conformance-startup
+                containers:
+                  - name: probe
+                    image: registry.k8s.io/pause:3.9
+      register: controller_probe
+      until: controller_probe.rc == 0
+      retries: 60
+      delay: 5
+
+    # Deployment readiness does not establish leadership or reconciliation.
+    # The gate keeps this probe from running or pulling its image.
+    - name: Wait for the controller-owned gated Pod
+      ansible.builtin.command:
+        argv:
+          - timeout
+          - 10s
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=5s
+          - --namespace=openshell
+          - get
+          - pod
+          - "{{ controller_probe_name }}"
+          - --output=json
+      register: controller_probe_pod
+      changed_when: false
+      until: >-
+        controller_probe_pod.rc == 0 and
+        ((controller_probe_pod.stdout | from_json).metadata.ownerReferences | default([]) |
+         selectattr('controller', 'defined') | selectattr('controller', 'equalto', true) |
+         selectattr('kind', 'equalto', 'Sandbox') |
+         selectattr('uid', 'equalto', (controller_probe.stdout | from_json).metadata.uid) | list | length) == 1 and
+        ((controller_probe_pod.stdout | from_json).spec.schedulingGates | default([]) |
+         selectattr('name', 'equalto', 'openshell.ai/conformance-startup') | list | length) == 1 and
+        not ((controller_probe_pod.stdout | from_json).spec.nodeName | default(''))
+      retries: 60
+      delay: 5
+
+  always:
+    - name: Remove the controller probe and its dependent resources
+      ansible.builtin.command:
+        argv:
+          - timeout
+          - 30s
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=5s
+          - --namespace=openshell
+          - delete
+          - sandboxes.agents.x-k8s.io
+          - "{{ controller_probe_name }}"
+          - --ignore-not-found=true
+          - --cascade=foreground
+          - --wait=true
+          - --timeout=20s
+      when: controller_probe_name is defined
diff --git a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh
@@ -16,6 +16,18 @@ collect() {
   printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
 }
 
+collect_log() {
+  printf '\n### %s\n' "$1"
+  shift
+  # Retain startup and recent evidence when later scenarios fill the log.
+  timeout 10s "$@" 2>&1 | {
+    head -c 131072
+    printf '\n### Remaining log tail\n'
+    tail -c 131072
+  }
+  printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
+}
+
 {
   date --utc --iso-8601=seconds
   collect 'K3s service state' systemctl show k3s.service \
@@ -34,6 +46,21 @@ collect() {
   collect 'K3s version' /usr/local/bin/k3s --version
   collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz
   collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide
+  collect 'Agent Sandbox controller deployment' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get deployment agent-sandbox-controller \
+    -o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,STATUS:.status'
+  collect 'Agent Sandbox controller Pods' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
+    -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
+  collect 'Agent Sandbox leader leases' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get leases \
+    -o 'custom-columns=NAME:.metadata.name,HOLDER:.spec.holderIdentity,ACQUIRED:.spec.acquireTime,RENEWED:.spec.renewTime,DURATION:.spec.leaseDurationSeconds,TRANSITIONS:.spec.leaseTransitions'
+  collect 'Agent Sandbox controller events' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get events --sort-by=.lastTimestamp
+  collect 'Sandbox reconciliation status' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get sandboxes.agents.x-k8s.io \
+    -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,CREATED:.metadata.creationTimestamp,GENERATION:.metadata.generation,STATUS:.status'
+  mapfile -t controller_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
+    --selector app=agent-sandbox-controller --output name 2>/dev/null)
+  for pod in "${controller_pods[@]}"; do
+    collect_log "Agent Sandbox current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --since=30m --timestamps
+    collect_log "Agent Sandbox previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --previous --since=30m --timestamps
+  done
   collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
     -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
   collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \