From a7d14a5fe2db3e3d31109a2986c20a31da670465 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 28 Oct 2026 11:24:38 -0700 Subject: [PATCH 1/3] CI: cover internal self-hosted runners in the runner trust probe Derive the probe's runner-label matrix from each branch's own workflows and guard on fork status, so the probe self-adapts across main and the private development branch without diverging. Reproduces the current main label set exactly (16 host, 17 Linux-GPU). --- .github/workflows/runner-trust-probe.yml | 67 ++++++++++++++++-------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index f144ae4483a..7c95fa08054 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -31,7 +31,10 @@ defaults: jobs: compute-matrix: name: Compute runner label sets - if: ${{ github.repository_owner == 'nvidia' }} + # Skip on forks (which cannot reach the self-hosted runners). This is true on + # both NVIDIA/cuda-python and the private development repo, so the one guard + # works unchanged on both and the workflow does not diverge between branches. + if: ${{ !github.event.repository.fork }} runs-on: ubuntu-latest timeout-minutes: 5 outputs: @@ -49,29 +52,47 @@ jobs: env: LABEL_FILTER: ${{ inputs.label-filter || '' }} run: | - # Hosted and self-hosted CPU labels are listed explicitly; GPU labels - # are derived from ci/test-matrix.yml. - labels_json="$(yq -o json '.' ci/test-matrix.yml | jq -c --arg filter "${LABEL_FILTER}" ' + # GitHub-hosted labels are a small, stable seed. Self-hosted labels are + # discovered from the runner labels referenced across this branch's own + # workflows, so the probe self-adapts per branch (for example the + # internal labels on the private development branch) without this file + # diverging between branches. GPU labels that are built from + # ci/test-matrix.yml at runtime are not literals in any workflow, so + # they are still derived from that file. + hosted='["ubuntu-latest","windows-latest","windows-2022","windows-11-arm"]' + + # Discover runner-label literals from the workflows. The grammar matches + # self-hosted labels (-cpuN, -gpu-, -internal, -1xgb10, riscv64) and any + # GPU labels hard-coded in a workflow; GitHub-hosted names and + # host-platform strings (e.g. 'linux-64') deliberately do not match. + scanned="$(grep -rhoE \ + '(linux|windows|wsl)-[a-z0-9-]*(cpu[0-9]+|gpu-[a-z0-9]|internal|1xgb10)[a-z0-9-]*|(linux|windows|wsl)-riscv64[a-z0-9-]*' \ + .github/workflows/ | sort -u | jq -Rsc 'split("\n") | map(select(length > 0))' || true)" + scanned="${scanned:-[]}" + + # GPU labels derived from the test matrix cover labels passed via matrix + # outputs that never appear as literals in a workflow. + gpu="$(yq -o json '.' ci/test-matrix.yml | jq -c ' { - host: ( - [ - "ubuntu-latest", - "linux-amd64-cpu8", - "linux-arm64-cpu8", - "linux-amd64-cpu4", - "windows-latest", - "windows-2022", - "windows-11-arm" - ] + [ - (.windows["pull-request"][]?, .windows["nightly"][]?) - | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" - ] - ), - linux_gpu: [ - (.linux["pull-request"][]?, .linux["nightly"][]?) - | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" - ] - } + windows: [ (.windows["pull-request"][]?, .windows["nightly"][]?) + | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" ], + linux: [ (.linux["pull-request"][]?, .linux["nightly"][]?) + | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" ] + }')" + + # A label runs in the Linux GPU container probe when it is a linux/wsl + # GPU label; every other discovered label runs on the host probe. + labels_json="$(jq -cn \ + --argjson hosted "${hosted}" \ + --argjson scanned "${scanned}" \ + --argjson gpu "${gpu}" \ + --arg filter "${LABEL_FILTER}" ' + def is_linux_gpu: test("^(linux|wsl)-.*-gpu-"); + (($scanned + $gpu.linux + $gpu.windows)) as $discovered + | { + host: ($hosted + ($discovered | map(select(is_linux_gpu | not)))), + linux_gpu: ($discovered | map(select(is_linux_gpu))) + } | map_values(unique | if $filter != "" then map(select(test($filter))) else . end) ')" From ff39a12183bbc563f67311e43e6da7cc64fb6417 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 28 Oct 2026 11:39:32 -0700 Subject: [PATCH 2/3] fixup! CI: cover internal self-hosted runners in the runner trust probe Tighten the riscv64 regex branch to require a hardware suffix (-[a-z0-9][a-z0-9-]*[a-z0-9]), preventing HOST_PLATFORM strings (linux-riscv64) and artifact name prefixes (linux-riscv64-) from matching as phantom runner labels and queuing until timeout. --- .github/workflows/runner-trust-probe.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 7c95fa08054..69b2c8a83d9 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -66,7 +66,7 @@ jobs: # GPU labels hard-coded in a workflow; GitHub-hosted names and # host-platform strings (e.g. 'linux-64') deliberately do not match. scanned="$(grep -rhoE \ - '(linux|windows|wsl)-[a-z0-9-]*(cpu[0-9]+|gpu-[a-z0-9]|internal|1xgb10)[a-z0-9-]*|(linux|windows|wsl)-riscv64[a-z0-9-]*' \ + '(linux|windows|wsl)-[a-z0-9-]*(cpu[0-9]+|gpu-[a-z0-9]|internal|1xgb10)[a-z0-9-]*|(linux|windows|wsl)-riscv64-[a-z0-9][a-z0-9-]*[a-z0-9]' \ .github/workflows/ | sort -u | jq -Rsc 'split("\n") | map(select(length > 0))' || true)" scanned="${scanned:-[]}" From a5ecf7589628901bfe04ca6ebfad0c9f27e2bb10 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 28 Oct 2026 11:57:48 -0700 Subject: [PATCH 3/3] fixup! CI: cover internal self-hosted runners in the runner trust probe Exclude the probe workflow itself from the grep scan so description-string examples do not produce phantom runner labels. Also filter extracted tokens ending in '-' as a defensive guard against artifact-name suffixes. --- .github/workflows/runner-trust-probe.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 69b2c8a83d9..8b46e6e0f67 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -67,7 +67,8 @@ jobs: # host-platform strings (e.g. 'linux-64') deliberately do not match. scanned="$(grep -rhoE \ '(linux|windows|wsl)-[a-z0-9-]*(cpu[0-9]+|gpu-[a-z0-9]|internal|1xgb10)[a-z0-9-]*|(linux|windows|wsl)-riscv64-[a-z0-9][a-z0-9-]*[a-z0-9]' \ - .github/workflows/ | sort -u | jq -Rsc 'split("\n") | map(select(length > 0))' || true)" + .github/workflows/ --exclude=runner-trust-probe.yml \ + | sort -u | jq -Rsc 'split("\n") | map(select(length > 0 and (endswith("-") | not)))' || true)" scanned="${scanned:-[]}" # GPU labels derived from the test matrix cover labels passed via matrix