CI: harden build-wheel.yml for the eMudhra/emSign CA incident (12.9.x backport)

.github/actions/fetch_ctk/action.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -76,19 +76,22 @@ runs:
         mkdir $CACHE_TMP_DIR
 
         # The binary archives (redist) are guaranteed to be updated as part of the release posting.
+        # Use the runtime workspace mount so this also works inside container jobs.
+        CTK_REDIST_TOOL="${GITHUB_WORKSPACE}/ci/tools/fetch_ctk_redistrib.py"
         CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
         CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
+        CTK_JSON_FILE="$CACHE_TMP_DIR/redistrib.json"
+        curl -LSs "$CTK_JSON_URL" -o "$CTK_JSON_FILE"
+        # Pin the metadata JSON: the per-archive sha256 checks below are only
+        # trustworthy if the JSON that carries those hashes is itself trusted.
+        python "$CTK_REDIST_TOOL" verify-metadata \
+          --cuda-version "${{ inputs.cuda-version }}" \
+          --metadata-path "$CTK_JSON_FILE"
         if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-          if [[ "${{ inputs.host-platform }}" == "linux-64" ]]; then
-            CTK_SUBDIR="linux-x86_64"
-          elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
-            CTK_SUBDIR="linux-sbsa"
-          fi
           function extract() {
             tar -xvf $1 -C $CACHE_TMP_DIR --strip-components=1
           }
         elif [[ "${{ inputs.host-platform }}" == "win-64" ]]; then
-          CTK_SUBDIR="windows-x86_64"
           function extract() {
             _TEMP_DIR_=$(mktemp -d)
             unzip $1 -d $_TEMP_DIR_
@@ -101,14 +104,23 @@ runs:
         function populate_cuda_path() {
           # take the component name as a argument
           function download() {
-            curl -kLSs $1 -o $2
+            curl -LSs $1 -o $2
           }
           CTK_COMPONENT=$1
-          CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL |
-              python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")"
+          CTK_COMPONENT_REL_PATH="$(python "$CTK_REDIST_TOOL" component-relative-path \
+            --host-platform "${{ inputs.host-platform }}" \
+            --component "$CTK_COMPONENT" \
+            --metadata-path "$CTK_JSON_FILE")"
           CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
           CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
           download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
+          # Verify the downloaded archive against the sha256 in the (pinned)
+          # metadata before extracting; fail closed on any mismatch.
+          CTK_COMPONENT_SHA256="$(python "$CTK_REDIST_TOOL" component-sha256 \
+            --host-platform "${{ inputs.host-platform }}" \
+            --component "$CTK_COMPONENT" \
+            --metadata-path "$CTK_JSON_FILE")"
+          echo "${CTK_COMPONENT_SHA256}  ${CTK_COMPONENT_COMPONENT_FILENAME}" | sha256sum --check --strict -
           extract $CTK_COMPONENT_COMPONENT_FILENAME
           rm $CTK_COMPONENT_COMPONENT_FILENAME
         }

.github/actions/setup-hardened-python/action.yml

@@ -0,0 +1,150 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Set up hardened Python
+description: >-
+  Install CPython with a pinned uv, create a venv from it, and put the venv
+  first on PATH. setup-uv verifies the uv binary and uv verifies the
+  python-build-standalone archive against embedded checksums, so the
+  interpreter's integrity does not depend on TLS. The job then fails unless
+  the interpreter's ssl module rejects the compromised eMudhra/emSign CA keys
+  (CVE-2026-115557; first fixed releases: 3.11.18, 3.12.16, 3.13.18, 3.14.10,
+  3.15.2), and pip is updated through that ssl module. Unlike setup-python,
+  this provides patched Windows builds of maintenance series, for which
+  python.org publishes no installers. Series that have no fixed release
+  (3.10 and older, which this branch still builds and tests) are installed
+  with a warning instead, and their pip is replaced by a sha256-pinned
+  wheel, so that the pip update does not depend on TLS. See
+  https://github.com/NVIDIA/cuda-python/issues/3195 and
+  https://discuss.python.org/t/109678.
+
+inputs:
+  python-version:
+    description: >-
+      uv Python request, e.g. "3.12" or "3.14t" (free-threaded). The newest
+      patch known to the pinned uv is installed. Series with a fixed release
+      listed in the interpreter check (3.11-3.15) must carry it. Series
+      without a fixed release (3.10 and older) are accepted with a warning,
+      and pip is installed from a sha256-pinned wheel.
+    required: true
+  venv:
+    description: Directory for the venv. Defaults to a new directory under RUNNER_TEMP.
+    default: ""
+
+outputs:
+  python-path:
+    description: Absolute path of the venv's interpreter.
+    value: ${{ steps.venv.outputs.python-path }}
+  python-version:
+    description: Full version of the interpreter, e.g. "3.14.10".
+    value: ${{ steps.venv.outputs.python-version }}
+
+runs:
+  using: composite
+  steps:
+    - name: Install uv
+      uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2  # v10.3.3
+      with:
+        # The newest uv whose checksum ships inside the pinned setup-uv, so
+        # the binary is verified against the SHA-pinned action itself. The uv
+        # release determines the newest patch of each CPython series, so
+        # bumping the setup-uv SHA also moves the CPython patch versions.
+        version: "latest-known"
+        enable-cache: false
+        # Some callers (e.g. release verification) have no checkout.
+        ignore-empty-workdir: true
+
+    - name: Install Python and create venv
+      id: venv
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        PY_REQUEST: ${{ inputs.python-version }}
+        VENV_DIR: ${{ inputs.venv }}
+        # A per-job install directory ensures the newest patch known to the
+        # pinned uv is used, never an older interpreter left on a persistent
+        # self-hosted runner.
+        UV_PYTHON_INSTALL_DIR: ${{ runner.temp }}/setup-hardened-python/interpreters
+        UV_NO_CONFIG: "1"
+      run: |
+        uv python install --no-bin --no-registry "${PY_REQUEST}"
+        base="$(uv python find --managed-python --no-project "${PY_REQUEST}")"
+        venv="${VENV_DIR}"
+        if [[ -z "${venv}" ]]; then
+          # A fresh directory per call, so a second call in the same job
+          # (e.g. build-wheel.yml) leaves the first venv intact.
+          venv="$("${base}" -c 'import sys, tempfile; print(tempfile.mkdtemp(prefix="venv-", dir=sys.argv[1]))' "${RUNNER_TEMP}")"
+        fi
+        # The stdlib venv seeds pip from the interpreter's bundled wheel, so
+        # nothing is downloaded before the ssl check below.
+        "${base}" -m venv --clear "${venv}"
+        if [[ "${RUNNER_OS}" == Windows ]]; then
+          py="${venv}/Scripts/python.exe"
+        else
+          py="${venv}/bin/python"
+        fi
+        # Let the interpreter report native absolute paths (Windows included).
+        python_path="$("${py}" -c 'import sys; print(sys.executable)')"
+        "${py}" -c 'import os, sys; print(os.path.dirname(sys.executable))' >> "${GITHUB_PATH}"
+        {
+          echo "python-path=${python_path}"
+          echo "python-version=$("${py}" -c 'import platform; print(platform.python_version())')"
+        } >> "${GITHUB_OUTPUT}"
+
+    - name: Check Python interpreter is patched (CVE-2026-115557)
+      id: check
+      # Runs the venv's python through the PATH entry added above.
+      shell: python {0}
+      env:
+        PYTHON: ${{ steps.venv.outputs.python-path }}
+      run: |
+        import os
+        import platform
+        import sys
+
+        if os.path.normcase(sys.executable) != os.path.normcase(os.environ["PYTHON"]):
+            print(f"::error::PATH resolves python to {sys.executable}, not the venv interpreter {os.environ['PYTHON']}.")
+            sys.exit(1)
+
+        FIXED = {(3, 11): 18, (3, 12): 16, (3, 13): 18, (3, 14): 10, (3, 15): 2}
+        # Series that will never get a fixed release (end of life before the
+        # fix). Their pip is installed from a pinned wheel instead.
+        UNPATCHABLE_BELOW = (3, 11)
+        ssl_patched = "true"
+        v = sys.version_info
+        series = (v.major, v.minor)
+        name = f"{v.major}.{v.minor}"
+        full = platform.python_version()
+        print("sys.version:", sys.version)
+        if series in FIXED:
+            floor = f"{name}.{FIXED[series]}"
+            if v.micro < FIXED[series]:
+                print(f"::error::Python {full} is older than {floor}, the first {name} release whose ssl module rejects the compromised eMudhra CA keys (CVE-2026-115557).")
+                sys.exit(1)
+            print(f"Python {full}: ssl module carries the fix (first fixed: {floor}).")
+        elif series < UNPATCHABLE_BELOW:
+            ssl_patched = "false"
+            print(f"::warning::Python {full}: no {name} release rejects the compromised eMudhra CA keys (CVE-2026-115557); pip will be installed from a sha256-pinned wheel instead.")
+        else:
+            # Fail closed: a newer series is only accepted once its fixed release is listed.
+            print(f"::error::Python {full}: no {name} release is known to reject the compromised eMudhra CA keys (CVE-2026-115557).")
+            sys.exit(1)
+
+        with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+            f.write(f"ssl-patched={ssl_patched}\n")
+
+    - name: Update pip
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      env:
+        PYTHON: ${{ steps.venv.outputs.python-path }}
+        SSL_PATCHED: ${{ steps.check.outputs.ssl-patched }}
+      run: |
+        if [[ "${SSL_PATCHED}" == "true" ]]; then
+          "${PYTHON}" -m pip install -U pip
+        else
+          # The interpreter's ssl module still trusts the compromised CA keys,
+          # so do not fetch pip through it. The sha256 fragment guarantees the
+          # wheel's integrity regardless of the transport.
+          "${PYTHON}" -m pip install \
+            "pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl#sha256=fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70"
+        fi

.github/actions/upgrade-ca-certificates/action.yml

@@ -0,0 +1,144 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Temporary stopgap for the eMudhra/emSign CA key compromise (CVE-2026-111943)
+# on GitHub-hosted runners until GitHub ships fixed images.
+#
+# On Linux: upgrades ca-certificates via apt so the compromised emSign ECC Root
+# CA - G3 anchor is removed from /etc/ssl/certs/ca-certificates.crt. Ubuntu's
+# bundle does not include AAA Certificate Services, so the Sectigo cross-signs
+# do not chain to a trusted root there.
+# On Windows: adds the eight compromised intermediates and their eight Sectigo
+# cross-signs (impacted-intermediates.json) to Cert:\LocalMachine\Disallowed so
+# schannel-based clients (git, gh, PowerShell/.NET) reject chains through them,
+# including chains to AAA Certificate Services, which Windows still trusts.
+#
+# This action is a no-op on self-hosted runners (runner.environment != 'github-hosted')
+# because self-hosted runner operators own and update their own host images.
+#
+# Removal criteria and implementation notes: security/emudhra-plan2.md
+# Tracked by: actions/runner-images#15018
+
+name: Harden OS certificate trust (eMudhra/emSign CA stopgap)
+description: >
+  Temporary per-job stopgap for CVE-2026-111943. Upgrades ca-certificates on
+  GitHub-hosted Linux runners and adds the compromised eMudhra intermediates
+  and cross-signs to the Disallowed store on GitHub-hosted Windows runners.
+  No-op on self-hosted.
+
+runs:
+  using: composite
+  steps:
+
+    # -------------------------------------------------------------------------
+    # Windows path: add the compromised intermediates to LocalMachine\Disallowed.
+    # Takes effect immediately for all schannel-based clients in subsequent steps
+    # (git, gh, PowerShell/.NET, WinHTTP). Must precede actions/checkout.
+    # -------------------------------------------------------------------------
+    - name: Distrust compromised eMudhra intermediates (Windows, GitHub-hosted)
+      if: ${{ runner.os == 'Windows' && runner.environment == 'github-hosted' }}
+      shell: pwsh
+      env:
+        ACTION_PATH: ${{ github.action_path }}
+      run: |
+        $ErrorActionPreference = 'Stop'
+        . (Join-Path $env:ACTION_PATH 'emudhra_intermediates.ps1')
+
+        $certs = @(Get-EmudhraIntermediates)
+        try {
+          # Adding is idempotent, so no early exit when the image is already fixed.
+          $store = [System.Security.Cryptography.X509Certificates.X509Store]::new('Disallowed', 'LocalMachine')
+          $store.Open('ReadWrite')
+          try {
+            foreach ($cert in $certs) { $store.Add($cert) }
+          } finally {
+            $store.Close()
+          }
+
+          $notDistrusted = @($certs | Where-Object { -not (Test-ExplicitlyDistrusted $_) } | ForEach-Object { $_.EmudhraLabel })
+          if ($notDistrusted.Count -gt 0) {
+            Write-Error "Not explicitly distrusted after adding to LocalMachine\Disallowed: $($notDistrusted -join '; ')"
+            exit 1
+          }
+          Write-Host "Distrusted $($certs.Count) compromised eMudhra intermediates and cross-signs."
+        } finally {
+          $certs | ForEach-Object { $_.Dispose() }
+        }
+
+    # -------------------------------------------------------------------------
+    # Linux path: upgrade ca-certificates via apt.
+    # The Ubuntu archive GPG check does not depend on the trust store being
+    # fixed, so this bootstrap is safe. Retries guard against transient mirror
+    # failures. Must precede actions/checkout.
+    # -------------------------------------------------------------------------
+    - name: Upgrade ca-certificates (Linux, GitHub-hosted)
+      if: ${{ runner.os == 'Linux' && runner.environment == 'github-hosted' }}
+      shell: bash
+      run: |
+        set -euo pipefail
+        EMSIGN_G3_FP="86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B"
+        CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
+        VERSION_ID="$(. /etc/os-release && echo "${VERSION_ID}")"
+
+        # Fail closed if openssl is missing or the bundle cannot be parsed
+        # (mirrors ci/tools/runner_trust_probe.sh).
+        if ! command -v openssl >/dev/null 2>&1; then
+          echo "::error::openssl not found; cannot verify CA bundle"
+          exit 1
+        fi
+        if [[ ! -f "${CA_BUNDLE}" ]]; then
+          echo "::error::CA bundle not found: ${CA_BUNDLE}"
+          exit 1
+        fi
+
+        fingerprints_in_bundle() {
+          awk '/-----BEGIN CERTIFICATE-----/{cert=""} {cert=cert $0 "\n"}
+               /-----END CERTIFICATE-----/{print cert | "openssl x509 -noout -fingerprint -sha256 2>/dev/null"; close("openssl x509 -noout -fingerprint -sha256 2>/dev/null")}' \
+            "${CA_BUNDLE}"
+        }
+
+        fps="$(fingerprints_in_bundle)"
+        cert_count="$(grep -c . <<< "${fps}" || true)"
+        if (( cert_count == 0 )); then
+          echo "::error::No certificates parsed from ${CA_BUNDLE}; openssl may have failed"
+          exit 1
+        fi
+
+        if grep -qiF "${EMSIGN_G3_FP}" <<< "${fps}"; then
+          before_state="not fixed"
+        else
+          before_state="already fixed"
+        fi
+        before_ver="$(dpkg-query -W -f='${Version}' ca-certificates 2>/dev/null || echo unknown)"
+        echo "::notice::emSign ECC Root CA - G3 on ${VERSION_ID}: ${before_state} (ca-certificates ${before_ver})"
+
+        if [ "${before_state}" = "already fixed" ]; then
+          exit 0
+        fi
+
+        for attempt in 1 2 3; do
+          if sudo apt-get update -q && sudo apt-get install --only-upgrade -y ca-certificates; then
+            break
+          fi
+          if [ "${attempt}" -lt 3 ]; then
+            echo "apt-get attempt ${attempt} failed; retrying in 10s..."
+            sleep 10
+          else
+            echo "::error::apt-get upgrade failed after ${attempt} attempts"
+            exit 1
+          fi
+        done
+
+        after_ver="$(dpkg-query -W -f='${Version}' ca-certificates 2>/dev/null || echo unknown)"
+        after_fps="$(fingerprints_in_bundle)"
+        after_count="$(grep -c . <<< "${after_fps}" || true)"
+        if (( after_count == 0 )); then
+          echo "::error::No certificates parsed after upgrade; cannot verify fix"
+          exit 1
+        fi
+        if grep -qiF "${EMSIGN_G3_FP}" <<< "${after_fps}"; then
+          echo "::error::emSign ECC Root CA - G3 still present after upgrade (ca-certificates ${after_ver} on ${VERSION_ID})"
+          exit 1
+        fi
+        echo "ca-certificates upgraded to ${after_ver}; emSign ECC Root CA - G3 absent."