diff --git a/.github/workflows/release-verification.yml b/.github/workflows/release-verification.yml new file mode 100644 index 00000000000..78ffd5cde60 --- /dev/null +++ b/.github/workflows/release-verification.yml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +name: "CI: Verify release" + +on: + workflow_run: + workflows: ["CI: Release"] + types: [completed] + workflow_dispatch: + inputs: + git-tag: + description: "Public release tag to audit, for example v12.9.11" + required: true + type: string + +permissions: {} + +jobs: + verify: + name: Verify public release + if: >- + github.repository == 'NVIDIA/cuda-python' && ( + github.event_name == 'workflow_dispatch' || + ( + github.event.workflow_run.event == 'workflow_dispatch' && + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name == github.repository + ) + ) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + contents: read + steps: + # workflow_run uses the default branch's verifier, not the producer's head. + - name: Checkout verifier + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: ./.github/actions/setup-hardened-python + + - name: Install CI tool dependencies + run: python -m pip install -e ./ci + + - name: Download release verification metadata + id: release-metadata + if: github.event_name == 'workflow_run' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + name: release-verification-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + path: ${{ runner.temp }}/release-verification + + - name: Explain unavailable release metadata + if: failure() && steps.release-metadata.outcome == 'failure' + run: | + echo "::error::Could not download staged wheel metadata for this release attempt." + echo "A failed-jobs rerun may reuse preparation from an earlier attempt." + echo "Once the GitHub release is public, dispatch this workflow with its tag for a public audit." + + - name: Verify component from completed release + if: github.event_name == 'workflow_run' + run: python -m ci.tools.verify_release verify --manifest "$RUNNER_TEMP/release-verification/release-verification.json" + + - name: Audit a public release tag + if: github.event_name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.git-tag }} + run: python -m ci.tools.verify_release verify --tag="$RELEASE_TAG" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0a30aee3ebf..2c355655f5e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -407,6 +407,28 @@ jobs: if-no-files-found: error retention-days: 3 + # A reporting failure belongs to the separate verification workflow. + - name: Record release verification metadata + id: record-verification + continue-on-error: true + env: + RELEASE_MODE: ${{ inputs.release-action }} + run: | + python -m ci.tools.verify_release make-manifest \ + --tag="$RELEASE_GIT_TAG" --component "$RELEASE_COMPONENT" \ + --mode "$RELEASE_MODE" --wheel-dir release/wheels \ + > "$RUNNER_TEMP/release-verification.json" + + - name: Upload release verification metadata + if: steps.record-verification.outcome == 'success' + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-verification-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/release-verification.json + if-no-files-found: error + retention-days: 3 + test-cuda-python-dependencies: name: Test cuda-python${{ matrix.extra }} dependencies (${{ matrix.os }}, Python ${{ matrix.python-version }}) if: ${{ inputs.component == 'cuda-python' }} diff --git a/ci/README.md b/ci/README.md index a928e6cf749..d2e1595422f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -367,6 +367,20 @@ workflow. Publication mode checks the requested component in the tagged source, retains the existing nonempty-note and post-release policies, and exports the version for downstream release jobs. +## Verify Public Releases + +[CI: Verify release](../.github/workflows/release-verification.yml) follows +successful **CI: Release** runs. For a full release it checks versioned docs +and compares the component's public PyPI wheels with the staged wheel names +and hashes. A dry run verifies the metadata handoff and skips the public +checks. + +After publishing the GitHub release, dispatch **CI: Verify release** with the +release tag to audit the public result. For a shared bindings tag such as +`v12.9.11`, it checks both bindings and metapackage wheels and docs. It also +checks the GitHub release wheels against PyPI and verifies the published source +archive checksum. The workflow summary lists any missing or mismatched results. + ## Nightly Artifact Sources `ci-nightly.yml` downloads wheels from the latest successful main CI run and diff --git a/ci/tools/tests/test_verify_release.py b/ci/tools/tests/test_verify_release.py new file mode 100644 index 00000000000..4d123f358ef --- /dev/null +++ b/ci/tools/tests/test_verify_release.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import io +import json +from urllib.error import HTTPError + +import pytest + +from ci.tools import verify_release + +TAG = "v12.9.11" +VERSION = "12.9.11" +BINDINGS = "cuda_bindings-12.9.11-cp311-cp311-win_amd64.whl" +METAPACKAGE = "cuda_python-12.9.11-py3-none-any.whl" +DIGEST = hashlib.sha256(b"staged wheel").hexdigest() +ARCHIVE = f"cuda-python-{TAG}.tar.gz" +RELEASE_API = f"https://api.github.com/repos/NVIDIA/cuda-python/releases/tags/{TAG}" +DOWNLOAD = f"https://github.com/NVIDIA/cuda-python/releases/download/{TAG}" + + +def _pypi_url(component): + return f"https://pypi.org/pypi/{component}/{VERSION}/json" + + +def _pypi_response(name, digest=DIGEST): + return { + "urls": [ + {"filename": name, "digests": {"sha256": digest}, "packagetype": "bdist_wheel", "yanked": False}, + ], + } + + +@pytest.fixture +def public_server(monkeypatch): + responses = { + _pypi_url("cuda-bindings"): _pypi_response(BINDINGS), + _pypi_url("cuda-python"): _pypi_response(METAPACKAGE), + RELEASE_API: { + "draft": False, + "tag_name": TAG, + "assets": [ + {"name": BINDINGS, "digest": f"sha256:{DIGEST}"}, + {"name": METAPACKAGE, "digest": f"sha256:{DIGEST}"}, + {"name": ARCHIVE}, + {"name": f"{ARCHIVE}.sha256sum"}, + ], + }, + f"{DOWNLOAD}/{ARCHIVE}": b"archive content", + f"{DOWNLOAD}/{ARCHIVE}.sha256sum": hashlib.sha256(b"archive content").hexdigest().encode(), + } + requested = [] + + def urlopen(request, *, timeout): + url = request.full_url + requested.append(url) + if url.startswith("https://nvidia.github.io/"): + return io.BytesIO() + payload = responses.get(url) + if payload is None: + raise HTTPError(url, 404, "Not found", {}, None) + if isinstance(payload, list): + payload = payload.pop(0) + return io.BytesIO(payload if isinstance(payload, bytes) else json.dumps(payload).encode()) + + monkeypatch.setattr(verify_release, "urlopen", urlopen) + monkeypatch.setattr(verify_release.time, "sleep", lambda _: None) + return responses, requested + + +@pytest.mark.agent_authored(model="GPT-6") +def test_dry_run_does_not_probe_public_surfaces(public_server, tmp_path): + (tmp_path / BINDINGS).write_bytes(b"staged wheel") + manifest = verify_release.make_manifest(TAG, "cuda-bindings", "dry-run", tmp_path) + + assert manifest["wheels"] == {BINDINGS: DIGEST} + _, errors = verify_release.verify_manifest(manifest) + assert errors == [] + assert public_server[1] == [] + + +@pytest.mark.agent_authored(model="GPT-6") +@pytest.mark.parametrize("digest", [DIGEST, "a" * 64]) +def test_automatic_report_compares_selected_release_to_staged_wheels(public_server, digest): + manifest = {"tag": TAG, "component": "cuda-bindings", "mode": "full-release", "wheels": {BINDINGS: DIGEST}} + responses, requested = public_server + responses[_pypi_url("cuda-bindings")] = _pypi_response(BINDINGS, digest) + + _, errors = verify_release.verify_manifest(manifest) + if digest == DIGEST: + assert errors == [] + else: + assert errors == [f"PyPI cuda-bindings {VERSION}: SHA-256 mismatch for {BINDINGS}"] + assert requested == [ + _pypi_url("cuda-bindings"), + f"https://nvidia.github.io/cuda-python/cuda-bindings/{VERSION}/", + f"https://nvidia.github.io/cuda-python/cuda-bindings/{VERSION}/release/{VERSION}-notes.html", + ] + + +@pytest.mark.agent_authored(model="GPT-6") +def test_automatic_report_retries_missing_wheels_and_fails_if_still_missing(public_server): + manifest = {"tag": TAG, "component": "cuda-bindings", "mode": "full-release", "wheels": {BINDINGS: DIGEST}} + responses, requested = public_server + url = _pypi_url("cuda-bindings") + responses[url] = [{"urls": []}, _pypi_response(BINDINGS)] + + _, errors = verify_release.verify_manifest(manifest) + assert errors == [] + assert requested.count(url) == 2 + + responses[url] = {"urls": []} + _, errors = verify_release.verify_manifest(manifest) + assert len(errors) == 1 + assert f"missing non-yanked wheels: {BINDINGS}" in errors[0] + + +@pytest.mark.agent_authored(model="GPT-6") +def test_manual_report_checks_both_components_and_downloaded_archive(public_server): + _, requested = public_server + passed, errors = verify_release.verify_tag(TAG) + assert errors == [] + assert _pypi_url("cuda-bindings") in requested + assert _pypi_url("cuda-python") in requested + assert f"{DOWNLOAD}/{ARCHIVE}" in requested + assert f"{DOWNLOAD}/{ARCHIVE}.sha256sum" in requested + assert f"GitHub release {TAG}: 2 wheels match PyPI" in passed + + +@pytest.mark.agent_authored(model="GPT-6") +def test_manual_report_fails_for_partial_publication_and_corrupt_archive(public_server): + responses, _ = public_server + del responses[_pypi_url("cuda-python")] + responses[f"{DOWNLOAD}/{ARCHIVE}"] = b"wrong archive content" + + _, errors = verify_release.verify_tag(TAG) + assert len(errors) == 2 + assert any("PyPI cuda-python" in error and "HTTP 404" in error for error in errors) + assert any("does not match its published SHA-256 checksum" in error for error in errors) + + +@pytest.mark.agent_authored(model="GPT-6") +def test_pathfinder_post_release_uses_its_versioned_landing_page(public_server): + filename = "cuda_pathfinder-1.9.1.post1-py3-none-any.whl" + pypi_url = "https://pypi.org/pypi/cuda-pathfinder/1.9.1.post1/json" + manifest = { + "tag": "cuda-pathfinder-v1.9.1.post1", + "component": "cuda-pathfinder", + "mode": "full-release", + "wheels": {filename: DIGEST}, + } + responses, requested = public_server + responses[pypi_url] = _pypi_response(filename) + + _, errors = verify_release.verify_manifest(manifest) + assert errors == [] + assert requested == [pypi_url, "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.9.1/"] diff --git a/ci/tools/verify_release.py b/ci/tools/verify_release.py new file mode 100644 index 00000000000..ef001aa1e26 --- /dev/null +++ b/ci/tools/verify_release.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Record staged wheel hashes and verify the public results of a CUDA Python release.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +from collections.abc import Callable +from http.client import HTTPException, HTTPResponse +from pathlib import Path +from typing import TypeVar +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +from packaging.utils import parse_wheel_filename +from packaging.version import Version + +from .bindings_config import parse_prefixed_version +from .check_tagged_release_notes import COMPONENTS + +_PYPI = "https://pypi.org/pypi" +_DOCS = "https://nvidia.github.io/cuda-python" +_REPO = "NVIDIA/cuda-python" +_RELEASES_API = f"https://api.github.com/repos/{_REPO}/releases/tags" +_ASSET_DOWNLOAD = f"https://github.com/{_REPO}/releases/download" +_RETRY_DELAYS = (0, 10, 20, 30) +_MODES = ("dry-run", "full-release") +_T = TypeVar("_T") + + +class VerificationError(Exception): + """The public release or verification metadata failed a required check.""" + + +class PublicationPendingError(VerificationError): + """A registry response may still be waiting for publication to propagate.""" + + +def _tag_components(tag: str) -> tuple[Version, tuple[str, ...]]: + for _, prefix in COMPONENTS.values(): + version = parse_prefixed_version(tag, prefix) + if version is not None: + return version, tuple(name for name, (_, candidate) in COMPONENTS.items() if candidate == prefix) + raise ValueError(f"Invalid CUDA Python release tag: {tag!r}") + + +def _wheel_hashes(wheels: dict[str, str], component: str, version: Version) -> dict[str, str]: + if not isinstance(wheels, dict) or not wheels: + raise ValueError(f"No wheels for {component} {version}") + for name, digest in wheels.items(): + distribution, wheel_version, _, _ = parse_wheel_filename(name) + if distribution != component or wheel_version != version: + raise ValueError(f"{name!r} is not a {component} {version} wheel") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise ValueError(f"{name!r} has no valid SHA-256 digest") + return wheels + + +def make_manifest(tag: str, component: str, mode: str, wheel_dir: Path) -> dict: + version, components = _tag_components(tag) + if component not in components or mode not in _MODES: + raise ValueError(f"Invalid release component or mode for {tag!r}: {component!r}, {mode!r}") + wheels = {} + for wheel in sorted(wheel_dir.glob("*.whl")): + with wheel.open("rb") as stream: + wheels[wheel.name] = hashlib.file_digest(stream, "sha256").hexdigest() + return {"tag": tag, "component": component, "mode": mode, "wheels": _wheel_hashes(wheels, component, version)} + + +def _read(url: str, read: Callable[[HTTPResponse], _T], *, github_token: str = "") -> _T: + # All requests use fixed HTTPS endpoints and normalized version or wheel identifiers. + request = Request(url, headers={"User-Agent": "cuda-python-release-verification"}) # noqa: S310 + if github_token: + request.add_unredirected_header("Authorization", f"Bearer {github_token}") + detail = "" + for delay in _RETRY_DELAYS: + if delay: + time.sleep(delay) + try: + with urlopen(request, timeout=15) as response: # noqa: S310 + return read(response) + except HTTPError as exc: + detail = f"HTTP {exc.code}" + if exc.code not in (404, 429) and exc.code < 500: + break + except (OSError, HTTPException, PublicationPendingError) as exc: + detail = str(exc) + raise VerificationError(f"{url}: {detail}") + + +def _compare_wheels(actual: dict[str, str], expected: dict[str, str]) -> list[str]: + return [ + *(f"missing wheel {name}" for name in sorted(expected.keys() - actual.keys())), + *(f"unexpected wheel {name}" for name in sorted(actual.keys() - expected.keys())), + *( + f"SHA-256 mismatch for {name}" + for name in sorted(expected.keys() & actual.keys()) + if actual[name] != expected[name] + ), + ] + + +def _read_pypi(component: str, version: Version, expected: dict[str, str] | None) -> dict[str, str]: + def parse(response): + files = json.load(response)["urls"] + wheels = { + entry["filename"]: entry["digests"]["sha256"] + for entry in files + if entry["packagetype"] == "bdist_wheel" and not entry["yanked"] + } + missing = sorted(expected.keys() - wheels.keys()) if expected is not None else [] + if missing: + raise PublicationPendingError("missing non-yanked wheels: " + ", ".join(missing)) + return _wheel_hashes(wheels, component, version) + + return _read(f"{_PYPI}/{component}/{version}/json", parse) + + +def _docs_urls(component: str, version: Version) -> list[str]: + # Core uses the exact version directory. The other builds reuse the + # corresponding base directory for post releases. + docs_version = str(version) + if version.is_postrelease and component != "cuda-core": + docs_version = docs_version.split(".post", 1)[0] + root = _DOCS if component == "cuda-python" else f"{_DOCS}/{component}" + landing = f"{root}/{docs_version}/" + # The Pathfinder notes check allows empty placeholders. + if component == "cuda-pathfinder" or version.is_postrelease: + return [landing] + return [landing, f"{landing}release/{version}-notes.html"] + + +def _github_assets(tag: str) -> dict: + release = _read(f"{_RELEASES_API}/{tag}", json.load, github_token=os.environ.get("GH_TOKEN", "")) + if release["draft"] or release["tag_name"] != tag: + raise VerificationError(f"GitHub does not show a public release for {tag}") + return {asset["name"]: asset for asset in release["assets"]} + + +def _check_archive(tag: str, assets: dict) -> str: + archive = f"cuda-python-{tag}.tar.gz" + checksum = f"{archive}.sha256sum" + missing = {archive, checksum} - assets.keys() + if missing: + raise VerificationError("GitHub release is missing: " + ", ".join(sorted(missing))) + + download = f"{_ASSET_DOWNLOAD}/{tag}" + expected = _read(f"{download}/{checksum}", lambda response: response.read(1024).decode("ascii").strip()) + if re.fullmatch(r"[0-9a-f]{64}", expected) is None: + raise VerificationError(f"GitHub release asset {checksum} does not contain a SHA-256 digest") + actual = _read(f"{download}/{archive}", lambda response: hashlib.file_digest(response, "sha256").hexdigest()) + if actual != expected: + raise VerificationError(f"GitHub release asset {archive} does not match its published SHA-256 checksum") + return archive + + +def _audit( + tag: str, version: Version, components: tuple[str, ...], staged_wheels: dict[str, str] | None = None +) -> tuple[list[str], list[str]]: + passed = [] + errors = [] + public_wheels = {} + for component in components: + try: + wheels = _read_pypi(component, version, staged_wheels) + differences = _compare_wheels(wheels, staged_wheels) if staged_wheels is not None else [] + if differences: + errors.extend(f"PyPI {component} {version}: {difference}" for difference in differences) + else: + public_wheels[component] = wheels + passed.append(f"PyPI {component} {version}: {len(wheels)} non-yanked wheels") + except (VerificationError, ValueError, KeyError) as exc: + errors.append(f"PyPI {component} {version}: {exc}") + + for url in _docs_urls(component, version): + try: + _read(url, lambda _: None) + passed.append(f"Documentation: {url}") + except VerificationError as exc: + errors.append(f"Documentation: {exc}") + + if staged_wheels is not None: + passed.append("Use a manual audit after the GitHub release is public to check its source archive and assets.") + return passed, errors + + try: + assets = _github_assets(tag) + except (VerificationError, ValueError, KeyError) as exc: + errors.append(f"GitHub release {tag}: {exc}") + return passed, errors + + if len(public_wheels) == len(components): + pypi = {name: digest for wheels in public_wheels.values() for name, digest in wheels.items()} + github = { + name: (asset.get("digest") or "").removeprefix("sha256:") + for name, asset in assets.items() + if name.endswith(".whl") + } + differences = _compare_wheels(github, pypi) + if differences: + errors.extend(f"GitHub release {tag}: {difference}" for difference in differences) + else: + passed.append(f"GitHub release {tag}: {len(github)} wheels match PyPI") + + try: + archive = _check_archive(tag, assets) + passed.append(f"GitHub release {tag}: {archive} matches its published SHA-256 checksum") + except (VerificationError, ValueError) as exc: + errors.append(f"GitHub release {tag}: {exc}") + return passed, errors + + +def verify_manifest(manifest: dict) -> tuple[list[str], list[str]]: + if not isinstance(manifest, dict): + raise ValueError("Expected a JSON object for release verification metadata") + tag, component, mode, wheels = (manifest[key] for key in ("tag", "component", "mode", "wheels")) + if not isinstance(tag, str): + raise ValueError("Release tag must be a string") + version, components = _tag_components(tag) + if component not in components or mode not in _MODES: + raise ValueError(f"Invalid release component or mode for {tag!r}: {component!r}, {mode!r}") + wheels = _wheel_hashes(wheels, component, version) + if mode == "dry-run": + return [f"Dry run for {component} {tag}. Public verification skipped."], [] + return _audit(tag, version, (component,), staged_wheels=wheels) + + +def verify_tag(tag: str) -> tuple[list[str], list[str]]: + version, components = _tag_components(tag) + return _audit(tag, version, components) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subcommands = parser.add_subparsers(dest="command", required=True) + manifest = subcommands.add_parser("make-manifest", help="record the staged release metadata as JSON") + manifest.add_argument("--tag", required=True) + manifest.add_argument("--component", required=True, choices=COMPONENTS) + manifest.add_argument("--mode", required=True, choices=_MODES) + manifest.add_argument("--wheel-dir", required=True, type=Path) + verify = subcommands.add_parser("verify", help="check the public results of a release") + target = verify.add_mutually_exclusive_group(required=True) + target.add_argument("--manifest", type=Path) + target.add_argument("--tag") + args = parser.parse_args(argv) + + try: + if args.command == "make-manifest": + print(json.dumps(make_manifest(args.tag, args.component, args.mode, args.wheel_dir))) + return 0 + if args.manifest is not None: + passed, errors = verify_manifest(json.loads(args.manifest.read_text(encoding="utf-8"))) + else: + passed, errors = verify_tag(args.tag) + except (ValueError, KeyError, TypeError, OSError) as exc: + print(f"Invalid release verification input: {exc}", file=sys.stderr) + return 1 + + for line in passed: + print(line) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with Path(summary).open("a", encoding="utf-8") as stream: + stream.write("## Public release verification\n\n") + stream.writelines(f"- {line}\n" for line in passed) + stream.writelines(f"- **Error** {error}\n" for error in errors) + return int(bool(errors)) + + +if __name__ == "__main__": + sys.exit(main())