Support llama-cpp-python 0.3.36 with optimized runtime wheels

.github/workflows/runtime-wheels.yml

@@ -11,16 +11,20 @@ permissions:
   contents: read
 
 jobs:
-  macos-intel:
-    runs-on: macos-15-intel
+  macos:
+    runs-on: ${{ matrix.platform.runner }}
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        platform:
+          - {name: intel, runner: macos-15-intel, arch: x64, wheel-arch: x86_64, deployment-target: "11.0"}
+          - {name: arm64, runner: macos-15, arch: arm64, wheel-arch: arm64, deployment-target: "11.0"}
+          - {name: arm64-modern, runner: macos-15, arch: arm64, wheel-arch: arm64, deployment-target: "14.0"}
     env:
-      CMAKE_ARGS: -C "${{ github.workspace }}/scripts/runtime-wheels/macos-intel.cmake"
-      MACOSX_DEPLOYMENT_TARGET: "11.0"
-      ARCHFLAGS: -arch x86_64
+      CMAKE_ARGS: -C "${{ github.workspace }}/scripts/runtime-wheels/macos-${{ matrix.platform.name }}.cmake"
+      MACOSX_DEPLOYMENT_TARGET: ${{ matrix.platform.deployment-target }}
+      ARCHFLAGS: -arch ${{ matrix.platform.wheel-arch }}
       CMAKE_BUILD_PARALLEL_LEVEL: "4"
       CC: /usr/bin/clang
       CXX: /usr/bin/clang++
@@ -29,7 +33,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          architecture: ${{ matrix.platform.arch }}
       - name: Install build tools
         run: python -m pip install -r scripts/runtime-wheels/macos-intel-requirements.txt
       - name: Build pinned upstream source
@@ -39,11 +43,36 @@ jobs:
       - name: Verify installation and offline inference
         shell: python
         run: |
-          import pathlib, subprocess, sys, tempfile, venv
+          import pathlib, re, subprocess, tempfile, venv, zipfile
           wheel, = pathlib.Path("wheelhouse").glob("*.whl")
-          tag = "cp%d%d" % sys.version_info[:2]
-          assert wheel.name == f"llama_cpp_python-0.3.19-{tag}-{tag}-macosx_11_0_x86_64.whl"
+          target = "${{ matrix.platform.deployment-target }}".replace(".", "_")
+          assert wheel.name == f"llama_cpp_python-0.3.36-py3-none-macosx_{target}_${{ matrix.platform.wheel-arch }}.whl"
           with tempfile.TemporaryDirectory() as root:
+              native_root = pathlib.Path(root) / "native"
+              with zipfile.ZipFile(wheel) as archive:
+                  archive.extractall(native_root)
+              libraries = [
+                  p for p in native_root.rglob("*")
+                  if p.is_file() and p.suffix in (".dylib", ".so")
+              ]
+              assert libraries, "Wheel contains no native libraries"
+              assert any("libggml-blas" in p.name for p in libraries) == (target == "14_0")
+              minimum_target = target.replace("_", ".")
+              for library in libraries:
+                  architecture = subprocess.check_output(
+                      ["lipo", "-archs", str(library)], text=True
+                  ).strip()
+                  assert architecture == "${{ matrix.platform.wheel-arch }}", (library, architecture)
+                  commands = subprocess.check_output(
+                      ["otool", "-l", str(library)], text=True
+                  )
+                  minima = re.findall(r"^\s*minos ([\d.]+)$", commands, re.MULTILINE)
+                  assert minima in ([minimum_target], [minimum_target + ".0"]), (library, minima)
+                  if target == "11_0":
+                      imports = subprocess.check_output(
+                          ["nm", "-u", str(library)], text=True
+                      )
+                      assert not re.search(r"NEWLAPACK|ILP64", imports, re.IGNORECASE), library
               venv.create(root, with_pip=True)
               python = str(pathlib.Path(root) / "bin/python")
               fixtures = str(pathlib.Path(root) / "fixtures")
@@ -57,7 +86,7 @@ jobs:
                   subprocess.run([python, *args], check=True)
       - uses: actions/upload-artifact@v4
         with:
-          name: llama-cpp-macos-intel-py${{ matrix.python-version }}
+          name: llama-cpp-macos-${{ matrix.platform.name }}-py${{ matrix.python-version }}
           path: wheelhouse/*.whl
           if-no-files-found: error
 
@@ -89,8 +118,7 @@ jobs:
         run: |
           import pathlib, subprocess, sys, tempfile, venv
           wheel, = pathlib.Path("wheelhouse").glob("*.whl")
-          tag = "cp%d%d" % sys.version_info[:2]
-          assert wheel.name == f"llama_cpp_python-0.3.19-{tag}-{tag}-win_amd64.whl"
+          assert wheel.name == "llama_cpp_python-0.3.36-py3-none-win_amd64.whl"
           subprocess.run([
               sys.executable, "-I", "scripts/runtime-wheels/audit_windows_dependencies.py",
               str(wheel.resolve()),
@@ -124,25 +152,51 @@ jobs:
       - name: Build and repair the Python 3.8 wheel
         run: |
           docker run --rm --volume "$PWD:/io" --workdir /io \
-            --env CMAKE_GENERATOR="Unix Makefiles" \
-            --env CMAKE_ARGS="-C /io/scripts/runtime-wheels/linux.cmake" \
             --env CMAKE_BUILD_PARALLEL_LEVEL=4 \
             quay.io/pypa/manylinux2014_x86_64@sha256:f09bf382e4bcbd5b3cab130f8d0f12dbe474833f95f54182e80e8c0aaa269ff0 \
-            bash -euxo pipefail -c '
-              export PATH="/opt/python/cp38-cp38/bin:$PATH"
-              python -m pip install -r scripts/runtime-wheels/linux-requirements.txt
-              python -m pip wheel --no-cache-dir --no-deps --no-build-isolation \
-                -r scripts/runtime-wheels/source.txt --wheel-dir /tmp/raw-wheels
-              auditwheel repair --plat manylinux2014_x86_64 --only-plat \
-                --wheel-dir wheelhouse /tmp/raw-wheels/*.whl
-            '
+            bash scripts/runtime-wheels/build-linux-wheel.sh /io/wheelhouse
       - name: Verify installation and offline inference
         shell: python
         run: |
-          import pathlib, subprocess, tempfile, venv
+          import os, pathlib, re, subprocess, tempfile, venv, zipfile
           wheel, = pathlib.Path("wheelhouse").glob("*.whl")
-          assert wheel.name == "llama_cpp_python-0.3.19-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
+          assert wheel.name == "llama_cpp_python-0.3.36-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
+          with zipfile.ZipFile(wheel) as archive:
+              members = archive.namelist()
+          assert len(members) == len(set(members)), "Duplicate wheel archive members"
+          variants = (
+              "x64 sse42 sandybridge ivybridge piledriver haswell skylakex "
+              "cannonlake cascadelake icelake cooperlake zen4 alderlake sapphirerapids"
+          ).split()
+          expected = {"llama_cpp/lib/libggml-cpu-" + name + ".so" for name in variants}
+          modules = {name for name in members
+                     if pathlib.PurePosixPath(name).name.startswith("libggml-cpu-")}
+          assert modules == expected, {"missing": sorted(expected - modules),
+                                       "unexpected": sorted(modules - expected)}
+          for name in ("libggml.so", "libggml-base.so", "libllama.so", "libmtmd.so"):
+              assert "llama_cpp/lib/" + name in members, "Missing runtime library: " + name
           with tempfile.TemporaryDirectory() as root:
+              native_root = pathlib.Path(root) / "native"
+              readelf_env = dict(os.environ, LC_ALL="C")
+              with zipfile.ZipFile(wheel) as archive:
+                  libraries = [name for name in archive.namelist() if re.search(r"\.so(?:\.\d+)*$", name)]
+                  assert libraries, "Wheel contains no native libraries"
+                  for name in libraries:
+                      library = archive.extract(name, native_root)
+                      with open(library, "rb") as binary:
+                          header = binary.read(20)
+                      assert header[:6] == b"\x7fELF\x02\x01" and int.from_bytes(header[18:20], "little") == 62, name
+                      dynamic = subprocess.check_output(["readelf", "-dW", library], text=True, env=readelf_env)
+                      rpaths = re.findall(r"\((?:RPATH|RUNPATH)\).*?\[(.*?)\]", dynamic)
+                      assert all(re.fullmatch(r"\$(?:ORIGIN|\{ORIGIN\})(?:/.*)?", path)
+                                 for entry in rpaths for path in entry.split(":")), (name, rpaths)
+                      versions = subprocess.check_output(["readelf", "--version-info", "-W", library], text=True, env=readelf_env)
+                      needs = re.search(r"(?ms)^Version needs section.*?(?=^Version |\Z)", versions)
+                      for namespace, maximum in (("GLIBC", (2, 17)), ("GLIBCXX", (3, 4, 19)), ("CXXABI", (1, 3, 7))):
+                          required = re.findall(r"\bName: " + namespace + r"_([^\s]+)", needs.group() if needs else "")
+                          assert all(re.fullmatch(r"\d+(?:\.\d+)+", version) and
+                                     tuple(map(int, version.split("."))) <= maximum
+                                     for version in required), (name, namespace, required)
               venv.create(root, with_pip=True)
               python = str(pathlib.Path(root) / "bin/python")
               fixtures = str(pathlib.Path(root) / "fixtures")
@@ -152,6 +206,7 @@ jobs:
                   ["-I", "-c", "import llama_cpp; print(llama_cpp.llama_print_system_info().decode())"],
                   ["-I", "scripts/runtime-wheels/prepare_native_fixtures.py", fixtures],
                   ["-I", "scripts/runtime-wheels/check_native_inference.py", fixtures],
+                  ["-I", "scripts/runtime-wheels/check_native_inference.py", fixtures, "--cpu-variant", "x64"],
               ):
                   subprocess.run([python, *args], check=True)
       - uses: actions/upload-artifact@v4

.github/workflows/test.yml

@@ -144,7 +144,7 @@ jobs:
         platform:
           - {runner: ubuntu-24.04, arch: x64}
           - {runner: windows-2022, arch: x64}
-          - {runner: macos-14, arch: arm64}
+          - {runner: macos-15, arch: arm64}
           - {runner: macos-15-intel, arch: x64}
     steps:
       - uses: actions/checkout@v4

pyproject.toml

@@ -31,7 +31,7 @@ classifiers = [
 ]
 dependencies = [
   "httpx>=0.27.0,<1.0",
-  "llama-cpp-python>=0.3.0,<=0.3.19",
+  "llama-cpp-python>=0.3.0,<=0.3.36",
 ]
 
 [project.urls]

scripts/runtime-wheels/audit_windows_dependencies.py

@@ -8,7 +8,10 @@
 import pefile
 
 # Windows 10+ components. MSVC and OpenMP redistributables are not OS libraries.
-SYSTEM_DLLS = {"advapi32.dll", "kernel32.dll", "ntdll.dll", "ucrtbase.dll"}
+SYSTEM_DLLS = {
+    "advapi32.dll", "kernel32.dll", "ntdll.dll",
+    "shell32.dll", "ucrtbase.dll", "ws2_32.dll",
+}
 IMPORT_TABLES = ("IMPORT", "DELAY_IMPORT")
 REQUIRED_DLLS = {"llama.dll", "ggml.dll", "ggml-base.dll", "ggml-cpu.dll", "mtmd.dll"}
 

scripts/runtime-wheels/build-linux-wheel.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+mkdir -p -- "${1:-wheelhouse}"
+wheelhouse=$(cd -- "${1:-wheelhouse}" && pwd)
+build_dir=$(mktemp -d)
+trap 'rm -rf -- "$build_dir"' EXIT
+
+read -r source_spec < "$script_dir/source.txt"
+curl --fail --location --retry 3 --output "$build_dir/source.tar.gz" "${source_spec%%#sha256=*}"
+printf '%s  %s\n' "${source_spec##*#sha256=}" "$build_dir/source.tar.gz" | sha256sum --check
+tar -xzf "$build_dir/source.tar.gz" -C "$build_dir"
+cd "$build_dir/llama_cpp_python-0.3.36"
+for patch_name in linux-backend-install-dir linux-backend-packaging linux-backend-search-path \
+    linux-cpu-os-state linux-amx-permission linux-amx-gcc11 linux-q6k-avx512; do
+    patch --batch --fuzz=0 -p1 < "$script_dir/$patch_name.patch"
+done
+
+bash "$script_dir/install-linux-toolchain.sh"
+export PATH=/opt/python/cp38-cp38/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export LD_LIBRARY_PATH=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/usr/local/lib64
+export CC=/opt/rh/devtoolset-11/root/usr/bin/gcc
+export CXX=/opt/rh/devtoolset-11/root/usr/bin/g++
+export LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+export SSL_CERT_FILE=/opt/_internal/certs.pem
+export CMAKE_GENERATOR='Unix Makefiles'
+export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-2}
+export CMAKE_ARGS="-C $script_dir/linux-dispatch.cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
+export SKBUILD_BUILD_DIR="$build_dir/build"
+
+python "$script_dir/check_linux_cpu_eligibility.py" \
+    vendor/llama.cpp/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp
+
+python -m pip install -r "$script_dir/linux-requirements.txt"
+python -m pip wheel --no-cache-dir --no-deps --no-build-isolation \
+    . --wheel-dir "$build_dir/raw-wheels"
+python - "$build_dir/build/compile_commands.json" <<'PY'
+import json, pathlib, re, shlex, sys
+commands = json.loads(pathlib.Path(sys.argv[1]).read_text())
+variants = (
+    "x64 sse42 sandybridge ivybridge piledriver haswell skylakex "
+    "cannonlake cascadelake icelake cooperlake zen4 alderlake sapphirerapids"
+).split()
+optimized = {"ggml-cpu-" + name for name in variants if name != "x64"}
+baseline = {"-march=x86-64", "-mtune=generic"}
+scorers = []
+for command in commands:
+    args = command.get("arguments") or shlex.split(command["command"])
+    assert not any(arg.startswith("@") for arg in args), command
+    assert not any(arg in ("-march=native", "-mtune=native", "-mcpu=native") or arg.startswith("-flto")
+                   for arg in args), command
+    output = args[args.index("-o") + 1]
+    target = re.search(r"(?:^|/)CMakeFiles/([^/]+)\.dir/", output).group(1)
+    if target not in optimized:
+        assert baseline <= set(args), command
+        assert all(arg in baseline for arg in args if arg.startswith("-m")), command
+    if command["file"].endswith("/arch/x86/cpu-feats.cpp"):
+        scorers.append(target)
+        assert "-fno-lto" in args, command
+assert sorted(scorers) == sorted("ggml-cpu-" + name + "-feats" for name in variants), scorers
+PY
+auditwheel repair --plat manylinux2014_x86_64 --only-plat \
+    --wheel-dir "$wheelhouse" "$build_dir"/raw-wheels/*.whl

scripts/runtime-wheels/check_linux_cpu_eligibility.py

@@ -0,0 +1,64 @@
+"""Check scorer OS-state gates using injected hardware/kernel responses."""
+import ctypes
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+import tempfile
+
+source = Path(sys.argv[1]).read_text()
+start = "static int ggml_backend_cpu_x86_score() {"
+end = "GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_x86_score)"
+assert source.count(start) == source.count(end) == 1
+scorer = start + source.split(start, 1)[1].split(end, 1)[0]
+xgetbv = '    __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));'
+assert scorer.count(xgetbv) == 1
+assert scorer.count("syscall(SYS_arch_prctl, ") == 3
+scorer = scorer.replace(xgetbv, "    test_xgetbv(eax, edx);")
+scorer = scorer.replace("syscall(SYS_arch_prctl, ", "test_arch_prctl(")
+variants = {"x64": [], "avx": ["AVX"], "avx512": ["AVX", "AVX512"],
+            "amx": ["AVX", "AVX512", "AMX_TILE", "AMX_INT8"]}
+tile = (1 << 17) | (1 << 18)
+full = tile | 0xe7
+defaults = [0, full, tile, tile, 0]
+cases = [("all available", defaults, set(variants))]
+
+def add_case(name, index, value, eligible):
+    inputs = defaults.copy()
+    inputs[index] = value
+    cases.append((name, inputs, set(eligible)))
+
+for missing, name in enumerate(("AVX", "XSAVE", "OSXSAVE"), 1):
+    add_case("missing " + name, 0, missing, ["x64"])
+for bit in (1, 2, 5, 6, 7, 17, 18):
+    eligible = ["x64"] if bit < 3 else ["x64", "avx"]
+    if bit > 7:
+        eligible.append("avx512")
+    add_case("missing XCR0 bit " + str(bit), 1, full & ~(1 << bit), eligible)
+without_amx = ["x64", "avx", "avx512"]
+add_case("missing AMX_TILE", 0, 4, without_amx)
+for index, name in ((2, "kernel support"), (3, "kernel permission")):
+    for bit in (17, 18):
+        add_case(name + " missing bit " + str(bit), index, tile & ~(1 << bit), without_amx)
+for request in (0x1021, 0x1022, 0x1023):
+    add_case("failed arch_prctl " + hex(request), 4, request, without_amx)
+
+with tempfile.TemporaryDirectory(prefix="cpu-eligibility-") as temporary:
+    output = Path(temporary)
+    (output / "scorer-under-test.inc").write_text(scorer)
+    for name, features in variants.items():
+        library = output / (name + ".so")
+        command = shlex.split(os.environ.get("CXX", "c++")) + [
+            "-std=c++17", "-O2", "-shared", "-fPIC", "-fno-lto",
+            "-march=x86-64", "-mtune=generic", "-I", str(output),
+            *["-DGGML_" + feature for feature in features],
+            str(Path(__file__).with_name("linux_eligibility_probe.cpp")), "-o", str(library)]
+        subprocess.run(command, check=True)
+        score = ctypes.CDLL(str(library)).score_case
+        score.argtypes = [ctypes.c_int, ctypes.c_uint64, ctypes.c_uint64, ctypes.c_uint64, ctypes.c_int]
+        score.restype = ctypes.c_int
+        for case, inputs, eligible in cases:
+            actual = score(*inputs)
+            assert (actual > 0) == (name in eligible), (name, case, actual)
+        print("Passed", name, len(cases), "OS-state eligibility cases", flush=True)