From d9790a363b80d6a67b24129f117cfd7e47be821f Mon Sep 17 00:00:00 2001 From: da03 Date: Tue, 6 Oct 2026 17:53:20 -0400 Subject: [PATCH 1/5] Add local Qwen3.5 vision functions with ordered text and image inputs --- .github/workflows/test.yml | 9 +- PYPI_README.md | 29 ++ README.md | 29 ++ docs/api-reference/python-sdk.md | 77 +++- programasweights/__init__.py | 42 ++- programasweights/_image_content.py | 107 ++++++ programasweights/_inputs.py | 99 +++++ programasweights/_runtime_vision.py | 369 +++++++++++++++++++ programasweights/_vision_assets.py | 194 ++++++++++ programasweights/_vision_contract.py | 194 ++++++++++ programasweights/cache.py | 62 +++- programasweights/runtime_llamacpp.py | 1 + pyproject.toml | 4 + tests/test_image_content.py | 389 ++++++++++++++++++++ tests/test_inputs.py | 265 ++++++++++++++ tests/test_vision_assets.py | 390 ++++++++++++++++++++ tests/test_vision_metadata.py | 359 ++++++++++++++++++ tests/test_vision_runtime.py | 530 +++++++++++++++++++++++++++ 18 files changed, 3128 insertions(+), 21 deletions(-) create mode 100644 programasweights/_image_content.py create mode 100644 programasweights/_inputs.py create mode 100644 programasweights/_runtime_vision.py create mode 100644 programasweights/_vision_assets.py create mode 100644 programasweights/_vision_contract.py create mode 100644 tests/test_image_content.py create mode 100644 tests/test_inputs.py create mode 100644 tests/test_vision_assets.py create mode 100644 tests/test_vision_metadata.py create mode 100644 tests/test_vision_runtime.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21d5bfe..46c00f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,12 +74,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install (hermetic deps only) - # Install httpx + pytest and the package itself without pulling the heavy + # Install test dependencies without pulling the heavy # llama-cpp-python build. Runtime tests inject a fake llama_cpp module, # so CI needs neither the native extension nor a model download. run: | python -m pip install --upgrade pip - python -m pip install httpx pytest + python -m pip install httpx pytest Pillow python -m pip install -e . --no-deps - name: Run hermetic tests @@ -96,7 +96,12 @@ jobs: tests/test_cli_auth.py \ tests/test_remote_inference.py \ tests/test_desktop_sdk.py \ + tests/test_inputs.py \ + tests/test_image_content.py \ tests/test_runtime_registry_sdk.py \ + tests/test_vision_metadata.py \ + tests/test_vision_assets.py \ + tests/test_vision_runtime.py \ tests/test_sdk.py::TestInstallAndImport local-files-windows: diff --git a/PYPI_README.md b/PYPI_README.md index b0fbc39..a204306 100644 --- a/PYPI_README.md +++ b/PYPI_README.md @@ -116,6 +116,35 @@ Advanced adapter-free inference is available with `paw.function(None, interpreter="gpt2")`; see the Python API reference for its intentionally strict semantics. +## Local image functions + +Install the optional image dependencies: + +```bash +pip install "programasweights[vision]" --extra-index-url https://pypi.programasweights.com/simple/ +``` + +Prompt Qwen3.5-0.8B without an adapter: + +```python +import programasweights as paw + +with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") as describe: + print(describe("Describe this image.", paw.Image("photo.png"), max_tokens=128)) +``` + +Or load a compatible image `.paw` bundle with `paw.function("./locator.paw")`. +Image functions accept ordered positional strings and `paw.Image` objects; +generation options are keyword-only. `paw.Image` snapshots a local path, +encoded bytes, or a Pillow image. The SDK keeps native image dimensions and +composites transparency over white; the model backend constructs image patches. + +The first load downloads the exact model and projector. Use `offline=True` +after caching both files. Matching model/projector hashes and runtime settings +share one local runtime; calls reset model state and select the correct LoRA. +Use context managers or `close()` to release it. Image inference is local only; +this SDK change does not add a hosted image compiler or remote image inference. + ## Browser SDK Programs compiled with GPT-2 also run in the browser via WebAssembly. The initial model and program assets download automatically; inference then runs client-side. diff --git a/README.md b/README.md index 86ae140..a77f090 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,35 @@ Advanced adapter-free inference is available with [Python API reference](docs/api-reference/python-sdk.md#advanced-adapter-free-base-interpreter) for its intentionally strict semantics. +## Local image functions + +Install the optional image dependencies: + +```bash +pip install "programasweights[vision]" --extra-index-url https://pypi.programasweights.com/simple/ +``` + +Prompt Qwen3.5-0.8B without an adapter: + +```python +import programasweights as paw + +with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") as describe: + print(describe("Describe this image.", paw.Image("photo.png"), max_tokens=128)) +``` + +Or load a compatible image `.paw` bundle with `paw.function("./locator.paw")`. +Image functions accept ordered positional strings and `paw.Image` objects; +generation options are keyword-only. `paw.Image` snapshots a local path, +encoded bytes, or a Pillow image. The SDK keeps native image dimensions and +composites transparency over white; the model backend constructs image patches. + +The first load downloads the exact model and projector. Use `offline=True` +after caching both files. Matching model/projector hashes and runtime settings +share one local runtime; calls reset model state and select the correct LoRA. +Use context managers or `close()` to release it. Image inference is local only; +this SDK change does not add a hosted image compiler or remote image inference. + ## Browser SDK Programs compiled with GPT-2 also run in the browser via WebAssembly. The initial model and program assets download automatically; inference then runs client-side. diff --git a/docs/api-reference/python-sdk.md b/docs/api-reference/python-sdk.md index e991ff2..8ee7197 100644 --- a/docs/api-reference/python-sdk.md +++ b/docs/api-reference/python-sdk.md @@ -42,9 +42,9 @@ bundle directly. Required runtime metadata and base models are cached for reuse. | `verbose` | Enable verbose logging (default `False`). | | `offline` | Use only local files/cache and make zero network calls; fail if required validated assets are missing. `PAW_OFFLINE=1` has the same effect. | | `remote` | Run hosted inference without downloading model assets (default `False`). Accepts a `Program` object, ID, or slug. Cannot be combined with offline mode, local file paths, `interpreter`, or non-default local runtime options. | -| `interpreter` | Advanced adapter-free mode only. Must be passed by keyword and only when `program_id` is explicitly `None`. Supported values are `Qwen/Qwen3-0.6B` and `gpt2`. | +| `interpreter` | Advanced adapter-free mode only. Must be passed by keyword and only when `program_id` is explicitly `None`. Supported values are `Qwen/Qwen3-0.6B`, `gpt2`, and `Qwen/Qwen3.5-0.8B`. | -For local inference, the returned callable accepts: +For local text-only programs, the returned callable accepts: ```python output: str = fn(input_text, max_tokens=None, temperature=0.0, logits_processor=None) @@ -63,7 +63,9 @@ This advanced hook is not built-in regex or JSON-schema validation. Each process Compiled mode is strict: the adapter, prompt template, matching metadata, runtime manifest, and runtime-compatible base-model file must all validate. Version 0.4.5 -accepts runtime manifest version 1 with `adapter_format="gguf_lora"`. +introduced runtime manifest version 1 with `adapter_format="gguf_lora"`. +Image bundles use version 2 and additionally pin the projector, ordered-content +prompt contract, and preprocessing settings. Built-in models are checked against pinned size/SHA-256 metadata and GGUF magic. Historical manifests for those known runtime IDs are normalized to the same canonical integrity metadata, so missing server-side checksum fields @@ -144,10 +146,10 @@ This mode is intentionally explicit: - `program_id=""` raises `ValueError` and explains that base mode requires explicit `None`. - A non-empty program reference together with `interpreter` raises `ValueError`. - No PAW API, slug lookup, program download, adapter load, or disk prefix cache is used. -- Online mode may download only the selected base GGUF from its built-in runtime manifest. Offline mode never downloads. -- Every invocation resets model state, renders the complete prompt, and tokenizes that complete rendered prompt in one call. +- Online mode may download the selected base GGUF and, for image interpreters, its projector from the built-in runtime manifest. Offline mode never downloads. +- Every invocation resets model state. Text interpreters render and tokenize the complete text prompt; the image interpreter processes ordered text/image content. -The built-in prompt contract is versioned with each runtime manifest and must contain exactly one `{INPUT_PLACEHOLDER}`: +The built-in text prompt contracts are versioned with each runtime manifest and contain exactly one `{INPUT_PLACEHOLDER}`: ```text # Qwen/Qwen3-0.6B @@ -168,6 +170,67 @@ The Qwen bytes are the exact raw-user rendering of Zero-token prompts and prompts that consume the full context window raise `ValueError`. +### Local text/image calls + +Install `programasweights[vision]` to use Qwen3.5 image inference. The public +call accepts ordered strings and `paw.Image` objects: + +```python +with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") as compare: + answer = compare( + "Before:", paw.Image("before.png"), + "After:", paw.Image("after.png"), + "What changed?", + max_tokens=128, + ) +``` + +The adapter-free function sends one user message containing these parts, with +no injected system message. The pinned model chat template adds the assistant +prefix with thinking disabled. It also accepts text-only calls. Each call is +independent; no conversation, image, or recurrent-state cache carries across calls. + +A compatible image bundle loads through `paw.function("./locator.paw")` and +uses the same call interface. Its `prompt_template.txt` provides literal system +instructions; `{INPUT_PLACEHOLDER}` has no substitution meaning in that file. +Bundle loading still requires a valid adapter and matching runtime metadata. + +```python +output: str = fn( + *parts, + max_tokens=None, + temperature=0.0, + logits_processor=None, + response_format=None, +) +``` + +Generation options are keyword-only for image functions. `response_format` +passes a llama.cpp-compatible JSON response-format specification to the backend; +always validate returned data, especially when `max_tokens` truncates the output. +`max_tokens=0` returns an empty string without generation. Empty input lists and +parts other than strings or `paw.Image` are rejected; unpack an existing list +with `fn(*parts)`. + +`paw.Image(source)` accepts a regular local path, encoded image bytes, or a +Pillow image and snapshots the supplied content. It does not fetch URLs. Images +are decoded locally and converted to RGB PNG without SDK resizing, EXIF rotation, +or ICC conversion. Transparency is composited over white. The backend uses the +manifest's image-token limits to construct patches; high-bit-depth and floating +point inputs require explicit conversion by the caller. + +Live functions share a model/projector only when the exact asset hashes, +preprocessing, context size, GPU settings, and verbosity match. Shared calls +serialize and switch adapters; closing one function leaves other users of that +runtime usable. Close the last function to release native resources. Native +runtimes must not be inherited through `fork`; use a spawned process instead. + +The built-in image interpreter pins a Q8_0 model and BF16 projector by repository +revision, size, and SHA-256. Existing bundles retain their own exact asset +identities; naming the same upstream model does not make different GGUF files +interchangeable. Offline mode requires both verified files. Remote image +inference and hosted image compilation are not introduced by this interface. + ## Preparing programs for offline use ```python @@ -178,7 +241,7 @@ ready = paw.is_offline_ready("da03/my-classifier") # local check; no network cached = paw.list_cached_programs() ``` -`prepare_program` resolves and downloads the program, runtime manifest, and shared base model without retaining a loaded `PawFunction`. Pass `offline=True` to require an already complete local cache and prohibit network access. +`prepare_program` resolves and downloads the program, runtime manifest, and shared base model (plus the projector for image programs) without retaining a loaded function. Pass `offline=True` to require an already complete local cache and prohibit network access. Desktop applications can receive structured progress without parsing stderr: diff --git a/programasweights/__init__.py b/programasweights/__init__.py index c9adb2a..a80015f 100644 --- a/programasweights/__init__.py +++ b/programasweights/__init__.py @@ -33,6 +33,7 @@ except Exception: __version__ = "0.4.11" +from ._inputs import Image from ._output import ProgressCallback, ProgressEvent, report_progress from .cache import CachedProgram from .client import ( @@ -332,7 +333,11 @@ def prepare_program( }, ) - if offline: + if cache.declares_vision(meta): + from ._vision_assets import get_vision_asset_paths + + get_vision_asset_paths(runtime_manifest, offline=offline, progress=progress) + elif offline: base_model_path = cache.get_cached_base_model_path(runtime_manifest) if base_model_path is None: raise RuntimeError( @@ -445,11 +450,15 @@ def function( program IDs, slugs, pinned versions, and Program objects. Cannot be combined with offline mode or non-default local runtime options. interpreter: Advanced adapter-free mode. This is only valid when - ``program_id`` is explicitly ``None``. Initially supported values - are ``"Qwen/Qwen3-0.6B"`` and ``"gpt2"``. + ``program_id`` is explicitly ``None``. Supported values are + ``"Qwen/Qwen3-0.6B"``, ``"gpt2"``, and ``"Qwen/Qwen3.5-0.8B"``. + Qwen3.5 accepts ordered text/image parts as one user message, + without a system prompt or an adapter. Returns: - A callable that takes an input string and returns an output string. + A callable returning an output string. Text programs accept one input + string. Image programs accept ordered positional strings and paw.Image + objects, with generation options passed by keyword. Example: >>> fn = paw.function("email-triage") @@ -463,6 +472,9 @@ def function( >>> fn = paw.function("./classifier.paw") # local bundle >>> base = paw.function(None, interpreter="gpt2") + + >>> vision = paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") + >>> vision("Describe this image.", paw.Image("photo.png")) """ import os from . import cache @@ -492,10 +504,16 @@ def function( raise ValueError( "program_id=None requires an explicit interpreter." ) - cache.get_base_runtime_manifest(interpreter) - from .runtime_llamacpp import PawFunction + runtime_manifest = cache.get_base_runtime_manifest(interpreter) + if cache.declares_vision({"runtime": runtime_manifest}): + from ._runtime_vision import VisionFunction - return PawFunction.from_base( + function_type = VisionFunction + else: + from .runtime_llamacpp import PawFunction + + function_type = PawFunction + return function_type.from_base( interpreter, n_ctx=n_ctx, n_gpu_layers=n_gpu_layers, @@ -549,7 +567,15 @@ def function( ) program_dir = cache.get_program_dir(resolved_id) - return PawFunction( + import json + + function_type = PawFunction + meta = json.loads((program_dir / "meta.json").read_text(encoding="utf-8")) + if cache.declares_vision(meta): + from ._runtime_vision import VisionFunction + + function_type = VisionFunction + return function_type( program_dir, n_ctx=n_ctx, n_gpu_layers=n_gpu_layers, diff --git a/programasweights/_image_content.py b/programasweights/_image_content.py new file mode 100644 index 0000000..1996164 --- /dev/null +++ b/programasweights/_image_content.py @@ -0,0 +1,107 @@ +"""Convert typed input snapshots to local llama.cpp chat content.""" + +from __future__ import annotations + +import base64 +from io import BytesIO +from typing import Any, Dict, List, Union + +from ._inputs import Image, _normalize_parts + +# Restrict byte decoding to raster formats; other Pillow plugins may invoke +# external programs. PIL objects have already been decoded by the caller. +_RASTER_FORMATS = ("PNG", "JPEG", "WEBP", "GIF", "TIFF", "BMP", "ICO") +_PIXEL_MODES = frozenset({"1", "L", "LA", "P", "RGB", "RGBA", "CMYK", "YCbCr", "HSV"}) + + +def _rgb_on_white(pixels): + from PIL import Image as PILImage + + palette_has_alpha = pixels.palette is not None and pixels.palette.mode == "RGBA" + if ("A" not in pixels.getbands() and "transparency" not in pixels.info + and not palette_has_alpha): + return pixels.convert("RGB") + # Follow qwen-vl-utils' white-background compositing for RGBA, also + # normalizing grayscale and palette/color-key transparency to RGBA first. + with pixels.convert("RGBA") as rgba, rgba.getchannel("A") as alpha: + background = PILImage.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=alpha) + return background + + +def _png_url(pixels) -> str: + if pixels.width < 1 or pixels.height < 1: + raise ValueError("Image dimensions must both be positive.") + if pixels.mode not in _PIXEL_MODES: + raise ValueError( + f"Unsupported image mode {pixels.mode!r}; explicitly convert to an " + "8-bit image before wrapping it in paw.Image." + ) + # Resolve transparency before encoding the model's three RGB channels. + with _rgb_on_white(pixels) as rgb, BytesIO() as output: + # Model input contains pixels only. In particular, don't attach EXIF + # orientation or caller metadata to the newly encoded pixel grid. + rgb.info.clear() + rgb.save(output, format="PNG") + payload = base64.b64encode(output.getvalue()).decode("ascii") + return "data:image/png;base64," + payload + + +def _image_url(image: Image, index: int) -> str: + try: + from PIL import Image as PILImage + except ModuleNotFoundError as error: + if error.name not in ("PIL", "PIL.Image"): + raise + raise ImportError( + "Image conversion requires Pillow; install programasweights[vision]." + ) from error + + source = image._copy_source() + try: + if isinstance(source, bytes): + # verify() checks container integrity (including PNG checksums). + # Reopen afterwards because verification consumes the decoder. + with BytesIO(source) as stream: + with PILImage.open(stream, formats=_RASTER_FORMATS) as pixels: + pixels.verify() + with BytesIO(source) as stream: + with PILImage.open(stream, formats=_RASTER_FORMATS) as pixels: + return _png_url(pixels) + with source: + return _png_url(source) + except (OSError, ValueError, SyntaxError, PILImage.DecompressionBombError) as error: + raise ValueError( + f"Input part {index}: image could not be decoded or encoded. {error}" + ) from error + + +def _to_chat_content(*parts: Union[str, Image]) -> List[Dict[str, Any]]: + """Prepare ordered local text/image parts without loading an inference backend. + + Text is passed through exactly, including empty strings and adjacent text + parts. Images become RGB PNG data URLs at their native dimensions. + Transparency is composited onto white, including alpha channels and + palette/color-key transparency. No EXIF rotation or color-profile transform + is applied. Metadata is omitted from the generated PNG. + + Encoded PNG, JPEG, WebP, GIF, TIFF, BMP, and ICO inputs are decoded locally. + Encoded animations use their first frame; PIL inputs use the frame already + snapshotted by Image. Floating-point and high-bit-depth pixel modes must be + explicitly converted by the caller rather than silently clipped here. + + The supplied snapshots are unchanged. Pillow is imported only when an + image is converted. This helper does not alter text-function call behavior + or perform model-specific resizing, tokenization, or inference. + """ + normalized = _normalize_parts(*parts) + content = [] + for index, part in enumerate(normalized, start=1): + if isinstance(part, str): + content.append({"type": "text", "text": part}) + else: + content.append({ + "type": "image_url", + "image_url": {"url": _image_url(part, index)}, + }) + return content diff --git a/programasweights/_inputs.py b/programasweights/_inputs.py new file mode 100644 index 0000000..3e6a98e --- /dev/null +++ b/programasweights/_inputs.py @@ -0,0 +1,99 @@ +"""Typed input snapshots, independent of model loading and preprocessing.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +import os +import stat +from typing import TYPE_CHECKING, Tuple, Union + +if TYPE_CHECKING: + from PIL.Image import Image as _PILImage + + +def _read_file(path: Union[str, os.PathLike]) -> bytes: + # Nonblocking open lets us reject FIFOs without waiting for a writer. + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError("Image source must be a regular local file.") + with os.fdopen(descriptor, "rb", closefd=False) as source: + return source.read() + finally: + os.close(descriptor) + + +def _copy_pixels(source): + snapshot = source.copy() + # Pillow's copy() copies pixels and palette, but only shallow-copies info. + snapshot.info = deepcopy(source.info) + return snapshot + + +@dataclass(frozen=True, init=False, repr=False, eq=False) +class Image: + """Snapshot an image supplied as a local path, encoded bytes, or PIL image. + + Paths are read immediately and encoded bytes are retained unchanged. Pillow + images are copied in their current mode and at their current frame, including + palette and metadata. Later changes to the caller's file or image do not + affect this value. URLs, file handles, and mutable byte buffers are not input + types; a plain string outside Image remains text. + + Encoded data is not decoded or validated here. No resizing, color conversion, + or model preprocessing is performed. Copying a lazily opened Pillow image + may decode its current frame. Paths and encoded bytes do not require Pillow. + + This input type does not itself enable image inference in text functions. + """ + + _source: Union[bytes, _PILImage] + + def __init__(self, source: Union[str, os.PathLike, bytes, _PILImage]): + if isinstance(source, bytes): + snapshot = bytes(source) + elif isinstance(source, (str, os.PathLike)): + path = os.fspath(source) + if not isinstance(path, str): + raise TypeError("Image paths must resolve to str; bytes are encoded image data.") + snapshot = _read_file(path) + else: + # A PIL object normally means Pillow is already imported. Keep it + # optional for text users and for callers supplying paths or bytes. + try: + from PIL.Image import Image as PILImage + except ModuleNotFoundError as error: + if error.name not in ("PIL", "PIL.Image"): + raise + PILImage = None + if PILImage is None or not isinstance(source, PILImage): + raise TypeError( + "Image source must be a local path, encoded bytes, or a PIL.Image.Image." + ) + snapshot = _copy_pixels(source) + object.__setattr__(self, "_source", snapshot) + + def _copy_source(self): + """Give preprocessing its own copy without exposing mutable pixels.""" + if isinstance(self._source, bytes): + return self._source + return _copy_pixels(self._source) + + def __repr__(self) -> str: + kind = "encoded bytes" if isinstance(self._source, bytes) else "PIL image" + return f"Image(<{kind} snapshot>)" + + +def _normalize_parts(*parts: Union[str, Image]) -> Tuple[Union[str, Image], ...]: + """Validate positional content without merging text or interpreting paths.""" + if not parts: + raise ValueError("At least one input part is required.") + for index, part in enumerate(parts, start=1): + if not isinstance(part, (str, Image)): + raise TypeError( + f"Input part {index} must be str or paw.Image, got {type(part).__name__}. " + "Wrap image sources in paw.Image(...); unpack a list with *parts." + ) + return parts diff --git a/programasweights/_runtime_vision.py b/programasweights/_runtime_vision.py new file mode 100644 index 0000000..872290b --- /dev/null +++ b/programasweights/_runtime_vision.py @@ -0,0 +1,369 @@ +"""Local ordered-content inference; shared resources are an internal detail. + +Live functions with matching model/projector hashes and inference settings +share one serialized runtime. Closing the last function releases it. Up to four +LoRAs are retained per runtime; eviction never frees an attached adapter. +There is no prompt, image, or recurrent-state reuse between calls. +""" + +from __future__ import annotations + +from collections import OrderedDict +import ctypes +import hashlib +import json +import math +import os +from pathlib import Path +import re +import threading + +from . import cache +from ._image_content import _to_chat_content +from ._vision_assets import get_vision_asset_paths +from ._vision_contract import VISION_BASE_INFERENCE, declares_vision + +_POOL_LOCK = threading.RLock() +_POOL = {} +_MAX_ADAPTERS = 4 +_ACTIVE = threading.local() +_PROCESS_ID = os.getpid() + + +def _outside_call(): + if os.getpid() != _PROCESS_ID: + raise RuntimeError("Vision inference cannot be inherited through fork; use a spawned process.") + if getattr(_ACTIVE, "running", False): + raise RuntimeError("Cannot load, call, or close vision functions inside a vision callback.") + + +def _digest(path: Path) -> str: + result = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + result.update(block) + return result.hexdigest() + + +def _backend(): + import llama_cpp + version = re.match(r"^(\d+)\.(\d+)\.(\d+)", llama_cpp.__version__) + if version is None or tuple(map(int, version.groups())) < (0, 3, 35): + raise ImportError( + "Qwen3.5 image functions require llama-cpp-python>=0.3.35. " + "Install programasweights[vision]." + ) + from llama_cpp.llama_chat_format import MTMDChatHandler + return llama_cpp, MTMDChatHandler + + +class _Runtime: + def __init__(self, paths, preprocessing, *, n_ctx, n_gpu_layers, verbose): + self.lock = threading.RLock() + self.running = False + self.closed = False + self.references = 0 + self.adapters = OrderedDict() + self.llm = None + self.lib, handler_type = _backend() + from .runtime_llamacpp import _suppress_native_stderr + + class Handler(handler_type): + def __call__(handler, **kwargs): + kwargs["enable_thinking"] = False + return super().__call__(**kwargs) + + def _init_mtmd_context(handler, model): + if handler.mtmd_ctx is not None: + return + with _suppress_native_stderr(not verbose): + params = handler._mtmd_cpp.mtmd_context_params_default() + params.use_gpu = n_gpu_layers != 0 + params.print_timings = verbose + params.n_threads = model.n_threads + params.image_min_tokens = preprocessing["image_min_tokens"] + params.image_max_tokens = preprocessing["image_max_tokens"] + params.flash_attn_type = self.lib.LLAMA_FLASH_ATTN_TYPE_ENABLED + handler.mtmd_ctx = handler._mtmd_cpp.mtmd_init_from_file( + os.fsencode(handler.clip_model_path), model.model, params, + ) + if not handler.mtmd_ctx: + handler.mtmd_ctx = None + raise RuntimeError("Failed to initialize the vision projector.") + + def free_projector(): + if handler.mtmd_ctx is not None: + handler._mtmd_cpp.mtmd_free(handler.mtmd_ctx) + handler.mtmd_ctx = None + + # Register before checking capabilities, including failures. + model._stack.callback(free_projector) + if not handler._mtmd_cpp.mtmd_support_vision(handler.mtmd_ctx): + raise RuntimeError("This projector does not support images.") + + try: + handler = Handler( + clip_model_path=str(paths.projector), verbose=verbose, + use_gpu=n_gpu_layers != 0, + ) + with _suppress_native_stderr(not verbose): + self.llm = self.lib.Llama( + model_path=str(paths.base_model), chat_handler=handler, + n_ctx=n_ctx, n_gpu_layers=n_gpu_layers, verbose=verbose, + n_batch=128, n_threads=2, n_threads_batch=2, + flash_attn=True, + ) + handler._init_mtmd_context(self.llm) + except BaseException: + try: + self.close() + except Exception: + pass # Keep the initialization failure as the primary error. + raise + + def _reset(self): + self.llm.reset() + # Llama.reset() alone does not clear Qwen3.5's recurrent memory. + memory = self.lib.llama_get_memory(self.llm.ctx) + if memory: + self.lib.llama_memory_clear(memory, True) + + def _detach(self): + result = self.lib.llama_set_adapters_lora(self.llm.ctx, None, 0, None) + if result not in (0, None): + raise RuntimeError("Failed to detach the previous vision LoRA.") + + def _select(self, adapter): + self._reset() + self._detach() + if adapter is None: + return + path, digest, size = adapter + handle = self.adapters.get(digest) + if handle is None: + if not cache._valid_gguf_file(path, expected_size=size, expected_sha256=digest): + raise ValueError("Vision adapter changed after the function was loaded.") + while len(self.adapters) >= _MAX_ADAPTERS: + _, old = self.adapters.popitem(last=False) + self.lib.llama_adapter_lora_free(old) + handle = self.lib.llama_adapter_lora_init(self.llm.model, os.fsencode(path)) + if not handle: + raise RuntimeError("Failed to load the vision LoRA adapter.") + self.adapters[digest] = handle + self.adapters.move_to_end(digest) + handles = (self.lib.llama_adapter_lora_p_ctypes * 1)(handle) + # The adapter GGUF already contains its training alpha/rank scaling. + scales = (ctypes.c_float * 1)(1.0) + result = self.lib.llama_set_adapters_lora(self.llm.ctx, handles, 1, scales) + if result not in (0, None): + raise RuntimeError("Failed to select the vision LoRA adapter.") + + def run(self, adapter, messages, *, max_tokens, temperature, + logits_processor=None, response_format=None): + with self.lock: + if self.closed: + raise RuntimeError("This vision runtime has been closed.") + if self.running: + raise RuntimeError("Reentrant calls to a shared vision runtime are unsupported.") + self.running = True + _ACTIVE.running = True + error = None + try: + self._select(adapter) + processor_error = None + options = {} + if logits_processor is not None: + def guarded_processor(input_ids, scores): + nonlocal processor_error + if processor_error is None: + try: + for processor in logits_processor: + scores[:] = processor(input_ids, scores) + return scores + except BaseException as exc: + processor_error = exc + # ctypes swallows callback exceptions. Force EOS, then + # re-raise the original error on the Python side. + fallback = [0.0] * len(scores) + fallback[self.llm.token_eos()] = 1e10 + return fallback + options["logits_processor"] = self.lib.LogitsProcessorList([guarded_processor]) + if response_format is not None: + options["response_format"] = response_format + response = self.llm.create_chat_completion( + messages=messages, max_tokens=max_tokens, + temperature=temperature, **options, + ) + if processor_error is not None: + raise processor_error + content = response["choices"][0]["message"]["content"] + if not isinstance(content, str): + raise RuntimeError("The vision function returned no text content.") + return content.strip() + except BaseException as exc: + error = exc + raise + finally: + try: + self._reset() + self._detach() + except BaseException: + # Never reuse a runtime whose state could not be cleared. + try: + self.close() + except Exception: + pass + if error is None: + raise + finally: + self.running = False + _ACTIVE.running = False + + def close(self): + with self.lock: + if self.closed: + return + self.closed = True + if self.llm is not None: + try: + self._detach() + finally: + try: + for handle in self.adapters.values(): + self.lib.llama_adapter_lora_free(handle) + self.adapters.clear() + finally: + self.llm.close() + + +class VisionFunction: + """A compiled image program or base interpreter, with ordered parts. + + Generation options are keyword-only. Calls return text (including partial + text when max_tokens is exhausted); callers validate structured outputs. + Close functions, or use a context manager, to release shared model memory. + """ + + def __init__(self, program_dir, n_ctx=2048, n_gpu_layers=-1, verbose=False, + api_url=None, api_key=None, offline=False): + self._initialize(n_ctx) + directory = Path(program_dir) + self._meta = json.loads((directory / "meta.json").read_text(encoding="utf-8")) + if not cache.validate_program_assets_dir(directory, self._meta.get("program_id")): + raise ValueError("Invalid compiled image-program assets.") + if not declares_vision(self._meta): + raise ValueError("VisionFunction requires an image program.") + manifest = cache.resolve_runtime_manifest(self._meta, offline=offline) + self._prompt = (directory / "prompt_template.txt").read_text(encoding="utf-8") + path = directory / "adapter.gguf" + self._adapter = (path, _digest(path), path.stat().st_size) + self._acquire_runtime(manifest, n_ctx, n_gpu_layers, verbose, offline) + + @classmethod + def from_base(cls, interpreter, *, n_ctx=2048, n_gpu_layers=-1, + verbose=False, offline=False): + self = cls.__new__(cls) + self._initialize(n_ctx) + manifest = cache.get_base_runtime_manifest(interpreter) + if ( + not declares_vision({"runtime": manifest}) + or manifest.get("base_inference") != VISION_BASE_INFERENCE + ): + raise ValueError("This interpreter has no adapter-free vision contract.") + self._meta = {"mode": "base", "interpreter": interpreter} + self._prompt = None + self._adapter = None + self._acquire_runtime(manifest, n_ctx, n_gpu_layers, verbose, offline) + return self + + def _initialize(self, n_ctx): + _outside_call() + self._runtime = None + self._lock = threading.RLock() + self._pid = os.getpid() + if type(n_ctx) is not int or n_ctx <= 0: + raise ValueError("n_ctx must be a positive integer.") + + def _acquire_runtime(self, manifest, n_ctx, n_gpu_layers, verbose, offline): + _backend() # Fail before downloading assets on an older text-only backend. + paths = get_vision_asset_paths(manifest, offline=offline) + local = manifest["local_sdk"] + preprocessing = local["vision"]["preprocessing"] + self._key = ( + self._pid, local["base_model"]["sha256"].lower(), + local["vision"]["projector"]["sha256"].lower(), + json.dumps(preprocessing, sort_keys=True), n_ctx, n_gpu_layers, verbose, + ) + with _POOL_LOCK: + runtime = _POOL.get(self._key) + if runtime is None or runtime.closed: + runtime = _Runtime(paths, preprocessing, n_ctx=n_ctx, + n_gpu_layers=n_gpu_layers, verbose=verbose) + _POOL[self._key] = runtime + runtime.references += 1 + self._runtime = runtime + + def __call__(self, *parts, max_tokens=None, temperature=0.0, + logits_processor=None, response_format=None) -> str: + _outside_call() + if os.getpid() != self._pid: + raise RuntimeError("Load a new vision function in each child process.") + with self._lock: + if self._runtime is None: + raise RuntimeError("This vision function has been closed.") + if max_tokens is not None and (type(max_tokens) is not int or max_tokens < 0): + raise ValueError("max_tokens must be None or a non-negative integer.") + if not isinstance(temperature, (int, float)) or not math.isfinite(temperature) or temperature < 0: + raise ValueError("temperature must be a finite non-negative number.") + content = _to_chat_content(*parts) + if max_tokens == 0: + return "" + messages = [] + if self._prompt is not None: + messages.append({"role": "system", "content": self._prompt}) + messages.append({"role": "user", "content": content}) + return self._runtime.run( + self._adapter, messages, max_tokens=max_tokens, temperature=temperature, + logits_processor=logits_processor, response_format=response_format, + ) + + def close(self): + if os.getpid() != self._pid: + return # Never free inherited native resources in a forked child. + _outside_call() + with self._lock, _POOL_LOCK: + runtime = self._runtime + if runtime is None: + return + with runtime.lock: + self._runtime = None + runtime.references -= 1 + if runtime.references == 0: + if _POOL.get(self._key) is runtime: + del _POOL[self._key] + runtime.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def __del__(self): + try: + self.close() + except Exception: + pass + + @property + def spec(self): + return self._meta.get("spec", "") + + @property + def interpreter(self): + return self._meta["interpreter"] + + def __repr__(self): + if self._meta.get("mode") == "base": + return f"VisionFunction(base interpreter={self.interpreter!r})" + return f"VisionFunction(interpreter={self.interpreter!r})" diff --git a/programasweights/_vision_assets.py b/programasweights/_vision_assets.py new file mode 100644 index 0000000..4070dee --- /dev/null +++ b/programasweights/_vision_assets.py @@ -0,0 +1,194 @@ +"""Resolve verified GGUF model/projector files; never instantiate a model. + +V2 assets use base_models/sha256/.gguf, independent of source filename +or runtime ID. Existing filename-based cache entries may be adopted via a +verified hard link (or a copy if links are unavailable). Text caching is unchanged. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import shutil +import stat +import tempfile +from typing import NamedTuple + +import httpx + +from . import cache, config +from ._output import ProgressCallback, report_progress +from ._vision_contract import VISION_MANIFEST_VERSION + + +class VisionAssetPaths(NamedTuple): + base_model: Path + projector: Path + + +def _manifest(value: dict) -> dict: + normalized = cache._normalize_runtime_manifest(value, require_local_model=True) + if ( + normalized is None + or normalized["manifest_version"] != VISION_MANIFEST_VERSION + or not normalized["local_sdk"]["supported"] + ): + raise ValueError("A valid, locally supported vision runtime manifest is required.") + return normalized + + +def _asset_path(asset: dict) -> Path: + return config.get_base_models_dir() / "sha256" / (asset["sha256"].lower() + ".gguf") + + +def _valid(path: Path, asset: dict) -> bool: + return cache._valid_gguf_file( + path, expected_size=asset["size_bytes"], expected_sha256=asset["sha256"], + ) + + +def get_cached_vision_asset_paths(runtime_manifest: dict) -> VisionAssetPaths | None: + """Return both verified content-addressed files, or None; never download.""" + local = _manifest(runtime_manifest)["local_sdk"] + base, projector = local["base_model"], local["vision"]["projector"] + paths = VisionAssetPaths(_asset_path(base), _asset_path(projector)) + if _valid(paths.base_model, base) and _valid(paths.projector, projector): + return paths + return None + + +def _ensure_directory(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + if not stat.S_ISDIR(path.lstat().st_mode): + raise ValueError(f"Unsafe vision asset cache directory: {path}") + + +def _adopt_existing(asset: dict, destination: Path) -> bool: + source = config.get_base_models_dir() / asset["file"] + if not _valid(source, asset): + return False + # The content-addressed path remains stable if a legacy downloader later + # atomically replaces its filename-based entry. Do not move/remove that entry. + fd, name = tempfile.mkstemp(prefix=".adopt-", suffix=".tmp", dir=str(destination.parent)) + os.close(fd) + temporary = Path(name) + try: + temporary.unlink() + try: + os.link(str(source), str(temporary)) + except OSError: + shutil.copyfile(str(source), str(temporary)) + if not _valid(temporary, asset): + raise RuntimeError("Cached vision asset changed while being adopted.") + os.replace(str(temporary), str(destination)) + return True + finally: + temporary.unlink(missing_ok=True) + + +def _download_verified( + asset: dict, destination: Path, stage: str, runtime_id: str, + progress: ProgressCallback | None, +) -> None: + url = asset.get("url") or cache._build_hf_url(asset["repo"], asset["file"]) + expected_size = asset["size_bytes"] + fd, name = tempfile.mkstemp(prefix=".download-", suffix=".tmp", dir=str(destination.parent)) + temporary = Path(name) + try: + with os.fdopen(fd, "wb") as output: + digest = hashlib.sha256() + size = 0 + magic = b"" + with httpx.stream("GET", url, follow_redirects=True, timeout=300.0) as response: + response.raise_for_status() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + size += len(chunk) + if size > expected_size: + raise RuntimeError(f"{stage} download exceeds its declared size.") + magic += chunk[:max(0, len(cache.GGUF_MAGIC) - len(magic))] + digest.update(chunk) + output.write(chunk) + if progress is not None: + progress({ + "stage": stage, "status": "downloading", + "runtime_id": runtime_id, "path": str(destination), + "downloaded_bytes": size, "total_bytes": expected_size, + }) + if ( + size != expected_size or magic != cache.GGUF_MAGIC + or digest.hexdigest() != asset["sha256"].lower() + ): + raise RuntimeError( + f"{stage} download failed GGUF magic, file size, or SHA-256 validation." + ) + output.flush() + os.fsync(output.fileno()) + # Only a fully verified file becomes visible at its shared cache path. + os.replace(str(temporary), str(destination)) + finally: + temporary.unlink(missing_ok=True) + + +def _resolve_asset( + asset: dict, stage: str, runtime_id: str, *, + offline: bool, progress: ProgressCallback | None, +) -> Path: + destination = _asset_path(asset) + + def report(status: str, message: str | None = None) -> None: + report_progress(progress, { + "stage": stage, "status": status, + "runtime_id": runtime_id, "path": str(destination), + }, message) + + if _valid(destination, asset): + report("cached") + return destination + + _ensure_directory(destination.parent) + lock_dir = cache._locks_dir() / "vision_assets" + _ensure_directory(lock_dir) + lock = lock_dir / (asset["sha256"].lower() + ".lock") + if os.path.lexists(lock) and not stat.S_ISREG(lock.lstat().st_mode): + raise ValueError(f"Unsafe vision asset cache lock: {lock}") + with cache._cross_process_lock(lock): + if _valid(destination, asset) or _adopt_existing(asset, destination): + report("cached") + return destination + if offline: + raise RuntimeError( + f"{stage} for runtime {runtime_id!r} is missing or invalid; " + "offline mode prohibits network downloads." + ) + report("downloading", f"Downloading {stage} for {runtime_id} (one-time download)...") + _download_verified(asset, destination, stage, runtime_id, progress) + report("ready", f"Saved to {destination}") + return destination + + +def get_vision_asset_paths( + runtime_manifest: dict, *, offline: bool = False, + progress: ProgressCallback | None = None, +) -> VisionAssetPaths: + """Resolve both exact files, reusing verified cache entries before downloading. + + Explicit offline mode and PAW_OFFLINE prohibit network access even if only + one asset is missing. A completed asset is retained when the other fails, + so retries fetch only what is still missing. This does not enable inference. + """ + manifest = _manifest(runtime_manifest) + local = manifest["local_sdk"] + offline = offline or os.environ.get("PAW_OFFLINE", "").strip().lower() in ( + "1", "true", "yes", + ) + base = _resolve_asset( + local["base_model"], "base_model", manifest["runtime_id"], + offline=offline, progress=progress, + ) + projector = _resolve_asset( + local["vision"]["projector"], "vision_projector", manifest["runtime_id"], + offline=offline, progress=progress, + ) + return VisionAssetPaths(base, projector) + diff --git a/programasweights/_vision_contract.py b/programasweights/_vision_contract.py new file mode 100644 index 0000000..0865b35 --- /dev/null +++ b/programasweights/_vision_contract.py @@ -0,0 +1,194 @@ +"""Inert v2 runtime metadata for local text/image functions. + +The existing prompt_template.txt member holds literal system instructions. +Each call appends one user message containing its ordered text/image parts; +there are no string placeholders or named image slots. The qwen3.5 chat +profile adds the assistant generation prompt with thinking disabled. + +Preprocessing v1 uses native input dimensions, RGB with transparency over +white, stored pixel orientation, and no color-profile transform. Resizing and +patch construction belong to the backend, with explicit image-token limits. +This contract describes assets and rendering; it does not load or execute them. +""" + +from __future__ import annotations + +import re +from urllib.parse import urlsplit + + +VISION_MANIFEST_VERSION = 2 +_SHA256 = re.compile(r"^[a-fA-F0-9]{64}$") +_INPUT = {"format": "content_parts", "types": ["text", "image"]} +_PROMPT = { + "format": "chat_messages", + "system_prompt_file": "prompt_template.txt", + "chat_format": "qwen3.5", + "enable_thinking": False, +} +_PREPROCESSING = { + "version": 1, + "color_mode": "RGB", + "alpha": "composite_white", + "orientation": "stored", + "color_profile": "ignore", + "resize": "backend", +} + +# A base call is one user message containing the ordered input parts. The +# pinned GGUF chat template adds the assistant prefix with thinking disabled; +# no system message or compiled prompt is inserted in adapter-free mode. +VISION_BASE_INFERENCE = {"contract_version": 1, "format": "chat_messages"} + +BUILTIN_VISION_RUNTIMES = { + "qwen3.5-0.8b-q8_0": { + "runtime_id": "qwen3.5-0.8b-q8_0", + "manifest_version": VISION_MANIFEST_VERSION, + "interpreter": "Qwen/Qwen3.5-0.8B", + "adapter_format": "gguf_lora", + "input": dict(_INPUT), + "prompt_template": dict(_PROMPT), + "program_assets": {"adapter_filename": "adapter.gguf"}, + "base_inference": dict(VISION_BASE_INFERENCE), + "local_sdk": { + "supported": True, + "n_ctx": 4096, + "base_model": { + "provider": "huggingface", + "repo": "ggml-org/Qwen3.5-0.8B-GGUF", + "file": "Qwen3.5-0.8B-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q8_0.gguf", + "size_bytes": 833592096, + "sha256": "37ae482d336108d23516fa35e8e0c4126688d81018b87178a18d752a1357814f", + }, + "vision": { + "projector": { + "provider": "huggingface", + "repo": "unsloth/Qwen3.5-0.8B-GGUF", + "file": "mmproj-BF16.gguf", + "url": "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/6ab461498e2023f6e3c1baea90a8f0fe38ab64d0/mmproj-BF16.gguf", + "size_bytes": 207346528, + "sha256": "d312c4d02fd46eea7a16e4f3bbb58840e6222209322ca1e33ca03247ad8935d6", + }, + "preprocessing": { + **_PREPROCESSING, + "image_min_tokens": 1, + "image_max_tokens": 512, + }, + }, + }, + "js_sdk": {"supported": False}, + }, +} + + +def _positive_int(value) -> bool: + return type(value) is int and value > 0 + + +def _asset_valid(asset) -> bool: + if not isinstance(asset, dict): + return False + file = asset.get("file") + if ( + not isinstance(file, str) or not file or file in (".", "..") + or any(char in file for char in ("/", "\\", ":", "\x00")) + ): + return False + digest = asset.get("sha256") + if ( + not isinstance(digest, str) or not _SHA256.fullmatch(digest) + or not _positive_int(asset.get("size_bytes")) + ): + return False + provider, repo, url = (asset.get(key) for key in ("provider", "repo", "url")) + if any(value is not None and not isinstance(value, str) + for value in (provider, repo, url)): + return False + if url: + try: + parsed = urlsplit(url) + if ( + parsed.scheme != "https" or not parsed.hostname + or parsed.username or parsed.password + ): + return False + except ValueError: + return False + elif provider != "huggingface" or not repo: + return False + return True + + +def valid_vision_contract(manifest: dict) -> bool: + """Validate v2's required fields after the common runtime identity checks.""" + if manifest.get("input") != _INPUT: + return False + prompt = manifest.get("prompt_template") + if prompt != _PROMPT or prompt.get("enable_thinking") is not False: + return False + assets = manifest.get("program_assets") + if ( + not isinstance(assets, dict) or assets.get("adapter_filename") != "adapter.gguf" + or assets.get("prefix_cache_required", False) is not False + or assets.get("prefix_cache_filename") is not None + or assets.get("prefix_tokens_filename") is not None + ): + return False + local = manifest.get("local_sdk") + if ( + not isinstance(local, dict) or not isinstance(local.get("supported"), bool) + or not _positive_int(local.get("n_ctx")) + ): + return False + vision = local.get("vision") + if not isinstance(vision, dict) or set(vision) != {"projector", "preprocessing"}: + return False + if not _asset_valid(local.get("base_model")) or not _asset_valid(vision.get("projector")): + return False + if local["base_model"]["file"].casefold() == vision["projector"]["file"].casefold(): + return False + preprocessing = vision.get("preprocessing") + if not isinstance(preprocessing, dict): + return False + if set(preprocessing) != set(_PREPROCESSING) | {"image_min_tokens", "image_max_tokens"}: + return False + if any(preprocessing.get(key) != value for key, value in _PREPROCESSING.items()): + return False + if type(preprocessing["version"]) is not int: + return False + minimum, maximum = (preprocessing[key] for key in ("image_min_tokens", "image_max_tokens")) + if not _positive_int(minimum) or not _positive_int(maximum) or minimum > maximum: + return False + base = manifest.get("base_inference") + if base is not None and ( + base != VISION_BASE_INFERENCE or type(base.get("contract_version")) is not int + ): + return False + js = manifest.get("js_sdk") + return js is None or (isinstance(js, dict) and js.get("supported") is False) + + +def declares_vision(program_meta: dict) -> bool: + """Recognize image intent, including malformed or downgraded manifests.""" + if program_meta.get("runtime_manifest_version") == VISION_MANIFEST_VERSION: + return True + runtime = program_meta.get("runtime") + if not isinstance(runtime, dict): + return False + local = runtime.get("local_sdk") + prompt = runtime.get("prompt_template") + return ( + runtime.get("manifest_version") == VISION_MANIFEST_VERSION + or "input" in runtime + or (isinstance(local, dict) and "vision" in local) + or (isinstance(prompt, dict) and prompt.get("format") == "chat_messages") + ) + + +def require_text_execution(program_meta: dict) -> None: + """Prevent validated image metadata from reaching the legacy text runtime.""" + if declares_vision(program_meta): + raise ValueError( + "Image programs require the vision runtime; load them with paw.function()." + ) diff --git a/programasweights/cache.py b/programasweights/cache.py index 8565999..b9e26c0 100644 --- a/programasweights/cache.py +++ b/programasweights/cache.py @@ -31,6 +31,13 @@ from . import config from ._output import ProgressCallback, report_progress +from ._vision_contract import ( + BUILTIN_VISION_RUNTIMES, + VISION_MANIFEST_VERSION, + declares_vision, + require_text_execution, + valid_vision_contract, +) BASE_MODEL_URLS = { "qwen3-0.6b-q6_k": "https://huggingface.co/programasweights/Qwen3-0.6B-GGUF-Q6_K/resolve/main/qwen3-0.6b-q6_k.gguf", @@ -39,6 +46,7 @@ INTERPRETER_TO_GGUF = { "Qwen/Qwen3-0.6B": "qwen3-0.6b-q6_k", + "Qwen/Qwen3.5-0.8B": "qwen3.5-0.8b-q8_0", "gpt2": "gpt2-q8_0", } @@ -57,7 +65,7 @@ _PROGRAM_ID_RE = re.compile(r"^[a-f0-9]{16,64}$") _RUNTIME_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") _SHA256_RE = re.compile(r"^[a-fA-F0-9]{64}$") -SUPPORTED_RUNTIME_MANIFEST_VERSIONS = frozenset({1}) +SUPPORTED_RUNTIME_MANIFEST_VERSIONS = frozenset({1, VISION_MANIFEST_VERSION}) GGUF_MAGIC = b"GGUF" MIN_ADAPTER_GGUF_SIZE = 1024 # A waiter may be behind a slow first download of the 622 MB Qwen GGUF. @@ -345,6 +353,11 @@ def _runtime_manifest_has_valid_shape( ): return False + if manifest_version == VISION_MANIFEST_VERSION: + return valid_vision_contract(runtime_manifest) + if declares_vision({"runtime": runtime_manifest}): + return False + prompt_template = runtime_manifest.get("prompt_template") if prompt_template is not None and ( not isinstance(prompt_template, dict) @@ -457,6 +470,17 @@ def _normalize_runtime_manifest( normalized = json.loads(json.dumps(runtime_manifest)) runtime_id = normalized["runtime_id"] + canonical_vision = BUILTIN_VISION_RUNTIMES.get(runtime_id) + if canonical_vision is not None: + # V2 already requires complete asset identities. A named built-in + # cannot silently substitute a projector, prompt, or preprocessing. + for field in ( + "manifest_version", "interpreter", "adapter_format", "input", + "prompt_template", "program_assets", "local_sdk", "js_sdk", + ): + if normalized.get(field) != canonical_vision[field]: + return None + return normalized canonical = LEGACY_RUNTIME_MANIFESTS.get(runtime_id) if canonical is None: return normalized @@ -688,7 +712,7 @@ def get_base_runtime_manifest(interpreter: str) -> dict: f"Unknown interpreter: {interpreter!r}. " f"Supported: {list(INTERPRETER_TO_GGUF.keys())}" ) - manifest = LEGACY_RUNTIME_MANIFESTS.get(runtime_id) + manifest = BUILTIN_VISION_RUNTIMES.get(runtime_id) or LEGACY_RUNTIME_MANIFESTS.get(runtime_id) if ( not _is_runtime_manifest_complete(manifest) or manifest.get("interpreter") != interpreter @@ -707,6 +731,7 @@ def get_base_prompt_template(runtime_manifest: dict) -> str: ) if normalized is None: raise ValueError("Invalid runtime manifest shape.") + require_text_execution({"runtime": normalized}) contract = normalized.get("base_inference") if not isinstance(contract, dict): raise ValueError( @@ -742,6 +767,13 @@ def _normalize_runtime_manifest_for_program( ) if normalized is None: return None + if normalized["manifest_version"] == VISION_MANIFEST_VERSION and ( + program_meta.get("runtime_id") != normalized["runtime_id"] + or program_meta.get("interpreter") != normalized["interpreter"] + or type(program_meta.get("runtime_manifest_version")) is not int + or program_meta["runtime_manifest_version"] != VISION_MANIFEST_VERSION + ): + return None runtime_id = program_meta.get("runtime_id") if runtime_id and normalized.get("runtime_id") != runtime_id: return None @@ -773,6 +805,10 @@ def _runtime_manifest_matches_program( def get_offline_runtime_manifest(program_meta: dict) -> dict | None: """Resolve the exact runtime manifest without performing network I/O.""" embedded = program_meta.get("runtime") + if declares_vision(program_meta): + # V2 is self-contained: never replace missing/invalid image settings + # with a cached, fetched, or legacy text contract. + return _normalize_runtime_manifest_for_program(embedded, program_meta) if isinstance(embedded, dict): normalized_embedded = _normalize_runtime_manifest_for_program( embedded, @@ -851,6 +887,8 @@ def resolve_runtime_manifest( offline: bool = False, ) -> dict | None: embedded = program_meta.get("runtime") + if declares_vision(program_meta): + return _normalize_runtime_manifest_for_program(embedded, program_meta) if isinstance(embedded, dict): normalized_embedded = _normalize_runtime_manifest_for_program( embedded, @@ -924,6 +962,11 @@ def get_cached_base_model_path(runtime_manifest: dict) -> Path | None: local_sdk = normalized.get("local_sdk") if not isinstance(local_sdk, dict) or not local_sdk.get("supported", False): return None + if normalized["manifest_version"] == VISION_MANIFEST_VERSION: + from ._vision_assets import get_cached_vision_asset_paths + + paths = get_cached_vision_asset_paths(normalized) + return paths.base_model if paths is not None else None base_model = _base_model_info_from_runtime(normalized) file_name = base_model.get("file") if base_model else None if ( @@ -980,6 +1023,11 @@ def get_base_model_path( f"{manifest_interpreter!r}, not {interpreter!r}." ) + if manifest["manifest_version"] == VISION_MANIFEST_VERSION: + from ._vision_assets import get_vision_asset_paths + + return get_vision_asset_paths(manifest, offline=offline, progress=progress).base_model + local_sdk = manifest["local_sdk"] base_model = local_sdk["base_model"] file_name = base_model["file"] @@ -1162,10 +1210,16 @@ def validate_program_assets_dir( except (UnicodeDecodeError, json.JSONDecodeError, OSError): return False - if template.count(INPUT_PLACEHOLDER) != 1: - return False if not isinstance(meta, dict) or meta.get("program_id") != expected_program_id: return False + if declares_vision(meta): + # The same archive member now holds literal system instructions. + # It must never be interpreted as a rendered-text placeholder template. + return _normalize_runtime_manifest_for_program( + meta.get("runtime"), meta, + ) is not None + if template.count(INPUT_PLACEHOLDER) != 1: + return False interpreter = meta.get("interpreter") if not isinstance(interpreter, str) or not interpreter: return False diff --git a/programasweights/runtime_llamacpp.py b/programasweights/runtime_llamacpp.py index fec6d32..fb7adb5 100644 --- a/programasweights/runtime_llamacpp.py +++ b/programasweights/runtime_llamacpp.py @@ -117,6 +117,7 @@ def __init__( self._meta = json.load(handle) if not isinstance(self._meta, dict): raise ValueError(f"Invalid program metadata: {meta_path}") + cache.require_text_execution(self._meta) interpreter = self._meta.get("interpreter") if not isinstance(interpreter, str) or not interpreter: raise ValueError( diff --git a/pyproject.toml b/pyproject.toml index c35a685..865cb91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,10 @@ Documentation = "https://programasweights.readthedocs.io" [project.optional-dependencies] test = ["pytest"] +vision = [ + "Pillow>=9.1.1", + "llama-cpp-python>=0.3.35", +] [project.scripts] paw = "programasweights.cli:main" diff --git a/tests/test_image_content.py b/tests/test_image_content.py new file mode 100644 index 0000000..dc15914 --- /dev/null +++ b/tests/test_image_content.py @@ -0,0 +1,389 @@ +"""Decode emitted model inputs to verify pixels, order, and isolation.""" + +import base64 +from io import BytesIO +from pathlib import Path +import struct +import subprocess +import sys + +import pytest + +from programasweights import Image +from programasweights._image_content import _to_chat_content + + +@pytest.fixture +def pil(): + return pytest.importorskip("PIL.Image") + + +def encoded(pixels, format="PNG", **options): + with BytesIO() as stream: + pixels.save(stream, format=format, **options) + return stream.getvalue() + + +def decoded(part, pil): + assert set(part) == {"type", "image_url"} + assert part["type"] == "image_url" + assert set(part["image_url"]) == {"url"} + header, payload = part["image_url"]["url"].split(",", 1) + assert header == "data:image/png;base64" + with BytesIO(base64.b64decode(payload, validate=True)) as stream: + with pil.open(stream) as pixels: + assert pixels.format == "PNG" + pixels.load() + return pixels.copy() + + +@pytest.mark.parametrize("size", [(1, 1), (7, 5), (166, 164), (1025, 3)]) +def test_preserves_native_dimensions_and_rgb_pixels(pil, size): + source = pil.new("RGB", size, (12, 34, 56)) + source.putpixel((size[0] - 1, size[1] - 1), (210, 180, 70)) + result = decoded(_to_chat_content(Image(encoded(source)))[0], pil) + assert result.mode == "RGB" + assert result.size == size + assert result.tobytes() == source.tobytes() + + +def test_preserves_text_and_image_order(pil): + first = Image(pil.new("RGB", (3, 2), "red")) + second = Image(pil.new("RGB", (2, 3), "blue")) + result = _to_chat_content("Before:\n", first, "", "After:", "\t é\n", second, first) + assert result[0] == {"type": "text", "text": "Before:\n"} + assert result[2:5] == [ + {"type": "text", "text": ""}, + {"type": "text", "text": "After:"}, + {"type": "text", "text": "\t é\n"}, + ] + for index, color, size in [(1, (255, 0, 0), (3, 2)), (5, (0, 0, 255), (2, 3)), + (6, (255, 0, 0), (3, 2))]: + pixels = decoded(result[index], pil) + assert pixels.getpixel((0, 0)) == color + assert pixels.size == size + + +@pytest.mark.parametrize("format", ["PNG", "JPEG", "WEBP", "GIF", "TIFF", "BMP", "ICO"]) +def test_encoded_raster_formats_use_decoded_pixels_without_resize(pil, format): + source = pil.new("RGB", (32, 32), (51, 102, 153)) + source.putpixel((1, 2), (200, 50, 90)) + payload = encoded(source, format, **({"sizes": [(32, 32)]} if format == "ICO" else {})) + with pil.open(BytesIO(payload)) as opened: + expected = opened.convert("RGB") + result = decoded(_to_chat_content(Image(payload))[0], pil) + assert result.size == expected.size + assert result.tobytes() == expected.tobytes() + + +@pytest.mark.parametrize("mode,color,expected", [ + ("1", 1, (255, 255, 255)), + ("L", 17, (17, 17, 17)), + ("LA", (17, 0), (255, 255, 255)), + ("RGB", (12, 34, 56), (12, 34, 56)), + ("RGBA", (12, 34, 56, 0), (255, 255, 255)), + ("CMYK", (0, 255, 255, 0), (255, 0, 0)), + ("YCbCr", (128, 128, 128), (128, 128, 128)), + ("HSV", (0, 255, 255), (255, 0, 0)), +]) +def test_pixel_modes_follow_explicit_rgb_and_alpha_policy(pil, mode, color, expected): + result = decoded(_to_chat_content(Image(pil.new(mode, (7, 5), color)))[0], pil) + assert result.mode == "RGB" + assert result.size == (7, 5) + assert result.getpixel((0, 0)) == expected + + +@pytest.mark.parametrize("kind", ["pil", "bytes", "path"]) +@pytest.mark.parametrize("mode,pixels,expected", [ + ("RGBA", [(255, 0, 0, 0), (255, 0, 0, 128), (12, 34, 56, 255)], + [(255, 255, 255), (255, 127, 127), (12, 34, 56)]), + ("LA", [(17, 0), (17, 128), (17, 255)], + [(255, 255, 255), (136, 136, 136), (17, 17, 17)]), +]) +def test_alpha_composites_onto_white(pil, tmp_path, kind, mode, pixels, expected): + source = pil.new(mode, (3, 1)) + source.putdata(pixels) + original = source.tobytes() + if kind == "pil": + value = source + elif kind == "bytes": + value = encoded(source) + else: + value = tmp_path / "transparent.png" + source.save(value) + image = Image(value) + first = decoded(_to_chat_content(image)[0], pil) + second = decoded(_to_chat_content(image)[0], pil) + assert first.mode == "RGB" + assert first.size == (3, 1) + assert [first.getpixel((x, 0)) for x in range(3)] == expected + assert second.tobytes() == first.tobytes() + assert first.info == {} + assert source.mode == mode + assert source.tobytes() == original + if kind == "pil": + with image._copy_source() as snapshot: + assert snapshot.mode == mode + assert snapshot.tobytes() == original + + +@pytest.mark.parametrize("as_bytes", [False, True]) +@pytest.mark.parametrize("transparency,expected", [ + (0, [(255, 255, 255), (255, 0, 0), (12, 34, 56)]), + (bytes([0, 128, 255]), [(255, 255, 255), (255, 127, 127), (12, 34, 56)]), +]) +def test_palette_transparency_composites_onto_white(pil, as_bytes, transparency, expected): + source = pil.new("P", (3, 1)) + source.putpalette([0, 255, 0, 255, 0, 0, 12, 34, 56] + [0] * 759) + source.putdata([0, 1, 2]) + source.info["transparency"] = transparency + image = Image(encoded(source) if as_bytes else source) + result = decoded(_to_chat_content(image)[0], pil) + assert [result.getpixel((x, 0)) for x in range(3)] == expected + assert result.info == {} + assert source.info["transparency"] == transparency + if not as_bytes: + with image._copy_source() as snapshot: + assert snapshot.info["transparency"] == transparency + assert snapshot.tobytes() == source.tobytes() + + +@pytest.mark.parametrize("as_bytes", [False, True]) +def test_rgba_palette_composites_without_transparency_metadata(pil, as_bytes): + source = pil.new("P", (3, 1)) + source.putpalette([0, 255, 0, 0, 255, 0, 0, 128, 12, 34, 56, 255] + + [0, 0, 0, 255] * 253, rawmode="RGBA") + source.putdata([0, 1, 2]) + assert "transparency" not in source.info + result = decoded(_to_chat_content(Image(encoded(source) if as_bytes else source))[0], pil) + assert [result.getpixel((x, 0)) for x in range(3)] == [ + (255, 255, 255), (255, 127, 127), (12, 34, 56), + ] + + +@pytest.mark.parametrize("as_bytes", [False, True]) +@pytest.mark.parametrize("mode,key,opaque,expected", [ + ("RGB", (12, 34, 56), (12, 34, 57), (12, 34, 57)), + ("L", 17, 18, (18, 18, 18)), +]) +def test_color_key_transparency_composites_onto_white(pil, as_bytes, mode, key, opaque, expected): + source = pil.new(mode, (2, 1), key) + source.putpixel((1, 0), opaque) + source.info["transparency"] = key + result = decoded(_to_chat_content(Image(encoded(source) if as_bytes else source))[0], pil) + assert result.getpixel((0, 0)) == (255, 255, 255) + assert result.getpixel((1, 0)) == expected + assert result.info == {} + + +def test_preserves_snapshot_and_omits_metadata_without_exif_rotation(pil): + source = pil.new("RGB", (7, 5), "red") + source.putpixel((6, 0), (0, 0, 255)) + exif = pil.Exif() + exif[274] = 6 # Rotate 90 degrees when interpreted by an EXIF-aware viewer. + source.info["exif"] = exif.tobytes() + source.info["icc_profile"] = b"caller color profile" + source.info["private"] = {"values": ["caller metadata"]} + image = Image(source) + expected = source.tobytes() + source.paste("green", (0, 0, 7, 5)) + source.close() + result = decoded(_to_chat_content(image)[0], pil) + assert result.size == (7, 5) + assert result.tobytes() == expected + assert result.info == {} + snapshot = image._copy_source() + assert snapshot.tobytes() == expected + assert snapshot.info["exif"] == exif.tobytes() + assert snapshot.info["private"] == {"values": ["caller metadata"]} + + +def test_file_snapshot_can_be_converted_after_file_removal(pil, tmp_path): + path = tmp_path / "source.png" + pil.new("RGB", (7, 5), "blue").save(path) + image = Image(path) + path.unlink() + assert decoded(_to_chat_content(image)[0], pil).getpixel((0, 0)) == (0, 0, 255) + + +def test_encoded_exif_does_not_rotate_pixels_or_survive_conversion(pil): + source = pil.new("RGB", (7, 5), "red") + source.putpixel((6, 0), (0, 0, 255)) + exif = pil.Exif() + exif[274] = 6 + payload = encoded(source, exif=exif.tobytes(), icc_profile=b"caller profile") + result = decoded(_to_chat_content(Image(payload))[0], pil) + assert result.size == source.size + assert result.tobytes() == source.tobytes() + assert result.info == {} + + +def test_encoded_animation_uses_first_frame_and_pil_uses_snapshotted_frame(pil): + first = pil.new("RGB", (7, 5), "red") + payload = encoded(first, "GIF", save_all=True, + append_images=[pil.new("RGB", (7, 5), "blue")], duration=100) + with pil.open(BytesIO(payload)) as source: + source.seek(1) + second = Image(source) + source.seek(0) + a, b = _to_chat_content(Image(payload), second) + assert decoded(a, pil).getpixel((0, 0)) == (255, 0, 0) + assert decoded(b, pil).getpixel((0, 0)) == (0, 0, 255) + + +@pytest.mark.parametrize("mode", ["I", "F", "I;16"]) +def test_high_bit_depth_requires_explicit_conversion(pil, mode): + with pytest.raises(ValueError, match="Input part 2:.*explicitly convert"): + _to_chat_content("Frame:", Image(pil.new(mode, (7, 5)))) + + +@pytest.mark.parametrize("size", [(0, 1), (1, 0), (0, 0)]) +def test_empty_pixel_grid_has_clear_error(pil, size): + with pytest.raises(ValueError, match="Input part 2:.*dimensions must both be positive"): + _to_chat_content("Frame:", Image(pil.new("RGB", size))) + + +@pytest.mark.parametrize("payload", [b"", b"not an image", b"https://example.invalid/image.png"]) +def test_invalid_bytes_report_part_index(pil, payload): + with pytest.raises(ValueError, match="Input part 2: image could not be decoded or encoded"): + _to_chat_content("Frame:", Image(payload)) + + +@pytest.mark.parametrize("format", ["PNG", "JPEG"]) +def test_truncated_image_fails_during_conversion(pil, format): + payload = encoded(pil.new("RGB", (32, 32), "red"), format) + with pytest.raises(ValueError, match="Input part 1: image could not be decoded or encoded"): + _to_chat_content(Image(payload[:len(payload) // 2])) + + +def test_png_with_corrupt_checksum_is_rejected(pil): + payload = bytearray(encoded(pil.new("RGB", (7, 5), "red"))) + offset = 8 + while offset < len(payload): + length = struct.unpack_from(">I", payload, offset)[0] + if payload[offset + 4:offset + 8] == b"IDAT": + payload[offset + 8 + length] ^= 1 + break + offset += 12 + length + else: + raise AssertionError("Fixture contains no IDAT chunk") + with pytest.raises(ValueError, match="Input part 1: image could not be decoded or encoded"): + _to_chat_content(Image(bytes(payload))) + + +def test_conversion_obeys_pillow_decompression_limit(pil, monkeypatch): + payload = encoded(pil.new("RGB", (10, 10), "red")) + monkeypatch.setattr(pil, "MAX_IMAGE_PIXELS", 10) + with pytest.raises(ValueError, match="Input part 1: image could not be decoded or encoded"): + _to_chat_content(Image(payload)) + assert pil.MAX_IMAGE_PIXELS == 10 + + +def test_returned_content_is_independent_between_calls(pil): + image = Image(pil.new("RGB", (7, 5), "red")) + first = _to_chat_content("Frame:", image) + first[0]["text"] = "changed" + first[1]["image_url"]["url"] = "changed" + second = _to_chat_content("Frame:", image) + assert second[0]["text"] == "Frame:" + assert decoded(second[1], pil).getpixel((0, 0)) == (255, 0, 0) + + +def run_isolated(code): + result = subprocess.run([sys.executable, "-c", code], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr + + +def test_text_and_invalid_arguments_do_not_import_pillow_or_backend(): + run_isolated(r''' +import sys, importlib.abc +class BlockImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in ('PIL', 'llama_cpp', 'numpy', 'torch'): + raise AssertionError('Unexpected import: ' + fullname) +sys.meta_path.insert(0, BlockImports()) +from programasweights import Image +from programasweights._image_content import _to_chat_content +parts = ('', 'https://example.invalid/image.png', '\t é\n', 'frame.png') +assert _to_chat_content(*parts) == [{'type': 'text', 'text': p} for p in parts] +for parts in [(), (['text'],), (Image(b'invalid'), 3)]: + try: + _to_chat_content(*parts) + except (ValueError, TypeError): + pass + else: + raise AssertionError('Invalid arguments accepted') +''') + + +def test_missing_pillow_has_actionable_error(): + run_isolated(r''' +import sys, importlib.abc +class NoPillow(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] == 'PIL': + raise ModuleNotFoundError('No Pillow', name='PIL') +sys.meta_path.insert(0, NoPillow()) +from programasweights import Image +from programasweights._image_content import _to_chat_content +try: + _to_chat_content(Image(b'bytes')) +except ImportError as error: + assert 'programasweights[vision]' in str(error) +else: + raise AssertionError('Missing Pillow accepted') +''') + + +def test_broken_pillow_import_is_not_misreported_as_missing_extra(): + run_isolated(r''' +import sys, importlib.abc +class BrokenPillow(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'PIL': + raise ModuleNotFoundError('Missing codec dependency', name='broken_codec') +sys.meta_path.insert(0, BrokenPillow()) +from programasweights import Image +from programasweights._image_content import _to_chat_content +try: + _to_chat_content(Image(b'bytes')) +except ModuleNotFoundError as error: + assert error.name == 'broken_codec' +else: + raise AssertionError('Broken import hidden') +''') + + +def test_image_conversion_is_local_and_does_not_load_inference_backend(pil): + payload = encoded(pil.new("RGB", (7, 5), "red")) + run_isolated(r''' +import sys, importlib.abc, base64 +class BlockBackend(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] == 'numpy': + raise ModuleNotFoundError('NumPy is unavailable', name=fullname) + if fullname.split('.')[0] in ('llama_cpp', 'torch'): + raise AssertionError('Unexpected import: ' + fullname) +sys.meta_path.insert(0, BlockBackend()) +def offline(event, args): + if event in ('socket.connect', 'socket.getaddrinfo', 'socket.sendto', + 'subprocess.Popen', 'os.system', 'os.posix_spawn'): + raise AssertionError('Unexpected external operation: ' + event) +sys.addaudithook(offline) +from programasweights import Image +from programasweights._image_content import _to_chat_content +''' + f''' +payload = base64.b64decode({base64.b64encode(payload).decode('ascii')!r}) +assert _to_chat_content(Image(payload))[0]['type'] == 'image_url' +# EPS must not reach Pillow's Ghostscript-backed rasterizer. +eps = b'%!PS-Adobe-3.0 EPSF-3.0\\n%%BoundingBox: 0 0 7 5\\n%%EndComments\\nshowpage\\n' +try: + _to_chat_content(Image(eps)) +except ValueError: + pass +else: + raise AssertionError('Unsupported vector input accepted') +assert not any(name.split('.')[0] == 'numpy' for name in sys.modules) +''') diff --git a/tests/test_inputs.py b/tests/test_inputs.py new file mode 100644 index 0000000..5324d9b --- /dev/null +++ b/tests/test_inputs.py @@ -0,0 +1,265 @@ +"""Image snapshots and ordered content do not depend on an inference backend.""" + +from copy import copy, deepcopy +from dataclasses import FrozenInstanceError +from io import BytesIO +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +import programasweights as paw +from programasweights._inputs import _normalize_parts +from programasweights import _inputs + + +@pytest.fixture +def pil(): + return pytest.importorskip("PIL.Image") + + +@pytest.mark.parametrize("as_string", [False, True]) +def test_file_is_snapshotted_before_replacement_and_deletion(tmp_path, as_string): + source = tmp_path / "frame.png" + source.write_bytes(b"original encoded data") + image = paw.Image(str(source) if as_string else source) + source.write_bytes(b"replacement encoded data") + assert image._copy_source() == b"original encoded data" + source.unlink() + assert image._copy_source() == b"original encoded data" + + +def test_custom_text_pathlike_is_supported(tmp_path): + source = tmp_path / "frame.png" + source.write_bytes(b"image bytes") + + class SourcePath: + def __fspath__(self): + return str(source) + + assert paw.Image(SourcePath())._copy_source() == b"image bytes" + + +def test_binary_pathlike_is_not_confused_with_encoded_bytes(): + class BinaryPath: + def __fspath__(self): + return b"frame.png" + + with pytest.raises(TypeError, match="paths must resolve to str"): + paw.Image(BinaryPath()) + + +@pytest.mark.parametrize("payload", [b"", b"not decoded yet", b"frame.png"]) +def test_bytes_are_retained_without_decoding_or_path_interpretation(payload): + assert paw.Image(payload)._copy_source() == payload + + +def test_missing_file_fails_at_construction(tmp_path): + with pytest.raises(FileNotFoundError): + paw.Image(tmp_path / "missing.png") + + +def test_directory_is_rejected(tmp_path): + # os.open/fdopen reject directories differently across platforms. + with pytest.raises((OSError, ValueError)): + paw.Image(tmp_path) + + +@pytest.mark.parametrize("fail_read", [False, True]) +def test_file_descriptor_is_closed_on_success_and_failure(tmp_path, monkeypatch, fail_read): + source = tmp_path / "frame.png" + source.write_bytes(b"encoded data") + descriptors = [] + real_open = os.open + + def record_open(*args, **kwargs): + descriptor = real_open(*args, **kwargs) + descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(_inputs.os, "open", record_open) + if fail_read: + def failed_stream(*args, **kwargs): + raise OSError("Cannot create stream") + + monkeypatch.setattr(_inputs.os, "fdopen", failed_stream) + with pytest.raises(OSError, match="Cannot create stream"): + paw.Image(source) + else: + assert paw.Image(source)._copy_source() == b"encoded data" + assert len(descriptors) == 1 + with pytest.raises(OSError): + os.fstat(descriptors[0]) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFO test requires POSIX") +def test_fifo_is_rejected_without_waiting_for_a_writer(tmp_path): + fifo = tmp_path / "not-an-image" + os.mkfifo(fifo) + result = subprocess.run( + [sys.executable, "-c", "import programasweights as paw; paw.Image(__import__('sys').argv[1])", str(fifo)], + cwd=Path(__file__).resolve().parents[1], capture_output=True, text=True, timeout=10, + ) + assert result.returncode != 0 + assert "regular local file" in result.stderr + + +@pytest.mark.parametrize("mode,color", [ + ("RGB", (12, 34, 56)), ("RGBA", (12, 34, 56, 78)), + ("L", 17), ("CMYK", (12, 34, 56, 78)), ("F", 0.5), +]) +def test_pil_pixels_mode_dimensions_and_metadata_are_snapshotted(pil, mode, color): + source = pil.new(mode, (7, 5), color) + source.info["exif"] = b"exif snapshot" + source.info["nested"] = {"values": [1, 2]} + expected = source.tobytes() + image = paw.Image(source) + source.paste(0, (0, 0, 7, 5)) + source.info["nested"]["values"].append(3) + source.close() + snapshot = image._copy_source() + assert snapshot.mode == mode + assert snapshot.size == (7, 5) + assert snapshot.tobytes() == expected + assert snapshot.info == {"exif": b"exif snapshot", "nested": {"values": [1, 2]}} + # A consumer may mutate its copy without changing subsequent calls. + snapshot.paste(0, (0, 0, 7, 5)) + snapshot.info["nested"]["values"].clear() + again = image._copy_source() + assert again.tobytes() == expected + assert again.info["nested"]["values"] == [1, 2] + + +def test_palette_is_snapshotted(pil): + source = pil.new("P", (2, 2), 0) + source.putpalette([255, 0, 0] + [0] * 765) + image = paw.Image(source) + source.putpalette([0, 0, 255] + [0] * 765) + assert image._copy_source().getpalette()[:3] == [255, 0, 0] + + +def test_lazy_pil_image_survives_closed_input_stream(pil): + buffer = BytesIO() + pil.new("RGB", (7, 5), "red").save(buffer, format="PNG") + with BytesIO(buffer.getvalue()) as stream: + with pil.open(stream) as source: + image = paw.Image(source) + assert image._copy_source().getpixel((0, 0)) == (255, 0, 0) + + +def test_pil_snapshot_uses_current_animation_frame(pil): + buffer = BytesIO() + first = pil.new("RGB", (4, 4), "red") + first.save(buffer, format="GIF", save_all=True, + append_images=[pil.new("RGB", (4, 4), "blue")], duration=100) + with pil.open(BytesIO(buffer.getvalue())) as source: + source.seek(1) + expected = source.copy() + image = paw.Image(source) + source.seek(0) + snapshot = image._copy_source() + assert snapshot.mode == expected.mode + assert snapshot.tobytes() == expected.tobytes() + + +@pytest.mark.parametrize("source", [None, 42, True, bytearray(b"image"), memoryview(b"image"), [], {}, BytesIO(b"image")]) +def test_unsupported_image_sources_fail_clearly(source): + with pytest.raises(TypeError, match="Image source must be"): + paw.Image(source) + + +def test_image_wrapper_cannot_be_reassigned_and_repr_omits_payload(): + image = paw.Image(b"private image content") + with pytest.raises(FrozenInstanceError): + image._source = b"replacement" + assert "private image content" not in repr(image) + assert "snapshot" in repr(image) + + +@pytest.mark.parametrize("copy_value", [copy, deepcopy]) +def test_image_value_can_be_copied(copy_value): + image = paw.Image(b"encoded data") + copied = copy_value(image) + assert copied._copy_source() == b"encoded data" + with pytest.raises(FrozenInstanceError): + copied._source = b"replacement" + + +def test_parts_preserve_interleaving_empty_strings_and_exact_text(): + before, after = paw.Image(b"before"), paw.Image(b"after") + parts = ("Before:\n", before, "", "After:", "\t é\n", after, after) + result = _normalize_parts(*parts) + assert result == parts + assert result[1] is before + assert result[-1] is after + + +def test_plain_strings_are_never_opened_or_interpreted_as_images(): + parts = ("nonexistent.png", "https://example.invalid/image.png", "", "second text") + assert _normalize_parts(*parts) == parts + + +def test_dynamic_input_list_must_be_unpacked(): + parts = ["Frame:", paw.Image(b"image")] + assert _normalize_parts(*parts) == tuple(parts) + with pytest.raises(TypeError, match=r"unpack a list with \*parts"): + _normalize_parts(parts) + + +@pytest.mark.parametrize("part", [None, 1, False, b"image", Path("frame.png"), [], ["nested"], ("nested",), {}]) +def test_invalid_content_reports_argument_position(part): + with pytest.raises(TypeError, match="Input part 2 must be str or paw.Image"): + _normalize_parts("first", part) + + +def test_zero_parts_are_rejected(): + with pytest.raises(ValueError, match="At least one input part"): + _normalize_parts() + + +def test_import_and_encoded_inputs_do_not_import_heavy_optional_modules(tmp_path): + source = tmp_path / "frame.png" + source.write_bytes(b"encoded data") + code = r''' +import sys +import importlib.abc +class BlockHeavyImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in ('PIL', 'numpy', 'llama_cpp', 'torch'): + raise AssertionError('Unexpected import: ' + fullname) +sys.meta_path.insert(0, BlockHeavyImports()) +import programasweights as paw +from programasweights._inputs import _normalize_parts +assert paw.Image(b'bytes')._copy_source() == b'bytes' +assert paw.Image(sys.argv[1])._copy_source() == b'encoded data' +assert _normalize_parts('text') == ('text',) +''' + result = subprocess.run([sys.executable, "-c", code, str(source)], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + + +def test_invalid_source_has_clear_error_without_pillow(): + code = r''' +import sys +import importlib.abc +class NoPillow(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] == 'PIL': + raise ModuleNotFoundError('No Pillow installed', name='PIL') +sys.meta_path.insert(0, NoPillow()) +import programasweights as paw +try: + paw.Image(object()) +except TypeError as error: + assert 'Image source must be' in str(error) +else: + raise AssertionError('Invalid source accepted') +''' + result = subprocess.run([sys.executable, "-c", code], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr diff --git a/tests/test_vision_assets.py b/tests/test_vision_assets.py new file mode 100644 index 0000000..8341c01 --- /dev/null +++ b/tests/test_vision_assets.py @@ -0,0 +1,390 @@ +"""Verified two-asset caching, with no real HTTP or model initialization.""" + +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +import copy +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time + +import httpx +import pytest + +from programasweights import config +from programasweights import _vision_assets as assets +from test_vision_metadata import MODEL, PROJECTOR, asset, manifest # shared inert fixture + + +@pytest.fixture(autouse=True) +def isolated(tmp_path, monkeypatch): + monkeypatch.setenv("PAW_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setenv("PAW_QUIET", "1") + monkeypatch.delenv("PAW_OFFLINE", raising=False) + + def forbidden(*args, **kwargs): + pytest.fail("Unexpected HTTP request") + + monkeypatch.setattr(httpx, "stream", forbidden) + + +@pytest.fixture +def serve(monkeypatch, manifest): + calls = [] + payloads = { + manifest["local_sdk"]["base_model"]["url"]: MODEL, + manifest["local_sdk"]["vision"]["projector"]["url"]: PROJECTOR, + } + + def install(handler=None): + def respond(request): + calls.append(str(request.url)) + if handler is not None: + response = handler(request) + if response is not None: + return response + return httpx.Response(200, content=payloads[str(request.url)]) + + @contextmanager + def stream(method, url, **kwargs): + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + with client.stream(method, url, **kwargs) as response: + yield response + + monkeypatch.setattr(httpx, "stream", stream) + return calls, payloads + return install + + +def location(data): + return config.get_base_models_dir() / "sha256" / (hashlib.sha256(data).hexdigest() + ".gguf") + + +def assert_no_temporary_files(): + assert not list(config.get_base_models_dir().rglob("*.tmp")) + + +def test_download_then_offline_reuse_and_progress(manifest, serve): + calls, _ = serve() + events = [] + assert assets.get_cached_vision_asset_paths(manifest) is None + paths = assets.get_vision_asset_paths(manifest, progress=events.append) + assert paths == (location(MODEL), location(PROJECTOR)) + assert paths.base_model.read_bytes() == MODEL + assert paths.projector.read_bytes() == PROJECTOR + assert len(calls) == 2 + assert assets.get_vision_asset_paths(manifest, offline=True) == paths + assert assets.get_cached_vision_asset_paths(manifest) == paths + assert len(calls) == 2 + for stage in ("base_model", "vision_projector"): + selected = [e for e in events if e["stage"] == stage] + assert selected[0]["status"] == "downloading" + assert selected[-1]["status"] == "ready" + assert selected[-2]["downloaded_bytes"] == 4096 + assert selected[-2]["total_bytes"] == 4096 + assert all(e["runtime_id"] == manifest["runtime_id"] for e in selected) + assert all(e["path"] == str(paths.base_model if stage == "base_model" else paths.projector) + for e in selected) + assert_no_temporary_files() + + +@pytest.mark.parametrize("environment", [None, "1", " true ", "YES"]) +def test_offline_never_downloads_missing_assets(manifest, monkeypatch, environment): + if environment is not None: + monkeypatch.setenv("PAW_OFFLINE", environment) + with pytest.raises(RuntimeError, match="base_model.*offline mode"): + assets.get_vision_asset_paths(manifest, offline=environment is None) + assert assets.get_cached_vision_asset_paths(manifest) is None + assert_no_temporary_files() + + +def test_offline_can_adopt_legacy_files_without_deleting_or_duplicating_them(manifest): + directory = config.get_base_models_dir() + base, projector = directory / "base.gguf", directory / "mmproj.gguf" + base.write_bytes(MODEL) + projector.write_bytes(PROJECTOR) + paths = assets.get_vision_asset_paths(manifest, offline=True) + assert paths.base_model.read_bytes() == MODEL + assert paths.projector.read_bytes() == PROJECTOR + assert base.read_bytes() == MODEL + assert projector.read_bytes() == PROJECTOR + assert os.path.samefile(base, paths.base_model) + assert os.path.samefile(projector, paths.projector) + # A legacy cache replacement must not change the content-addressed file. + replacement = directory / "new.gguf" + replacement.write_bytes(b"different legacy version") + os.replace(str(replacement), str(base)) + assert paths.base_model.read_bytes() == MODEL + + +def test_adoption_falls_back_to_verified_copy_when_links_unavailable(manifest, monkeypatch): + directory = config.get_base_models_dir() + (directory / "base.gguf").write_bytes(MODEL) + (directory / "mmproj.gguf").write_bytes(PROJECTOR) + + def no_link(*args): + raise OSError("Hard links unavailable") + monkeypatch.setattr(assets.os, "link", no_link) + paths = assets.get_vision_asset_paths(manifest, offline=True) + assert paths.base_model.read_bytes() == MODEL + assert not os.path.samefile(directory / "base.gguf", paths.base_model) + assert paths.projector.read_bytes() == PROJECTOR + assert_no_temporary_files() + + +def test_changed_legacy_source_is_not_published(manifest, monkeypatch): + source = config.get_base_models_dir() / "base.gguf" + source.write_bytes(MODEL) + + def changed_link(source_path, target): + Path(target).write_bytes(b"changed") + monkeypatch.setattr(assets.os, "link", changed_link) + with pytest.raises(RuntimeError, match="changed while being adopted"): + assets.get_vision_asset_paths(manifest, offline=True) + assert not location(MODEL).exists() + assert source.read_bytes() == MODEL + assert_no_temporary_files() + + +def test_same_filenames_different_hashes_remain_independent(manifest, serve): + calls, payloads = serve() + first = assets.get_vision_asset_paths(manifest) + other = copy.deepcopy(manifest) + second_model = b"GGUF" + b"N" * 4092 + other["runtime_id"] = "other-runtime" + other["local_sdk"]["base_model"] = asset("base.gguf", second_model) + payloads[other["local_sdk"]["base_model"]["url"]] = second_model + second = assets.get_vision_asset_paths(other) + assert first.base_model != second.base_model + assert first.base_model.read_bytes() == MODEL + assert second.base_model.read_bytes() == second_model + assert first.projector == second.projector + assert len(calls) == 3 + + +def test_different_filenames_same_content_share_cache(manifest, serve): + calls, _ = serve() + original = assets.get_vision_asset_paths(manifest) + other = copy.deepcopy(manifest) + other["runtime_id"] = "renamed-runtime" + for descriptor in [other["local_sdk"]["base_model"], other["local_sdk"]["vision"]["projector"]]: + descriptor["file"] = "renamed-" + descriptor["file"] + descriptor["url"] = "https://unused.invalid/" + descriptor["file"] + descriptor["sha256"] = descriptor["sha256"].upper() + assert assets.get_vision_asset_paths(other, offline=True) == original + assert len(calls) == 2 + + +def test_missing_projector_does_not_redownload_cached_base(manifest, serve): + calls, _ = serve() + paths = assets.get_vision_asset_paths(manifest) + paths.projector.unlink() + with pytest.raises(RuntimeError, match="vision_projector.*offline mode"): + assets.get_vision_asset_paths(manifest, offline=True) + assert len(calls) == 2 + assert assets.get_cached_vision_asset_paths(manifest) is None + assert assets.get_vision_asset_paths(manifest) == paths + assert len(calls) == 3 + + +@pytest.mark.parametrize("failure", ["hash", "short", "oversize", "magic", "http", "interrupt"]) +def test_failed_download_never_publishes_and_can_retry(manifest, serve, failure): + candidate = copy.deepcopy(manifest) + if failure == "magic": + bad = b"NOPE" + b"M" * 4092 + candidate["local_sdk"]["base_model"] = asset("base.gguf", bad) + else: + bad = {"hash": b"GGUF" + b"X" * 4092, "short": MODEL[:-1], + "oversize": MODEL + b"extra"}.get(failure, MODEL) + closed = [] + + class Broken(httpx.SyncByteStream): + def __iter__(self): + yield MODEL[:100] + raise httpx.ReadError("Connection lost") + def close(self): + closed.append(True) + + def handler(request): + if failure == "http": + return httpx.Response(503) + if failure == "interrupt": + return httpx.Response(200, stream=Broken()) + return httpx.Response(200, content=bad) + + calls, _ = serve(handler) + with pytest.raises((RuntimeError, httpx.HTTPError)): + assets.get_vision_asset_paths(candidate) + target = config.get_base_models_dir() / "sha256" / ( + candidate["local_sdk"]["base_model"]["sha256"] + ".gguf") + assert not target.exists() + assert assets.get_cached_vision_asset_paths(manifest) is None + assert_no_temporary_files() + if failure == "interrupt": + assert closed + # The failed attempt releases its lock and cleans up its own temporary file. + serve() + assert assets.get_vision_asset_paths(manifest).base_model.read_bytes() == MODEL + + +def test_projector_failure_retains_completed_base(manifest, serve): + projector_url = manifest["local_sdk"]["vision"]["projector"]["url"] + calls, _ = serve(lambda request: httpx.Response(503) if str(request.url) == projector_url else None) + with pytest.raises(httpx.HTTPStatusError): + assets.get_vision_asset_paths(manifest) + assert location(MODEL).read_bytes() == MODEL + assert not location(PROJECTOR).exists() + assert_no_temporary_files() + serve() + paths = assets.get_vision_asset_paths(manifest) + assert paths.projector.read_bytes() == PROJECTOR + assert calls.count(manifest["local_sdk"]["base_model"]["url"]) == 1 + assert calls.count(projector_url) == 2 + + +def test_corrupt_cache_is_never_reused_and_failed_repair_does_not_delete_it(manifest, serve): + serve() + paths = assets.get_vision_asset_paths(manifest) + corrupted = b"GGUF" + b"X" * 4092 + paths.base_model.write_bytes(corrupted) + assert assets.get_cached_vision_asset_paths(manifest) is None + with pytest.raises(RuntimeError, match="offline mode"): + assets.get_vision_asset_paths(manifest, offline=True) + serve(lambda request: httpx.Response(503)) + with pytest.raises(httpx.HTTPStatusError): + assets.get_vision_asset_paths(manifest) + assert paths.base_model.read_bytes() == corrupted + assert paths.projector.read_bytes() == PROJECTOR + serve() + assert assets.get_vision_asset_paths(manifest).base_model.read_bytes() == MODEL + assert_no_temporary_files() + + +def test_progress_exception_cleans_up_before_publish(manifest, serve): + serve() + def callback(event): + if event.get("downloaded_bytes"): + assert not Path(event["path"]).exists() + raise RuntimeError("Caller cancelled") + with pytest.raises(RuntimeError, match="Caller cancelled"): + assets.get_vision_asset_paths(manifest, progress=callback) + assert not location(MODEL).exists() + assert_no_temporary_files() + + +def test_http_content_length_does_not_override_declared_size(manifest, serve): + payloads = {"base.gguf": MODEL, "mmproj.gguf": PROJECTOR} + serve(lambda request: httpx.Response( + 200, content=payloads[request.url.path.rsplit("/", 1)[-1]], + headers={"content-length": "1"}, + )) + paths = assets.get_vision_asset_paths(manifest) + assert paths.base_model.read_bytes() == MODEL + assert paths.projector.read_bytes() == PROJECTOR + + +def test_threads_share_one_download_per_asset(manifest, serve): + entered = threading.Event() + release = threading.Event() + def handler(request): + entered.set() + assert release.wait(5) + calls, _ = serve(handler) + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [executor.submit(assets.get_vision_asset_paths, manifest) for _ in range(4)] + assert entered.wait(5) + release.set() + paths = [f.result(timeout=10) for f in futures] + assert len(calls) == 2 + assert paths == [paths[0]] * 4 + assert_no_temporary_files() + + +def test_two_processes_share_one_download_per_asset(tmp_path, manifest): + script = r''' +import sys, os, json, time +from contextlib import contextmanager +from pathlib import Path +import httpx +from programasweights._vision_assets import get_vision_asset_paths +manifest = json.loads(sys.argv[1]) +ready, go, requests = map(Path, sys.argv[2:]) +def respond(request): + with requests.open("a") as log: + log.write(str(request.url) + "\n") + time.sleep(0.1) + data = b"GGUF" + (b"M" if request.url.path.endswith("base.gguf") else b"P") * 4092 + return httpx.Response(200, content=data) +@contextmanager +def stream(method, url, **kwargs): + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + with client.stream(method, url, **kwargs) as response: + yield response +httpx.stream = stream +ready.write_text("ready") +deadline = time.monotonic() + 15 +while not go.exists(): + if time.monotonic() > deadline: + raise RuntimeError("Parent did not release start barrier") + time.sleep(0.01) +result = get_vision_asset_paths(manifest) +assert not any(name.split(".")[0] in ("PIL", "llama_cpp", "torch", "numpy") for name in sys.modules) +print(json.dumps([str(p) for p in result])) +''' + go, requests = tmp_path / "go", tmp_path / "requests" + ready = [tmp_path / ("ready-" + str(i)) for i in range(2)] + env = os.environ.copy() + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1]) + workers = [subprocess.Popen( + [sys.executable, "-c", script, json.dumps(manifest), str(path), str(go), str(requests)], + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) for path in ready] + try: + deadline = time.monotonic() + 15 + while not all(path.exists() for path in ready): + assert time.monotonic() < deadline + assert all(worker.poll() is None for worker in workers) + time.sleep(0.01) + go.write_text("go") + results = [] + for worker in workers: + stdout, stderr = worker.communicate(timeout=20) + assert worker.returncode == 0, stderr + results.append(json.loads(stdout)) + assert results[0] == results[1] + assert len(requests.read_text().splitlines()) == 2 + finally: + for worker in workers: + if worker.poll() is None: + worker.terminate() + worker.wait(timeout=5) + + +@pytest.mark.parametrize("damage", ["legacy", "unsupported", "missing_hash"]) +def test_invalid_manifest_fails_before_cache_or_network(manifest, tmp_path, damage): + if damage == "legacy": + manifest["manifest_version"] = 1 + elif damage == "unsupported": + manifest["local_sdk"]["supported"] = False + else: + del manifest["local_sdk"]["vision"]["projector"]["sha256"] + with pytest.raises(ValueError, match="vision runtime manifest"): + assets.get_vision_asset_paths(manifest) + assert not (tmp_path / "cache").exists() + + +def test_huggingface_repo_source_resolves_using_existing_convention(manifest, serve): + calls, payloads = serve() + for descriptor, data in [(manifest["local_sdk"]["base_model"], MODEL), + (manifest["local_sdk"]["vision"]["projector"], PROJECTOR)]: + del descriptor["url"] + descriptor.update(provider="huggingface", repo="example/model") + payloads["https://huggingface.co/example/model/resolve/main/" + descriptor["file"]] = data + paths = assets.get_vision_asset_paths(manifest) + assert paths.base_model.read_bytes() == MODEL + assert all(url.startswith("https://huggingface.co/example/model/resolve/main/") for url in calls) + diff --git a/tests/test_vision_metadata.py b/tests/test_vision_metadata.py new file mode 100644 index 0000000..6aca5fa --- /dev/null +++ b/tests/test_vision_metadata.py @@ -0,0 +1,359 @@ +"""Image artifacts carry an exact, inert contract and cannot run as text.""" + +import copy +import hashlib +import importlib +import json +from pathlib import Path +import subprocess +import sys +import types +import zipfile + +import httpx +import pytest + +import programasweights as paw +from programasweights import cache, config +from programasweights.local_program import import_local_program + + +PID = "c" * 20 +MODEL = b"GGUF" + b"M" * 4092 +PROJECTOR = b"GGUF" + b"P" * 4092 +ADAPTER = b"GGUF" + b"A" * (cache.MIN_ADAPTER_GGUF_SIZE - 4) + + +def asset(file, data): + return { + "file": file, "url": "https://example.invalid/" + file, + "size_bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(), + } + + +@pytest.fixture +def manifest(): + return { + "runtime_id": "qwen3.5-0.8b-test", + "manifest_version": 2, + "interpreter": "Qwen/Qwen3.5-0.8B", + "adapter_format": "gguf_lora", + "input": {"format": "content_parts", "types": ["text", "image"]}, + "prompt_template": { + "format": "chat_messages", + "system_prompt_file": "prompt_template.txt", + "chat_format": "qwen3.5", + "enable_thinking": False, + }, + "program_assets": {"adapter_filename": "adapter.gguf"}, + "local_sdk": { + "supported": True, + "n_ctx": 4096, + "base_model": asset("base.gguf", MODEL), + "vision": { + "projector": asset("mmproj.gguf", PROJECTOR), + "preprocessing": { + "version": 1, "color_mode": "RGB", + "alpha": "composite_white", "orientation": "stored", + "color_profile": "ignore", "resize": "backend", + "image_min_tokens": 64, "image_max_tokens": 512, + }, + }, + }, + "js_sdk": {"supported": False}, + } + + +def metadata(manifest): + return { + "version": 4, "program_id": PID, + "interpreter": manifest["interpreter"], + "runtime_id": manifest["runtime_id"], + "runtime_manifest_version": manifest["manifest_version"], + "runtime": manifest, + } + + +def write_program(directory, meta, prompt="Locate the target in the supplied images."): + directory.mkdir(parents=True, exist_ok=True) + (directory / "meta.json").write_text(json.dumps(meta), encoding="utf-8") + (directory / "prompt_template.txt").write_text(prompt, encoding="utf-8") + (directory / "adapter.gguf").write_bytes(ADAPTER) + return directory + + +@pytest.fixture(autouse=True) +def isolated(tmp_path, monkeypatch): + monkeypatch.setenv("PAW_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.delenv("PAW_OFFLINE", raising=False) + + def forbidden(*args, **kwargs): + pytest.fail("Metadata validation must not download assets or make API calls") + + monkeypatch.setattr(httpx.Client, "send", forbidden) + monkeypatch.setattr(httpx.AsyncClient, "send", forbidden) + monkeypatch.setattr(cache, "_download_file", forbidden) + + +def test_valid_manifest_roundtrip_preserves_contract_and_copies_it(manifest): + expected = copy.deepcopy(manifest) + resolved = cache.resolve_runtime_manifest(metadata(manifest)) + assert resolved == expected + resolved["local_sdk"]["vision"]["preprocessing"]["image_max_tokens"] = 256 + assert manifest == expected + cache.save_runtime_manifest(manifest) + assert cache.get_cached_runtime_manifest(manifest["runtime_id"], 2) == expected + assert cache.get_offline_runtime_manifest(metadata(manifest)) == expected + + +@pytest.mark.parametrize("path", [ + ("input",), ("prompt_template",), ("program_assets",), + ("local_sdk", "vision"), ("local_sdk", "base_model"), + ("local_sdk", "n_ctx"), + ("local_sdk", "vision", "projector"), + ("local_sdk", "vision", "preprocessing"), +]) +def test_required_contract_sections_cannot_be_omitted(manifest, path): + current = manifest + for key in path[:-1]: + current = current[key] + del current[path[-1]] + assert cache._normalize_runtime_manifest(manifest) is None + + +@pytest.mark.parametrize("path,value", [ + (("manifest_version",), True), (("manifest_version",), 2.0), + (("manifest_version",), 3), (("manifest_version",), 1), + (("input", "types"), ["text"]), (("input", "types"), ["image", "audio"]), + (("input", "format"), "named_slots"), (("input", "extra"), True), + (("prompt_template", "format"), "rendered_text"), + (("prompt_template", "chat_format"), "unknown"), + (("prompt_template", "system_prompt_file"), "../prompt.txt"), + (("prompt_template", "enable_thinking"), True), + (("prompt_template", "enable_thinking"), 0), + (("prompt_template", "placeholder"), cache.INPUT_PLACEHOLDER), + (("program_assets", "adapter_filename"), "other.gguf"), + (("program_assets", "prefix_cache_required"), True), + (("program_assets", "prefix_tokens_filename"), "prefix_tokens.json"), + (("local_sdk", "supported"), "yes"), + (("local_sdk", "n_ctx"), True), (("local_sdk", "n_ctx"), 0), + (("local_sdk", "vision"), []), + (("local_sdk", "vision", "extra"), True), + (("local_sdk", "vision", "preprocessing", "version"), True), + (("local_sdk", "vision", "preprocessing", "version"), 2), + (("local_sdk", "vision", "preprocessing", "alpha"), "discard"), + (("local_sdk", "vision", "preprocessing", "color_mode"), "RGBA"), + (("local_sdk", "vision", "preprocessing", "orientation"), "exif"), + (("local_sdk", "vision", "preprocessing", "color_profile"), "apply"), + (("local_sdk", "vision", "preprocessing", "resize"), "512x512"), + (("local_sdk", "vision", "preprocessing", "image_min_tokens"), 513), + (("local_sdk", "vision", "preprocessing", "image_min_tokens"), 0), + (("local_sdk", "vision", "preprocessing", "image_max_tokens"), True), + (("local_sdk", "vision", "preprocessing", "image_max_tokens"), 512.0), + (("local_sdk", "vision", "preprocessing", "crop"), "center"), + (("js_sdk", "supported"), True), + (("base_inference",), {"format": "rendered_text"}), +]) +def test_incompatible_or_ambiguous_settings_rejected(manifest, path, value): + current = manifest + for key in path[:-1]: + current = current[key] + current[path[-1]] = value + assert cache._normalize_runtime_manifest(manifest) is None + with pytest.raises(ValueError): + cache.save_runtime_manifest(manifest) + + +@pytest.mark.parametrize("which", ["base", "projector"]) +@pytest.mark.parametrize("key,value", [ + ("sha256", None), ("sha256", "a" * 63), ("sha256", "z" * 64), + ("size_bytes", None), ("size_bytes", True), ("size_bytes", 0), ("size_bytes", 4.0), + ("file", "../file.gguf"), ("file", "..\\file.gguf"), + ("file", "C:file.gguf"), ("file", ".."), ("file", "file\x00.gguf"), + ("url", "file:///tmp/model.gguf"), ("url", "https://"), + ("url", "https://user:secret@example.invalid/a"), +]) +def test_asset_identity_is_required_for_both_files(manifest, which, key, value): + local = manifest["local_sdk"] + model = local["base_model"] if which == "base" else local["vision"]["projector"] + if value is None: + del model[key] + else: + model[key] = value + assert cache._normalize_runtime_manifest(manifest) is None + + +def test_huggingface_source_can_use_existing_repo_file_convention(manifest): + for model in [manifest["local_sdk"]["base_model"], manifest["local_sdk"]["vision"]["projector"]]: + del model["url"] + model.update(provider="huggingface", repo="example/model") + assert cache._normalize_runtime_manifest(manifest) == manifest + + +@pytest.mark.parametrize("file", ["base.gguf", "BASE.GGUF"]) +def test_base_and_projector_cannot_collide_by_filename(manifest, file): + manifest["local_sdk"]["vision"]["projector"]["file"] = file + assert cache._normalize_runtime_manifest(manifest) is None + + +@pytest.mark.parametrize("prompt", [ + "Locate the target.", "Treat {INPUT_PLACEHOLDER} literally.", + "{INPUT_PLACEHOLDER} and {INPUT_PLACEHOLDER} are examples.", +]) +def test_image_archive_import_preserves_literal_instructions(tmp_path, manifest, prompt): + source = write_program(tmp_path / "source", metadata(manifest), prompt) + archive = tmp_path / "function.paw" + with zipfile.ZipFile(archive, "w") as output: + for path in source.iterdir(): + output.write(path, path.name) + before = archive.read_bytes() + imported = import_local_program(archive) + assert cache.validate_program_assets_dir(imported, PID) + assert (imported / "prompt_template.txt").read_text() == prompt + assert json.loads((imported / "meta.json").read_text()) == metadata(manifest) + assert archive.read_bytes() == before + + +@pytest.mark.parametrize("field,value", [ + ("runtime_id", None), ("runtime_id", "different-runtime"), + ("interpreter", None), ("interpreter", "gpt2"), + ("runtime_manifest_version", None), ("runtime_manifest_version", 1), + ("runtime_manifest_version", 2.0), +]) +def test_program_and_embedded_runtime_must_agree(tmp_path, manifest, field, value): + meta = metadata(manifest) + if value is None: + del meta[field] + else: + meta[field] = value + path = write_program(tmp_path / "program", meta) + assert not cache.validate_program_assets_dir(path, PID) + assert cache.get_offline_runtime_manifest(meta) is None + assert cache.resolve_runtime_manifest(meta) is None + + +@pytest.mark.parametrize("offline", [False, True]) +@pytest.mark.parametrize("damage", ["missing", "bad_alpha", "wrong_version"]) +def test_bad_embedded_image_contract_never_falls_back(manifest, offline, damage, monkeypatch): + cache.save_runtime_manifest(manifest) + meta = metadata(copy.deepcopy(manifest)) + if damage == "missing": + del meta["runtime"] + elif damage == "bad_alpha": + meta["runtime"]["local_sdk"]["vision"]["preprocessing"]["alpha"] = "discard" + else: + meta["runtime"]["manifest_version"] = 1 + + def forbidden(*args, **kwargs): + pytest.fail("Must not substitute a different contract for image metadata") + + monkeypatch.setattr(cache, "get_cached_runtime_manifest", forbidden) + monkeypatch.setattr(cache, "fetch_runtime_manifest", forbidden) + monkeypatch.setattr(cache, "_legacy_runtime_manifest", forbidden) + assert cache.resolve_runtime_manifest(meta, offline=offline) is None + assert cache.get_offline_runtime_manifest(meta) is None + + +def test_image_readiness_requires_both_verified_assets(manifest): + write_program(config.get_programs_dir() / PID, metadata(manifest)) + (config.get_base_models_dir() / "base.gguf").write_bytes(MODEL) + (config.get_base_models_dir() / "mmproj.gguf").write_bytes(PROJECTOR) + assert cache.has_valid_program_assets(PID) + assert not paw.is_offline_ready(PID) + info = cache.get_cached_program_metadata(PID) + assert info["base_model_path"] is None + assert info["offline_ready"] is False + info = paw.prepare_program(PID, offline=True) + assert info["offline_ready"] is True + assert paw.is_offline_ready(PID) + path = cache.get_base_model_path(manifest["interpreter"], runtime_manifest=manifest, offline=True) + assert str(path) == info["base_model_path"] + from programasweights._vision_assets import get_cached_vision_asset_paths + paths = get_cached_vision_asset_paths(manifest) + paths.projector.write_bytes(b"corrupt") + assert not paw.is_offline_ready(PID) + + +@pytest.mark.parametrize("prompt", ["Locate the target.", "Input: {INPUT_PLACEHOLDER}"]) +def test_text_runtime_rejects_image_program_before_loading_model( + tmp_path, manifest, monkeypatch, prompt, +): + directory = write_program(config.get_programs_dir() / PID, metadata(manifest), prompt) + + def forbidden(*args, **kwargs): + pytest.fail("Image program must not create a text-only model") + + fake = types.ModuleType("llama_cpp") + fake.Llama = forbidden + monkeypatch.setitem(sys.modules, "llama_cpp", fake) + previous = sys.modules.pop("programasweights.runtime_llamacpp", None) + try: + runtime = importlib.import_module("programasweights.runtime_llamacpp") + with pytest.raises(ValueError, match="vision runtime"): + runtime.PawFunction(directory, offline=True) + finally: + sys.modules.pop("programasweights.runtime_llamacpp", None) + if previous is not None: + sys.modules["programasweights.runtime_llamacpp"] = previous + + +def test_validation_does_not_import_vision_or_native_libraries(manifest): + code = ''' +import sys, importlib.abc, json +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in ('PIL', 'llama_cpp', 'torch', 'numpy'): + raise AssertionError('Unexpected import: ' + fullname) +sys.meta_path.insert(0, Block()) +from programasweights import cache +manifest = json.loads(sys.argv[1]) +assert cache._normalize_runtime_manifest(manifest) == manifest +''' + result = subprocess.run([sys.executable, "-c", code, json.dumps(manifest)], + cwd=Path(__file__).resolve().parents[1], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_builtin_vision_base_contract_is_complete_and_defensively_copied(): + original = cache.get_base_runtime_manifest("Qwen/Qwen3.5-0.8B") + assert original["base_inference"] == {"contract_version": 1, "format": "chat_messages"} + assert cache._normalize_runtime_manifest(original) == original + clone = cache.get_base_runtime_manifest("Qwen/Qwen3.5-0.8B") + clone["local_sdk"]["vision"]["preprocessing"]["image_max_tokens"] = 64 + assert cache.get_base_runtime_manifest("Qwen/Qwen3.5-0.8B") == original + with pytest.raises(ValueError, match="vision runtime"): + cache.get_base_prompt_template(original) + + +@pytest.mark.parametrize("which", ["base_model", "projector"]) +@pytest.mark.parametrize("field,value", [ + ("sha256", "a" * 64), ("size_bytes", 12345), + ("url", "https://example.invalid/substituted.gguf"), +]) +def test_named_builtin_pins_both_asset_identities(which, field, value): + manifest = cache.get_base_runtime_manifest("Qwen/Qwen3.5-0.8B") + local = manifest["local_sdk"] + asset = local["base_model"] if which == "base_model" else local["vision"]["projector"] + asset[field] = value + assert cache._normalize_runtime_manifest(manifest) is None + + +@pytest.mark.parametrize("contract", [ + {"contract_version": 1, "format": "chat_messages"}, + None, +]) +def test_vision_base_contract_optional_for_compiled_artifacts(manifest, contract): + manifest["base_inference"] = contract + assert cache._normalize_runtime_manifest(manifest) == manifest + + +@pytest.mark.parametrize("contract", [ + [], "chat_messages", {}, + {"contract_version": True, "format": "chat_messages"}, + {"contract_version": 2, "format": "chat_messages"}, + {"contract_version": 1, "format": "rendered_text"}, + {"contract_version": 1, "format": "chat_messages", "system_prompt": "extra"}, +]) +def test_unsupported_vision_base_contract_rejected(manifest, contract): + manifest["base_inference"] = contract + assert cache._normalize_runtime_manifest(manifest) is None diff --git a/tests/test_vision_runtime.py b/tests/test_vision_runtime.py new file mode 100644 index 0000000..1634505 --- /dev/null +++ b/tests/test_vision_runtime.py @@ -0,0 +1,530 @@ +"""Hermetic native-boundary tests; real-model qualification is separate.""" + +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +import copy +import ctypes +import importlib +import json +from pathlib import Path +import sys +import threading +import types +import zipfile + +import httpx +from PIL import Image as PILImage +import pytest + +import programasweights as paw +from programasweights import cache, config +from test_vision_metadata import ADAPTER, MODEL, PROJECTOR, PID, manifest, metadata, write_program + + +@pytest.fixture +def backend(tmp_path, monkeypatch): + monkeypatch.setenv("PAW_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setenv("PAW_OFFLINE", "1") + monkeypatch.setenv("PAW_QUIET", "1") + def forbidden(*args, **kwargs): + pytest.fail("Runtime test attempted network access") + monkeypatch.setattr(httpx, "stream", forbidden) + (config.get_base_models_dir() / "base.gguf").write_bytes(MODEL) + (config.get_base_models_dir() / "mmproj.gguf").write_bytes(PROJECTOR) + state = types.SimpleNamespace( + models=[], calls=[], events=[], selected=None, handle=0, + support=True, init_fail=False, completion=None, create_error=None, + ) + lib = types.ModuleType("llama_cpp") + lib.__version__ = "0.3.35" + lib.LLAMA_FLASH_ATTN_TYPE_ENABLED = 1 + lib.llama_adapter_lora_p_ctypes = ctypes.c_void_p + lib.LogitsProcessorList = list + def event(name, *args): + state.events.append((name, *args)) + lib.llama_get_memory = lambda ctx: ctx + lib.llama_memory_clear = lambda ctx, full: event("memory_clear", full) + def select(ctx, handles, count, scales): + state.selected = handles[0] if count else None + event("select", state.selected, scales[0] if count else None) + return 0 + lib.llama_set_adapters_lora = select + def load(model, path): + state.handle += 1 + event("load", state.handle, path) + return state.handle + lib.llama_adapter_lora_init = load + def free(handle): + assert handle != state.selected, "Freed an attached adapter" + event("free", handle) + lib.llama_adapter_lora_free = free + class Handler: + def __init__(self, clip_model_path, **kwargs): + self.clip_model_path = clip_model_path + self.mtmd_ctx = None + self._mtmd_cpp = types.SimpleNamespace( + mtmd_context_params_default=types.SimpleNamespace, + mtmd_init_from_file=self.init, + mtmd_free=lambda ctx: event("projector_free"), + mtmd_support_vision=lambda ctx: state.support, + ) + def init(self, path, model, params): + event("projector_init", vars(params)) + return None if state.init_fail else object() + def __call__(self, **kwargs): + state.calls.append(kwargs) + if state.completion: + return state.completion(kwargs) + return {"choices": [{"message": {"content": " result "}}]} + class Llama: + def __init__(self, **kwargs): + if state.create_error: + raise state.create_error + self.options = kwargs + self.verbose = kwargs["verbose"] + self.n_threads = kwargs["n_threads"] + self.handler = kwargs["chat_handler"] + self.ctx = self + self.model = object() + self._stack = ExitStack() + self.closed = False + state.models.append(self) + def reset(self): + event("reset") + def create_chat_completion(self, **kwargs): + assert not self.closed + return self.handler(llama=self, **kwargs) + def token_eos(self): + return 1 + def close(self): + assert not self.closed, "Double close" + self._stack.close() + self.closed = True + event("model_free") + lib.Llama = Llama + chat = types.ModuleType("llama_cpp.llama_chat_format") + chat.MTMDChatHandler = Handler + monkeypatch.setitem(sys.modules, "llama_cpp", lib) + monkeypatch.setitem(sys.modules, "llama_cpp.llama_chat_format", chat) + for name in ("programasweights.runtime_llamacpp", "programasweights._runtime_vision"): + monkeypatch.delitem(sys.modules, name, raising=False) + runtime = importlib.import_module("programasweights._runtime_vision") + yield state, lib, runtime + assert not runtime._POOL, "Test leaked a shared runtime" + sys.modules.pop("programasweights.runtime_llamacpp", None) + sys.modules.pop("programasweights._runtime_vision", None) + + +@pytest.fixture +def program(backend, manifest): + return write_program(config.get_programs_dir() / PID, metadata(manifest), + "Literal {INPUT_PLACEHOLDER}; do not substitute.") + + +def second_program(program, manifest): + meta = metadata(copy.deepcopy(manifest)) + meta["program_id"] = "d" * 20 + directory = write_program(program.parent / meta["program_id"], meta, "Different system prompt") + (directory / "adapter.gguf").write_bytes(ADAPTER[:-1] + b"B") + return directory + + +@pytest.mark.parametrize("entrypoint", ["cached", "local"]) +def test_public_ordered_parts_preserve_literal_prompt_and_native_dimensions(program, backend, tmp_path, entrypoint): + state, _, _ = backend + reference = PID + if entrypoint == "local": + reference = tmp_path / "images.paw" + with zipfile.ZipFile(reference, "w") as output: + for path in program.iterdir(): + output.write(path, path.name) + image = paw.Image(PILImage.new("RGB", (31, 27), "red")) + with paw.function(reference, offline=True) as fn: + assert fn("one", image, "", "two", image, max_tokens=32) == "result" + call = state.calls[-1] + assert call["messages"][0]["content"] == "Literal {INPUT_PLACEHOLDER}; do not substitute." + parts = call["messages"][1]["content"] + assert [part["type"] for part in parts] == ["text", "image_url", "text", "text", "image_url"] + assert [part["text"] for part in parts if part["type"] == "text"] == ["one", "", "two"] + import base64, io + with PILImage.open(io.BytesIO(base64.b64decode(parts[1]["image_url"]["url"].split(",", 1)[1]))) as actual: + assert actual.size == (31, 27) + assert call["enable_thinking"] is False + assert fn.interpreter == "Qwen/Qwen3.5-0.8B" + assert state.models[0].closed + + +def test_aba_shares_model_resets_memory_and_detaches_before_switch(program, backend, manifest): + state, _, module = backend + other = second_program(program, manifest) + first = paw.function(PID, offline=True) + second = paw.function(other.name, offline=True) + try: + assert first._runtime is second._runtime + assert len(state.models) == 1 + for fn in (first, second, first): + fn("read") + assert state.selected is None + selected = [e for e in state.events if e[0] == "select" and e[1] is not None] + assert selected == [("select", 1, 1.0), ("select", 2, 1.0), ("select", 1, 1.0)] + assert len([e for e in state.events if e[0] == "memory_clear"]) == 6 + first.close() + assert not state.models[0].closed + assert second("still open") == "result" + finally: + first.close() + second.close() + assert len([e for e in state.events if e[0] == "projector_free"]) == 1 + assert not module._POOL + + +@pytest.mark.parametrize("setting,value", [("n_ctx", 4096), ("n_gpu_layers", 0), ("verbose", True)]) +def test_different_runtime_settings_do_not_share(program, backend, setting, value): + state, _, _ = backend + with paw.function(PID, offline=True) as one, paw.function(PID, offline=True, **{setting: value}) as two: + assert one._runtime is not two._runtime + assert len(state.models) == 2 + + +def test_token_budget_and_gpu_settings_reach_projector(program, backend): + state, _, _ = backend + with paw.function(PID, offline=True, n_gpu_layers=0): + params = next(e[1] for e in state.events if e[0] == "projector_init") + assert params["image_min_tokens"] == 64 + assert params["image_max_tokens"] == 512 + assert params["use_gpu"] is False + + +def test_different_image_limits_do_not_share(program, backend, manifest): + other = second_program(program, manifest) + meta = json.loads((other / "meta.json").read_text()) + meta["runtime"]["local_sdk"]["vision"]["preprocessing"]["image_max_tokens"] = 128 + (other / "meta.json").write_text(json.dumps(meta)) + with paw.function(PID, offline=True) as one, paw.function(other.name, offline=True) as two: + assert one._runtime is not two._runtime + + +def test_bounded_adapter_eviction_and_reload(program, backend, manifest, monkeypatch): + state, _, module = backend + monkeypatch.setattr(module, "_MAX_ADAPTERS", 1) + other = second_program(program, manifest) + with paw.function(PID, offline=True) as one, paw.function(other.name, offline=True) as two: + for fn in (one, two, one): + fn("read") + assert len(fn._runtime.adapters) == 1 + assert len([e for e in state.events if e[0] == "load"]) == 3 + assert len([e for e in state.events if e[0] == "free"]) == 2 + + +def test_changed_adapter_is_rejected_before_native_load(program, backend): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + (program / "adapter.gguf").write_bytes(ADAPTER[:-1] + b"X") + with pytest.raises(ValueError, match="changed"): + fn("read") + assert not [e for e in state.events if e[0] == "load"] + + +@pytest.mark.parametrize("parts,error", [((), ValueError), ((["bad"],), TypeError), (("ok", 32), TypeError)]) +def test_invalid_content_does_not_touch_native_state(program, backend, parts, error): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + before = list(state.events) + with pytest.raises(error): + fn(*parts) + assert state.events == before + + +@pytest.mark.parametrize("tokens", [-1, True, 1.5]) +def test_invalid_max_tokens(program, tokens): + with paw.function(PID, offline=True) as fn: + with pytest.raises(ValueError, match="max_tokens"): + fn("read", max_tokens=tokens) + + +@pytest.mark.parametrize("temperature", [-1, float("nan"), float("inf"), "warm"]) +def test_invalid_temperature(program, temperature): + with paw.function(PID, offline=True) as fn: + with pytest.raises(ValueError, match="temperature"): + fn("read", temperature=temperature) + + +def test_zero_tokens_does_not_mean_unlimited_generation(program, backend): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + assert fn("read", max_tokens=0) == "" + assert not state.calls + + +def test_response_format_and_generation_options_pass_through(program, backend): + state, _, _ = backend + schema = {"type": "json_object", "schema": {"type": "object"}} + with paw.function(PID, offline=True) as fn: + fn("read", temperature=.25, max_tokens=16, response_format=schema) + assert state.calls[-1]["response_format"] == schema + assert state.calls[-1]["max_tokens"] == 16 + assert state.calls[-1]["temperature"] == .25 + + +@pytest.mark.parametrize("failure", [ValueError("context overflow"), KeyboardInterrupt()]) +def test_call_failure_cleans_state_and_next_call_works(program, backend, failure): + state, _, _ = backend + def fail(kwargs): + raise failure + with paw.function(PID, offline=True) as fn: + state.completion = fail + with pytest.raises(type(failure)): + fn("read") + assert state.selected is None + assert state.events[-2:] == [("memory_clear", True), ("select", None, None)] + state.completion = None + assert fn("retry") == "result" + + +def test_logit_callback_exception_is_not_swallowed(program, backend): + state, _, _ = backend + failure = ValueError("bad user callback") + def processor(ids, scores): + raise failure + def completion(kwargs): + fallback = kwargs["logits_processor"][0]([], [0., 0., 0.]) + assert fallback == [0., 1e10, 0.] + return {"choices": [{"message": {"content": "discarded"}}]} + state.completion = completion + with paw.function(PID, offline=True) as fn: + with pytest.raises(ValueError, match="bad user callback") as caught: + fn("read", logits_processor=[processor]) + assert caught.value is failure + + +@pytest.mark.parametrize("operation", ["call", "close", "load"]) +def test_reentrant_callback_fails_without_corrupting_model(program, backend, operation): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + def completion(kwargs): + if operation == "call": + return fn("nested") + if operation == "close": + return fn.close() + return paw.function(PID, offline=True) + state.completion = completion + with pytest.raises(RuntimeError, match="inside a vision callback"): + fn("read") + state.completion = None + assert fn("retry") == "result" + + +def test_concurrent_functions_serialize_and_close_waits(program, backend, manifest): + state, _, _ = backend + other = second_program(program, manifest) + one, two = paw.function(PID, offline=True), paw.function(other.name, offline=True) + started, release, attempted = threading.Event(), threading.Event(), threading.Event() + def completion(kwargs): + started.set() + assert release.wait(5) + return {"choices": [{"message": {"content": "ok"}}]} + state.completion = completion + def second_call(): + attempted.set() + return two("two") + try: + with ThreadPoolExecutor(2) as executor: + first = executor.submit(one, "one") + assert started.wait(5) + second = executor.submit(second_call) + assert attempted.wait(5) + assert len(state.calls) == 1 + release.set() + assert first.result(5) == second.result(5) == "ok" + started.clear(); release.clear(); attempted.clear() + with ThreadPoolExecutor(2) as executor: + first = executor.submit(one, "one") + assert started.wait(5) + def close_other(): + attempted.set() + two.close() + closing = executor.submit(close_other) + assert attempted.wait(5) + assert not closing.done() + release.set() + first.result(5); closing.result(5) + assert not state.models[0].closed + finally: + release.set() + one.close(); two.close() + + +@pytest.mark.parametrize("kind", ["projector", "unsupported", "model"]) +def test_initialization_failure_is_not_pooled(program, backend, kind): + state, _, module = backend + state.init_fail = kind == "projector" + state.support = kind != "unsupported" + state.create_error = RuntimeError("model init failed") if kind == "model" else None + with pytest.raises(RuntimeError): + paw.function(PID, offline=True) + assert not module._POOL + assert all(model.closed for model in state.models) + state.init_fail, state.support, state.create_error = False, True, None + with paw.function(PID, offline=True) as fn: + assert fn("retry") == "result" + + +def test_old_backend_fails_before_download(program, backend, monkeypatch): + state, lib, module = backend + lib.__version__ = "0.3.34" + def forbidden(*args, **kwargs): + pytest.fail("Should check version before asset resolution") + monkeypatch.setattr(module, "get_vision_asset_paths", forbidden) + with pytest.raises(ImportError, match="0.3.35"): + paw.function(PID, offline=True) + assert not state.models + + +def test_closed_function_rejects_calls(program): + fn = paw.function(PID, offline=True) + fn.close(); fn.close() + with pytest.raises(RuntimeError, match="closed"): + fn("read") + + +def test_partial_output_returns_text_but_nontext_output_errors(program, backend): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + state.completion = lambda _: {"choices": [{"finish_reason": "length", "message": {"content": "partial"}}]} + assert fn("read", max_tokens=1) == "partial" + state.completion = lambda _: {"choices": [{"message": {"content": None}}]} + with pytest.raises(RuntimeError, match="no text content"): + fn("read") + + +def test_failed_state_cleanup_retires_runtime_and_preserves_call_error(program, backend, monkeypatch): + state, lib, module = backend + failure = ValueError("original inference failure") + original_clear = lib.llama_memory_clear + count = 0 + def clear(ctx, full): + nonlocal count + count += 1 + if count == 2: + raise RuntimeError("cleanup failed") + return original_clear(ctx, full) + def completion(kwargs): + raise failure + with paw.function(PID, offline=True) as old: + monkeypatch.setattr(lib, "llama_memory_clear", clear) + state.completion = completion + with pytest.raises(ValueError) as caught: + old("read") + assert caught.value is failure + assert old._runtime.closed + state.completion = None + with paw.function(PID, offline=True) as new: + assert new._runtime is not old._runtime + old.close() + assert new("recovered") == "result" + assert len(module._POOL) == 1 + + +def test_forked_process_rejected_before_acquiring_inherited_locks(program, backend, monkeypatch): + _, _, module = backend + with paw.function(PID, offline=True) as fn: + with monkeypatch.context() as child: + child.setattr(module, "_PROCESS_ID", -1) + with pytest.raises(RuntimeError, match="spawned"): + fn("read") + with pytest.raises(RuntimeError, match="spawned"): + paw.function(PID, offline=True) + + +@pytest.fixture +def vision_base(backend, manifest, monkeypatch): + builtin = copy.deepcopy(manifest) + builtin["runtime_id"] = "qwen3.5-0.8b-q8_0" + builtin["base_inference"] = {"contract_version": 1, "format": "chat_messages"} + monkeypatch.setitem(cache.BUILTIN_VISION_RUNTIMES, builtin["runtime_id"], builtin) + def forbidden(*args, **kwargs): + pytest.fail("Base interpreter must not use program or Hub resolution") + monkeypatch.setattr(paw, "_resolve_program_id", forbidden) + monkeypatch.setattr(cache, "get_program_dir", forbidden) + monkeypatch.setattr(cache, "fetch_runtime_manifest", forbidden) + return builtin + + +def test_base_vision_accepts_ordered_parts_without_system_or_adapter(vision_base, backend): + state, _, _ = backend + with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True) as fn: + assert fn("first", paw.Image(PILImage.new("RGB", (31, 27))), "last") == "result" + messages = state.calls[-1]["messages"] + assert len(messages) == 1 and messages[0]["role"] == "user" + assert [p["type"] for p in messages[0]["content"]] == ["text", "image_url", "text"] + assert fn._adapter is None and fn.spec == "" + assert "base interpreter" in repr(fn) + assert not [e for e in state.events if e[0] == "load"] + assert not (config.get_cache_dir() / "programs").exists() + + +def test_base_vision_string_calls_remain_stateless(vision_base, backend): + state, _, _ = backend + with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True) as fn: + fn("old prompt") + fn("new prompt", max_tokens=3) + assert state.calls[-1]["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "new prompt"}]}, + ] + assert state.calls[-1]["enable_thinking"] is False + + +def test_base_and_compiled_share_and_cannot_leak_lora(program, vision_base, backend): + state, _, module = backend + # Call the internal constructor for the already validated compiled directory: + # vision_base deliberately forbids public Hub/program resolution. + with module.VisionFunction(program, offline=True) as compiled: + with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True) as base: + assert compiled._runtime is base._runtime + selected = [] + def completion(kwargs): + selected.append(state.selected) + return {"choices": [{"message": {"content": "ok"}}]} + state.completion = completion + for fn in (compiled, base, compiled, base): + fn("read") + assert selected == [1, None, 1, None] + compiled.close() + assert base("still open") == "ok" + assert len(state.models) == 1 and state.models[0].closed + + +@pytest.mark.parametrize("missing", ["base.gguf", "mmproj.gguf"]) +def test_base_vision_offline_missing_asset_fails_without_model_load(vision_base, backend, missing): + state, _, _ = backend + (config.get_base_models_dir() / missing).unlink() + with pytest.raises(RuntimeError, match="offline mode"): + paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") + assert not state.models + + +@pytest.mark.parametrize("n_ctx", [0, -1, True, 1.5]) +def test_base_vision_bad_context_fails_before_loading(vision_base, backend, n_ctx): + state, _, _ = backend + with pytest.raises(ValueError, match="n_ctx"): + paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", n_ctx=n_ctx) + assert not state.models + + +def test_base_vision_failed_init_does_not_leak(vision_base, backend): + state, _, module = backend + state.init_fail = True + with pytest.raises(RuntimeError, match="projector"): + paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") + assert not module._POOL and state.models[0].closed + + +def test_base_vision_old_backend_fails_before_asset_resolution(vision_base, backend, monkeypatch): + state, lib, module = backend + lib.__version__ = "0.3.34" + def forbidden(*args, **kwargs): + pytest.fail("Should reject old backend before asset resolution") + monkeypatch.setattr(module, "get_vision_asset_paths", forbidden) + with pytest.raises(ImportError, match="0.3.35"): + paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") + assert not state.models From 3be4ff1fffc592da9b78d0c354b2786a2a4b4e2b Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 11:05:36 -0400 Subject: [PATCH 2/5] Add opt-in result metadata for local image functions --- .github/workflows/test.yml | 1 + docs/api-reference/python-sdk.md | 27 ++++ programasweights/__init__.py | 2 + programasweights/_result.py | 24 ++++ programasweights/_runtime_vision.py | 42 +++++- tests/test_vision_result.py | 216 ++++++++++++++++++++++++++++ 6 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 programasweights/_result.py create mode 100644 tests/test_vision_result.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46c00f4..f3c5386 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,6 +102,7 @@ jobs: tests/test_vision_metadata.py \ tests/test_vision_assets.py \ tests/test_vision_runtime.py \ + tests/test_vision_result.py \ tests/test_sdk.py::TestInstallAndImport local-files-windows: diff --git a/docs/api-reference/python-sdk.md b/docs/api-reference/python-sdk.md index 8ee7197..6438117 100644 --- a/docs/api-reference/python-sdk.md +++ b/docs/api-reference/python-sdk.md @@ -202,6 +202,7 @@ output: str = fn( temperature=0.0, logits_processor=None, response_format=None, + return_info=False, ) ``` @@ -212,6 +213,32 @@ always validate returned data, especially when `max_tokens` truncates the output parts other than strings or `paw.Image` are rejected; unpack an existing list with `fn(*parts)`. +Pass `return_info=True` to receive a `paw.FunctionResult` instead of a string: + +```python +result = fn("Find the target:", paw.Image("scene.png"), return_info=True) +print(result.text) +print(result.finish_reason) # Backend reason, e.g. "stop" or "length"; otherwise None. +print(result.usage) # Backend token counts, or None when unavailable. +print(result.elapsed_seconds) # Full call duration through native cleanup. +``` + +The result is immutable and belongs to that call. `usage`, when present, is a +copied, read-only mapping; use `dict(result.usage)` when logging it as JSON. +Counts have the backend's semantics; they are not an estimate of image patches +or billable tokens. `elapsed_seconds` uses a monotonic clock and includes input +preparation, waiting for function/runtime locks, adapter selection, inference, +and native cleanup. It excludes loading the function and constructing input +`paw.Image` objects before the call. + +`return_info` is a keyword-only boolean and defaults to `False`, preserving +string returns. With `max_tokens=0`, the opt-in result has empty text and `None` +for both backend fields, because no generation ran. Truncated text is still +returned with the backend's `"length"` reason; errors still raise normally. +This option applies to the local Qwen3.5 image runtime, including text-only calls +to that interpreter and compiled image programs. Existing text runtimes and +remote inference do not accept this option. + `paw.Image(source)` accepts a regular local path, encoded image bytes, or a Pillow image and snapshots the supplied content. It does not fetch URLs. Images are decoded locally and converted to RGB PNG without SDK resizing, EXIF rotation, diff --git a/programasweights/__init__.py b/programasweights/__init__.py index a80015f..0a2b6d6 100644 --- a/programasweights/__init__.py +++ b/programasweights/__init__.py @@ -34,6 +34,7 @@ __version__ = "0.4.11" from ._inputs import Image +from ._result import FunctionResult from ._output import ProgressCallback, ProgressEvent, report_progress from .cache import CachedProgram from .client import ( @@ -730,6 +731,7 @@ def list_compilers() -> list[dict]: "CompileJob", "CompilePrecheck", "CompileStatus", + "FunctionResult", "Program", "ProgressCallback", "ProgressEvent", diff --git a/programasweights/_result.py b/programasweights/_result.py new file mode 100644 index 0000000..ee823c4 --- /dev/null +++ b/programasweights/_result.py @@ -0,0 +1,24 @@ +"""Per-call inference results without mutable state on a function.""" + +from dataclasses import dataclass +from types import MappingProxyType +from typing import Mapping, Optional + + +@dataclass(frozen=True) +class FunctionResult: + """Text and optional backend metadata from one successful image-runtime call. + + ``usage`` is a copied, read-only mapping of backend token counts, or None + when unavailable. ``elapsed_seconds`` covers the call through native cleanup, + including input preparation and lock waits, but not function loading. + """ + + text: str + finish_reason: Optional[str] + usage: Optional[Mapping[str, int]] + elapsed_seconds: float + + def __post_init__(self): + if self.usage is not None: + object.__setattr__(self, "usage", MappingProxyType(dict(self.usage))) diff --git a/programasweights/_runtime_vision.py b/programasweights/_runtime_vision.py index 872290b..3faa01d 100644 --- a/programasweights/_runtime_vision.py +++ b/programasweights/_runtime_vision.py @@ -17,9 +17,12 @@ from pathlib import Path import re import threading +from time import perf_counter +from typing import Literal, Union, overload from . import cache from ._image_content import _to_chat_content +from ._result import FunctionResult from ._vision_assets import get_vision_asset_paths from ._vision_contract import VISION_BASE_INFERENCE, declares_vision @@ -159,7 +162,7 @@ def _select(self, adapter): raise RuntimeError("Failed to select the vision LoRA adapter.") def run(self, adapter, messages, *, max_tokens, temperature, - logits_processor=None, response_format=None): + logits_processor=None, response_format=None, return_info=False): with self.lock: if self.closed: raise RuntimeError("This vision runtime has been closed.") @@ -199,6 +202,12 @@ def guarded_processor(input_ids, scores): content = response["choices"][0]["message"]["content"] if not isinstance(content, str): raise RuntimeError("The vision function returned no text content.") + if return_info: + usage = response.get("usage") + return ( + content.strip(), response["choices"][0].get("finish_reason"), + None if usage is None else dict(usage), + ) return content.strip() except BaseException as exc: error = exc @@ -241,6 +250,7 @@ class VisionFunction: Generation options are keyword-only. Calls return text (including partial text when max_tokens is exhausted); callers validate structured outputs. + Opt into a per-call FunctionResult with return_info=True. Close functions, or use a context manager, to release shared model memory. """ @@ -303,8 +313,27 @@ def _acquire_runtime(self, manifest, n_ctx, n_gpu_layers, verbose, offline): runtime.references += 1 self._runtime = runtime + @overload + def __call__(self, *parts, max_tokens=None, temperature=0.0, + logits_processor=None, response_format=None, + return_info: Literal[False] = False) -> str: ... + + @overload + def __call__(self, *parts, max_tokens=None, temperature=0.0, + logits_processor=None, response_format=None, + return_info: Literal[True]) -> FunctionResult: ... + + @overload + def __call__(self, *parts, max_tokens=None, temperature=0.0, + logits_processor=None, response_format=None, + return_info: bool) -> Union[str, FunctionResult]: ... + def __call__(self, *parts, max_tokens=None, temperature=0.0, - logits_processor=None, response_format=None) -> str: + logits_processor=None, response_format=None, + return_info: bool = False) -> Union[str, FunctionResult]: + if type(return_info) is not bool: + raise TypeError("return_info must be a bool.") + started = perf_counter() if return_info else None _outside_call() if os.getpid() != self._pid: raise RuntimeError("Load a new vision function in each child process.") @@ -317,15 +346,22 @@ def __call__(self, *parts, max_tokens=None, temperature=0.0, raise ValueError("temperature must be a finite non-negative number.") content = _to_chat_content(*parts) if max_tokens == 0: + if return_info: + return FunctionResult("", None, None, perf_counter() - started) return "" messages = [] if self._prompt is not None: messages.append({"role": "system", "content": self._prompt}) messages.append({"role": "user", "content": content}) - return self._runtime.run( + output = self._runtime.run( self._adapter, messages, max_tokens=max_tokens, temperature=temperature, logits_processor=logits_processor, response_format=response_format, + return_info=return_info, ) + if return_info: + text, finish_reason, usage = output + return FunctionResult(text, finish_reason, usage, perf_counter() - started) + return output def close(self): if os.getpid() != self._pid: diff --git a/tests/test_vision_result.py b/tests/test_vision_result.py new file mode 100644 index 0000000..78a8d2d --- /dev/null +++ b/tests/test_vision_result.py @@ -0,0 +1,216 @@ +"""Opt-in result metadata at the native boundary; no models or network used.""" + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import FrozenInstanceError +import threading + +import pytest +import programasweights as paw +from test_vision_metadata import PID, manifest +from test_vision_runtime import backend, program, second_program, vision_base + + +def completion(text=" result ", reason="stop", usage=None): + return {"choices": [{"message": {"content": text}, "finish_reason": reason}], + "usage": usage} + + +@pytest.mark.parametrize("reason", ["stop", "length", "backend-specific", None]) +def test_result_is_opt_in_and_preserves_backend_metadata(program, backend, reason): + state, _, _ = backend + counts = {"prompt_tokens": 13, "completion_tokens": 2, "total_tokens": 15} + state.completion = lambda _: completion(reason=reason, usage=counts) + with paw.function(PID, offline=True) as fn: + assert fn("read") == "result" + assert fn("read", return_info=False) == "result" + result = fn("read", max_tokens=2, return_info=True) + assert isinstance(result, paw.FunctionResult) + assert result.text == "result" and result.finish_reason == reason + assert result.usage == counts + assert result.elapsed_seconds >= 0 + assert "return_info" not in state.calls[-1] + assert state.calls[-1]["max_tokens"] == 2 + counts["prompt_tokens"] = 999 + assert result.usage["prompt_tokens"] == 13 + with pytest.raises(FrozenInstanceError): + result.text = "changed" + with pytest.raises(TypeError): + result.usage["prompt_tokens"] = 0 + assert "FunctionResult" in paw.__all__ + + +def test_missing_metadata_and_empty_usage_are_distinct(program, backend): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + result = fn("read", return_info=True) + assert result.finish_reason is result.usage is None + state.completion = lambda _: completion(usage={}) + assert dict(fn("read", return_info=True).usage) == {} + + +def test_zero_tokens_returns_metadata_without_generation(program, backend): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + before = list(state.events) + result = fn("read", max_tokens=0, return_info=True) + assert result.text == "" + assert result.finish_reason is result.usage is None + assert result.elapsed_seconds >= 0 + assert not state.calls and state.events == before + + +@pytest.mark.parametrize("flag", [None, 0, 1, "yes"]) +def test_flag_is_boolean_and_validation_precedes_native_state(program, backend, flag): + state, _, _ = backend + with paw.function(PID, offline=True) as fn: + before = list(state.events) + with pytest.raises(TypeError, match="return_info"): + fn("read", return_info=flag) + assert not state.calls and state.events == before + + +@pytest.mark.parametrize("kind", ["generation", "callback", "nontext", "cleanup"]) +def test_opt_in_does_not_swallow_errors_or_publish_a_partial_result(program, backend, monkeypatch, kind): + state, _, module = backend + failure = ValueError("original failure") + with paw.function(PID, offline=True) as fn: + def fail(*args): + raise failure + if kind == "nontext": + state.completion = lambda _: completion(text=None) + elif kind == "callback": + def callback_response(kwargs): + scores = [0.0, 0.0] + # Simulate the native callback boundary, which finishes sampling + # before the wrapper re-raises the processor's error. + kwargs["logits_processor"][0]([], scores) + return completion() + state.completion = callback_response + elif kind == "cleanup": + reset = fn._runtime._reset + resets = [] + def fail_cleanup(): + resets.append(True) + if len(resets) == 2: + raise failure + reset() + monkeypatch.setattr(fn._runtime, "_reset", fail_cleanup) + else: + state.completion = fail + with pytest.raises(RuntimeError if kind == "nontext" else ValueError) as caught: + fn("read", return_info=True, + logits_processor=[fail] if kind == "callback" else None) + if kind != "nontext": + assert caught.value is failure + if kind == "cleanup": + assert fn._runtime.closed + else: + state.completion = None + assert fn("retry", return_info=True).text == "result" + + +def test_elapsed_includes_preparation_both_locks_and_cleanup(program, backend, monkeypatch): + state, _, module = backend + clock = [100.0] + monkeypatch.setattr(module, "perf_counter", lambda: clock[0]) + prepare = module._to_chat_content + def prepare_input(*parts): + clock[0] += 1 + return prepare(*parts) + monkeypatch.setattr(module, "_to_chat_content", prepare_input) + class TimedLock: + def __init__(self, lock, seconds): + self.lock, self.seconds = lock, seconds + def __enter__(self): + self.lock.acquire() + clock[0] += self.seconds + def __exit__(self, *_): + self.lock.release() + with paw.function(PID, offline=True) as fn: + fn._lock = TimedLock(fn._lock, 2) + fn._runtime.lock = TimedLock(fn._runtime.lock, 3) + reset = fn._runtime._reset + def reset_state(): + clock[0] += 4 + reset() + monkeypatch.setattr(fn._runtime, "_reset", reset_state) + def infer(kwargs): + clock[0] += 5 + return completion() + state.completion = infer + result = fn("read", return_info=True) + assert result.elapsed_seconds == 1 + 2 + 3 + 4 + 5 + 4 + + +@pytest.mark.parametrize("same_function", [False, True]) +def test_serialized_calls_keep_their_own_results(program, backend, manifest, same_function): + state, _, _ = backend + other = second_program(program, manifest) + started, release, attempted = threading.Event(), threading.Event(), threading.Event() + shared_counts = {"completion_tokens": 0} + def infer(kwargs): + text = kwargs["messages"][-1]["content"][0]["text"] + if text == "first": + started.set() + assert release.wait(5) + shared_counts["completion_tokens"] = 1 + else: + shared_counts["completion_tokens"] = 2 + return completion(text=text, usage=shared_counts) + state.completion = infer + with paw.function(PID, offline=True) as one, paw.function(other.name, offline=True) as two: + def second_call(): + attempted.set() + return (one if same_function else two)("second", return_info=True) + with ThreadPoolExecutor(2) as executor: + first = executor.submit(one, "first", return_info=True) + try: + assert started.wait(5) + second = executor.submit(second_call) + assert attempted.wait(5) + assert len(state.calls) == 1 + finally: + release.set() + a, b = first.result(5), second.result(5) + assert (a.text, a.usage["completion_tokens"]) == ("first", 1) + assert (b.text, b.usage["completion_tokens"]) == ("second", 2) + assert a is not b + + +def test_base_interpreter_supports_result_metadata(vision_base, backend): + with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True) as fn: + assert fn("read", return_info=True).text == "result" + assert fn("read") == "result" + + +def test_usage_is_copied_before_releasing_the_shared_runtime(program, backend, manifest, monkeypatch): + state, _, _ = backend + other = second_program(program, manifest) + counts = {"completion_tokens": 0} + first_done, second_done = threading.Event(), threading.Event() + def infer(kwargs): + text = kwargs["messages"][-1]["content"][0]["text"] + counts["completion_tokens"] = 1 if text == "first" else 2 + return completion(text=text, usage=counts) + state.completion = infer + with paw.function(PID, offline=True) as one, paw.function(other.name, offline=True) as two: + run = one._runtime.run + def interleave(*args, **kwargs): + output = run(*args, **kwargs) + if output[0] == "first": + first_done.set() + assert second_done.wait(5) + else: + second_done.set() + return output + monkeypatch.setattr(one._runtime, "run", interleave) + with ThreadPoolExecutor(2) as executor: + first = executor.submit(one, "first", return_info=True) + try: + assert first_done.wait(5) + second = executor.submit(two, "second", return_info=True) + a, b = first.result(5), second.result(5) + finally: + second_done.set() + assert a.usage["completion_tokens"] == 1 + assert b.usage["completion_tokens"] == 2 From c89113946e3e704f4bbc30d37bae2e8ffc5be3aa Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 11:41:04 -0400 Subject: [PATCH 3/5] Make inference results transferable between processes --- programasweights/_result.py | 11 +++++++++ tests/test_vision_result.py | 47 ++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/programasweights/_result.py b/programasweights/_result.py index ee823c4..a6cb614 100644 --- a/programasweights/_result.py +++ b/programasweights/_result.py @@ -22,3 +22,14 @@ class FunctionResult: def __post_init__(self): if self.usage is not None: object.__setattr__(self, "usage", MappingProxyType(dict(self.usage))) + + def __reduce__(self): + return ( + type(self), + ( + self.text, + self.finish_reason, + None if self.usage is None else dict(self.usage), + self.elapsed_seconds, + ), + ) diff --git a/tests/test_vision_result.py b/tests/test_vision_result.py index 78a8d2d..71cb09f 100644 --- a/tests/test_vision_result.py +++ b/tests/test_vision_result.py @@ -1,7 +1,9 @@ """Opt-in result metadata at the native boundary; no models or network used.""" -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from dataclasses import FrozenInstanceError +import multiprocessing +import pickle import threading import pytest @@ -15,6 +17,49 @@ def completion(text=" result ", reason="stop", usage=None): "usage": usage} +def _result_from_worker(usage): + return paw.FunctionResult("résultat", "stop", usage, 0.125) + + +def _assert_restored_result(result, usage): + assert isinstance(result, paw.FunctionResult) + assert (result.text, result.finish_reason, result.elapsed_seconds) == ( + "résultat", "stop", 0.125, + ) + if usage is None: + assert result.usage is None + else: + assert result.usage == usage + with pytest.raises(TypeError): + result.usage["prompt_tokens"] = 0 + for name in ("text", "finish_reason", "usage", "elapsed_seconds"): + with pytest.raises(FrozenInstanceError): + setattr(result, name, None) + + +@pytest.mark.parametrize("usage", [None, {}, { + "prompt_tokens": 13, "completion_tokens": 2, "total_tokens": 15, +}]) +@pytest.mark.parametrize("protocol", range(pickle.HIGHEST_PROTOCOL + 1)) +def test_result_pickle_round_trip_preserves_metadata_and_immutability(usage, protocol): + original = _result_from_worker(usage) + restored = pickle.loads(pickle.dumps(original, protocol=protocol)) + assert restored is not original + _assert_restored_result(restored, usage) + + +def test_spawned_worker_can_return_results_with_immutable_metadata(): + usages = [None, {}, { + "prompt_tokens": 13, "completion_tokens": 2, "total_tokens": 15, + }] + with ProcessPoolExecutor( + max_workers=1, mp_context=multiprocessing.get_context("spawn"), + ) as executor: + for usage in usages: + result = executor.submit(_result_from_worker, usage).result(timeout=30) + _assert_restored_result(result, usage) + + @pytest.mark.parametrize("reason", ["stop", "length", "backend-specific", None]) def test_result_is_opt_in_and_preserves_backend_metadata(program, backend, reason): state, _, _ = backend From 5f243fff6329e9ddedb102a84be08d8aa20e98ea Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 11:51:33 -0400 Subject: [PATCH 4/5] Check vision extra and native image backend in install CI --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f3c5386..70f2e71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -181,3 +181,8 @@ jobs: run("-I", "-c", "import programasweights, llama_cpp; " "print(programasweights.__version__, llama_cpp.__version__); " "print(llama_cpp.llama_print_system_info().decode())") + run("-m", "pip", "install", str(wheel) + "[vision]", "--extra-index-url", + "https://pypi.programasweights.com/simple/", "--only-binary=llama-cpp-python") + run("-m", "pip", "check") + run("-I", "-c", "import PIL, llama_cpp.mtmd_cpp; " + "from llama_cpp.llama_chat_format import MTMDChatHandler") From 1f7bf133798daace200195ba43f612dfd92d5c9f Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 12:02:04 -0400 Subject: [PATCH 5/5] Prepare SDK 0.4.12 release --- CHANGELOG.md | 8 ++++++++ programasweights/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3070534..1e64d94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.4.12 (2026-10-07) + +- Add local Qwen3.5-0.8B functions with ordered text and `paw.Image` inputs, + including compatible image `.paw` bundles. +- Add image dependencies through `programasweights[vision]`. +- Add `return_info=True` for immutable results with finish reason, token counts, + and elapsed call time. Results can be returned from multiprocessing workers. + ## 0.4.11 (2026-10-04) - Support llama-cpp-python 0.3.36. diff --git a/programasweights/__init__.py b/programasweights/__init__.py index 0a2b6d6..f4820a7 100644 --- a/programasweights/__init__.py +++ b/programasweights/__init__.py @@ -31,7 +31,7 @@ from importlib.metadata import version as _meta_version __version__ = _meta_version("programasweights") except Exception: - __version__ = "0.4.11" + __version__ = "0.4.12" from ._inputs import Image from ._result import FunctionResult diff --git a/pyproject.toml b/pyproject.toml index 865cb91..f0e8d32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "programasweights" -version = "0.4.11" +version = "0.4.12" description = "Compile natural language specifications into neural programs that run locally via llama.cpp." readme = "PYPI_README.md" requires-python = ">=3.8"