From 7c0e9fe3198983764bde5e828881d0e7d1fb6644 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 19:37:14 -0400 Subject: [PATCH 1/7] Support complete prompt templates for local text and image functions --- .github/workflows/test.yml | 2 + CHANGELOG.md | 9 + PYPI_README.md | 25 ++- README.md | 25 ++- docs/api-reference/python-sdk.md | 130 +++++++++++-- programasweights/__init__.py | 39 +++- programasweights/_image_content.py | 29 +++ programasweights/_inputs.py | 4 +- programasweights/_prompt_template.py | 61 ++++++ programasweights/_runtime_vision.py | 162 ++++++++++++++-- programasweights/_vision_contract.py | 27 +-- programasweights/cache.py | 14 +- programasweights/runtime_llamacpp.py | 107 ++++++++--- tests/test_numbered_templates.py | 126 +++++++++++++ tests/test_vision_metadata.py | 38 +++- tests/test_vision_prompt.py | 268 +++++++++++++++++++++++++++ tests/test_vision_result.py | 14 +- tests/test_vision_runtime.py | 104 +++++++---- 18 files changed, 1050 insertions(+), 134 deletions(-) create mode 100644 programasweights/_prompt_template.py create mode 100644 tests/test_numbered_templates.py create mode 100644 tests/test_vision_prompt.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70f2e71..337553c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -93,6 +93,7 @@ jobs: tests/test_compile_timeouts.py \ tests/test_local_program.py \ tests/test_base_interpreter.py \ + tests/test_numbered_templates.py \ tests/test_cli_auth.py \ tests/test_remote_inference.py \ tests/test_desktop_sdk.py \ @@ -102,6 +103,7 @@ jobs: tests/test_vision_metadata.py \ tests/test_vision_assets.py \ tests/test_vision_runtime.py \ + tests/test_vision_prompt.py \ tests/test_vision_result.py \ tests/test_sdk.py::TestInstallAndImport diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e64d94..febdc62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Add `prompt_template=` for local adapter-free text and image interpreters, + using shared numbered positional placeholders with repeatable, ordered slots. +- Execute complete image prompt templates directly, without an added chat + wrapper. Older experimental image bundles must be re-exported with the + `rendered_text` / `{INPUT_N}` contract and a complete prompt template. +- Preserve existing compiled text templates, call conventions and prefix caches. + ## 0.4.12 (2026-10-07) - Add local Qwen3.5-0.8B functions with ordered text and `paw.Image` inputs, diff --git a/PYPI_README.md b/PYPI_README.md index 2b695c4..3933c04 100644 --- a/PYPI_README.md +++ b/PYPI_README.md @@ -125,17 +125,32 @@ Install the optional image dependencies: pip install "programasweights[vision]" --extra-index-url https://pypi.programasweights.com/simple/ ``` -Prompt Qwen3.5-0.8B locally with text and images (no adapter applied): +The unreleased numbered-template API lets you specify the complete prompt for +text-only or image-capable base models. For Qwen3.5-0.8B with no adapter: ```python import programasweights as paw -describe = paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") -print(describe("Describe this image.", paw.Image("photo.png"))) +prompt = ( + "<|im_start|>user\n{INPUT_0}\n{INPUT_1}<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" +) +with paw.function( + None, interpreter="Qwen/Qwen3.5-0.8B", prompt_template=prompt, +) as describe: + print(describe("Describe this image.", paw.Image("photo.png"))) ``` -To use a compiled image program, load its `.paw` file with -`paw.function("./locator.paw")`. +`{INPUT_0}`, `{INPUT_1}`, and subsequent numbered slots bind positional arguments +where they appear in the template. Slots may repeat or change order. The SDK +adds no chat wrapper; the template supplies roles and the assistant prefix. +The same option supports text-only models such as `Qwen/Qwen3-0.6B`, with string +arguments. Existing text calls without this option retain their behavior. + +Compiled image programs read the complete template from their `.paw` bundle: +load with `paw.function("./locator.paw")` and supply the arguments its slots +require. Older experimental image bundles using `chat_messages` metadata must +be re-exported with a complete numbered template; there is no automatic fallback. See the [image API reference](https://programasweights.readthedocs.io/en/latest/api-reference/python-sdk/#local-textimage-calls) for multiple images, input formats, and result metadata. diff --git a/README.md b/README.md index cde090b..55c9d19 100644 --- a/README.md +++ b/README.md @@ -125,17 +125,32 @@ Install the optional image dependencies: pip install "programasweights[vision]" --extra-index-url https://pypi.programasweights.com/simple/ ``` -Prompt Qwen3.5-0.8B locally with text and images (no adapter applied): +The unreleased numbered-template API lets you specify the complete prompt for +text-only or image-capable base models. For Qwen3.5-0.8B with no adapter: ```python import programasweights as paw -describe = paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") -print(describe("Describe this image.", paw.Image("photo.png"))) +prompt = ( + "<|im_start|>user\n{INPUT_0}\n{INPUT_1}<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" +) +with paw.function( + None, interpreter="Qwen/Qwen3.5-0.8B", prompt_template=prompt, +) as describe: + print(describe("Describe this image.", paw.Image("photo.png"))) ``` -To use a compiled image program, load its `.paw` file with -`paw.function("./locator.paw")`. +`{INPUT_0}`, `{INPUT_1}`, and subsequent numbered slots bind positional arguments +where they appear in the template. Slots may repeat or change order. The SDK +adds no chat wrapper; the template supplies roles and the assistant prefix. +The same option supports text-only models such as `Qwen/Qwen3-0.6B`, with string +arguments. Existing text calls without this option retain their behavior. + +Compiled image programs read the complete template from their `.paw` bundle: +load with `paw.function("./locator.paw")` and supply the arguments its slots +require. Older experimental image bundles using `chat_messages` metadata must +be re-exported with a complete numbered template; there is no automatic fallback. See the [image API reference](docs/api-reference/python-sdk.md#local-textimage-calls) for multiple images, input formats, and result metadata. diff --git a/docs/api-reference/python-sdk.md b/docs/api-reference/python-sdk.md index 55296c9..3b1e7d6 100644 --- a/docs/api-reference/python-sdk.md +++ b/docs/api-reference/python-sdk.md @@ -23,9 +23,9 @@ fn = paw.function( n_gpu_layers=None, verbose=False, offline=False, - *, remote=False, interpreter=None, + prompt_template=None, ) ``` @@ -43,8 +43,10 @@ bundle directly. Required runtime metadata and base models are cached for reuse. | `offline` | Use only local files/cache and make zero network calls; fail if required validated assets are missing. `PAW_OFFLINE=1` has the same effect. | | `remote` | Run hosted inference without downloading model assets (default `False`). Accepts a `Program` object, ID, or slug. Cannot be combined with offline mode, local file paths, `interpreter`, or non-default local runtime options. | | `interpreter` | Advanced adapter-free mode only. Must be passed by keyword and only when `program_id` is explicitly `None`. Supported values are `Qwen/Qwen3-0.6B`, `gpt2`, and `Qwen/Qwen3.5-0.8B`. | +| `prompt_template` | Unreleased. Optional complete numbered template for a local base interpreter. Pass by keyword with `program_id=None`. Works with text-only and image-capable models. Cannot override a compiled bundle or be used with `remote=True`. | -For local text-only programs, the returned callable accepts: +For existing local text-only programs and text base interpreters without an +explicit `prompt_template`, the returned callable accepts: ```python output: str = fn(input_text, max_tokens=None, temperature=0.0, logits_processor=None) @@ -148,35 +150,137 @@ The text interpreters format prompts as follows: ``` The Qwen3 interpreter uses its chat template with thinking disabled. GPT-2 uses -the input as a raw prompt. +the input as a raw prompt. These defaults are unchanged when `prompt_template` +is omitted. + +#### Complete numbered templates (unreleased) + +For a local base model, pass `prompt_template=` when loading the function to +specify the complete prompt. This option works for both text-only and +image-capable interpreters: + +```python +prompt = ( + "<|im_start|>system\n{INPUT_0}<|im_end|>\n" + "<|im_start|>user\n{INPUT_1}<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" +) +with paw.function( + None, interpreter="Qwen/Qwen3-0.6B", prompt_template=prompt, +) as answer: + output = answer( + "Answer using only the facts in the question.", + "Alice owns three cats. How many cats does Alice own?", + max_tokens=32, + ) +``` + +The template controls role delimiters, examples, whitespace, thinking markers +and the assistant prefix. The SDK adds no chat wrapper. Use the format expected +by the selected model. Text is tokenized with special tokens recognized and +without an automatically added BOS prefix. Image-capable models insert the +required vision tokens and embeddings at image slots. + +Numbered templates share these rules across text and image models: + +- `{INPUT_0}`, `{INPUT_1}`, etc. refer to positional arguments. Used indices must + start at zero without gaps; `{INPUT_1}/{INPUT_0}/{INPUT_1}` is valid. +- Pass exactly the number of distinct indices. Repeating a slot repeats its + content at that position. A nonempty constant template may have no slots and + be called with `fn()`. +- Input strings are inserted literally and are not parsed again for slots. + Only canonical numbered placeholders are interpreted. Other brace content is + literal; this is not Python `str.format`. Doubled braces do not escape a slot: + `{{INPUT_0}}` surrounds the inserted value with literal braces. +- Text-only models accept strings. Image-capable models accept strings or + `paw.Image` in any slot, including calls containing only strings. +- Generation options are keyword-only for numbered-template calls: + `fn("first", "second", max_tokens=32)`. + +For an already rendered text prompt, use `prompt_template="{INPUT_0}"`. +Qwen3.5's default base template is this identity template: it accepts one +argument unchanged and supplies no conversation wrapper. GPT-2 and Qwen3-0.6B +keep the default text templates shown above when the option is omitted. + +`prompt_template=` is available only with `program_id=None` and local inference. +Compiled programs use their bundled `prompt_template.txt`; passing an override +with a compiled program or `remote=True` raises an error. + +Existing compiled text programs still require exactly one +`{INPUT_PLACEHOLDER}`. Their literal text, tokenization boundaries and prefix +cache behavior are unchanged. Existing text calls such as `fn(input_text="x")` +and `fn("x", 32)` remain valid; explicit numbered templates use the positional +inputs and keyword generation options described above. ### Local text/image calls -Install `programasweights[vision]` to prompt Qwen3.5-0.8B locally -(no adapter applied). Pass text and `paw.Image` objects in the order -the model should read them: +Install `programasweights[vision]` to use Qwen3.5-0.8B. With the unreleased +numbered-template API, order images and text through the complete template: ```python -compare = paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") -answer = compare( - "Before:", paw.Image("before.png"), - "After:", paw.Image("after.png"), - "What changed?", +prompt = ( + "<|im_start|>user\n" + "Before: {INPUT_1}\nAfter: {INPUT_2}\n{INPUT_0}<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" ) +with paw.function( + None, interpreter="Qwen/Qwen3.5-0.8B", prompt_template=prompt, +) as compare: + answer = compare( + "What changed?", paw.Image("before.png"), paw.Image("after.png"), + ) print(answer) ``` -Use the same inputs with a compiled image program: +Here the question is argument zero, but the model receives both images before +that question. Repeating `{INPUT_1}` would place the first image at each +occurrence. There is no separate public image marker to insert manually. + +For a compiled image program, `prompt_template.txt` contains the complete +prompt, for example: + +```text +<|im_start|>system +Locate the requested object.<|im_end|> +<|im_start|>user +{INPUT_0} +{INPUT_1}<|im_end|> +<|im_start|>assistant + + + + +``` + +With that template, the call is: ```python fn = paw.function("./locator.paw") answer = fn("Find the red cup.", paw.Image("scene.png")) ``` +The image runtime manifest declares the template contract as: + +```json +"prompt_template": { + "format": "rendered_text", + "placeholder": "{INPUT_N}" +} +``` + +This is the template field inside the full runtime manifest, not a standalone +manifest. Keep the required model, projector, preprocessing and adapter metadata. +Older experimental image bundles with `chat_messages`, `system_prompt_file`, +`chat_format` or `enable_thinking` prompt metadata are rejected. Re-export those +bundles with the new contract and a full template matching the adapter's training +prompt. Changing the metadata alone does not reconstruct the missing roles or +assistant prefix. Existing text bundles do not need this migration. + `paw.Image(source)` accepts a local file path, encoded image bytes, or a Pillow image. For an image URL, download the file first. Plain strings are text inputs. -Image functions return a string. Generation options are passed by keyword: +Image functions return a string. Their positional argument count and order +come from the template. Generation options are passed by keyword: ```python output: str = fn( diff --git a/programasweights/__init__.py b/programasweights/__init__.py index 7599c41..95f7f37 100644 --- a/programasweights/__init__.py +++ b/programasweights/__init__.py @@ -426,6 +426,7 @@ def function( *, remote: bool = False, interpreter: str | None = None, + prompt_template: str | None = None, ): """Load a compiled program, or explicitly load a bare base interpreter. @@ -453,13 +454,22 @@ def function( interpreter: Advanced adapter-free mode. This is only valid when ``program_id`` is explicitly ``None``. Supported values are ``"Qwen/Qwen3-0.6B"``, ``"gpt2"``, and ``"Qwen/Qwen3.5-0.8B"``. - Qwen3.5 processes text and ``paw.Image`` arguments in the order - provided. + Qwen3.5 places text and ``paw.Image`` arguments according to + its complete numbered prompt template. + prompt_template: Complete numbered template for a local adapter-free + interpreter, including text-only models. {INPUT_0}, {INPUT_1}, ... + bind positional arguments; generation options are keyword-only + when this is supplied. Text-only models require string arguments; + image-capable models also accept paw.Image. No chat wrapper is + added. Omitting this option preserves the interpreter's default + template and call convention. Compiled functions use their bundled + prompt_template.txt and do not accept this override. Returns: A callable returning an output string. Text programs accept one input - string. Image programs accept ordered positional strings and paw.Image - objects, with generation options passed by keyword. + string by default. Explicit numbered templates accept positional + strings; image programs also accept paw.Image objects. Numbered + template calls take generation options by keyword. Example: >>> fn = paw.function("email-triage") @@ -475,11 +485,26 @@ def function( >>> base = paw.function(None, interpreter="gpt2") >>> vision = paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") - >>> vision("Describe this image.", paw.Image("photo.png")) + >>> vision(paw.Image("photo.png")) # default template is {INPUT_0} + + >>> prompted = paw.function( + ... None, interpreter="gpt2", + ... prompt_template="Context: {INPUT_0}. Question: {INPUT_1}. Answer:", + ... ) + >>> prompted("Context goes here.", "What happened?", max_tokens=32) """ import os from . import cache + if prompt_template is not None: + if remote or program_id is not None: + raise ValueError( + "prompt_template requires local adapter-free inference; " + "compiled functions use their bundled template." + ) + if not isinstance(prompt_template, str): + raise TypeError("prompt_template must be a string.") + offline = _offline_requested(offline) if remote: _validate_remote_options( @@ -514,12 +539,16 @@ def function( from .runtime_llamacpp import PawFunction function_type = PawFunction + template_options = ( + {} if prompt_template is None else {"prompt_template": prompt_template} + ) return function_type.from_base( interpreter, n_ctx=n_ctx, n_gpu_layers=n_gpu_layers, verbose=verbose, offline=offline, + **template_options, ) from ._program_reference import local_program_path diff --git a/programasweights/_image_content.py b/programasweights/_image_content.py index 1996164..63ade8b 100644 --- a/programasweights/_image_content.py +++ b/programasweights/_image_content.py @@ -105,3 +105,32 @@ def _to_chat_content(*parts: Union[str, Image]) -> List[Dict[str, Any]]: "image_url": {"url": _image_url(part, index)}, }) return content + + +def _to_prompt(*parts: Union[str, Image], image_marker: str) -> tuple: + """Convert bound parts into exact prompt text and ordered image URLs.""" + if not isinstance(image_marker, str) or not image_marker: + raise ValueError("The backend image marker must be a nonempty string.") + + text = [] + images = [] + for index, part in enumerate(_normalize_parts(*parts), start=1): + if isinstance(part, str): + if image_marker in part: + raise ValueError( + "The backend image marker is reserved; use paw.Image." + ) + text.append(part) + else: + text.append(image_marker) + images.append((index, part)) + + prompt = "".join(text) + # Also catch a reserved marker formed across adjacent text parts. + if prompt.count(image_marker) != len(images): + raise ValueError( + "The backend image marker is reserved; use paw.Image." + ) + + urls = [_image_url(image, index) for index, image in images] + return prompt, urls diff --git a/programasweights/_inputs.py b/programasweights/_inputs.py index 0a5ba09..ebf2f13 100644 --- a/programasweights/_inputs.py +++ b/programasweights/_inputs.py @@ -36,7 +36,9 @@ def _copy_pixels(source): class Image: """An image to pass to a local image function. - Accepts a file path, encoded image bytes, or a Pillow image. For example: + Accepts a file path, encoded image bytes, or a Pillow image. Pass it as a + positional argument to the function's numbered prompt template. For example, + when the template uses {INPUT_0} for the question and {INPUT_1} for an image: ``fn("Describe this image.", paw.Image("photo.png"))``. """ diff --git a/programasweights/_prompt_template.py b/programasweights/_prompt_template.py new file mode 100644 index 0000000..1937bec --- /dev/null +++ b/programasweights/_prompt_template.py @@ -0,0 +1,61 @@ +import re + +from ._inputs import _normalize_parts + + +_NUMBERED_SLOT = re.compile(r"\{INPUT_(0|[1-9][0-9]*)\}") + + +def parse_template(template: str, placeholder: str) -> tuple: + """Return literal strings interleaved with positional argument indices.""" + + if placeholder == "{INPUT_PLACEHOLDER}": + # Preserve the existing text-template semantics exactly. + if template.count(placeholder) != 1: + raise ValueError( + "Compiled prompt template must contain exactly one " + "{INPUT_PLACEHOLDER} placeholder." + ) + + prefix, suffix = template.split(placeholder) + return prefix, 0, suffix + + if placeholder != "{INPUT_N}": + raise ValueError(f"Unsupported placeholder syntax: {placeholder!r}") + + segments = [] + indices = set() + end = 0 + + for match in _NUMBERED_SLOT.finditer(template): + index = int(match.group(1)) + segments.extend((template[end:match.start()], index)) + indices.add(index) + end = match.end() + + segments.append(template[end:]) + + if sorted(indices) != list(range(len(indices))): + raise ValueError("Template indices must start at 0 without gaps.") + + return tuple(segments) + + +def bind_template(segments: tuple, *args) -> tuple: + """Bind arguments to segments returned by parse_template.""" + arity = 1 + max( + (segment for segment in segments if isinstance(segment, int)), + default=-1, + ) + if len(args) != arity: + raise TypeError( + f"Template requires {arity} positional arguments; got {len(args)}." + ) + + if args: + _normalize_parts(*args) + + return tuple( + args[segment] if isinstance(segment, int) else segment + for segment in segments + ) diff --git a/programasweights/_runtime_vision.py b/programasweights/_runtime_vision.py index 75a0313..2f0a4a6 100644 --- a/programasweights/_runtime_vision.py +++ b/programasweights/_runtime_vision.py @@ -21,7 +21,8 @@ from typing import Literal, Union, overload from . import cache -from ._image_content import _to_chat_content +from ._image_content import _to_prompt +from ._prompt_template import bind_template, parse_template from ._result import FunctionResult from ._vision_assets import get_vision_asset_paths from ._vision_contract import VISION_BASE_INFERENCE, declares_vision @@ -72,10 +73,6 @@ def __init__(self, paths, preprocessing, *, n_ctx, n_gpu_layers, verbose): from .runtime_llamacpp import _suppress_native_stderr class Handler(handler_type): - def __call__(handler, **kwargs): - kwargs["enable_thinking"] = False - return super().__call__(**kwargs) - def _init_mtmd_context(handler, model): if handler.mtmd_ctx is not None: return @@ -161,7 +158,133 @@ def _select(self, adapter): if result not in (0, None): raise RuntimeError("Failed to select the vision LoRA adapter.") - def run(self, adapter, messages, *, max_tokens, temperature, + def _eval_prompt(self, parts): + """Evaluate bound parts after _select has reset the model.""" + model = self.llm + handler = model.chat_handler + mtmd = handler._mtmd_cpp + + if model.n_tokens != 0: + raise RuntimeError("Prompt evaluation requires a reset runtime.") + + marker = mtmd.mtmd_default_marker().decode("utf-8") + prompt, urls = _to_prompt(*parts, image_marker=marker) + encoded = prompt.encode("utf-8") + + input_text = mtmd.mtmd_input_text() + input_text.text = encoded + input_text.text_len = len(encoded) + input_text.add_special = False + input_text.parse_special = True + + bitmaps = [] + chunks = None + try: + for url in urls: + bitmap = handler._create_bitmap_from_bytes( + handler.load_image(url) + ) + if not bitmap: + raise RuntimeError("Failed to create image bitmap.") + bitmaps.append(bitmap) + + chunks = mtmd.mtmd_input_chunks_init() + if not chunks: + raise RuntimeError("Failed to allocate prompt chunks.") + + bitmap_array = ( + mtmd.mtmd_bitmap_p_ctypes * len(bitmaps) + )(*bitmaps) + result = mtmd.mtmd_tokenize( + handler.mtmd_ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + len(bitmaps), + ) + if result != 0: + raise ValueError(f"Prompt tokenization failed: {result}.") + + for index in range(mtmd.mtmd_input_chunks_size(chunks)): + chunk = mtmd.mtmd_input_chunks_get(chunks, index) + if not chunk: + raise RuntimeError("Missing prompt chunk.") + + size = mtmd.mtmd_input_chunk_get_n_tokens(chunk) + if model.n_tokens + size >= model.n_ctx(): + raise ValueError("Prompt leaves no room for output.") + + kind = mtmd.mtmd_input_chunk_get_type(chunk) + if kind == mtmd.MTMD_INPUT_CHUNK_TYPE_TEXT: + count = ctypes.c_size_t() + tokens = mtmd.mtmd_input_chunk_get_tokens_text( + chunk, ctypes.byref(count) + ) + if count.value: + if not tokens: + raise RuntimeError("Missing text tokens.") + model.eval([tokens[i] for i in range(count.value)]) + + elif kind == mtmd.MTMD_INPUT_CHUNK_TYPE_IMAGE: + start = model.n_tokens + end = self.lib.llama_pos(0) + result = mtmd.mtmd_helper_eval_chunk_single( + handler.mtmd_ctx, + model._ctx.ctx, + chunk, + self.lib.llama_pos(start), + self.lib.llama_seq_id(0), + model.n_batch, + True, + ctypes.byref(end), + ) + if result != 0: + raise RuntimeError( + f"Image evaluation failed: {result}." + ) + if not start < end.value < model.n_ctx(): + raise RuntimeError("Invalid image end position.") + + # Image embeddings occupy these positions. These zeros + # are Python bookkeeping, not tokens sent to the model. + model.input_ids[start:end.value] = 0 + model.n_tokens = end.value + model._requires_eval = False + else: + raise ValueError("Unsupported prompt chunk type.") + + if model.n_tokens == 0: + raise ValueError("The prompt produced no tokens.") + + return model.input_ids[:model.n_tokens].tolist() + finally: + if chunks: + mtmd.mtmd_input_chunks_free(chunks) + for bitmap in bitmaps: + mtmd.mtmd_bitmap_free(bitmap) + + def _complete_prompt( + self, parts, *, max_tokens, temperature, + logits_processor=None, response_format=None, + ): + """Complete an exact prompt inside the caller's reset, locked runtime.""" + options = {} + if logits_processor is not None: + options["logits_processor"] = logits_processor + if response_format is not None: + from llama_cpp.llama_chat_format import _grammar_for_response_format + + options["grammar"] = _grammar_for_response_format(response_format) + + prompt_tokens = self._eval_prompt(parts) + return self.llm.create_completion( + prompt=prompt_tokens, + max_tokens=max_tokens, + temperature=temperature, + **options, + ) + + def run(self, adapter, parts, *, max_tokens, temperature, logits_processor=None, response_format=None, return_info=False): with self.lock: if self.closed: @@ -193,13 +316,13 @@ def guarded_processor(input_ids, scores): options["logits_processor"] = self.lib.LogitsProcessorList([guarded_processor]) if response_format is not None: options["response_format"] = response_format - response = self.llm.create_chat_completion( - messages=messages, max_tokens=max_tokens, + response = self._complete_prompt( + parts, max_tokens=max_tokens, temperature=temperature, **options, ) if processor_error is not None: raise processor_error - content = response["choices"][0]["message"]["content"] + content = response["choices"][0]["text"] if not isinstance(content, str): raise RuntimeError("The vision function returned no text content.") if return_info: @@ -263,14 +386,17 @@ def __init__(self, program_dir, n_ctx=2048, n_gpu_layers=-1, verbose=False, if not declares_vision(self._meta): raise ValueError("VisionFunction requires an image program.") manifest = cache.resolve_runtime_manifest(self._meta, offline=offline) - self._prompt = (directory / "prompt_template.txt").read_text(encoding="utf-8") + template = (directory / "prompt_template.txt").read_text(encoding="utf-8") + self._segments = parse_template( + template, manifest["prompt_template"]["placeholder"], + ) path = directory / "adapter.gguf" self._adapter = (path, _digest(path), path.stat().st_size) self._acquire_runtime(manifest, n_ctx, n_gpu_layers, verbose, offline) @classmethod def from_base(cls, interpreter, *, n_ctx=2048, n_gpu_layers=-1, - verbose=False, offline=False): + verbose=False, offline=False, prompt_template=None): self = cls.__new__(cls) self._initialize(n_ctx) manifest = cache.get_base_runtime_manifest(interpreter) @@ -280,7 +406,11 @@ def from_base(cls, interpreter, *, n_ctx=2048, n_gpu_layers=-1, ): raise ValueError("This interpreter has no adapter-free vision contract.") self._meta = {"mode": "base", "interpreter": interpreter} - self._prompt = None + base = manifest["base_inference"] + template = base["template"] if prompt_template is None else prompt_template + if not isinstance(template, str): + raise TypeError("prompt_template must be a string.") + self._segments = parse_template(template, base["placeholder"]) self._adapter = None self._acquire_runtime(manifest, n_ctx, n_gpu_layers, verbose, offline) return self @@ -343,17 +473,13 @@ def __call__(self, *parts, max_tokens=None, temperature=0.0, raise ValueError("max_tokens must be None or a non-negative integer.") if not isinstance(temperature, (int, float)) or not math.isfinite(temperature) or temperature < 0: raise ValueError("temperature must be a finite non-negative number.") - content = _to_chat_content(*parts) + bound = bind_template(self._segments, *parts) if max_tokens == 0: if return_info: return FunctionResult("", None, None, perf_counter() - started) return "" - messages = [] - if self._prompt is not None: - messages.append({"role": "system", "content": self._prompt}) - messages.append({"role": "user", "content": content}) output = self._runtime.run( - self._adapter, messages, max_tokens=max_tokens, temperature=temperature, + self._adapter, bound, max_tokens=max_tokens, temperature=temperature, logits_processor=logits_processor, response_format=response_format, return_info=return_info, ) diff --git a/programasweights/_vision_contract.py b/programasweights/_vision_contract.py index 0865b35..b3148c8 100644 --- a/programasweights/_vision_contract.py +++ b/programasweights/_vision_contract.py @@ -1,9 +1,8 @@ """Inert v2 runtime metadata for local text/image functions. -The existing prompt_template.txt member holds literal system instructions. -Each call appends one user message containing its ordered text/image parts; -there are no string placeholders or named image slots. The qwen3.5 chat -profile adds the assistant generation prompt with thinking disabled. +The prompt_template.txt member holds the complete rendered prompt template. +Numbered placeholders bind positional text/image arguments. The template owns +roles, separators, and the assistant prefix; no chat template is applied. Preprocessing v1 uses native input dimensions, RGB with transparency over white, stored pixel orientation, and no color-profile transform. Resizing and @@ -21,10 +20,8 @@ _SHA256 = re.compile(r"^[a-fA-F0-9]{64}$") _INPUT = {"format": "content_parts", "types": ["text", "image"]} _PROMPT = { - "format": "chat_messages", - "system_prompt_file": "prompt_template.txt", - "chat_format": "qwen3.5", - "enable_thinking": False, + "format": "rendered_text", + "placeholder": "{INPUT_N}", } _PREPROCESSING = { "version": 1, @@ -35,10 +32,14 @@ "resize": "backend", } -# A base call is one user message containing the ordered input parts. The -# pinned GGUF chat template adds the assistant prefix with thinking disabled; -# no system message or compiled prompt is inserted in adapter-free mode. -VISION_BASE_INFERENCE = {"contract_version": 1, "format": "chat_messages"} +# The default base template passes one argument through unchanged. +# Callers supply the complete prompt; no conversation wrapper is inserted. +VISION_BASE_INFERENCE = { + "contract_version": 1, + "format": "rendered_text", + "placeholder": "{INPUT_N}", + "template": "{INPUT_0}", +} BUILTIN_VISION_RUNTIMES = { "qwen3.5-0.8b-q8_0": { @@ -125,7 +126,7 @@ def valid_vision_contract(manifest: dict) -> bool: if manifest.get("input") != _INPUT: return False prompt = manifest.get("prompt_template") - if prompt != _PROMPT or prompt.get("enable_thinking") is not False: + if prompt != _PROMPT: return False assets = manifest.get("program_assets") if ( diff --git a/programasweights/cache.py b/programasweights/cache.py index b9e26c0..6fef2ba 100644 --- a/programasweights/cache.py +++ b/programasweights/cache.py @@ -30,6 +30,7 @@ import httpx from . import config +from ._prompt_template import parse_template from ._output import ProgressCallback, report_progress from ._vision_contract import ( BUILTIN_VISION_RUNTIMES, @@ -1213,11 +1214,16 @@ def validate_program_assets_dir( if not isinstance(meta, dict) or meta.get("program_id") != expected_program_id: return False if declares_vision(meta): - # The same archive member now holds literal system instructions. - # It must never be interpreted as a rendered-text placeholder template. - return _normalize_runtime_manifest_for_program( + runtime = _normalize_runtime_manifest_for_program( meta.get("runtime"), meta, - ) is not None + ) + if runtime is None: + return False + try: + parse_template(template, runtime["prompt_template"]["placeholder"]) + except ValueError: + return False + return True if template.count(INPUT_PLACEHOLDER) != 1: return False interpreter = meta.get("interpreter") diff --git a/programasweights/runtime_llamacpp.py b/programasweights/runtime_llamacpp.py index fb7adb5..630c906 100644 --- a/programasweights/runtime_llamacpp.py +++ b/programasweights/runtime_llamacpp.py @@ -24,6 +24,7 @@ from llama_cpp import Llama from . import cache +from ._prompt_template import bind_template, parse_template _NATIVE_STDERR_LOCK = threading.RLock() @@ -84,6 +85,7 @@ def __init__( self._adapter = None self._closed = False self._mode = "compiled" + self._template_segments = None self._program_dir = Path(program_dir) self._verbose = verbose self._n_ctx = n_ctx @@ -178,8 +180,8 @@ def __init__( ) self._apply_adapter(1.0) - prefix_text, self._suffix_text = self._template.split( - cache.INPUT_PLACEHOLDER, + prefix_text, _, self._suffix_text = parse_template( + self._template, cache.INPUT_PLACEHOLDER, ) self._prefix_tokens = self._llm.tokenize( prefix_text.encode("utf-8"), @@ -202,13 +204,15 @@ def from_base( n_gpu_layers: int = -1, verbose: bool = False, offline: bool = False, + prompt_template: str | None = None, ) -> "PawFunction": - """Create an adapter-free base interpreter with a built-in prompt.""" + """Create a base interpreter with its default or a numbered template.""" self = cls.__new__(cls) self._llm = None self._adapter = None self._closed = False self._mode = "base" + self._template_segments = None self._program_dir = None self._verbose = verbose self._n_ctx = n_ctx @@ -217,7 +221,15 @@ def from_base( try: runtime_manifest = cache.get_base_runtime_manifest(interpreter) - self._template = cache.get_base_prompt_template(runtime_manifest) + if prompt_template is None: + self._template = cache.get_base_prompt_template(runtime_manifest) + else: + if not isinstance(prompt_template, str): + raise TypeError("prompt_template must be a string.") + self._template = prompt_template + self._template_segments = parse_template( + prompt_template, "{INPUT_N}", + ) self._interpreter = interpreter self._meta = { "mode": "base", @@ -393,7 +405,43 @@ def _save_prefix_cache_atomically(self, cache_path: Path) -> None: except OSError: pass - def __call__( + def __call__(self, *args, **kwargs) -> str: + """Run with the input convention selected when the function was loaded. + + Explicit numbered templates bind positional strings and take generation + options by keyword. Existing functions retain input_text= and positional + max_tokens, temperature and logits_processor arguments. + """ + if self._template_segments is not None: + return self._call_numbered(*args, **kwargs) + return self._call_single_input(*args, **kwargs) + + def _call_numbered( + self, + *inputs: str, + max_tokens: int | None = None, + temperature: float = 0.0, + logits_processor: llama_cpp.LogitsProcessorList | None = None, + ) -> str: + if self._closed or self._llm is None: + raise RuntimeError("This PawFunction has been closed.") + if any(not isinstance(value, str) for value in inputs): + raise TypeError("Text-only interpreters require string arguments.") + bound = bind_template(self._template_segments, *inputs) + if max_tokens is not None and ( + not isinstance(max_tokens, int) + or isinstance(max_tokens, bool) + or max_tokens < 0 + ): + raise ValueError("max_tokens must be None or a non-negative integer.") + return self._complete_base_prompt( + "".join(bound), + max_tokens=max_tokens, + temperature=temperature, + logits_processor=logits_processor, + ) + + def _call_single_input( self, input_text: str, max_tokens: int | None = None, @@ -427,32 +475,16 @@ def __call__( raise ValueError("max_tokens must be None or a non-negative integer.") if self._mode == "base": - reset = getattr(self._llm, "reset", None) - if not callable(reset): - raise RuntimeError( - "The installed llama-cpp runtime cannot reset model state." - ) - reset() rendered_prompt = self._template.replace( cache.INPUT_PLACEHOLDER, input_text, 1, ) - prompt_tokens = self._llm.tokenize( - rendered_prompt.encode("utf-8"), - add_bos=False, - special=True, - ) - if not prompt_tokens: - raise ValueError( - "The rendered base prompt tokenized to zero tokens." - ) - return self._generate( - prompt_tokens, + return self._complete_base_prompt( + rendered_prompt, max_tokens=max_tokens, temperature=temperature, logits_processor=logits_processor, - token_description="prompt", ) self._llm.n_tokens = self._n_prefix @@ -472,6 +504,35 @@ def __call__( token_description="input", ) + def _complete_base_prompt( + self, + rendered_prompt: str, + *, + max_tokens: int | None, + temperature: float, + logits_processor: llama_cpp.LogitsProcessorList | None, + ) -> str: + reset = getattr(self._llm, "reset", None) + if not callable(reset): + raise RuntimeError( + "The installed llama-cpp runtime cannot reset model state." + ) + reset() + prompt_tokens = self._llm.tokenize( + rendered_prompt.encode("utf-8"), + add_bos=False, + special=True, + ) + if not prompt_tokens: + raise ValueError("The rendered base prompt tokenized to zero tokens.") + return self._generate( + prompt_tokens, + max_tokens=max_tokens, + temperature=temperature, + logits_processor=logits_processor, + token_description="prompt", + ) + def _generate( self, prompt_tokens: list[int], diff --git a/tests/test_numbered_templates.py b/tests/test_numbered_templates.py new file mode 100644 index 0000000..536d11d --- /dev/null +++ b/tests/test_numbered_templates.py @@ -0,0 +1,126 @@ +"""Numbered text templates share binding while existing text calls stay intact.""" +import pytest +import programasweights as paw +from programasweights._prompt_template import bind_template, parse_template +from test_base_interpreter import fake_runtime, _isolated_cache, _write_base_model, _write_compiled_program + +@pytest.mark.parametrize('model', ['gpt2', 'Qwen/Qwen3-0.6B']) +def test_exact_numbered_prompt(model, fake_runtime): + _, state = fake_runtime + _write_base_model(model) + template = ' {INPUT_1}\n{INPUT_0} / {INPUT_1} \nAnswer:' + with paw.function(None, interpreter=model, prompt_template=template, offline=True) as fn: + assert fn('A{INPUT_1}', 'B', max_tokens=1) == 'A' + assert state.instances[-1].tokenize_calls == [dict(data=b' B\nA{INPUT_1} / B \nAnswer:', add_bos=False, special=True)] + assert state.instances[-1].reset_calls == 1 + assert fn('C', 'D', max_tokens=0) == '' + assert state.instances[-1].reset_calls == 2 + +@pytest.mark.parametrize('template,args,expected', [ + ('{INPUT_0}', ('raw <|im_start|> prompt',), 'raw <|im_start|> prompt'), + ('Constant prompt', (), 'Constant prompt'), + ('{{INPUT_0}}', ('x',), '{x}'), + ('{INPUT_0}\n{INPUT_PLACEHOLDER}', ('X',), 'X\n{INPUT_PLACEHOLDER}'), +]) +def test_template_forms(template, args, expected, fake_runtime): + _, state = fake_runtime + _write_base_model('gpt2') + with paw.function(None, interpreter='gpt2', prompt_template=template, offline=True) as fn: + fn(*args, max_tokens=0) + assert state.instances[-1].tokenize_calls[0]['data'] == expected.encode() + +@pytest.mark.parametrize('template,error', [('{INPUT_1}', ValueError), ('{INPUT_0}{INPUT_2}', ValueError), (12, TypeError), (False, TypeError)]) +def test_invalid_template_precedes_native_load(template, error, fake_runtime): + _, state = fake_runtime + with pytest.raises(error): + paw.function(None, interpreter='gpt2', prompt_template=template, offline=True) + assert state.instances == [] + +@pytest.mark.parametrize('args,kwargs,error', [ + ((), {}, TypeError), (('a','b','c'), {}, TypeError), + (('a',32), {}, TypeError), (('a','b'), {'input_text':'c'}, TypeError), + (('a','b'), {'max_tokens':True}, ValueError), + (('a','b'), {'max_tokens':-1}, ValueError), +]) +def test_invalid_call_does_not_touch_native_state(args, kwargs, error, fake_runtime): + _, state = fake_runtime + _write_base_model('gpt2') + with paw.function(None, interpreter='gpt2', prompt_template='{INPUT_0}{INPUT_1}', offline=True) as fn: + with pytest.raises(error): + fn(*args, **kwargs) + assert state.instances[-1].reset_calls == 0 + assert state.instances[-1].tokenize_calls == [] + +@pytest.mark.parametrize('compiled', [False, True]) +def test_legacy_keyword_positional_calls_and_literal_slots(compiled, fake_runtime, tmp_path): + runtime, state = fake_runtime + if compiled: + directory = _write_compiled_program(tmp_path) + (directory/'prompt_template.txt').write_text('P{INPUT_0}{{INPUT_PLACEHOLDER}}S') + fn = runtime.PawFunction(directory, offline=True) + else: + _write_base_model('gpt2') + fn = paw.function(None, interpreter='gpt2', offline=True) + with fn: + fn(input_text='alpha', max_tokens=0) + fn('beta', 0, 0.0, None) + if compiled: + assert [c['data'] for c in state.instances[-1].tokenize_calls] == [b'P{INPUT_0}{', b'alpha}S', b'beta}S'] + else: + assert [c['data'] for c in state.instances[-1].tokenize_calls] == [b'alpha', b'beta'] + +@pytest.mark.parametrize('reference,kwargs', [('compiled-program', {}), (None, {'remote':True})]) +def test_override_not_allowed_for_compiled_or_remote(reference, kwargs): + with pytest.raises(ValueError, match='local adapter-free'): + paw.function(reference, prompt_template='{INPUT_0}', **kwargs) + + +@pytest.mark.parametrize("template,expected", [ + ("{INPUT_PLACEHOLDER}", ("", 0, "")), + ("{INPUT_0}{{INPUT_PLACEHOLDER}}", ("{INPUT_0}{", 0, "}")), +]) +def test_old_placeholder_parser_preserves_literal_braces(template, expected): + assert parse_template(template, "{INPUT_PLACEHOLDER}") == expected + + +@pytest.mark.parametrize("template", ["", "{INPUT_0}", "{INPUT_PLACEHOLDER}{INPUT_PLACEHOLDER}"]) +def test_old_placeholder_still_requires_exactly_one(template): + with pytest.raises(ValueError, match="exactly one"): + parse_template(template, "{INPUT_PLACEHOLDER}") + + +def test_numbered_binding_preserves_images_and_does_not_reparse_values(): + picture = paw.Image(b"snapshot not decoded at binding") + segments = parse_template("{INPUT_1}|{INPUT_0}|{INPUT_1}", "{INPUT_N}") + bound = bind_template(segments, "literal {INPUT_1}", picture) + assert bound == ("", picture, "|", "literal {INPUT_1}", "|", picture, "") + assert bound[1] is picture and bound[5] is picture + + +@pytest.mark.parametrize("template,args,error", [ + ("{INPUT_0}", (), TypeError), ("constant", ("extra",), TypeError), + ("{INPUT_0}", (["not a string"],), TypeError), +]) +def test_shared_binder_rejects_wrong_arity_and_types(template, args, error): + with pytest.raises(error): + bind_template(parse_template(template, "{INPUT_N}"), *args) + + +def test_unknown_placeholder_contract_rejected(): + with pytest.raises(ValueError, match="Unsupported placeholder"): + parse_template("{INPUT_0}", "unknown") + + +def test_custom_text_template_rejects_an_image(fake_runtime): + _write_base_model("gpt2") + with paw.function(None, interpreter="gpt2", prompt_template="{INPUT_0}", offline=True) as fn: + with pytest.raises(TypeError, match="Text-only"): + fn(paw.Image(b"encoded snapshot")) + + +def test_closed_numbered_text_function_rejects_calls(fake_runtime): + _write_base_model("gpt2") + fn = paw.function(None, interpreter="gpt2", prompt_template="{INPUT_0}", offline=True) + fn.close() + with pytest.raises(RuntimeError, match="closed"): + fn("input") diff --git a/tests/test_vision_metadata.py b/tests/test_vision_metadata.py index 6aca5fa..212a38f 100644 --- a/tests/test_vision_metadata.py +++ b/tests/test_vision_metadata.py @@ -40,10 +40,8 @@ def manifest(): "adapter_format": "gguf_lora", "input": {"format": "content_parts", "types": ["text", "image"]}, "prompt_template": { - "format": "chat_messages", - "system_prompt_file": "prompt_template.txt", - "chat_format": "qwen3.5", - "enable_thinking": False, + "format": "rendered_text", + "placeholder": "{INPUT_N}", }, "program_assets": {"adapter_filename": "adapter.gguf"}, "local_sdk": { @@ -126,7 +124,7 @@ def test_required_contract_sections_cannot_be_omitted(manifest, path): (("manifest_version",), 3), (("manifest_version",), 1), (("input", "types"), ["text"]), (("input", "types"), ["image", "audio"]), (("input", "format"), "named_slots"), (("input", "extra"), True), - (("prompt_template", "format"), "rendered_text"), + (("prompt_template", "format"), "chat_messages"), (("prompt_template", "chat_format"), "unknown"), (("prompt_template", "system_prompt_file"), "../prompt.txt"), (("prompt_template", "enable_thinking"), True), @@ -316,7 +314,8 @@ def find_spec(self, fullname, path=None, target=None): def test_builtin_vision_base_contract_is_complete_and_defensively_copied(): original = cache.get_base_runtime_manifest("Qwen/Qwen3.5-0.8B") - assert original["base_inference"] == {"contract_version": 1, "format": "chat_messages"} + assert original["base_inference"] == {"contract_version": 1, "format": "rendered_text", + "placeholder": "{INPUT_N}", "template": "{INPUT_0}"} assert cache._normalize_runtime_manifest(original) == original clone = cache.get_base_runtime_manifest("Qwen/Qwen3.5-0.8B") clone["local_sdk"]["vision"]["preprocessing"]["image_max_tokens"] = 64 @@ -339,7 +338,8 @@ def test_named_builtin_pins_both_asset_identities(which, field, value): @pytest.mark.parametrize("contract", [ - {"contract_version": 1, "format": "chat_messages"}, + {"contract_version": 1, "format": "rendered_text", + "placeholder": "{INPUT_N}", "template": "{INPUT_0}"}, None, ]) def test_vision_base_contract_optional_for_compiled_artifacts(manifest, contract): @@ -357,3 +357,27 @@ def test_vision_base_contract_optional_for_compiled_artifacts(manifest, contract def test_unsupported_vision_base_contract_rejected(manifest, contract): manifest["base_inference"] = contract assert cache._normalize_runtime_manifest(manifest) is None + + +@pytest.mark.parametrize("template", ["{INPUT_0}", "{INPUT_0}{INPUT_0}", "{INPUT_1} then {INPUT_0}"]) +def test_numbered_image_templates_import_without_rewriting(tmp_path, manifest, template): + path = write_program(tmp_path / "source", metadata(manifest), template) + assert cache.validate_program_assets_dir(path, PID) + assert (path / "prompt_template.txt").read_text() == template + + +@pytest.mark.parametrize("template", ["{INPUT_1}", "{INPUT_0}{INPUT_2}"]) +def test_image_template_gaps_rejected_before_execution(tmp_path, manifest, template): + path = write_program(tmp_path / "source", metadata(manifest), template) + assert not cache.validate_program_assets_dir(path, PID) + + +def test_old_image_chat_contract_is_rejected_without_text_fallback(tmp_path, manifest): + manifest["prompt_template"] = { + "format": "chat_messages", "system_prompt_file": "prompt_template.txt", + "chat_format": "qwen3.5", "enable_thinking": False, + } + path = write_program(tmp_path / "source", metadata(manifest), "{INPUT_0}") + assert cache.declares_vision(metadata(manifest)) + assert cache.resolve_runtime_manifest(metadata(manifest)) is None + assert not cache.validate_program_assets_dir(path, PID) diff --git a/tests/test_vision_prompt.py b/tests/test_vision_prompt.py new file mode 100644 index 0000000..e6c2e9e --- /dev/null +++ b/tests/test_vision_prompt.py @@ -0,0 +1,268 @@ +"""Exercise actual prompt/chunk code with a deterministic native API double.""" + +import ctypes +from types import SimpleNamespace as NS + +import pytest + +import base64 +from io import BytesIO +from PIL import Image as PILImage +from programasweights._inputs import Image +from programasweights._runtime_vision import _Runtime + +def evaluate_prompt(handler, model, content, lib): + model.chat_handler = handler + return _Runtime._eval_prompt(NS(llm=model, lib=lib), content) + + +def load_image(url): + with PILImage.open(BytesIO(base64.b64decode(url.split(',', 1)[1]))) as pixels: + return 'A' if pixels.getpixel((0, 0)) == (255, 0, 0) else 'B' + + +class InputText(ctypes.Structure): + _fields_ = [("text", ctypes.c_char_p), ("text_len", ctypes.c_size_t), + ("add_special", ctypes.c_bool), ("parse_special", ctypes.c_bool)] + + +class Vector(list): + def __getitem__(self, key): + result = super().__getitem__(key) + return Vector(result) if isinstance(key, slice) else result + + def __setitem__(self, key, value): + if isinstance(key, slice) and isinstance(value, int): + value = [value] * (key.stop - key.start) + super().__setitem__(key, value) + + def tolist(self): + return list(self) + + +@pytest.fixture +def native(): + state = NS(chunks=[], flags=None, loaded=[], freed=[], chunk_freed=False, + images=[], fail_tokenize=False, fail_image=False) + model = NS(n_tokens=0, input_ids=Vector([0] * 256), n_ctx=lambda: 256, + _ctx=NS(ctx=None), n_batch=128) + def evaluate(tokens): + model.input_ids[model.n_tokens:model.n_tokens + len(tokens)] = tokens + model.n_tokens += len(tokens) + model.eval = evaluate + def tokenize(ctx, chunks, ptr, bitmaps, count): + prompt = ptr._obj + state.flags = (prompt.add_special, prompt.parse_special) + state.text = prompt.text[:prompt.text_len].decode() + state.chunks = [] + for i, piece in enumerate(state.text.split("<__media__>")): + if i: + state.chunks.append(NS(kind=2, tokens=[0, 0], image=bitmaps[i - 1])) + if piece: + state.chunks.append(NS(kind=1, tokens=list(piece.encode()))) + return 1 if state.fail_tokenize else 0 + def image_eval(ctx, model_ctx, chunk, past, seq, batch, logits_last, new): + state.images.append((chunk.image, logits_last)) + new._obj.value = past.value + len(chunk.tokens) + return 1 if state.fail_image else 0 + def text_tokens(chunk, count): + count._obj.value = len(chunk.tokens) + return chunk.tokens + mtmd = NS( + mtmd_default_marker=lambda: b"<__media__>", mtmd_input_text=InputText, + mtmd_bitmap_p_ctypes=ctypes.c_void_p, + mtmd_input_chunks_init=lambda: 1, + mtmd_input_chunks_free=lambda chunks: setattr(state, "chunk_freed", True), + mtmd_bitmap_free=state.freed.append, mtmd_tokenize=tokenize, + mtmd_input_chunks_size=lambda chunks: len(state.chunks), + mtmd_input_chunks_get=lambda chunks, i: state.chunks[i], + mtmd_input_chunk_get_n_tokens=lambda c: len(c.tokens), + mtmd_input_chunk_get_type=lambda c: c.kind, + mtmd_input_chunk_get_tokens_text=text_tokens, + mtmd_helper_eval_chunk_single=image_eval, + MTMD_INPUT_CHUNK_TYPE_TEXT=1, MTMD_INPUT_CHUNK_TYPE_IMAGE=2, + ) + def bitmap(data): + state.loaded.append(data) + return len(state.loaded) + handler = NS(_mtmd_cpp=mtmd, mtmd_ctx=1, load_image=load_image, + _create_bitmap_from_bytes=bitmap) + lib = NS(llama_pos=ctypes.c_int, llama_seq_id=ctypes.c_int) + return state, handler, model, lib + + +def text(value): + return value + + +def image(value): + return Image(PILImage.new("RGB", (3, 5), "red" if value == "A" else "blue")) + + +def test_exact_text_and_image_order_no_wrapping_or_special_token_insertion(native): + state, handler, model, lib = native + prompt = evaluate_prompt(handler, model, [text(" pre"), text("fix "), image("B"), + text("mid"), image("A"), image("B"), text("end\n ")], lib) + assert state.text == " prefix <__media__>mid<__media__><__media__>end\n " + assert state.flags == (False, True) + assert state.loaded == ["B", "A", "B"] + assert state.images == [(1, True), (2, True), (3, True)] + assert state.freed == [1, 2, 3] and state.chunk_freed + assert len(prompt) == len(" prefix midend\n ") + 6 + + +def test_text_only_calls_take_the_same_path_without_bitmaps(native): + state, handler, model, lib = native + assert evaluate_prompt(handler, model, [text("he"), text("llo")], lib) == list(b"hello") + assert state.loaded == [] and state.flags == (False, True) + + +def test_image_can_end_prompt_and_requests_logits(native): + state, handler, model, lib = native + assert evaluate_prompt(handler, model, [image("A")], lib) == [0, 0] + assert state.images == [(1, True)] + + +@pytest.mark.parametrize("parts", [[text("<__media__>")], [text("<__"), text("media__>")]]) +def test_literal_media_marker_cannot_create_unbound_images(native, parts): + state, handler, model, lib = native + with pytest.raises(ValueError, match="reserved"): + evaluate_prompt(handler, model, parts, lib) + assert not state.loaded + + +@pytest.mark.parametrize("failure", ["fail_tokenize", "fail_image", "overflow"]) +def test_native_errors_release_all_allocations(native, failure): + state, handler, model, lib = native + if failure == "overflow": + model.n_ctx = lambda: 2 + else: + setattr(state, failure, True) + with pytest.raises((ValueError, RuntimeError)): + evaluate_prompt(handler, model, [image("A"), image("B")], lib) + assert state.freed == [1, 2] and state.chunk_freed + + +def test_nonempty_runtime_rejected_before_allocating(native): + state, handler, model, lib = native + model.n_tokens = 1 + with pytest.raises(RuntimeError, match='reset runtime'): + evaluate_prompt(handler, model, ['hello'], lib) + assert not state.loaded and not state.chunk_freed + + +def test_empty_prompt_releases_chunks(native): + state, handler, model, lib = native + with pytest.raises(ValueError, match='no tokens'): + evaluate_prompt(handler, model, [''], lib) + assert state.chunk_freed + + +def test_partial_bitmap_allocation_releases_previous_image(native): + state, handler, model, lib = native + original = handler._create_bitmap_from_bytes + handler._create_bitmap_from_bytes = lambda data: original(data) if not state.loaded else None + with pytest.raises(RuntimeError, match='bitmap'): + evaluate_prompt(handler, model, [image('A'), image('B')], lib) + assert state.freed == [1] and not state.chunk_freed + + +def test_chunk_allocation_failure_releases_images(native): + state, handler, model, lib = native + handler._mtmd_cpp.mtmd_input_chunks_init = lambda: None + with pytest.raises(RuntimeError, match='allocate'): + evaluate_prompt(handler, model, [image('A')], lib) + assert state.freed == [1] + + +@pytest.mark.parametrize('end', [0, 256]) +def test_invalid_image_position_releases_resources(native, end): + state, handler, model, lib = native + def bad_position(*args): + args[-1]._obj.value = end + return 0 + handler._mtmd_cpp.mtmd_helper_eval_chunk_single = bad_position + with pytest.raises(RuntimeError, match='end position'): + evaluate_prompt(handler, model, [image('A')], lib) + assert state.freed == [1] and state.chunk_freed + + +def test_image_ending_prompt_marks_live_logits(native): + state, handler, model, lib = native + model._requires_eval = True + evaluate_prompt(handler, model, [image('A')], lib) + assert model._requires_eval is False + + +from types import SimpleNamespace +from unittest.mock import Mock +import sys +import types +import pytest +from programasweights._runtime_vision import _Runtime + + +from contextlib import contextmanager + + +@contextmanager +def fake_grammar(convert): + chat = types.ModuleType("llama_cpp.llama_chat_format") + chat._grammar_for_response_format = convert + with pytest.MonkeyPatch.context() as patcher: + patcher.setitem(sys.modules, "llama_cpp.llama_chat_format", chat) + yield convert + + +def runtime(): + result = {'choices': [{'text': ' OK ', 'finish_reason': 'stop'}], + 'usage': {'prompt_tokens': 3, 'completion_tokens': 1}} + native = Mock(return_value=result) + return SimpleNamespace(llm=SimpleNamespace(create_completion=native), + _eval_prompt=Mock(return_value=[1, 2, 3])), result + + +def test_exact_prompt_tokens_and_generation_options_preserved(): + owner, result = runtime() + parts = ('one', 'two') + actual = _Runtime._complete_prompt(owner, parts, max_tokens=17, temperature=0.25) + owner._eval_prompt.assert_called_once_with(parts) + owner.llm.create_completion.assert_called_once_with(prompt=[1, 2, 3], max_tokens=17, temperature=0.25) + assert actual is result + + +def test_logits_processor_passed_unchanged(): + owner, _ = runtime() + processor = object() + _Runtime._complete_prompt(owner, ('input',), max_tokens=None, temperature=0, + logits_processor=processor) + assert owner.llm.create_completion.call_args.kwargs['logits_processor'] is processor + + +@pytest.mark.parametrize('format', [{'type': 'json_object'}, {'type': 'json_object', 'schema': {'type': 'object'}}, {'type': 'text'}]) +def test_existing_backend_grammar_conversion_reused(format): + owner, _ = runtime() + grammar = object() + with fake_grammar(Mock(return_value=grammar)) as convert: + _Runtime._complete_prompt(owner, ('input',), max_tokens=8, temperature=0, + response_format=format) + convert.assert_called_once_with(format) + assert owner.llm.create_completion.call_args.kwargs['grammar'] is grammar + + +def test_grammar_error_precedes_model_evaluation(): + owner, _ = runtime() + with fake_grammar(Mock(side_effect=ValueError('bad schema'))): + with pytest.raises(ValueError, match='bad schema'): + _Runtime._complete_prompt(owner, ('input',), max_tokens=8, temperature=0, + response_format={'type': 'json_object'}) + owner._eval_prompt.assert_not_called() + owner.llm.create_completion.assert_not_called() + + +def test_evaluation_error_prevents_generation(): + owner, _ = runtime() + owner._eval_prompt.side_effect = RuntimeError('native failure') + with pytest.raises(RuntimeError, match='native failure'): + _Runtime._complete_prompt(owner, ('input',), max_tokens=8, temperature=0) + owner.llm.create_completion.assert_not_called() diff --git a/tests/test_vision_result.py b/tests/test_vision_result.py index 71cb09f..1f0d28e 100644 --- a/tests/test_vision_result.py +++ b/tests/test_vision_result.py @@ -13,7 +13,7 @@ def completion(text=" result ", reason="stop", usage=None): - return {"choices": [{"message": {"content": text}, "finish_reason": reason}], + return {"choices": [{"text": text, "finish_reason": reason}], "usage": usage} @@ -158,11 +158,11 @@ def test_elapsed_includes_preparation_both_locks_and_cleanup(program, backend, m state, _, module = backend clock = [100.0] monkeypatch.setattr(module, "perf_counter", lambda: clock[0]) - prepare = module._to_chat_content - def prepare_input(*parts): + prepare = module._to_prompt + def prepare_input(*parts, **kwargs): clock[0] += 1 - return prepare(*parts) - monkeypatch.setattr(module, "_to_chat_content", prepare_input) + return prepare(*parts, **kwargs) + monkeypatch.setattr(module, "_to_prompt", prepare_input) class TimedLock: def __init__(self, lock, seconds): self.lock, self.seconds = lock, seconds @@ -194,7 +194,7 @@ def test_serialized_calls_keep_their_own_results(program, backend, manifest, sam started, release, attempted = threading.Event(), threading.Event(), threading.Event() shared_counts = {"completion_tokens": 0} def infer(kwargs): - text = kwargs["messages"][-1]["content"][0]["text"] + text = bytes(kwargs["prompt"]).decode("utf-8") if text == "first": started.set() assert release.wait(5) @@ -234,7 +234,7 @@ def test_usage_is_copied_before_releasing_the_shared_runtime(program, backend, m counts = {"completion_tokens": 0} first_done, second_done = threading.Event(), threading.Event() def infer(kwargs): - text = kwargs["messages"][-1]["content"][0]["text"] + text = bytes(kwargs["prompt"]).decode("utf-8") counts["completion_tokens"] = 1 if text == "first" else 2 return completion(text=text, usage=counts) state.completion = infer diff --git a/tests/test_vision_runtime.py b/tests/test_vision_runtime.py index 1634505..ce90bb9 100644 --- a/tests/test_vision_runtime.py +++ b/tests/test_vision_runtime.py @@ -32,7 +32,7 @@ def forbidden(*args, **kwargs): (config.get_base_models_dir() / "base.gguf").write_bytes(MODEL) (config.get_base_models_dir() / "mmproj.gguf").write_bytes(PROJECTOR) state = types.SimpleNamespace( - models=[], calls=[], events=[], selected=None, handle=0, + models=[], calls=[], prompts=[], events=[], selected=None, handle=0, support=True, init_fail=False, completion=None, create_error=None, ) lib = types.ModuleType("llama_cpp") @@ -72,10 +72,7 @@ def init(self, path, model, params): event("projector_init", vars(params)) return None if state.init_fail else object() def __call__(self, **kwargs): - state.calls.append(kwargs) - if state.completion: - return state.completion(kwargs) - return {"choices": [{"message": {"content": " result "}}]} + raise AssertionError("A complete prompt must not use the chat handler") class Llama: def __init__(self, **kwargs): if state.create_error: @@ -93,7 +90,13 @@ def reset(self): event("reset") def create_chat_completion(self, **kwargs): assert not self.closed - return self.handler(llama=self, **kwargs) + raise AssertionError("A complete prompt must not use chat completion") + def create_completion(self, **kwargs): + assert not self.closed + state.calls.append(kwargs) + if state.completion: + return state.completion(kwargs) + return {"choices": [{"text": " result "}]} def token_eos(self): return 1 def close(self): @@ -104,11 +107,19 @@ def close(self): lib.Llama = Llama chat = types.ModuleType("llama_cpp.llama_chat_format") chat.MTMDChatHandler = Handler + chat._grammar_for_response_format = lambda value: ("grammar", value) monkeypatch.setitem(sys.modules, "llama_cpp", lib) monkeypatch.setitem(sys.modules, "llama_cpp.llama_chat_format", chat) for name in ("programasweights.runtime_llamacpp", "programasweights._runtime_vision"): monkeypatch.delitem(sys.modules, name, raising=False) runtime = importlib.import_module("programasweights._runtime_vision") + # Lifecycle tests stop at the native tokenizer boundary. Chunk evaluation + # and allocation cleanup are exercised in test_vision_prompt.py. + def evaluate_prompt(owner, parts): + prompt, urls = runtime._to_prompt(*parts, image_marker="<__media__>") + state.prompts.append((prompt, urls)) + return list(prompt.encode("utf-8")) + monkeypatch.setattr(runtime._Runtime, "_eval_prompt", evaluate_prompt) yield state, lib, runtime assert not runtime._POOL, "Test leaked a shared runtime" sys.modules.pop("programasweights.runtime_llamacpp", None) @@ -118,20 +129,22 @@ def close(self): @pytest.fixture def program(backend, manifest): return write_program(config.get_programs_dir() / PID, metadata(manifest), - "Literal {INPUT_PLACEHOLDER}; do not substitute.") + "{INPUT_0}") def second_program(program, manifest): meta = metadata(copy.deepcopy(manifest)) meta["program_id"] = "d" * 20 - directory = write_program(program.parent / meta["program_id"], meta, "Different system prompt") + directory = write_program(program.parent / meta["program_id"], meta, "{INPUT_0}") (directory / "adapter.gguf").write_bytes(ADAPTER[:-1] + b"B") return directory @pytest.mark.parametrize("entrypoint", ["cached", "local"]) -def test_public_ordered_parts_preserve_literal_prompt_and_native_dimensions(program, backend, tmp_path, entrypoint): +def test_public_template_orders_parts_without_wrapping_or_resizing(program, backend, tmp_path, entrypoint): state, _, _ = backend + template = "Literal {INPUT_PLACEHOLDER}; {INPUT_1}/{INPUT_0}/{INPUT_1} End" + (program / "prompt_template.txt").write_text(template) reference = PID if entrypoint == "local": reference = tmp_path / "images.paw" @@ -140,16 +153,17 @@ def test_public_ordered_parts_preserve_literal_prompt_and_native_dimensions(prog output.write(path, path.name) image = paw.Image(PILImage.new("RGB", (31, 27), "red")) with paw.function(reference, offline=True) as fn: - assert fn("one", image, "", "two", image, max_tokens=32) == "result" + assert fn("one", image, max_tokens=32) == "result" call = state.calls[-1] - assert call["messages"][0]["content"] == "Literal {INPUT_PLACEHOLDER}; do not substitute." - parts = call["messages"][1]["content"] - assert [part["type"] for part in parts] == ["text", "image_url", "text", "text", "image_url"] - assert [part["text"] for part in parts if part["type"] == "text"] == ["one", "", "two"] + prompt, urls = state.prompts[-1] + assert prompt == "Literal {INPUT_PLACEHOLDER}; <__media__>/one/<__media__> End" + assert call["prompt"] == list(prompt.encode("utf-8")) + assert len(urls) == 2 and urls[0] == urls[1] import base64, io - with PILImage.open(io.BytesIO(base64.b64decode(parts[1]["image_url"]["url"].split(",", 1)[1]))) as actual: + with PILImage.open(io.BytesIO(base64.b64decode(urls[0].split(",", 1)[1]))) as actual: assert actual.size == (31, 27) - assert call["enable_thinking"] is False + assert actual.getpixel((0, 0)) == (255, 0, 0) + assert "messages" not in call and "enable_thinking" not in call assert fn.interpreter == "Qwen/Qwen3.5-0.8B" assert state.models[0].closed @@ -225,7 +239,7 @@ def test_changed_adapter_is_rejected_before_native_load(program, backend): assert not [e for e in state.events if e[0] == "load"] -@pytest.mark.parametrize("parts,error", [((), ValueError), ((["bad"],), TypeError), (("ok", 32), TypeError)]) +@pytest.mark.parametrize("parts,error", [((), TypeError), ((["bad"],), TypeError), (("ok", 32), TypeError)]) def test_invalid_content_does_not_touch_native_state(program, backend, parts, error): state, _, _ = backend with paw.function(PID, offline=True) as fn: @@ -261,7 +275,8 @@ def test_response_format_and_generation_options_pass_through(program, backend): schema = {"type": "json_object", "schema": {"type": "object"}} with paw.function(PID, offline=True) as fn: fn("read", temperature=.25, max_tokens=16, response_format=schema) - assert state.calls[-1]["response_format"] == schema + assert state.calls[-1]["grammar"] == ("grammar", schema) + assert "response_format" not in state.calls[-1] assert state.calls[-1]["max_tokens"] == 16 assert state.calls[-1]["temperature"] == .25 @@ -289,7 +304,7 @@ def processor(ids, scores): def completion(kwargs): fallback = kwargs["logits_processor"][0]([], [0., 0., 0.]) assert fallback == [0., 1e10, 0.] - return {"choices": [{"message": {"content": "discarded"}}]} + return {"choices": [{"text": "discarded"}]} state.completion = completion with paw.function(PID, offline=True) as fn: with pytest.raises(ValueError, match="bad user callback") as caught: @@ -322,7 +337,7 @@ def test_concurrent_functions_serialize_and_close_waits(program, backend, manife def completion(kwargs): started.set() assert release.wait(5) - return {"choices": [{"message": {"content": "ok"}}]} + return {"choices": [{"text": "ok"}]} state.completion = completion def second_call(): attempted.set() @@ -390,9 +405,9 @@ def test_closed_function_rejects_calls(program): def test_partial_output_returns_text_but_nontext_output_errors(program, backend): state, _, _ = backend with paw.function(PID, offline=True) as fn: - state.completion = lambda _: {"choices": [{"finish_reason": "length", "message": {"content": "partial"}}]} + state.completion = lambda _: {"choices": [{"finish_reason": "length", "text": "partial"}]} assert fn("read", max_tokens=1) == "partial" - state.completion = lambda _: {"choices": [{"message": {"content": None}}]} + state.completion = lambda _: {"choices": [{"text": None}]} with pytest.raises(RuntimeError, match="no text content"): fn("read") @@ -440,7 +455,8 @@ def test_forked_process_rejected_before_acquiring_inherited_locks(program, backe def vision_base(backend, manifest, monkeypatch): builtin = copy.deepcopy(manifest) builtin["runtime_id"] = "qwen3.5-0.8b-q8_0" - builtin["base_inference"] = {"contract_version": 1, "format": "chat_messages"} + builtin["base_inference"] = {"contract_version": 1, "format": "rendered_text", + "placeholder": "{INPUT_N}", "template": "{INPUT_0}"} monkeypatch.setitem(cache.BUILTIN_VISION_RUNTIMES, builtin["runtime_id"], builtin) def forbidden(*args, **kwargs): pytest.fail("Base interpreter must not use program or Hub resolution") @@ -450,13 +466,14 @@ def forbidden(*args, **kwargs): return builtin -def test_base_vision_accepts_ordered_parts_without_system_or_adapter(vision_base, backend): +def test_base_vision_uses_explicit_numbered_template_without_adapter(vision_base, backend): state, _, _ = backend - with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True) as fn: + with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True, + prompt_template="{INPUT_2}{INPUT_1}{INPUT_0}{INPUT_1}") as fn: assert fn("first", paw.Image(PILImage.new("RGB", (31, 27))), "last") == "result" - messages = state.calls[-1]["messages"] - assert len(messages) == 1 and messages[0]["role"] == "user" - assert [p["type"] for p in messages[0]["content"]] == ["text", "image_url", "text"] + prompt, urls = state.prompts[-1] + assert prompt == "last<__media__>first<__media__>" + assert len(urls) == 2 and urls[0] == urls[1] assert fn._adapter is None and fn.spec == "" assert "base interpreter" in repr(fn) assert not [e for e in state.events if e[0] == "load"] @@ -468,10 +485,9 @@ def test_base_vision_string_calls_remain_stateless(vision_base, backend): with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True) as fn: fn("old prompt") fn("new prompt", max_tokens=3) - assert state.calls[-1]["messages"] == [ - {"role": "user", "content": [{"type": "text", "text": "new prompt"}]}, - ] - assert state.calls[-1]["enable_thinking"] is False + assert state.prompts == [("old prompt", []), ("new prompt", [])] + assert state.calls[-1]["prompt"] == list(b"new prompt") + assert "enable_thinking" not in state.calls[-1] def test_base_and_compiled_share_and_cannot_leak_lora(program, vision_base, backend): @@ -484,7 +500,7 @@ def test_base_and_compiled_share_and_cannot_leak_lora(program, vision_base, back selected = [] def completion(kwargs): selected.append(state.selected) - return {"choices": [{"message": {"content": "ok"}}]} + return {"choices": [{"text": "ok"}]} state.completion = completion for fn in (compiled, base, compiled, base): fn("read") @@ -528,3 +544,25 @@ def forbidden(*args, **kwargs): with pytest.raises(ImportError, match="0.3.35"): paw.function(None, interpreter="Qwen/Qwen3.5-0.8B") assert not state.models + + +@pytest.mark.parametrize("template,inputs,expected", [ + ("{INPUT_1}|{INPUT_0}|{INPUT_1}", ("A", "B"), "B|A|B"), + ("constant", (), "constant"), + ("{INPUT_0}", ("<|im_start|>literal {INPUT_1}",), "<|im_start|>literal {INPUT_1}"), +]) +def test_text_only_inputs_on_vision_base_use_same_template_contract(vision_base, backend, template, inputs, expected): + state, _, _ = backend + with paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True, + prompt_template=template) as fn: + assert fn(*inputs) == "result" + assert state.prompts == [(expected, [])] + + +@pytest.mark.parametrize("template,error", [(False, TypeError), ("{INPUT_1}", ValueError)]) +def test_bad_vision_template_precedes_model_load(vision_base, backend, template, error): + state, _, _ = backend + with pytest.raises(error): + paw.function(None, interpreter="Qwen/Qwen3.5-0.8B", offline=True, + prompt_template=template) + assert not state.models From e3a4332c7646193aa20dfed209f3472975d01530 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 20:13:20 -0400 Subject: [PATCH 2/7] Simplify local image examples in READMEs --- PYPI_README.md | 6 +++--- README.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PYPI_README.md b/PYPI_README.md index 3933c04..b6ceb48 100644 --- a/PYPI_README.md +++ b/PYPI_README.md @@ -135,10 +135,10 @@ prompt = ( "<|im_start|>user\n{INPUT_0}\n{INPUT_1}<|im_end|>\n" "<|im_start|>assistant\n\n\n\n\n" ) -with paw.function( +describe = paw.function( None, interpreter="Qwen/Qwen3.5-0.8B", prompt_template=prompt, -) as describe: - print(describe("Describe this image.", paw.Image("photo.png"))) +) +print(describe("Describe this image.", paw.Image("photo.png"))) ``` `{INPUT_0}`, `{INPUT_1}`, and subsequent numbered slots bind positional arguments diff --git a/README.md b/README.md index 55c9d19..acdaffe 100644 --- a/README.md +++ b/README.md @@ -135,10 +135,10 @@ prompt = ( "<|im_start|>user\n{INPUT_0}\n{INPUT_1}<|im_end|>\n" "<|im_start|>assistant\n\n\n\n\n" ) -with paw.function( +describe = paw.function( None, interpreter="Qwen/Qwen3.5-0.8B", prompt_template=prompt, -) as describe: - print(describe("Describe this image.", paw.Image("photo.png"))) +) +print(describe("Describe this image.", paw.Image("photo.png"))) ``` `{INPUT_0}`, `{INPUT_1}`, and subsequent numbered slots bind positional arguments From 195b7addf6ea6fd39fe15c8169176e9ccfdb2b36 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 20:14:49 -0400 Subject: [PATCH 3/7] Focus compiled image guidance on usage --- PYPI_README.md | 6 ++---- README.md | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/PYPI_README.md b/PYPI_README.md index b6ceb48..45c909a 100644 --- a/PYPI_README.md +++ b/PYPI_README.md @@ -147,10 +147,8 @@ adds no chat wrapper; the template supplies roles and the assistant prefix. The same option supports text-only models such as `Qwen/Qwen3-0.6B`, with string arguments. Existing text calls without this option retain their behavior. -Compiled image programs read the complete template from their `.paw` bundle: -load with `paw.function("./locator.paw")` and supply the arguments its slots -require. Older experimental image bundles using `chat_messages` metadata must -be re-exported with a complete numbered template; there is no automatic fallback. +Load a compiled image program with `paw.function("./locator.paw")`, then pass +the arguments required by its prompt template. See the [image API reference](https://programasweights.readthedocs.io/en/latest/api-reference/python-sdk/#local-textimage-calls) for multiple images, input formats, and result metadata. diff --git a/README.md b/README.md index acdaffe..d23ecbc 100644 --- a/README.md +++ b/README.md @@ -147,10 +147,8 @@ adds no chat wrapper; the template supplies roles and the assistant prefix. The same option supports text-only models such as `Qwen/Qwen3-0.6B`, with string arguments. Existing text calls without this option retain their behavior. -Compiled image programs read the complete template from their `.paw` bundle: -load with `paw.function("./locator.paw")` and supply the arguments its slots -require. Older experimental image bundles using `chat_messages` metadata must -be re-exported with a complete numbered template; there is no automatic fallback. +Load a compiled image program with `paw.function("./locator.paw")`, then pass +the arguments required by its prompt template. See the [image API reference](docs/api-reference/python-sdk.md#local-textimage-calls) for multiple images, input formats, and result metadata. From d1a263089eab6e2bf61cd29769fbce50eec7efe3 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 20:16:18 -0400 Subject: [PATCH 4/7] Clarify numbered template usage in READMEs --- PYPI_README.md | 9 ++++----- README.md | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/PYPI_README.md b/PYPI_README.md index 45c909a..f9821b5 100644 --- a/PYPI_README.md +++ b/PYPI_README.md @@ -141,11 +141,10 @@ describe = paw.function( print(describe("Describe this image.", paw.Image("photo.png"))) ``` -`{INPUT_0}`, `{INPUT_1}`, and subsequent numbered slots bind positional arguments -where they appear in the template. Slots may repeat or change order. The SDK -adds no chat wrapper; the template supplies roles and the assistant prefix. -The same option supports text-only models such as `Qwen/Qwen3-0.6B`, with string -arguments. Existing text calls without this option retain their behavior. +`{INPUT_0}`, `{INPUT_1}`, etc. insert positional arguments into the prompt. +Placeholders can repeat or appear in any order. The template supplies the +model's role markers and assistant prefix. For text-only models such as +`Qwen/Qwen3-0.6B`, pass string inputs. Load a compiled image program with `paw.function("./locator.paw")`, then pass the arguments required by its prompt template. diff --git a/README.md b/README.md index d23ecbc..f4e25c0 100644 --- a/README.md +++ b/README.md @@ -141,11 +141,10 @@ describe = paw.function( print(describe("Describe this image.", paw.Image("photo.png"))) ``` -`{INPUT_0}`, `{INPUT_1}`, and subsequent numbered slots bind positional arguments -where they appear in the template. Slots may repeat or change order. The SDK -adds no chat wrapper; the template supplies roles and the assistant prefix. -The same option supports text-only models such as `Qwen/Qwen3-0.6B`, with string -arguments. Existing text calls without this option retain their behavior. +`{INPUT_0}`, `{INPUT_1}`, etc. insert positional arguments into the prompt. +Placeholders can repeat or appear in any order. The template supplies the +model's role markers and assistant prefix. For text-only models such as +`Qwen/Qwen3-0.6B`, pass string inputs. Load a compiled image program with `paw.function("./locator.paw")`, then pass the arguments required by its prompt template. From ec322717372dc9a34f3291d3864d7f309bf32ee3 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 20:34:09 -0400 Subject: [PATCH 5/7] Simplify text template example in API reference --- docs/api-reference/python-sdk.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/api-reference/python-sdk.md b/docs/api-reference/python-sdk.md index 3b1e7d6..b760ae7 100644 --- a/docs/api-reference/python-sdk.md +++ b/docs/api-reference/python-sdk.md @@ -165,14 +165,13 @@ prompt = ( "<|im_start|>user\n{INPUT_1}<|im_end|>\n" "<|im_start|>assistant\n\n\n\n\n" ) -with paw.function( +answer = paw.function( None, interpreter="Qwen/Qwen3-0.6B", prompt_template=prompt, -) as answer: - output = answer( - "Answer using only the facts in the question.", - "Alice owns three cats. How many cats does Alice own?", - max_tokens=32, - ) +) +output = answer( + "Answer using only the facts in the question.", + "Alice owns three cats. How many cats does Alice own?", +) ``` The template controls role delimiters, examples, whitespace, thinking markers From aa499f6e8b3096a36e1712a593ce651f2eaa7261 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 20:45:01 -0400 Subject: [PATCH 6/7] Simplify image template example in API reference --- docs/api-reference/python-sdk.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/api-reference/python-sdk.md b/docs/api-reference/python-sdk.md index b760ae7..ea41dd9 100644 --- a/docs/api-reference/python-sdk.md +++ b/docs/api-reference/python-sdk.md @@ -222,12 +222,12 @@ prompt = ( "Before: {INPUT_1}\nAfter: {INPUT_2}\n{INPUT_0}<|im_end|>\n" "<|im_start|>assistant\n\n\n\n\n" ) -with paw.function( +compare = paw.function( None, interpreter="Qwen/Qwen3.5-0.8B", prompt_template=prompt, -) as compare: - answer = compare( - "What changed?", paw.Image("before.png"), paw.Image("after.png"), - ) +) +answer = compare( + "What changed?", paw.Image("before.png"), paw.Image("after.png"), +) print(answer) ``` From 3a0e336ddaa947ff6e58e5263da5d0bcbc0a3177 Mon Sep 17 00:00:00 2001 From: da03 Date: Wed, 7 Oct 2026 21:02:47 -0400 Subject: [PATCH 7/7] Remove obsolete image migration guidance --- docs/api-reference/python-sdk.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/api-reference/python-sdk.md b/docs/api-reference/python-sdk.md index ea41dd9..e598a30 100644 --- a/docs/api-reference/python-sdk.md +++ b/docs/api-reference/python-sdk.md @@ -269,11 +269,6 @@ The image runtime manifest declares the template contract as: This is the template field inside the full runtime manifest, not a standalone manifest. Keep the required model, projector, preprocessing and adapter metadata. -Older experimental image bundles with `chat_messages`, `system_prompt_file`, -`chat_format` or `enable_thinking` prompt metadata are rejected. Re-export those -bundles with the new contract and a full template matching the adapter's training -prompt. Changing the metadata alone does not reconstruct the missing roles or -assistant prefix. Existing text bundles do not need this migration. `paw.Image(source)` accepts a local file path, encoded image bytes, or a Pillow image. For an image URL, download the file first. Plain strings are text inputs.