From 276aee7662d858faeb02228d4f561fff64d12321 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 14:29:31 +0100 Subject: [PATCH 1/8] Price runs from the user's own price list, not a table in the code A standard run's cost estimate came from 13 OpenAI models written in run.py, matched by name prefix. The list is now the user's: fetched on request from a public list, with the user's own rows over it, and a model is priced by exactly its name. A run never fetches a price. --- AGENTS.md | 6 +- app/.env.example | 1 + app/nullius/cli.py | 2 + app/nullius/execution.py | 13 +- app/nullius/paths.py | 7 +- app/nullius/prices.py | 278 +++++++++++++++++++++++ app/nullius/settings.py | 1 + app/nullius/standard/cli.py | 20 +- app/nullius/standard/fidelity.py | 3 +- app/nullius/standard/job.py | 7 +- app/nullius/standard/relay.py | 6 +- app/nullius/standard/run.py | 54 +++-- app/packaging/verify.py | 2 +- app/site/mkdocs.yml | 1 + app/static/settings/prices.js | 63 +++++ app/static/settings/settings.css | 3 +- app/templates/settings/settings.html | 28 +++ app/tests/parallel.py | 2 +- app/tests/test_execution.py | 16 +- app/tests/test_prices.py | 203 +++++++++++++++++ app/tests/test_standard_bench_options.py | 1 - app/tests/test_standard_relay.py | 9 + app/tests/test_standard_run.py | 28 +++ app/ui_tests/test_settings_page.py | 37 ++- app/web/settings.py | 38 +++- docs/cli/README.md | 1 + docs/cli/run.md | 2 + docs/field-guide.md | 4 +- docs/settings/README.md | 3 + docs/settings/model-prices.md | 37 +++ 30 files changed, 827 insertions(+), 49 deletions(-) create mode 100644 app/nullius/prices.py create mode 100644 app/static/settings/prices.js create mode 100644 app/tests/test_prices.py create mode 100644 docs/settings/model-prices.md diff --git a/AGENTS.md b/AGENTS.md index 691ad15..1661928 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,7 +70,7 @@ and its resources. The source layout stays under `app/`. (the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities//` (the keyword-memory `capability`); `workspace` (and `workspace_cli`, its shell command), `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), - `paths` (user config, data and cache locations), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine + `paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request, never by a run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine says is done, and where each step is done), `gitrefs` (a repository's branches and tags, by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository), `sandbox` (Settings' docker: its Docker Hub login and the images runs keep) and `redact` at the top. - `app/templates/`, `app/static/` — the interface, grouped by the same capability @@ -110,8 +110,8 @@ remain available; the executable is `nullius` and storage paths keep that name. Harbor a recipe runs is its pin's, run through `uvx` (or the target's own, for a recipe without a pin), and the viewer never imports it. -`nullius paths` lists persistent locations. On Linux, settings and benchmark -licences live under `~/.config/nullius/`, user catalogue entries, workspaces, +`nullius paths` lists persistent locations. On Linux, settings, the model price +list and benchmark licences live under `~/.config/nullius/`, user catalogue entries, workspaces, runs and standard installs under `~/.local/share/nullius/`, and caches under `~/.cache/nullius/`. XDG variables move those roots; `NULLIUS_HOME` selects a separate profile with `config/`, `data/` and `cache/`. Existing storage overrides diff --git a/app/.env.example b/app/.env.example index fccb609..ddfd9fc 100644 --- a/app/.env.example +++ b/app/.env.example @@ -6,6 +6,7 @@ # Start-up options, not on the Settings page: # NULLIUS_HOME: a separate profile with config/, data/ and cache/ directories # NULLIUS_CATALOGUE: the user catalogue directory (default catalogue/ in the user data directory) +# NULLIUS_PRICES: the model price list file (default prices.json in the user configuration directory); set for a run on a host # APP_HOST: the address the server binds (default 127.0.0.1, or --host) # APP_PORT: the port the server listens on (default 5057, or --port) # JOB_ROOTS: job roots to open at start (or the command-line argument) diff --git a/app/nullius/cli.py b/app/nullius/cli.py index e569c4f..36a1d9a 100644 --- a/app/nullius/cli.py +++ b/app/nullius/cli.py @@ -20,6 +20,7 @@ "standard": "inspect and run standard definitions", "workspace": "manage workspaces", "paths": "show persistent storage locations", + "prices": "show and update model prices", "migrate": "copy legacy storage into the user profile", } MODULES = { @@ -29,6 +30,7 @@ "standard": "nullius.standard.cli", "workspace": "nullius.workspace_cli", "paths": "nullius.paths", + "prices": "nullius.prices", "migrate": "nullius.migrate", } diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 90b08ad..0fb6b76 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -120,7 +120,7 @@ from datetime import datetime, timezone from typing import Callable, Dict, List, Optional -from nullius import hosts, paths, progress, redact, secrets, settings +from nullius import hosts, paths, prices as price_list, progress, redact, secrets, settings from nullius.harbor import source as harbor_source from nullius.standard import payload as standard_payload @@ -1450,6 +1450,16 @@ def _launch(self, part: dict) -> Optional[bool]: if rc != 0: self._fail("run", f"the script did not land on {where}: " + out.strip()[-400:]) return False + prices_on_target = "" + if self.plan.runner == STANDARD and self.on_host: + # the price list is this machine's user's: a host holds none, so the run takes it along (nullius.prices) + doc = price_list.load() + if doc["models"] or doc["own"]: + prices_on_target = run_dir + "/prices.json" + rc, out = self._sh(f"cat > {self._q(prices_on_target)}", 60, stdin=json.dumps(doc) + "\n") + if rc != 0: + self._fail("run", f"the price list did not land on {where}: " + out.strip()[-400:]) + return False exports = " ".join(f"{k}={self._q(v) if v.startswith('~') else shlex.quote(v)}" for k, v in [("NULLIUS_HARBOR", self.harbor), ("NULLIUS_TASKS", self.tasks_on_target), ("NULLIUS_JOBS", run_dir + "/jobs"), ("NULLIUS_PROGRESS", "1")] @@ -1457,6 +1467,7 @@ def _launch(self, part: dict) -> Optional[bool]: if self.plan.runner == STANDARD else []) + ([("NULLIUS_CATALOGUE", self.catalogue_on_target)] if self.plan.runner == STANDARD and self.catalogue_on_target else []) + + ([("NULLIUS_PRICES", prices_on_target)] if prices_on_target else []) + ([("NULLIUS_PYTHON", sys.executable)] if self.plan.runner == STANDARD and not self.on_host else []) + ([("NULLIUS_LICENCE_DIR", self._licence_dir())] if self._licence_dir() else []) + ([("API_MODEL_NAME", self.plan.model)] if self.plan.model else [])) diff --git a/app/nullius/paths.py b/app/nullius/paths.py index 84c601c..4377a55 100644 --- a/app/nullius/paths.py +++ b/app/nullius/paths.py @@ -46,6 +46,11 @@ def settings_file(env=None) -> str: return os.path.join(config_dir(env), "settings.json") +def prices_file(env=None) -> str: + env = os.environ if env is None else env + return env.get("NULLIUS_PRICES") or os.path.join(config_dir(env), "prices.json") + + def catalogue_dir(env=None) -> str: env = os.environ if env is None else env return env.get("NULLIUS_CATALOGUE") or os.path.join(data_dir(env), "catalogue") @@ -70,7 +75,7 @@ def main(argv=None) -> int: parser = argparse.ArgumentParser(description="Show Nullius's persistent user directories (no secret values).") parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) - values = {"settings": settings_file(), "secrets": secrets_dir(), "catalogue": catalogue_dir(), + values = {"settings": settings_file(), "secrets": secrets_dir(), "prices": prices_file(), "catalogue": catalogue_dir(), "workspaces": workspaces_dir(), "runs": runs_dir(), "cache": cache_dir()} if args.json: print(json.dumps(values, indent=2)) diff --git a/app/nullius/prices.py b/app/nullius/prices.py new file mode 100644 index 0000000..e500c92 --- /dev/null +++ b/app/nullius/prices.py @@ -0,0 +1,278 @@ +"""Model prices, the user's own: what a call's cost is estimated from when +the provider's reply does not carry one. + +Nullius ships no price. The list lives in one file of the user's +configuration, ``prices.json`` (``NULLIUS_PRICES`` names another: the file a +host is sent), in two parts: + +``models`` a public list, fetched on request (``update``) and kept whole + with where it came from and when: LiteLLM's + ``model_prices_and_context_window.json``, keyed by the model + names the providers themselves use +``own`` the user's rows, which win over the list: a price it lacks, a + negotiated one, a model served under a name of one's own + +A row is USD per million tokens: ``input`` and ``output``, optionally +``cached`` (input a cache served) and ``cache_write``, and ``max_output``, +the model's output limit in tokens. A model is priced by exactly its name or +not at all: a name that only starts like another's is another model. + +Nothing here runs during a run: a run reads the file, never the network. + + nullius prices # where the list came from, how many models, the user's own rows + nullius prices list MODEL ... # those models' rows + nullius prices update [--from URL-or-file] + nullius prices set MODEL INPUT OUTPUT [--cached X] [--cache-write X] [--max-output N] + nullius prices remove MODEL +""" +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.request +from typing import Callable, Dict, Iterable, Optional, Tuple + +from nullius import paths + +SOURCE = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +RATES = ("input", "output", "cached", "cache_write") # USD per million tokens +_LISTED = {"input": "input_cost_per_token", "output": "output_cost_per_token", + "cached": "cache_read_input_token_cost", "cache_write": "cache_creation_input_token_cost"} +_MOST = 64 * 1024 * 1024 # a list larger than this is not a price list + + +def path(env=None) -> str: + return paths.prices_file(env) + + +def load(env=None) -> dict: + """The file as ``{from, on, models, own}``; missing or unreadable, empty: + no price is better than a run that cannot start for its price list.""" + doc = {"from": "", "on": "", "models": {}, "own": {}} + try: + with open(path(env), encoding="utf-8") as f: + got = json.load(f) + except (OSError, ValueError): + return doc + if isinstance(got, dict): + for k in ("from", "on"): + if isinstance(got.get(k), str): + doc[k] = got[k] + for k in ("models", "own"): + if isinstance(got.get(k), dict): + doc[k] = {n: r for n, r in got[k].items() if _usable(r)} + return doc + + +def _usable(r) -> bool: + return isinstance(r, dict) and all(_number(r.get(k)) for k in ("input", "output")) + + +def _number(v) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) and v >= 0 + + +def save(doc: dict, env=None) -> None: + """Written whole beside the file, then renamed: a reader never finds half a list.""" + out = path(env) + os.makedirs(os.path.dirname(out) or ".", exist_ok=True) + part = f"{out}.{os.getpid()}.part" + with open(part, "w", encoding="utf-8") as f: + json.dump(doc, f, indent=1, sort_keys=True) + f.write("\n") + os.replace(part, out) + + +def table(env=None) -> Dict[str, dict]: + """Every priced model: the list, then the user's own rows over it.""" + doc = load(env) + return {**doc["models"], **doc["own"]} + + +def given(names: Iterable[Optional[str]], prices: Optional[Dict[str, dict]]) -> Dict[str, dict]: + """The rows of ``prices`` for exactly ``names``: what a relay started as + a process of its own is handed, which cannot take the whole list.""" + prices = prices or {} + return {n: prices[n] for n in dict.fromkeys(n for n in names if n) if n in prices} + + +def row(input, output, cached=None, cache_write=None, max_output=None) -> dict: + """One row, checked: rates are numbers not below zero, the limit a whole number above it.""" + out = {} + for name, v in (("input", input), ("output", output), ("cached", cached), ("cache_write", cache_write)): + if v is None: + continue + try: + v = float(v) + except (TypeError, ValueError): + raise ValueError(f"{name}: a price in USD per million tokens, not {v!r}") + if not v >= 0 or v == float("inf"): + raise ValueError(f"{name}: a price in USD per million tokens, not {v!r}") + out[name] = v + if max_output is not None: + try: + n = int(max_output) + except (TypeError, ValueError): + n = 0 + if n <= 0: + raise ValueError(f"max_output: the model's output limit in tokens, not {max_output!r}") + out["max_output"] = n + return out + + +def parse(text: str) -> Tuple[str, dict]: + """``MODEL=INPUT,OUTPUT[,CACHED]``, a run's ``--price``.""" + name, eq, rest = str(text).partition("=") + bits = [b.strip() for b in rest.split(",")] if eq else [] + if not name.strip() or not 2 <= len(bits) <= 3 or not all(bits): + raise ValueError(f"--price {text!r}: MODEL=INPUT,OUTPUT[,CACHED], in USD per million tokens") + try: + return name.strip(), row(*bits) + except ValueError as e: + raise ValueError(f"--price {text!r}: {e}") + + +def convert(listing) -> Dict[str, dict]: + """LiteLLM's list as rows: its costs are per token, a row's per million. + An entry with no input price is not a priced model (the list's own + sample, an image model priced per picture) and is left out.""" + if not isinstance(listing, dict): + raise ValueError("not a price list: a JSON object of models was expected") + out = {} + for name, e in listing.items(): + if not isinstance(e, dict) or not _number(e.get(_LISTED["input"])): + continue + r = {"input": round(e[_LISTED["input"]] * 1e6, 6), + "output": round(e[_LISTED["output"]] * 1e6, 6) if _number(e.get(_LISTED["output"])) else 0.0} + for k in ("cached", "cache_write"): + if _number(e.get(_LISTED[k])): + r[k] = round(e[_LISTED[k]] * 1e6, 6) + limit = e.get("max_output_tokens") + if isinstance(limit, int) and not isinstance(limit, bool) and limit > 0: + r["max_output"] = limit + out[str(name)] = r + if not out: + raise ValueError("not a price list: no model in it has a price") + return out + + +def _fetch(source: str) -> bytes: + if source.startswith("https://"): + req = urllib.request.Request(source, headers={"User-Agent": "nullius"}) # nothing of the user or the machine + with urllib.request.urlopen(req, timeout=60) as r: + return r.read(_MOST + 1) + if "://" in source: + raise ValueError(f"{source}: an https address or a file") + with open(source, "rb") as f: + return f.read(_MOST + 1) + + +def update(source: Optional[str] = None, env=None, fetch: Optional[Callable[[str], bytes]] = None, + today: Optional[str] = None) -> dict: + """The public list fetched and kept in place of the last one; the user's + own rows are not touched. Nothing is written when the fetch or the list + is bad: the list held stays. Returns ``{n, from, on}``.""" + source = source or SOURCE + try: + raw = (fetch or _fetch)(source) + except (OSError, ValueError) as e: + raise ValueError(f"could not fetch the price list from {source}: {e}") + if len(raw) > _MOST: + raise ValueError(f"{source}: too large to be a price list") + try: + models = convert(json.loads(raw)) + except ValueError as e: + raise ValueError(f"{source}: {e}") + doc = load(env) + doc.update({"from": source, "on": today or time.strftime("%Y-%m-%d"), "models": models}) + save(doc, env) + return {"n": len(models), "from": doc["from"], "on": doc["on"]} + + +def set_own(name: str, r: dict, env=None) -> None: + name = str(name).strip() + if not name or not _usable(r): + raise ValueError("a model's name, and its input and output prices") + doc = load(env) + doc["own"][name] = r + save(doc, env) + + +def remove_own(name: str, env=None) -> bool: + doc = load(env) + if doc["own"].pop(str(name).strip(), None) is None: + return False + save(doc, env) + return True + + +def state(env=None) -> dict: + """What Settings shows: how many models the list prices, where it came + from and when, and the user's own rows.""" + doc = load(env) + return {"n": len(doc["models"]), "from": doc["from"], "on": doc["on"], "own": doc["own"]} + + +def said(name: str, r: dict) -> str: + bits = [f"input {r['input']:g}", f"output {r['output']:g}"] + bits += [f"{label} {r[k]:g}" for k, label in (("cached", "cached"), ("cache_write", "cache write")) if k in r] + if "max_output" in r: + bits.append(f"output limit {r['max_output']}") + return f"{name}: " + ", ".join(bits) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(prog="nullius prices", + description="The user's model prices, in USD per million tokens: what a run's cost is estimated from.") + sub = ap.add_subparsers(dest="cmd") + ls = sub.add_parser("list", help="the list's source and size, the user's own rows, and the rows of the models named") + ls.add_argument("models", nargs="*") + ls.add_argument("--json", action="store_true") + up = sub.add_parser("update", help="fetch the public list and keep it; the user's own rows stay") + up.add_argument("--from", dest="source", metavar="URL-or-file", help=f"another list in the same format (default: {SOURCE})") + st = sub.add_parser("set", help="a price of the user's own, which wins over the list") + st.add_argument("model") + st.add_argument("input") + st.add_argument("output") + st.add_argument("--cached") + st.add_argument("--cache-write") + st.add_argument("--max-output") + rm = sub.add_parser("remove", help="remove one of the user's own prices") + rm.add_argument("model") + a = ap.parse_args(argv) + try: + if a.cmd == "update": + got = update(a.source) + print(f"{got['n']} models priced, from {got['from']}") + return 0 + if a.cmd == "set": + r = row(a.input, a.output, a.cached, a.cache_write, a.max_output) + set_own(a.model, r) + print(said(a.model.strip(), r)) + return 0 + if a.cmd == "remove": + if not remove_own(a.model): + ap.exit(1, f"{a.model}: not one of your own prices\n") + return 0 + except ValueError as e: + ap.exit(1, f"{e}\n") + doc, names = load(), list(getattr(a, "models", None) or []) + both = {**doc["models"], **doc["own"]} + if getattr(a, "json", False): + print(json.dumps({"file": path(), "from": doc["from"], "on": doc["on"], "n": len(doc["models"]), "own": doc["own"], + "models": {n: both.get(n) for n in names}}, indent=2)) + return 0 + print(f"file: {path()}") + print(f"list: {len(doc['models'])} models, from {doc['from']} on {doc['on']}" if doc["models"] + else "list: none yet - nullius prices update") + for n, r in sorted(doc["own"].items()): + print("own " + said(n, r)) + for n in names: + print(said(n, both[n]) if n in both else f"{n}: no price") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/nullius/settings.py b/app/nullius/settings.py index 4796f35..7750c7d 100644 --- a/app/nullius/settings.py +++ b/app/nullius/settings.py @@ -150,6 +150,7 @@ def validate(self, value: str) -> Optional[str]: STARTUP: Dict[str, str] = { "NULLIUS_HOME": "a separate profile with config/, data/ and cache/ directories", "NULLIUS_CATALOGUE": "the user catalogue directory (default catalogue/ in the user data directory)", + "NULLIUS_PRICES": "the model price list file (default prices.json in the user configuration directory); set for a run on a host", "APP_HOST": "the address the server binds (default 127.0.0.1, or --host)", "APP_PORT": "the port the server listens on (default 5057, or --port)", "JOB_ROOTS": "job roots to open at start (or the command-line argument)", diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 143070e..efc0fb0 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,7 +6,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend docker] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--unsandboxed | --backend docker] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED] ...] [--unsandboxed | --backend docker] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -63,7 +63,7 @@ import sys import tempfile -from nullius import paths, progress, secrets, settings +from nullius import paths, prices as price_list, progress, secrets, settings # The legacy module command needs saved path overrides before install.HOME # and other runtime defaults are computed, just like the unified CLI does. if __name__ == "__main__" and "--from-run" not in sys.argv[1:]: @@ -416,6 +416,18 @@ def options(pairs) -> dict: return out +def prices(given) -> dict: + """``--price MODEL=INPUT,OUTPUT[,CACHED]``, repeatable: this run's own prices, over the user's list.""" + out = {} + for p in given or []: + try: + name, row = price_list.parse(p) + except ValueError as e: + sys.exit(str(e)) + out[name] = row + return out + + def cmd_conform(a): o = _other(a.other) try: @@ -457,7 +469,7 @@ def cmd_run(a): concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, - **({"env": run_env(a.provider)} if a.from_run else {})) + prices=prices(a.price), **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) return 2 @@ -610,6 +622,8 @@ def main(argv=None): p.add_argument("--launch", help="the launch id every shard records (default: the day, the benchmark, the agent, the model)") p.add_argument("--bench-option", action="append", metavar="NAME=VALUE", help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults") + p.add_argument("--price", action="append", metavar="MODEL=INPUT,OUTPUT[,CACHED]", + help="this run's price for a model in USD per million tokens, over your price list (repeatable; nullius prices)") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") v.add_argument("benchmark") v.add_argument("--dataset", help="one dataset (default: each)") diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index 9cdd8d2..bc60de2 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -29,6 +29,7 @@ import time from typing import Callable, Dict, List, Optional +from nullius import prices as price_list from nullius.standard import definitions, install, jsonl, relay, run SIDES = ("acp", "cli") @@ -171,7 +172,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa units = [str(u) for u in r.driver().request("units", dataset=r.ds["id"], timeout=300)["units"]] # One relay for the whole comparison, on this machine: every route's # records side by side, with the fields fidelity reads. - r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=run.PRICES, model=model, + r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 15e0198..95aaa0a 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -459,7 +459,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, - simulator: str = ""): + simulator: str = "", prices: Optional[dict] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -467,7 +467,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str several machines (``shard``), which ``merge`` reads back. ``repeat``: how many times each unit was run, said only when more than once. ``judge`` and ``simulator``: the models that graded and that played the user, when the - benchmark has them.""" + benchmark has them. ``prices``: the rows its models' costs are + estimated from and where the user's list came from, when it has any.""" with open(os.path.join(job_dir, "config.json"), "w") as f: json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius", "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, @@ -476,7 +477,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"image": image} if image else {}), **({"judge": judge} if judge else {}), **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other), **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), - **({"repeat": repeat} if repeat > 1 else {})}, + **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, "agents": [{"name": other["key"], "model_name": model}], diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index a525a99..f90dca3 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -11,8 +11,8 @@ the tokens, and an abridged request and reply. Three API shapes are read, Chat Completions, Responses and Anthropic's Messages, streamed or not. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an -estimate from ``prices`` when one is given for the model, marked as an -estimate. +estimate from ``prices`` when one is given for exactly that model (the +user's price list, ``nullius.prices``), marked as an estimate. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -380,7 +380,7 @@ def _cost(self, model: str, usage: Optional[dict]): return None, None if "cost" in usage: return usage["cost"], "provider" - p = self.prices.get(model) or next((v for k, v in self.prices.items() if model and model.startswith(k)), None) + p = self.prices.get(model) # exactly this model: one whose name only starts the same is another if not p: return None, None fresh = max(0, usage["input"] - usage["cached"]) diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index b9a8403..23259c3 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -61,7 +61,7 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from typing import Callable, Dict, List, Optional -from nullius import progress, providers +from nullius import prices as price_list, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, match, mcp_machine, relay, shard as shards @@ -80,24 +80,6 @@ "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"}, "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, } -# List prices in USD per million tokens, for an estimate of what a call cost -# when the provider does not say (OpenAI does not; OpenRouter does). A model -# not here has no cost estimate rather than a wrong one. -PRICES = { - "gpt-5-mini": {"input": 0.25, "cached": 0.025, "output": 2.0}, - "gpt-5-nano": {"input": 0.05, "cached": 0.005, "output": 0.4}, - "gpt-4.1-mini": {"input": 0.4, "cached": 0.1, "output": 1.6}, - "gpt-4.1": {"input": 2.0, "cached": 0.5, "output": 8.0}, - "gpt-4o-mini": {"input": 0.15, "cached": 0.075, "output": 0.6}, - "gpt-4o-mini-2024-07-18": {"input": 0.15, "cached": 0.075, "output": 0.6}, - "gpt-4o": {"input": 2.5, "cached": 1.25, "output": 10.0}, - "gpt-4o-2024-08-06": {"input": 2.5, "cached": 1.25, "output": 10.0}, - "gpt-4-turbo": {"input": 10.0, "cached": 10.0, "output": 30.0}, # a paper's reader (a setup's original) - "gpt-3.5-turbo": {"input": 0.5, "cached": 0.5, "output": 1.5}, - "text-embedding-3-large": {"input": 0.13, "cached": 0.13, "output": 0.0}, # a search option's embeddings (tau3-bench) - "text-embedding-3-small": {"input": 0.02, "cached": 0.02, "output": 0.0}, # a capability's own embedder (Mem0) - "text-embedding-ada-002": {"input": 0.1, "cached": 0.1, "output": 0.0}, -} TURN_SECONDS = 900 DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image) @@ -308,7 +290,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None, launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None, licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, - remove_images: bool = False): + remove_images: bool = False, prices: Optional[dict] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: the caller's explicit yes to running an agent on the @@ -326,6 +308,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas ``u0042r2``; once by default, tagged ``u0042`` as always. ``simulator_model``: a benchmark whose user a model plays (``simulator``): this model instead of its official one. + ``prices``: this run's own prices, by model, over the user's list + (``nullius.prices``): what a call's cost is estimated from. ``licences``: what the run holds of the benchmark's licences (:class:`Licences`); by default its store on this machine. ``bench_options``: the benchmark's own options for this run @@ -396,6 +380,9 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas raise Refused("; ".join(bad)) needs = definitions.option_needs(bench, self.bench_options) self.embed_model = needs["embeddings"][0] if needs["embeddings"] else None + # What a call's cost is estimated from when the provider does not say it: the user's price list + # (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price. + self.prices = {**price_list.table(), **(prices or {})} lacking = hostreq.missing(bench, needs["host"], which) if lacking: raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine " @@ -493,7 +480,7 @@ def start(self) -> str: granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend, image=self.image, options=self.options, launch=self.launch or "", bench_options=self.bench_options, shard=shards.record(*self.shard, len(everything), units, self.limit) if self.shard else None, - repeat=self.repeat, + repeat=self.repeat, prices=self.prices_record(), simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -620,7 +607,7 @@ def ready(self) -> None: # own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's. if self.embed_model and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), - prices=PRICES, model=self.embed_model, + prices=self.prices, model=self.embed_model, key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -878,6 +865,24 @@ def agent_servers(self, env_def: Optional[dict]) -> List[str]: out.append(h["value"]) return out + def prices_record(self) -> Optional[dict]: + """What the job's config keeps of its prices: the rows of the models + it calls, and where the user's list came from. None when it has none.""" + rows = price_list.given(self.models_called(), self.prices) + if not rows: + return None + doc = price_list.load() + return {**({"from": doc["from"], "on": doc["on"]} if doc["from"] else {}), "models": rows} + + def models_called(self) -> List[str]: + """Every model this run's relays may send a call to, as sent: the + run's, the judge's, the simulator's, an option's embeddings and the + agent's side models. Any other name a caller asks for is replaced by + the run's (``relay``), so these are all a run can be priced for.""" + side = [(b or a) for a, _, b in (str(m).partition("=") for m in (self.other.get("model") or {}).get("side_models") or ())] + mine = [getattr(self, n, None) for n in ("model", "judge_model", "simulator_model", "embed_model")] + return [m for m in dict.fromkeys([*mine, *side]) if m] + def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] = None, own: bool = False): """The unit's own relay, started: in this process on the local backend, a sidecar container on docker. Either answers ``url`` (the @@ -891,7 +896,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] side = tuple((self.other.get("model") or {}).get("side_models") or ()) key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": - rl = relay.Relay(self.base, self.key, record, prices=PRICES, model=self.model, side_models=side, + rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header).start() rl.unit_url = rl.url(tag) return rl @@ -910,7 +915,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}") door = getattr(self, "loopback", None) rl = container.Unit(self.name, tag, record, image=self.image, app=APP, base=door.base if door else self.base, key=self.key, - model=self.model, routes=routes, prices=PRICES, side_models=side, key_header=key_header, + model=self.model, routes=routes, side_models=side, key_header=key_header, + prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None) rl.mcp_listeners = listeners try: diff --git a/app/packaging/verify.py b/app/packaging/verify.py index b11edd9..ae740b3 100644 --- a/app/packaging/verify.py +++ b/app/packaging/verify.py @@ -182,7 +182,7 @@ def run(*command, expected=0): assert 'serve' in run(*prefix, '--help') assert set(json.loads(run(*prefix, 'import', '--list', '--json'))) == {'atif', 'otel'} assert run(*prefix, '--version').strip() == 'nullius ' + backend['version'] - for subcommand in ('runs', 'compare', 'standard', 'workspace', 'paths', 'migrate'): + for subcommand in ('runs', 'compare', 'standard', 'workspace', 'paths', 'prices', 'migrate'): assert 'usage:' in run(*prefix, subcommand, '--help') run(*prefix, 'import', root / 'missing', expected=3) locations = json.loads(run(command, 'paths', '--json')) diff --git a/app/site/mkdocs.yml b/app/site/mkdocs.yml index b702a9b..5034a18 100644 --- a/app/site/mkdocs.yml +++ b/app/site/mkdocs.yml @@ -111,6 +111,7 @@ nav: - System One provider: settings/system-one.md - LLM judge provider: settings/llm-judge.md - Agent keys: settings/agent-keys.md + - Model prices: settings/model-prices.md - Run hosts: settings/run-hosts.md - Sandbox: settings/sandbox.md - Other: settings/other.md diff --git a/app/static/settings/prices.js b/app/static/settings/prices.js new file mode 100644 index 0000000..1257b4c --- /dev/null +++ b/app/static/settings/prices.js @@ -0,0 +1,63 @@ +// The Model prices section: the price list's state, its Update, and the user's own prices. +// The server sends the state with the page and again after each change; this draws it. +(() => { + const section = document.getElementById('prices'); + if (!section) return; + const $ = sel => section.querySelector(sel); + const csrf = section.dataset.csrf; + const tell = (message, error = false) => window.nulliusNotify ? window.nulliusNotify(message, {error}) : null; + const post = async (url, fields) => { + try { + const r = await fetch(url, {method: 'POST', headers: {'X-CSRF-Token': csrf}, body: new URLSearchParams(fields)}); + const j = await r.json(); + return j.ok ? j : {ok: false, error: j.error || 'HTTP ' + r.status}; + } catch (err) { return {ok: false, error: String(err)}; } + }; + const rate = v => String(Number(v)); + + const show = state => { + $('[data-prices-state]').textContent = state.n + ? `${state.n.toLocaleString('en-US')} models · ${state.on}` : 'No price list yet'; + const own = Object.entries(state.own || {}).sort(([a], [b]) => a.localeCompare(b)); + $('[data-own-rows]').replaceChildren(...own.map(([model, r]) => { + const row = document.createElement('div'); row.className = 'st-row'; + const name = document.createElement('span'); name.className = 'st-row-label mono'; name.textContent = model; + const value = document.createElement('span'); value.className = 'st-row-value st-value'; + value.append(`input ${rate(r.input)} · output ${rate(r.output)}` + (r.cached === undefined ? '' : ` · cached ${rate(r.cached)}`)); + if (csrf) { + const button = document.createElement('button'); button.type = 'button'; button.className = 'btn sm ghost'; button.textContent = 'Remove'; + button.setAttribute('aria-label', `Remove the price of ${model}`); + button.addEventListener('click', async () => { + button.disabled = true; + const got = await post(section.dataset.remove, {model}); + if (got.ok) show(got); else { button.disabled = false; tell(got.error, true); } + }); + value.append(' ', button); + } + row.append(name, value); + return row; + })); + $('[data-own-none]').hidden = own.length > 0; + }; + + const update = $('[data-prices-update]'); + if (update) update.addEventListener('click', async () => { + update.disabled = true; update.textContent = 'Updating…'; + const got = await post(section.dataset.update, {}); + update.disabled = false; update.textContent = 'Update'; + tell(got.ok ? `${got.n.toLocaleString('en-US')} models priced` : got.error, !got.ok); + if (got.ok) show(got); + }); + + const form = $('[data-own-add]'); + if (form) form.addEventListener('submit', async e => { + e.preventDefault(); + const f = new FormData(form), button = form.querySelector('[type=submit]'); + button.disabled = true; + const got = await post(section.dataset.set, {model: f.get('model'), input: f.get('input'), output: f.get('output'), cached: f.get('cached')}); + button.disabled = false; + if (got.ok) { form.reset(); show(got); } else tell(got.error, true); + }); + + try { show(JSON.parse(section.dataset.state)); } catch (err) { show({n: 0, own: {}}); } +})(); diff --git a/app/static/settings/settings.css b/app/static/settings/settings.css index 65ab761..8be2006 100644 --- a/app/static/settings/settings.css +++ b/app/static/settings/settings.css @@ -225,5 +225,6 @@ .st-sandbox-rows { display: grid; gap: 14px; } .st-sandbox-rows > .st-row + .st-row { padding-top: 14px; border-top: 1px solid var(--border-soft); } .settings-page .st-sandbox-rows .st-input { height: 34px; min-height: 34px; padding-block: 0; } -.settings-page #sandbox .st-row-value.st-value { flex-direction: row; justify-content: flex-start; align-items: center; } +.settings-page #sandbox .st-row-value.st-value, .settings-page #prices .st-row-value.st-value { flex-direction: row; justify-content: flex-start; align-items: center; } +.st-prices-add { margin-top: 14px; padding-top: 14px; border-top: 1px solid var(--border-soft); } .st-input[list]::-webkit-calendar-picker-indicator { opacity: 1; display: block; cursor: pointer; } diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index c368d0d..e61f2da 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -74,6 +74,7 @@

Settings

{{ section_link('providers', 'Model providers', 'capability', 'openrouter openai anthropic local endpoint api key model_api_url') }} {{ section_link('agent-keys', 'Agent keys', 'agent', 'credentials cursor copilot token agent_keys') }} {{ section_link('judge', 'Judges', 'compare', 'typesafe system one llm model judgement') }} + {{ section_link('prices', 'Model prices', 'refresh', 'cost price estimate usd tokens list update') }}
Execution
@@ -239,6 +240,32 @@

Run hosts

+ {# the user's price list (nullius.prices), outside the form: prices.js draws the state and answers the buttons #} +
+

Model prices

What a run's cost is estimated from, in USD per million tokens.

+
+
Price list{% if writable %} {% endif %}
+
+
+ Your prices +
+ + {% if writable %} +
+
+
+
+
+
+
+
+
+
+
+ {% endif %} +
+
+ {# kept in this browser, not in the file: onboarding.js fills the state and answers the buttons #}
@@ -276,6 +303,7 @@

General

+ {% if writable %} {# a host: one of the ssh config's, or by hand — who and where, and how ssh gets in without a prompt. Test before Add; no password is ever stored, one is used once to put this machine's key on the host. #} diff --git a/app/tests/parallel.py b/app/tests/parallel.py index f35f1dd..b89faa6 100644 --- a/app/tests/parallel.py +++ b/app/tests/parallel.py @@ -80,7 +80,7 @@ def main(argv) -> int: if not docker: os.environ.pop("NULLIUS_TEST_DOCKER", None) for name in ("NULLIUS_CATALOGUE", "WORKSPACES_DIR", "RUNS_DIR", "SECRETS_DIR", - "STANDARD_DIR", "STANDARD_DATA_DIR", "HARBOR_PINS"): + "STANDARD_DIR", "STANDARD_DATA_DIR", "HARBOR_PINS", "NULLIUS_PRICES"): os.environ.pop(name, None) return _suite(argv) diff --git a/app/tests/test_execution.py b/app/tests/test_execution.py index e636a34..eedca47 100644 --- a/app/tests/test_execution.py +++ b/app/tests/test_execution.py @@ -12,7 +12,7 @@ from unittest import mock from unittest.mock import patch -from nullius import execution, hosts, settings +from nullius import execution, hosts, prices as price_list, settings from nullius.harbor import source from nullius import providers @@ -725,6 +725,20 @@ def test_app_is_sent_at_its_content_address_and_the_part_runs_from_it(self): self.assertEqual(json.loads((self.tmp / "runs" / "std-shard-2of2" / "run.json").read_text())["catalogue_on_target"], "~/.nullius/runs/std-shard-2of2/catalogue") + def test_the_price_list_goes_with_the_run_and_a_host_without_one_is_told_nothing(self): + ex = self.run_it() + launch = next(c for c, _ in self.calls if c[0] == "ssh" and "setsid nohup" in c[-1]) + self.assertNotIn("NULLIUS_PRICES", launch[-1]) # no list here: none sent + self.calls.clear(); self.polls = 0 + price_list.set_own("gpt-5-mini", price_list.row(1, 2)) + self.plan.name = "std-priced" + ex = self.run_it() + self.assertEqual((ex.state, ex.exit_code), ("done", 0), ex.lines) + sent = next(s for c, s in self.calls if c[0] == "ssh" and "cat > " in c[-1] and "/prices.json" in c[-1]) + self.assertEqual(json.loads(sent)["own"], {"gpt-5-mini": {"input": 1.0, "output": 2.0}}) + launch = next(c for c, _ in self.calls if c[0] == "ssh" and "setsid nohup" in c[-1]) + self.assertIn('NULLIUS_PRICES="$HOME/.nullius/runs/std-priced/prices.json"', launch[-1]) + def test_what_it_does_is_said_in_stages_its_own_and_the_cli_s(self): """Getting the host ready is said stage by stage in the run's log, and the cli's own stages come through the part's log: the page folds them.""" diff --git a/app/tests/test_prices.py b/app/tests/test_prices.py new file mode 100644 index 0000000..2841b9f --- /dev/null +++ b/app/tests/test_prices.py @@ -0,0 +1,203 @@ +"""The user's price list: nothing shipped, a public list fetched on request, +the user's own rows over it, a model priced by exactly its name.""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import shutil +import tempfile +import unittest +from unittest import mock + +from nullius import prices +from nullius.standard import run + +# LiteLLM's shape: costs per token, a sample entry and an image model without an input price. +LISTING = { + "sample_spec": {"input_cost_per_token": "a number", "max_output_tokens": "a number"}, + "model-a": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 1.25e-06, + "max_output_tokens": 16384, "litellm_provider": "one"}, + "model-b": {"input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, "max_output_tokens": 128000}, + "embedder": {"input_cost_per_token": 2e-08, "output_cost_per_token": 0.0}, + "picture-model": {"output_cost_per_image": 0.04}, +} + + +class Profile(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="nullius-prices-") + self.addCleanup(shutil.rmtree, self.tmp, True) + env = mock.patch.dict(os.environ, {"NULLIUS_HOME": self.tmp}) + env.start() + self.addCleanup(env.stop) + os.environ.pop("NULLIUS_PRICES", None) + + def read(self): + with open(prices.path()) as f: + return f.read() + + def fetch(self, listing=LISTING): + return lambda source: json.dumps(listing).encode() + + +class Nothing(Profile): + def test_no_price_is_written_in_the_code(self): + self.assertFalse(hasattr(run, "PRICES")) + self.assertEqual(prices.table(), {}) + self.assertEqual(prices.state(), {"n": 0, "from": "", "on": "", "own": {}}) + + def test_an_unreadable_file_is_no_prices_rather_than_a_run_that_cannot_start(self): + os.makedirs(os.path.dirname(prices.path())) + with open(prices.path(), "w") as f: + f.write("{not json") + self.assertEqual(prices.table(), {}) + + def test_reading_the_list_never_fetches(self): + prices.update(fetch=self.fetch()) + with mock.patch("urllib.request.urlopen", side_effect=AssertionError("the network was asked")): + self.assertEqual(len(prices.table()), 3) + prices.state() + prices.given(["model-a"], prices.table()) + + +class Update(Profile): + def test_the_list_is_kept_per_million_tokens_with_its_source_and_day(self): + got = prices.update(fetch=self.fetch(), today="2026-11-10") + self.assertEqual(got, {"n": 3, "from": prices.SOURCE, "on": "2026-11-10"}) + self.assertEqual(prices.table(), { + "model-a": {"input": 2.5, "output": 10.0, "cached": 1.25, "max_output": 16384}, + "model-b": {"input": 3.0, "output": 15.0, "cached": 0.3, "cache_write": 3.75, "max_output": 128000}, + "embedder": {"input": 0.02, "output": 0.0}}) + self.assertEqual(prices.path(), os.path.join(self.tmp, "config", "prices.json")) + + def test_a_list_that_is_not_one_changes_nothing(self): + prices.update(fetch=self.fetch(), today="2026-11-10") + before = self.read() + for bad in (b"", b"[]", json.dumps({"x": {"mode": "chat"}}).encode()): + with self.assertRaises(ValueError): + prices.update(fetch=lambda source, bad=bad: bad) + with self.assertRaises(ValueError): + prices.update(fetch=mock.Mock(side_effect=OSError("no route"))) + self.assertEqual(self.read(), before) + + def test_a_file_is_a_source_and_only_https_is_fetched(self): + listing = os.path.join(self.tmp, "listing.json") + with open(listing, "w") as f: + json.dump(LISTING, f) + self.assertEqual(prices.update(listing)["n"], 3) + with mock.patch("urllib.request.urlopen", side_effect=AssertionError("fetched")): + for source in ("http://example.invalid/prices.json", "ftp://example.invalid/prices.json"): + with self.assertRaises(ValueError): + prices.update(source) + + def test_the_fetch_says_nothing_of_the_user(self): + seen = [] + + class Reply(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def urlopen(req, timeout=None): + seen.append(req) + return Reply(json.dumps(LISTING).encode()) + with mock.patch("urllib.request.urlopen", urlopen): + prices.update() + self.assertEqual(seen[0].full_url, prices.SOURCE) + self.assertEqual(dict(seen[0].header_items()), {"User-agent": "nullius"}) + self.assertIsNone(seen[0].data) + + +class Own(Profile): + def test_the_users_own_price_wins_and_survives_an_update(self): + prices.update(fetch=self.fetch()) + prices.set_own("model-a", prices.row(1, 2)) + prices.set_own("my-gateway/model", prices.row("0.5", "1.5", cached="0.05", max_output="4096")) + prices.update(fetch=self.fetch()) + t = prices.table() + self.assertEqual(t["model-a"], {"input": 1.0, "output": 2.0}) + self.assertEqual(t["my-gateway/model"], {"input": 0.5, "output": 1.5, "cached": 0.05, "max_output": 4096}) + self.assertTrue(prices.remove_own("model-a")) + self.assertFalse(prices.remove_own("model-a")) + self.assertEqual(prices.table()["model-a"]["input"], 2.5) # the list's again + + def test_a_row_is_checked(self): + for bad in ((-1, 2), ("x", 2), (1, None), (float("inf"), 1), (float("nan"), 1)): + with self.assertRaises((ValueError, TypeError), msg=bad): + prices.set_own("m", prices.row(*bad)) + with self.assertRaises(ValueError): + prices.row(1, 2, max_output=0) + with self.assertRaises(ValueError): + prices.set_own(" ", prices.row(1, 2)) + + def test_a_runs_price_is_model_input_output_and_optionally_cached(self): + self.assertEqual(prices.parse("m=1,2"), ("m", {"input": 1.0, "output": 2.0})) + self.assertEqual(prices.parse("org/m:free = 0.5, 1.5, 0.05"), ("org/m:free", {"input": 0.5, "output": 1.5, "cached": 0.05})) + for bad in ("m", "m=1", "=1,2", "m=1,2,3,4", "m=1,x", "m=1,", "m=-1,2"): + with self.assertRaises(ValueError, msg=bad): + prices.parse(bad) + + def test_another_file_is_named_by_the_environment(self): + other = os.path.join(self.tmp, "sent", "prices.json") + with mock.patch.dict(os.environ, {"NULLIUS_PRICES": other}): + prices.set_own("m", prices.row(1, 2)) + self.assertEqual(prices.table(), {"m": {"input": 1.0, "output": 2.0}}) + self.assertTrue(os.path.isfile(other)) + self.assertEqual(prices.table(), {}) + + +class Exactly(Profile): + def test_only_the_models_named_are_given_and_only_by_their_exact_name(self): + t = {"model-a": {"input": 1.0, "output": 2.0}, "model-a-mini": {"input": 0.1, "output": 0.2}} + self.assertEqual(prices.given(["model-a", None, "", "model-a", "model-a-2026"], t), {"model-a": t["model-a"]}) + self.assertEqual(prices.given(["model-a"], None), {}) + + def test_a_run_is_priced_for_every_model_its_relays_send(self): + r = run.Run.__new__(run.Run) + r.model, r.judge_model, r.simulator_model, r.embed_model = "m", "judge", None, "embedder" + r.other = {"model": {"side_models": ["side", "asked=sent", "m"]}} + self.assertEqual(r.models_called(), ["m", "judge", "embedder", "side", "sent"]) + + +class Cli(Profile): + def say(self, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = prices.main(list(argv)) + return code, out.getvalue() + + def test_set_list_and_remove(self): + self.assertEqual(self.say("set", "m", "1", "2", "--cached", "0.1")[0], 0) + code, out = self.say("list", "m", "unknown") + self.assertEqual(code, 0) + self.assertIn("list: none yet - nullius prices update", out) + self.assertIn("own m: input 1, output 2, cached 0.1", out) + self.assertIn("unknown: no price", out) + got = json.loads(self.say("list", "m", "--json")[1]) + self.assertEqual((got["n"], got["models"]), (0, {"m": {"input": 1.0, "output": 2.0, "cached": 0.1}})) + self.assertEqual(self.say("remove", "m")[0], 0) + self.assertEqual(prices.table(), {}) + + def test_an_update_from_a_file_says_how_many(self): + listing = os.path.join(self.tmp, "listing.json") + with open(listing, "w") as f: + json.dump(LISTING, f) + code, out = self.say("update", "--from", listing) + self.assertEqual((code, out.strip()), (0, f"3 models priced, from {listing}")) + self.assertIn(f"list: 3 models, from {listing} on ", self.say()[1]) + + def test_a_bad_price_is_refused_with_its_reason(self): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit) as e: + prices.main(["set", "m", "free", "2"]) + self.assertEqual(e.exception.code, 1) + self.assertIn("input: a price in USD per million tokens", err.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index 25cfc96..9c0f86a 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -98,7 +98,6 @@ def test_an_embeddings_reply_is_recorded_by_its_usage_never_its_vectors(self): got = relay.read_reply("/embeddings", body, False) self.assertEqual((got["embeddings"], got["usage"]["input"]), (3, 1000)) self.assertNotIn("0.1", json.dumps(got)) - self.assertIn("text-embedding-3-large", run.PRICES) class Refused(unittest.TestCase): diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index a01ca91..3d79e5a 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -93,6 +93,15 @@ def test_plain_reply_passes_through_and_the_real_key_goes_out(self): self.assertAlmostEqual(rec["cost_usd"], (6 * 1.0 + 4 * 0.1 + 2 * 10.0) / 1e6) self.assertEqual(rec["cost_from"], "estimate") + def test_a_model_is_priced_by_exactly_its_name(self): + """``m`` has a price; a model whose name only starts with it is another model, and has no cost.""" + status, _ = self.post("u1x", "/chat/completions", {"model": "m-large", "messages": [{"role": "user", "content": "hi"}]}) + self.assertEqual(status, 200) + rec = self.relay.calls("u1x")[-1] + self.assertEqual(rec["model"], "m-large") + self.assertNotIn("cost_usd", rec) + self.assertNotIn("cost_from", rec) + def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): status, body = self.post("u2", "/chat/completions", {"model": "m", "stream": True, "messages": []}) self.assertEqual(body, CHAT_SSE) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 479ec4b..8c29308 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -254,6 +254,34 @@ def test_an_agent_run_reads_back_with_its_outcomes(self): self.assert_no_key(job_dir) self.assert_unit_relays(job_dir) + def relay_calls(self, job_dir): + out = [] + for root, _, files in os.walk(job_dir): + for n in files: + if n.endswith(".jsonl") and os.path.basename(root) == "relay": + with open(os.path.join(root, n)) as f: + out += [json.loads(line) for line in f if line.strip()] + return out + + def test_a_run_given_a_price_estimates_its_calls_and_records_the_price(self): + price = {"input": 1.0, "output": 10.0} + job_dir = self.go("fakebench", "fake", prices={"m-1": price, "another": {"input": 9.0, "output": 9.0}}) + calls = self.relay_calls(job_dir) + self.assertTrue(calls) + for c in calls: + self.assertAlmostEqual(c["cost_usd"], (11 * 1.0 + 3 * 10.0) / 1e6) + self.assertEqual(c["cost_from"], "estimate") + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["prices"], {"models": {"m-1": price}}) # the models it calls, no other + + def test_a_run_with_no_price_has_no_cost_and_says_nothing_of_prices(self): + job_dir = self.go("fakebench", "fake") + calls = self.relay_calls(job_dir) + self.assertTrue(calls) + self.assertFalse(any("cost_usd" in c for c in calls)) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertNotIn("prices", json.load(f)["standard"]) + def test_the_trajectory_carries_the_prompt_the_tool_call_and_the_answer(self): job_dir = self.go("fakebench", "fake", limit=1) trial = [d for d in os.listdir(job_dir) if d.startswith("good__")][0] diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index 3f6ddd6..d328c65 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -5,6 +5,8 @@ that refuses before anything is written.""" from __future__ import annotations +import html as html_text +import json import os import re import shutil @@ -14,7 +16,7 @@ from pathlib import Path from unittest.mock import patch -from nullius import hosts, sandbox, settings +from nullius import hosts, prices, sandbox, settings from .test_routes import web @@ -434,6 +436,39 @@ def test_images_are_removed_per_benchmark_only_from_this_machines_browser(self): environ_base={"REMOTE_ADDR": "10.0.0.7"}).status_code, 403) + def test_the_price_list_is_updated_and_priced_only_from_this_machines_browser(self): + self.stack.enter_context(patch.dict(os.environ, {"NULLIUS_PRICES": str(self.tmp / "prices.json")})) + page = self.page() + section = page[page.index('id="prices"'):page.index('id="general"')] + self.assertNotIn('name="value:', section) # outside Save: nothing of it lands in the file + state = json.loads(html_text.unescape(re.search(r"data-state='([^']*)'", section).group(1))) + self.assertEqual(state, {"n": 0, "from": "", "on": "", "own": {}}) # nothing ships with Nullius + headers = {"Origin": "http://localhost", "X-CSRF-Token": self.token()} + listing = json.dumps({"model-a": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}}).encode() + with patch.object(prices, "_fetch", lambda source: listing): + r = self.client.post("/settings/prices/update", headers=headers); self.addCleanup(r.close) + self.assertEqual((r.status_code, r.json["ok"], r.json["n"], r.json["from"]), (200, True, 1, prices.SOURCE)) + self.assertEqual(prices.table(), {"model-a": {"input": 2.5, "output": 10.0}}) + with patch.object(prices, "_fetch", lambda source: b"not a list"): + r = self.client.post("/settings/prices/update", headers=headers); self.addCleanup(r.close) + self.assertEqual((r.status_code, r.json["ok"]), (502, False)) + self.assertEqual(prices.state()["n"], 1) # the list held stays + r = self.client.post("/settings/prices/set", data={"model": "model-a", "input": "1", "output": "2", "cached": ""}, headers=headers); self.addCleanup(r.close) + self.assertEqual(r.json["own"], {"model-a": {"input": 1.0, "output": 2.0}}) + self.assertEqual(prices.table()["model-a"]["input"], 1.0) # the user's own, over the list + for bad in ({"model": "m", "input": "free", "output": "2"}, {"model": "m", "input": "1"}, {"model": "", "input": "1", "output": "2"}): + r = self.client.post("/settings/prices/set", data=bad, headers=headers); self.addCleanup(r.close) + self.assertEqual((r.status_code, r.json["ok"]), (400, False), bad) + r = self.client.post("/settings/prices/remove", data={"model": "model-a"}, headers=headers); self.addCleanup(r.close) + self.assertEqual(r.json["own"], {}) + self.assertEqual(self.client.post("/settings/prices/remove", data={"model": "model-a"}, headers=headers).status_code, 404) + # the same guards as Save: token, origin, loopback + for route in ("update", "set", "remove"): + for kw in ({"headers": {"Origin": "http://localhost"}}, {"headers": headers, "environ_base": {"REMOTE_ADDR": "10.0.0.7"}}): + self.assertEqual(self.client.post(f"/settings/prices/{route}", data={"model": "x", "input": "1", "output": "2"}, **kw).status_code, 403) + self.assertIsNone(self.file()) + + class StartUp(unittest.TestCase): """app.py's command line: the address and port come from APP_HOST and APP_PORT (app/.env is loaded before it is read), the flags win, and a diff --git a/app/web/settings.py b/app/web/settings.py index 1f3d4d5..c00ca8b 100644 --- a/app/web/settings.py +++ b/app/web/settings.py @@ -3,7 +3,7 @@ import os from flask import abort, flash, jsonify, redirect, render_template, request, url_for -from nullius import execution, hosts, paths, sandbox, settings +from nullius import execution, hosts, paths, prices, sandbox, settings from nullius.standard import container from nullius.judge import client @@ -30,9 +30,11 @@ def settings_page(): env_file=shared.ENV_FILE, storage_paths=[("Settings", shared.ENV_FILE), ("User catalogue", paths.catalogue_dir()), ("Workspaces", paths.workspaces_dir()), ("Runs", paths.runs_dir()), - ("Benchmark licences", paths.secrets_dir()), ("Cache", paths.cache_dir())], + ("Benchmark licences", paths.secrets_dir()), ("Model prices", paths.prices_file()), + ("Cache", paths.cache_dir())], unread=[dict(u, inherited=u["name"] not in withheld, startup=u["instead"] in settings.STARTUP) for u in settings.unread(shared.ENV_FILE)], + prices=prices.state(), ssh_hosts=hosts.scan_ssh_config(), judges=(client.system_one(os.environ), client.llm_judge(os.environ)), llm_providers=client.LLM_PROVIDERS, s1_providers=client.SYSTEM_ONE_PROVIDERS, agent_keys=settings.agent_key_names()) @@ -150,6 +152,38 @@ def settings_sandbox_remove(): return jsonify(got), (200 if got["ok"] else 409) +@app.route("/settings/prices/update", methods=["POST"]) +def settings_prices_update(): + """The public price list fetched and kept (``nullius.prices``): the one + fetch, asked for here and never by a run. Same guards as Save.""" + _sandbox_guard("Prices are updated") + try: + prices.update() + except ValueError as e: + return jsonify(ok=False, error=str(e)), 502 + return jsonify(ok=True, **prices.state()) + + +@app.route("/settings/prices/set", methods=["POST"]) +def settings_prices_set(): + """A price of the user's own for one model, over the list.""" + _sandbox_guard("Prices are set") + f = request.form + try: + prices.set_own(f.get("model", ""), prices.row(f.get("input") or None, f.get("output") or None, f.get("cached") or None)) + except (ValueError, TypeError) as e: + return jsonify(ok=False, error=str(e)), 400 + return jsonify(ok=True, **prices.state()) + + +@app.route("/settings/prices/remove", methods=["POST"]) +def settings_prices_remove(): + _sandbox_guard("Prices are removed") + if not prices.remove_own(request.form.get("model", "")): + return jsonify(ok=False, error="not one of your own prices"), 404 + return jsonify(ok=True, **prices.state()) + + def _settings_return(): return url_for("settings_page", embedded="1") if request.form.get("embedded") == "1" else url_for("settings_page") diff --git a/docs/cli/README.md b/docs/cli/README.md index 95f073c..d8b045e 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -24,6 +24,7 @@ also works. | [Fidelity](fidelity.md) | `nullius standard fidelity` | | [Shard and merge](shard-and-merge.md) | `nullius standard run --shard`, `merge` | | [Storage](../settings/README.md#storage) | `nullius paths [--json]` | +| [Model prices](../settings/model-prices.md) | `nullius prices [list\|update\|set\|remove]` | | [Migration](../getting-started.md#migrate-an-existing-checkout) | `nullius migrate --from CHECKOUT [--secrets PATH] [--apply]` | | [Pages from a shell](pages.md) | | | [Exit codes and output](exit-codes.md) | | diff --git a/docs/cli/run.md b/docs/cli/run.md index 71146ad..7b8c06b 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -5,6 +5,7 @@ One benchmark and one agent, capability, control or setup, into a job. ```bash title="Run" nullius standard run [@version] [--model M] [--dataset D] [--limit N] [-j N] \ [--unsandboxed | --backend docker] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ + [--price MODEL=INPUT,OUTPUT[,CACHED]]... \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` @@ -26,6 +27,7 @@ nullius standard run locomo model --model gpt-4o-mini | `--backend` | `local` (default) or `docker`, each unit in a container with its relay | | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | +| `--price MODEL=INPUT,OUTPUT[,CACHED]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | | `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | | `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--repeat`); by default they stay for the next run, listed and removed in Settings, [Sandbox](../settings/sandbox.md) | | `--judge-model` | Instead of the official judge | diff --git a/docs/field-guide.md b/docs/field-guide.md index c2fc57b..c0fede5 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -163,7 +163,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, the unit's setup, the benchmark's driver and its scorer. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | The cached part of the prompt tokens: a subset, never added to them. | measured | any | -| `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from a price table. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | +| `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | | `n_steps / n_tool_calls` | Trajectory steps and tool calls; empty placeholder calls are not counted. | measured | any | | `exception_type` | What ended the trial, if anything. It decides the infra / agent split. | measured | any | | `n_rate_limited_retries` | 429s counted in `agent/gateway.log`: on Harbor, the ones the agent's SDK retried and survived; on a standard run, every model call the relay saw refused with 429. No log reads *not recorded*, never 0. | inferred | any | @@ -237,7 +237,7 @@ Eleven classes. The three marked **yes** under Infrastructure leave the adjusted ## Caveats {#caveats} -- **Cost is an estimate, and only where one was recorded.** A Harbor agent prices its own usage; a standard run's relay keeps the provider's figure when the reply carries one and estimates otherwise. No bill is read. A trial without a cost shows —, and a run's total covers only the trials that have one. +- **Cost is an estimate, and only where one was recorded.** A Harbor agent prices its own usage; a standard run's relay keeps the provider's figure when the reply carries one and otherwise estimates from the user's price list, when it prices that exact model. No bill is read. A trial without a cost shows —, and a run's total covers only the trials that have one. - **The infra/agent split is an explicit list.** Which exceptions leave the adjusted denominator and which count against the agent is pinned in `classify.py` and tested against Harbor's own classes; anything else is *Unknown*. diff --git a/docs/settings/README.md b/docs/settings/README.md index 773558e..af155a7 100644 --- a/docs/settings/README.md +++ b/docs/settings/README.md @@ -9,6 +9,7 @@ results open without credentials. Changes save automatically. | [System One provider](system-one.md) | The System One judgement | | [LLM judge provider](llm-judge.md) | The second judge | | [Agent keys](agent-keys.md) | A Harbor agent's vendor credentials | +| [Model prices](model-prices.md) | What a run's cost is estimated from | | [Run hosts](run-hosts.md) | SSH machines | | [Sandbox](sandbox.md) | Docker: its Docker Hub login and the images runs keep | | [General](other.md) | Tour and checklist; start-up options remain in the environment | @@ -46,6 +47,7 @@ and job roots also have command-line options. The app does not load `app/.env`. | `RUNS_DIR`, `HARBOR_PINS` | `runs/` in the user data directory; `harbor-pins/` beside the runs | | `STANDARD_DIR`, `STANDARD_DATA_DIR`, `SECRETS_DIR` | `standard/` in the data directory; `datasets/` inside it; `secrets/` in the configuration directory | | `NULLIUS_CATALOGUE` | `catalogue/` in the user data directory; whole-entry overrides of the bundled catalogue | +| `NULLIUS_PRICES` | `prices.json` in the configuration directory; the [model price list](model-prices.md) | | `DEBUG` | Development debugger and reloader; off by default, enabled by `1` | ## Save @@ -71,6 +73,7 @@ directory and installation: | --- | --- | | Saved settings | `~/.config/nullius/settings.json` | | Benchmark licences | `~/.config/nullius/secrets//` | +| Model prices | `~/.config/nullius/prices.json` | | User catalogue | `~/.local/share/nullius/catalogue/` | | Workspaces | `~/.local/share/nullius/workspaces/` | | Runs started from Review | `~/.local/share/nullius/runs/` | diff --git a/docs/settings/model-prices.md b/docs/settings/model-prices.md new file mode 100644 index 0000000..96dcea5 --- /dev/null +++ b/docs/settings/model-prices.md @@ -0,0 +1,37 @@ +# Model prices + +What a run's cost is estimated from, in USD per million tokens. Under +**Evaluation**. Nullius ships no price: a new profile shows no estimate until +its list is filled once. + +| Card | Does | +| --- | --- | +| Price list | How many models are priced and the day the list was fetched; **Update** fetches it again | +| Your prices | A model's price of your own: input, output and optionally cached input; **Remove** takes one out | + +| Rule | | +| --- | --- | +| The list | LiteLLM's public `model_prices_and_context_window.json`, keyed by the model names the providers themselves use, kept whole with its address and day | +| Fetched | Only when **Update** or `nullius prices update` asks; a run reads the saved list and never the network | +| Your prices | Win over the list: a model it lacks, a negotiated price, a model served under your own name. An update leaves them as they are | +| One run | `--price MODEL=INPUT,OUTPUT[,CACHED]` on [`nullius standard run`](../cli/run.md) prices that run only, over both | +| A call's cost | The provider's own figure when its reply carries one (OpenRouter's does); else the price of exactly that model name; else none, shown as — | +| Exact names | `gpt-4.1-nano` is not priced as `gpt-4.1`: a model whose name only starts like another's is another model | +| Recorded | A job's `config.json` keeps the prices of the models it calls and where the list came from | +| Docker and hosts | A unit's relay in a container gets its models' prices from the run; a run on a host takes this machine's list with it | +| An estimate | List prices, not a bill: [Metrics](../field-guide.md) | +| Code | `app/nullius/prices.py` | + +## From a shell + +```bash +nullius prices # the list's source and size, and your own prices +nullius prices list gpt-4o --json # those models' prices +nullius prices update # fetch the public list +nullius prices update --from FILE # another list in the same format: a mirror, a saved copy +nullius prices set my-model 1.5 6 --cached 0.15 +nullius prices remove my-model +``` + +The list is `prices.json` in the configuration directory +([Storage](README.md#storage)); `NULLIUS_PRICES` names another file. From 1feedbba6659118edb1677ff6781aa0809ab291e Mon Sep 17 00:00:00 2001 From: aitianshi <35424809+aitianshi@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:32:59 +0100 Subject: [PATCH 2/8] Send the price list to a host even when it is empty, so a host's own list never prices a run --- app/nullius/execution.py | 15 +++++++-------- app/tests/test_execution.py | 6 ++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 0fb6b76..8fad11e 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -1452,14 +1452,13 @@ def _launch(self, part: dict) -> Optional[bool]: return False prices_on_target = "" if self.plan.runner == STANDARD and self.on_host: - # the price list is this machine's user's: a host holds none, so the run takes it along (nullius.prices) - doc = price_list.load() - if doc["models"] or doc["own"]: - prices_on_target = run_dir + "/prices.json" - rc, out = self._sh(f"cat > {self._q(prices_on_target)}", 60, stdin=json.dumps(doc) + "\n") - if rc != 0: - self._fail("run", f"the price list did not land on {where}: " + out.strip()[-400:]) - return False + # the price list is this machine's user's, so the run takes it along (nullius.prices) - an empty one + # too: a host's own list, if it has one, never prices a run started here + prices_on_target = run_dir + "/prices.json" + rc, out = self._sh(f"cat > {self._q(prices_on_target)}", 60, stdin=json.dumps(price_list.load()) + "\n") + if rc != 0: + self._fail("run", f"the price list did not land on {where}: " + out.strip()[-400:]) + return False exports = " ".join(f"{k}={self._q(v) if v.startswith('~') else shlex.quote(v)}" for k, v in [("NULLIUS_HARBOR", self.harbor), ("NULLIUS_TASKS", self.tasks_on_target), ("NULLIUS_JOBS", run_dir + "/jobs"), ("NULLIUS_PROGRESS", "1")] diff --git a/app/tests/test_execution.py b/app/tests/test_execution.py index eedca47..72e06e0 100644 --- a/app/tests/test_execution.py +++ b/app/tests/test_execution.py @@ -725,10 +725,12 @@ def test_app_is_sent_at_its_content_address_and_the_part_runs_from_it(self): self.assertEqual(json.loads((self.tmp / "runs" / "std-shard-2of2" / "run.json").read_text())["catalogue_on_target"], "~/.nullius/runs/std-shard-2of2/catalogue") - def test_the_price_list_goes_with_the_run_and_a_host_without_one_is_told_nothing(self): + def test_the_price_list_goes_with_the_run_even_empty_so_the_host_s_own_is_never_read(self): ex = self.run_it() + sent = next(s for c, s in self.calls if c[0] == "ssh" and "cat > " in c[-1] and "/prices.json" in c[-1]) + self.assertEqual((json.loads(sent)["models"], json.loads(sent)["own"]), ({}, {})) # no list here: an empty one launch = next(c for c, _ in self.calls if c[0] == "ssh" and "setsid nohup" in c[-1]) - self.assertNotIn("NULLIUS_PRICES", launch[-1]) # no list here: none sent + self.assertIn("NULLIUS_PRICES=", launch[-1]) self.calls.clear(); self.polls = 0 price_list.set_own("gpt-5-mini", price_list.row(1, 2)) self.plan.name = "std-priced" From 6b805dfeddf63f883d072e2617b37fd766f0aabb Mon Sep 17 00:00:00 2001 From: aitianshi <35424809+aitianshi@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:50:14 +0100 Subject: [PATCH 3/8] Give Model prices a dollar icon and the Update button the refresh icon, turning while it updates --- app/static/settings/prices.js | 5 +++-- app/static/settings/settings.css | 4 ++++ app/templates/base.html | 1 + app/templates/settings/settings.html | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/static/settings/prices.js b/app/static/settings/prices.js index 1257b4c..060e6d8 100644 --- a/app/static/settings/prices.js +++ b/app/static/settings/prices.js @@ -42,9 +42,10 @@ const update = $('[data-prices-update]'); if (update) update.addEventListener('click', async () => { - update.disabled = true; update.textContent = 'Updating…'; + const label = update.querySelector('span'); + update.disabled = true; update.classList.add('busy'); label.textContent = 'Updating…'; const got = await post(section.dataset.update, {}); - update.disabled = false; update.textContent = 'Update'; + update.disabled = false; update.classList.remove('busy'); label.textContent = 'Update'; tell(got.ok ? `${got.n.toLocaleString('en-US')} models priced` : got.error, !got.ok); if (got.ok) show(got); }); diff --git a/app/static/settings/settings.css b/app/static/settings/settings.css index 8be2006..a33a8e4 100644 --- a/app/static/settings/settings.css +++ b/app/static/settings/settings.css @@ -226,5 +226,9 @@ .st-sandbox-rows > .st-row + .st-row { padding-top: 14px; border-top: 1px solid var(--border-soft); } .settings-page .st-sandbox-rows .st-input { height: 34px; min-height: 34px; padding-block: 0; } .settings-page #sandbox .st-row-value.st-value, .settings-page #prices .st-row-value.st-value { flex-direction: row; justify-content: flex-start; align-items: center; } +#prices [data-prices-update] .ic { width: 15px; height: 15px; } +#prices [data-prices-update].busy .ic { animation: st-prices-spin .9s linear infinite; } +@keyframes st-prices-spin { to { transform: rotate(360deg); } } +@media (prefers-reduced-motion: reduce) { #prices [data-prices-update].busy .ic { animation: none; } } .st-prices-add { margin-top: 14px; padding-top: 14px; border-top: 1px solid var(--border-soft); } .st-input[list]::-webkit-calendar-picker-indicator { opacity: 1; display: block; cursor: pointer; } diff --git a/app/templates/base.html b/app/templates/base.html index 98e6f39..e61f298 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -44,6 +44,7 @@ + diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index e61f2da..d43f158 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -74,7 +74,7 @@

Settings

{{ section_link('providers', 'Model providers', 'capability', 'openrouter openai anthropic local endpoint api key model_api_url') }} {{ section_link('agent-keys', 'Agent keys', 'agent', 'credentials cursor copilot token agent_keys') }} {{ section_link('judge', 'Judges', 'compare', 'typesafe system one llm model judgement') }} - {{ section_link('prices', 'Model prices', 'refresh', 'cost price estimate usd tokens list update') }} + {{ section_link('prices', 'Model prices', 'dollar', 'cost price estimate usd tokens list update') }}
Execution
@@ -244,7 +244,7 @@

Run hosts

Model prices

What a run's cost is estimated from, in USD per million tokens.

-
Price list{% if writable %} {% endif %}
+
Price list{% if writable %} {% endif %}
Your prices From 8c58db5e22508faccf08e129397d18b11ab4049c Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 16:39:06 +0100 Subject: [PATCH 4/8] Record whose each price is, and name a refused price list once A job's config.json credited the public list (its address and day) with every recorded price, including a row of the user's own and a run's --price. Each row now carries its source (list, own, run), and the list is named only when a row is its. A refused update repeated its address or file in the message. --- app/nullius/prices.py | 5 +++-- app/nullius/standard/run.py | 13 +++++++++---- app/tests/test_prices.py | 6 +++++- app/tests/test_standard_run.py | 20 +++++++++++++++++++- docs/settings/model-prices.md | 2 +- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/app/nullius/prices.py b/app/nullius/prices.py index e500c92..cd4c5de 100644 --- a/app/nullius/prices.py +++ b/app/nullius/prices.py @@ -164,7 +164,7 @@ def _fetch(source: str) -> bytes: with urllib.request.urlopen(req, timeout=60) as r: return r.read(_MOST + 1) if "://" in source: - raise ValueError(f"{source}: an https address or a file") + raise ValueError("only an https address or a file is read") with open(source, "rb") as f: return f.read(_MOST + 1) @@ -178,7 +178,8 @@ def update(source: Optional[str] = None, env=None, fetch: Optional[Callable[[str try: raw = (fetch or _fetch)(source) except (OSError, ValueError) as e: - raise ValueError(f"could not fetch the price list from {source}: {e}") + why = e.strerror if isinstance(e, OSError) and e.filename else e # a file's error names the file again + raise ValueError(f"could not fetch the price list from {source}: {why}") if len(raw) > _MOST: raise ValueError(f"{source}: too large to be a price list") try: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index db1a010..b84d328 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -400,7 +400,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self.embed_model = needs["embeddings"][0] if needs["embeddings"] else None # What a call's cost is estimated from when the provider does not say it: the user's price list # (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price. - self.prices = {**price_list.table(), **(prices or {})} + self.run_prices = dict(prices or {}) + self.prices = {**price_list.table(), **self.run_prices} lacking = hostreq.missing(bench, needs["host"], which) if lacking: raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine " @@ -885,12 +886,16 @@ def agent_servers(self, env_def: Optional[dict]) -> List[str]: def prices_record(self) -> Optional[dict]: """What the job's config keeps of its prices: the rows of the models - it calls, and where the user's list came from. None when it has none.""" + it calls, each with its ``source`` (``list`` the public list, ``own`` + the user's own row, ``run`` this run's ``--price``), and where and + when the list was fetched when a row is its. None when it has none.""" rows = price_list.given(self.models_called(), self.prices) if not rows: return None - doc = price_list.load() - return {**({"from": doc["from"], "on": doc["on"]} if doc["from"] else {}), "models": rows} + doc, mine = price_list.load(), getattr(self, "run_prices", {}) + rows = {n: {**r, "source": "run" if n in mine else "own" if n in doc["own"] else "list"} for n, r in rows.items()} + listed = doc["from"] and any(r["source"] == "list" for r in rows.values()) + return {**({"from": doc["from"], "on": doc["on"]} if listed else {}), "models": rows} def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the diff --git a/app/tests/test_prices.py b/app/tests/test_prices.py index 2841b9f..5b6994c 100644 --- a/app/tests/test_prices.py +++ b/app/tests/test_prices.py @@ -90,8 +90,12 @@ def test_a_file_is_a_source_and_only_https_is_fetched(self): self.assertEqual(prices.update(listing)["n"], 3) with mock.patch("urllib.request.urlopen", side_effect=AssertionError("fetched")): for source in ("http://example.invalid/prices.json", "ftp://example.invalid/prices.json"): - with self.assertRaises(ValueError): + with self.assertRaises(ValueError) as e: prices.update(source) + self.assertEqual(str(e.exception).count(source), 1) # the address once, not twice + with self.assertRaises(ValueError) as e: + prices.update(os.path.join(self.tmp, "none.json")) + self.assertEqual(str(e.exception).count("none.json"), 1) def test_the_fetch_says_nothing_of_the_user(self): seen = [] diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 8c29308..c9856e8 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -272,7 +272,25 @@ def test_a_run_given_a_price_estimates_its_calls_and_records_the_price(self): self.assertAlmostEqual(c["cost_usd"], (11 * 1.0 + 3 * 10.0) / 1e6) self.assertEqual(c["cost_from"], "estimate") with open(os.path.join(job_dir, "config.json")) as f: - self.assertEqual(json.load(f)["standard"]["prices"], {"models": {"m-1": price}}) # the models it calls, no other + # the models it calls, no other; the row is this run's, so no list is credited with it + self.assertEqual(json.load(f)["standard"]["prices"], {"models": {"m-1": {**price, "source": "run"}}}) + + def test_the_record_says_whose_each_price_is_and_names_the_list_only_for_its_own(self): + listed, own = {"input": 1.0, "output": 10.0}, {"input": 2.0, "output": 4.0} + held = os.path.join(self.tmp, "prices.json") + + def recorded(doc): + with open(held, "w") as f: + json.dump({"from": "https://example.invalid/list.json", "on": "2026-11-10", **doc}, f) + with mock.patch.dict(os.environ, {"NULLIUS_PRICES": held}): + job_dir = self.go("fakebench", "fake") + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"]["prices"] + + self.assertEqual(recorded({"models": {"m-1": listed}}), + {"from": "https://example.invalid/list.json", "on": "2026-11-10", "models": {"m-1": {**listed, "source": "list"}}}) + # the user's own row over the list's: the list did not say this price, so it is not named + self.assertEqual(recorded({"models": {"m-1": listed}, "own": {"m-1": own}}), {"models": {"m-1": {**own, "source": "own"}}}) def test_a_run_with_no_price_has_no_cost_and_says_nothing_of_prices(self): job_dir = self.go("fakebench", "fake") diff --git a/docs/settings/model-prices.md b/docs/settings/model-prices.md index 96dcea5..5dec9ed 100644 --- a/docs/settings/model-prices.md +++ b/docs/settings/model-prices.md @@ -17,7 +17,7 @@ its list is filled once. | One run | `--price MODEL=INPUT,OUTPUT[,CACHED]` on [`nullius standard run`](../cli/run.md) prices that run only, over both | | A call's cost | The provider's own figure when its reply carries one (OpenRouter's does); else the price of exactly that model name; else none, shown as — | | Exact names | `gpt-4.1-nano` is not priced as `gpt-4.1`: a model whose name only starts like another's is another model | -| Recorded | A job's `config.json` keeps the prices of the models it calls and where the list came from | +| Recorded | A job's `config.json` keeps the prices of the models it calls, each with its source (`list`, `own`, or `run` for `--price`), and where and when the list was fetched when a price is the list's | | Docker and hosts | A unit's relay in a container gets its models' prices from the run; a run on a host takes this machine's list with it | | An estimate | List prices, not a bill: [Metrics](../field-guide.md) | | Code | `app/nullius/prices.py` | From eddfe3cd13783182650a9c38275ea8aedb99a45a Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 17:23:30 +0100 Subject: [PATCH 5/8] Stop a run at a spending cap nullius standard run --max-usd USD: every relay of the job, the units' own in this process or in their containers and the benchmark's, books each call in one ledger before sending it. A call is sent only while what was spent, what the calls in flight have booked and its own booking stay within the cap. The first that does not fit is refused by the relay itself (HTTP 402, recorded as refused, never as the provider's error), and so is every call after it. A unit cut off ends as SpendingCapError, an API error outside the agent's rate. Units not begun are not run. result.json carries the cap, what was spent, whether it stopped the run, the units cut off and those not run. A capped run is refused before it starts when a model it calls has no price. --- AGENTS.md | 2 +- app/nullius/runs/classify.py | 4 +- app/nullius/standard/cli.py | 6 +- app/nullius/standard/container.py | 19 ++- app/nullius/standard/job.py | 9 +- app/nullius/standard/ledger.py | 183 +++++++++++++++++++++++++++ app/nullius/standard/relay.py | 81 +++++++++++- app/nullius/standard/run.py | 85 ++++++++++++- app/tests/test_classify.py | 7 + app/tests/test_standard_container.py | 8 ++ app/tests/test_standard_ledger.py | 138 ++++++++++++++++++++ app/tests/test_standard_relay.py | 123 +++++++++++++++++- app/tests/test_standard_run.py | 59 ++++++++- docs/cli/run.md | 22 +++- docs/field-guide.md | 2 +- docs/settings/model-prices.md | 1 + docs/standard.md | 3 +- 17 files changed, 722 insertions(+), 30 deletions(-) create mode 100644 app/nullius/standard/ledger.py create mode 100644 app/tests/test_standard_ledger.py diff --git a/AGENTS.md b/AGENTS.md index 3aa76e5..199ab26 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ and its resources. The source layout stays under `app/`. `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), `standard/` (the Nullius standard, `docs/standard.md`: `ports`, `names` (capability vocabulary and legacy aliases), `definitions` (with an agent's or capability's versions), `match`, - `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `install`, `run`, `container` (the docker backend), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its + `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap: each call booked before it is sent), `install`, `run`, `container` (the docker backend), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its `payload` snapshots the running backend and effective catalogue for a remote host; conformance kits under `standard/kits/`, each a benchmark: a `driver` and, for the tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: diff --git a/app/nullius/runs/classify.py b/app/nullius/runs/classify.py index 2b3e817..2b37bf2 100644 --- a/app/nullius/runs/classify.py +++ b/app/nullius/runs/classify.py @@ -57,6 +57,7 @@ "modelnotfounderror", # the model name is wrong; the agent never ran "networkconnectionerror", # DNS, refused connection, TLS — transport, not skill "unknownapierror", # Harbor's own "some provider error"; an ApiError without the prefix + "spendingcaperror", # the run's own spending cap refused the call (the standard's relay): no attempt was had } # What Harbor's agents raise when the agent itself fails — its budget spent, @@ -131,7 +132,8 @@ ), API_ERROR: ( "The model API failed some other way — overloaded, unreachable, out of " - "quota, or the key was rejected. The agent never got a fair attempt, so " + "quota, or the key was rejected — or the run's own spending cap refused " + "the call. The agent never got a fair attempt, so " "this is infrastructure too and is excluded from the adjusted rate." ), SETUP_ERROR: ( diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index efc0fb0..837044b 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,7 +6,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend docker] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED] ...] [--unsandboxed | --backend docker] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED] ...] [--max-usd USD] [--unsandboxed | --backend docker] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -469,7 +469,7 @@ def cmd_run(a): concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, - prices=prices(a.price), **({"env": run_env(a.provider)} if a.from_run else {})) + prices=prices(a.price), max_usd=a.max_usd, **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) return 2 @@ -624,6 +624,8 @@ def main(argv=None): help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults") p.add_argument("--price", action="append", metavar="MODEL=INPUT,OUTPUT[,CACHED]", help="this run's price for a model in USD per million tokens, over your price list (repeatable; nullius prices)") + p.add_argument("--max-usd", type=float, metavar="USD", + help="the run's spending cap: no call goes out once the spend plus the calls in flight would pass it") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") v.add_argument("benchmark") v.add_argument("--dataset", help="one dataset (default: each)") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index 794f239..789c4df 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -467,15 +467,19 @@ def install_argv(image: str, where: str, cmd: List[str]) -> List[str]: def relay_argv(image: str, *, name: str, job: str, app: str, record: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), - key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None) -> List[str]: + key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, + ledger: Optional[str] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is named, never given: ``-e NULLIUS_RELAY_UPSTREAM_KEY``. ``hosts``: names it resolves - (``--add-host``), the machine's own for an endpoint on its loopback.""" + (``--add-host``), the machine's own for an endpoint on its loopback. + ``ledger``: the file of the run's spending cap, its folder mounted so + this relay books its calls where every other relay of the job does.""" a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", "--user", _user(), "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--init", *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")], - "-p", f"127.0.0.1::{RELAY_PORT}", "-v", f"{app}:{app}:ro", "-v", f"{record}:{record}:rw", "-w", app, + "-p", f"127.0.0.1::{RELAY_PORT}", "-v", f"{app}:{app}:ro", "-v", f"{record}:{record}:rw", + *(["-v", f"{os.path.dirname(ledger)}:{os.path.dirname(ledger)}:rw"] if ledger else []), "-w", app, "-e", "PYTHONPATH=" + app, "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "NULLIUS_RELAY_UPSTREAM_KEY", image, "python3", "-m", "nullius.standard.relay", "serve", "--listen", f"0.0.0.0:{RELAY_PORT}", "--base", base, "--record", record, "--public", f"http://{RELAY_ALIAS}:{RELAY_PORT}"] @@ -485,6 +489,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, base: a += ["--route", f"{tag}={m}"] if prices: a += ["--prices", json.dumps(prices, separators=(",", ":"))] + if ledger: + a += ["--ledger", ledger] for m in side_models: a += ["--side-model", m] if key_header != "authorization": @@ -646,8 +652,10 @@ class Unit: def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, base: str, key: str, model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), - doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None): + doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, + ledger: Optional[str] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app + self.ledger = ledger # the file of the run's spending cap, when it has one self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -665,7 +673,8 @@ def start(self) -> "Unit": self.network = True argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, base=self.base, model=self.model, routes=self.routes, prices=self.prices, - side_models=self.side_models, key_header=self.key_header, hosts=self.hosts) + side_models=self.side_models, key_header=self.key_header, hosts=self.hosts, + ledger=self.ledger) self.started.append(self.names["relay"]) _docker(argv[1:], env=client_env({"NULLIUS_RELAY_UPSTREAM_KEY": self.key})) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 95aaa0a..58cc1c3 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -486,7 +486,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str #: an error kind naming the machine or the benchmark's side rather than the agent: the live board's infra_error _INFRA = ("Setup", "Verifier", "Api", "RateLimit", "Environment", "Docker", "Build", "Connection", "Network", "Authentication", - "ModelNotFound") + "ModelNotFound", "SpendingCap") def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict: @@ -532,11 +532,14 @@ def pass_hat(results: List[dict]) -> Optional[dict]: "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}} -def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int): +def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): + """``budget``: a run with a spending cap (``run.Run.budget``): the cap, + what was spent, whether it stopped the run, the units it cut off and + those never begun.""" errored = sum(1 for r in results if r.get("exception_info")) hat = pass_hat(results) with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0}, - **({"pass_hat_k": hat} if hat else {})}, f, indent=1) + **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1) diff --git a/app/nullius/standard/ledger.py b/app/nullius/standard/ledger.py new file mode 100644 index 0000000..72f07a6 --- /dev/null +++ b/app/nullius/standard/ledger.py @@ -0,0 +1,183 @@ +"""A run's spending cap: one small file every relay of the job books its calls in. + +A capped run (``run --max-usd``) gives each of its relays - the units' own, +in this process or in their containers, and the benchmark's - the same +ledger. Before a call goes out, its relay books the most the call is likely +to cost (:func:`bound`). The call is sent only while what was spent, what the +calls in flight have booked and this booking stay within the cap. The first +call that does not fit is refused and closes the ledger: every later call is +refused too, so a run stops at one point instead of letting its small calls +through. + +A reply turns the booking into what the call cost (``settle``). A call the +provider answered with an error gives its booking back (``release``). One +whose cost was never known - a reply with no token count, a reply cut off - +keeps its whole booking, as ``unsettled``: it may have been billed. + +A booking is an estimate, not a bill: the prompt's characters over three as +input tokens, and as output the limit the call names, else the model's own +(its price row's ``max_output``), else ``OUTPUT``. A call that costs more +than was booked for it takes the spend past the cap by the difference, and +the next call is refused. + +The file is rewritten whole under a lock held for one read and one write +(``flock`` on a file beside it, as an install is locked): a relay that dies +holding it lets it go. +""" +from __future__ import annotations + +import contextlib +import fcntl +import json +import math +import os +import uuid +from typing import Optional, Tuple + +OUTPUT = 4096 # output tokens booked when neither the call nor the model's price row names a limit +ERROR = "SpendingCapError" # what a unit the cap ended carries: infrastructure, never a miss (runs.classify) +_LIMITS = ("max_completion_tokens", "max_output_tokens", "max_tokens") + + +class Unavailable(Exception): + """The ledger could not be read or written: a capped call is then not sent.""" + + +def bound(req: dict, raw: bytes, row: dict, path: str = "") -> float: + """The most a call is likely to cost, in USD, at ``row`` (a price row, + USD per million tokens). Input: the request's characters over three - + its JSON whole, less what it carries as base64 - at the dearer of the + input and the cache-write price. Output: the limit the call names, else + the row's, else ``OUTPUT``; an embeddings call has none.""" + tokens = math.ceil(max(0, len(raw) - _blobs(req)) / 3) + rate = max(row["input"], row.get("cache_write") or 0) + out = 0 + if not path.split("?")[0].endswith("/embeddings"): + named = [req.get(k) for k in _LIMITS if isinstance(req.get(k), int) and not isinstance(req.get(k), bool) and req.get(k) > 0] + out = named[0] if named else row.get("max_output") or OUTPUT + return (tokens * rate + out * row["output"]) / 1e6 + + +def _blobs(v) -> int: + """Characters of a request that are an encoded file, not a prompt: a + ``data:`` address, a block's base64 ``data``.""" + if isinstance(v, str): + return len(v) if v.startswith("data:") else 0 + if isinstance(v, list): + return sum(_blobs(x) for x in v) + if isinstance(v, dict): + if v.get("type") == "base64" and isinstance(v.get("data"), str): + return len(v["data"]) + return sum(_blobs(x) for x in v.values()) + return 0 + + +class Ledger: + def __init__(self, path: str): + self.path = path + + @classmethod + def create(cls, path: str, max_usd: float) -> "Ledger": + os.makedirs(os.path.dirname(path), exist_ok=True) + led = cls(path) + led._write({"max_usd": float(max_usd), "spent_usd": 0.0, "unsettled_usd": 0.0, "held": {}, "closed": False}) + return led + + @contextlib.contextmanager + def _locked(self): + try: + with open(self.path + ".lock", "a") as f: + fcntl.flock(f, fcntl.LOCK_EX) + try: + yield self._read() + finally: + fcntl.flock(f, fcntl.LOCK_UN) + except (OSError, ValueError, KeyError, TypeError) as e: + raise Unavailable(f"the spending cap's ledger could not be used: {type(e).__name__}: {e}") + + def _read(self) -> dict: + with open(self.path, encoding="utf-8") as f: + doc = json.load(f) + float(doc["max_usd"]), float(doc["spent_usd"]), float(doc["unsettled_usd"]), sum(doc["held"].values()) + return doc + + def _write(self, doc: dict) -> None: + """Written whole beside the file, then renamed: a reader that takes no lock never finds half of it.""" + part = f"{self.path}.{os.getpid()}.{uuid.uuid4().hex[:8]}.part" + with open(part, "w", encoding="utf-8") as f: + json.dump(doc, f) + os.replace(part, self.path) + + def reserve(self, amount: float) -> Tuple[Optional[str], dict]: + """(the booking's id, the state before it) when ``amount`` fits; + (None, the state) when it does not, or the ledger is closed already - + and it is closed from then on.""" + with self._locked() as doc: + seen = _state(doc) + if doc["closed"] or amount > seen["left_usd"]: + if not doc["closed"]: + doc["closed"] = True + self._write(doc) + return None, seen + booking = uuid.uuid4().hex[:12] + doc["held"][booking] = amount + self._write(doc) + return booking, seen + + def settle(self, booking: str, cost: Optional[float]) -> None: + """The call's cost in place of its booking; None, its cost was never + known: the whole booking is kept, as unsettled.""" + with self._locked() as doc: + amount = doc["held"].pop(booking, None) + if amount is None: + return + if cost is None: + doc["unsettled_usd"] += amount + else: + doc["spent_usd"] += max(0.0, float(cost)) + self._write(doc) + + def release(self, booking: str) -> None: + """The booking given back: the provider answered with an error, or was never reached.""" + with self._locked() as doc: + if doc["held"].pop(booking, None) is not None: + self._write(doc) + + def state(self) -> dict: + """``max_usd``, ``spent_usd``, ``unsettled_usd``, ``held_usd`` (the + calls in flight), ``left_usd`` and ``closed``, read without the lock.""" + try: + return _state(self._read()) + except (OSError, ValueError, KeyError, TypeError) as e: + raise Unavailable(f"the spending cap's ledger could not be read: {type(e).__name__}: {e}") + + def closed(self) -> bool: + """Whether a call was refused: nothing more of the run is begun. A + ledger that cannot be read is closed - no call it guards is sent.""" + try: + return bool(self.state()["closed"]) + except Unavailable: + return True + + +def _state(doc: dict) -> dict: + held = sum(doc["held"].values()) + return {"max_usd": doc["max_usd"], "spent_usd": doc["spent_usd"], "unsettled_usd": doc["unsettled_usd"], "held_usd": held, + "left_usd": doc["max_usd"] - doc["spent_usd"] - doc["unsettled_usd"] - held, "closed": bool(doc["closed"])} + + +def usd(v: float) -> str: + """An amount as a person reads it: cents, and up to two digits more when it has them.""" + whole, _, cents = f"{v:.4f}".rstrip("0").partition(".") + return f"{whole}.{cents.ljust(2, '0')}" + + +def refusal(state: dict, amount: float) -> str: + """Why a call was not sent, as its caller and the record read it.""" + if state["closed"]: + return (f"an earlier call did not fit the run's spending cap of {usd(state['max_usd'])} USD " + f"({usd(state['spent_usd'])} spent): no further call of this run is sent") + flight = state["held_usd"] + state["unsettled_usd"] + return (f"the run's spending cap of {usd(state['max_usd'])} USD: {usd(state['spent_usd'])} spent" + + (f", {usd(flight)} booked by calls in flight or never settled" if flight else "") + + f", and this call may cost up to {usd(amount)}; it was not sent, and no further call of this run is") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index f90dca3..aaa02b2 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -23,6 +23,11 @@ the provider's headers come back the same way (its request id, its rate limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. + +One answer is the relay's own: a run with a spending cap (``ledger``) books +each call before it goes out, and a call that does not fit is not sent. Its +caller gets a 402 that names the relay and the cap, and the record says +``refused``, so it never reads as the provider's error. """ from __future__ import annotations @@ -38,6 +43,7 @@ from typing import Dict, List, Optional from nullius.redact import redact +from nullius.standard import ledger as ledgers PLACEHOLDER_KEY = "nullius-relay" _TAG = re.compile(r"^[A-Za-z0-9_-]{1,64}$") @@ -330,7 +336,8 @@ def close(self): class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, - model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False): + model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, + ledger: Optional[str] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -345,8 +352,12 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s run's, recorded with ``model_asked``. ``key_header``: how the provider takes its key - a bearer ``authorization``, or Anthropic's ``x-api-key``. ``fields``: each record also holds the request's fields as sent - (:func:`request_fields`) - ``fidelity``'s, never a normal run's.""" + (:func:`request_fields`) - ``fidelity``'s, never a normal run's. + ``ledger``: the file of the run's spending cap (``ledger``), shared + by every relay of the job: a call is booked there before it is sent, + and not sent when it does not fit.""" self.fields = fields + self.ledger = ledgers.Ledger(ledger) if ledger else None self.base, self.key, self.dir = base.rstrip("/"), key, record_dir self.prices, self.model = prices or {}, model self.side_models = {a: (b or a) for a, _, b in (str(m).partition("=") for m in side_models or ())} @@ -434,8 +445,15 @@ def _forward(self, h: BaseHTTPRequestHandler): if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"): req.setdefault("stream_options", {})["include_usage"] = True raw = json.dumps(req).encode() + booking = booked = None + if self.ledger is not None and h.command == "POST": + booking, booked, why = self._book(req if isinstance(req, dict) else {}, raw, rest) + if why: + self._refuse(h, tag, rest, req, asked, why) + return t0 = time.time() conn, r, back, body = None, None, [], b"" + cut = False # the reply was cut off: the call may have been billed try: conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) @@ -448,7 +466,7 @@ def _forward(self, h: BaseHTTPRequestHandler): try: body, whole = r.read(), True except Exception as e: # cut off mid-reply - status, kind, back = 502, "application/json", [] + status, kind, back, cut = 502, "application/json", [], True body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode() def record(body): @@ -459,6 +477,7 @@ def record(body): rec["side"] = True if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) + cost = None if status == 200: rec["reply"] = read_reply(rest, body, streamed) cost, how = self._cost(rec["model"], rec["reply"].get("usage")) @@ -466,6 +485,15 @@ def record(body): rec["cost_usd"], rec["cost_from"] = cost, how else: rec["error"] = _said(status, body) + if booking is not None: + rec["booked_usd"] = booked + try: + if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole + self.ledger.settle(booking, cost) + else: # an error of the provider's, or never reached: not billed + self.ledger.release(booking) + except ledgers.Unavailable: + pass # the booking stays held, against the cap with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") @@ -515,6 +543,41 @@ def record(body): else: conn.close() + def _book(self, req: dict, raw: bytes, rest: str) -> tuple: + """(booking, its amount, None) for a call the cap lets out; (None, + its amount, why) for one it does not. A model with no price is not + sent either: a capped run sends no call it cannot price.""" + row = self.prices.get(req.get("model")) + if not row: + return None, None, f"no price for the model {req.get('model')!r}: a run with a spending cap sends no call it cannot price" + try: + amount = ledgers.bound(req, raw, row, rest) + booking, seen = self.ledger.reserve(amount) + except ledgers.Unavailable as e: + return None, None, str(e) + return booking, amount, None if booking else ledgers.refusal(seen, amount) + + def _refuse(self, h: BaseHTTPRequestHandler, tag: str, rest: str, req, asked, why: str) -> None: + """A call the run's spending cap did not let out: answered here and + recorded as the relay's own (``refused``), nothing sent.""" + t = time.time() + rec = {"t0": t, "t1": t, "path": rest, "model": req.get("model") if isinstance(req, dict) else None, "model_asked": asked, + "status": 402, "refused": "cap", "error": why, "request": _request_tail(req) if isinstance(req, dict) else {}, + "headers": sorted(k.lower() for k in forwarded(h.headers))} + with self.lock: + with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: + f.write(json.dumps(rec) + "\n") + body = json.dumps({"type": "error", "error": {"type": "spending_cap", "code": "spending_cap", + "message": "nullius relay: " + why}}).encode() + h.send_response(402) + for n, v in (("Content-Type", "application/json"), ("X-Nullius-Refused", "cap"), ("Content-Length", str(len(body)))): + h.send_header(n, v) + h.end_headers() + try: + h.wfile.write(body) + except OSError: + pass + # The address ``url`` hands out: the listening one unless the caller sets # another - a sidecar listens on 0.0.0.0 and is reached as ``relay``. public: Optional[str] = None @@ -553,7 +616,12 @@ def failure(calls: List[dict]) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the infrastructure it was - and the endpoint's words. None when the last call - succeeded (a failure the agent recovered from is not the unit's).""" + succeeded (a failure the agent recovered from is not the unit's). A call + the run's spending cap refused is the unit's wherever it falls: once one + is refused, every later one is.""" + cut = next((c for c in calls if c.get("refused") == "cap"), None) + if cut: + return ledgers.ERROR, cut.get("error") or "" if not calls or calls[-1].get("status") == 200: return None last = calls[-1] @@ -576,7 +644,7 @@ def failure(calls: List[dict]) -> Optional[tuple]: def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME] - [--public URL] [--prices JSON]``. + [--public URL] [--prices JSON] [--ledger FILE]``. ``--base`` is the provider's endpoint, ``--route`` a route sent as another model (a benchmark's judge), ``--public`` the address ``url`` hands out. @@ -596,6 +664,7 @@ def serve(argv=None) -> int: ap.add_argument("--key-header", default="authorization", choices=("authorization", "x-api-key")) ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") + ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") a = ap.parse_args(argv) key = os.environ.get(KEY_VAR) if not key: @@ -603,7 +672,7 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index b84d328..e99e51b 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -65,7 +65,7 @@ from nullius import paths, prices as price_list, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed -from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, match, mcp_machine, relay, shard as shards +from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, relay, shard as shards # the totals a unit's end carries onto the live board, from its trial's agent_result FEED_TOTALS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd") @@ -308,7 +308,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None, launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None, licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, - remove_images: bool = False, prices: Optional[dict] = None): + remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: the caller's explicit yes to running an agent on the @@ -328,6 +328,11 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas (``simulator``): this model instead of its official one. ``prices``: this run's own prices, by model, over the user's list (``nullius.prices``): what a call's cost is estimated from. + ``max_usd``: the run's spending cap (``ledger``): every relay of the + job books a call before sending it, and once one does not fit no + further call goes out and nothing more is begun. Refused when a model + the run calls has no price: a call that cannot be priced cannot be + booked. ``licences``: what the run holds of the benchmark's licences (:class:`Licences`); by default its store on this machine. ``bench_options``: the benchmark's own options for this run @@ -402,6 +407,20 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price. self.run_prices = dict(prices or {}) self.prices = {**price_list.table(), **self.run_prices} + self.max_usd, self.ledger = None, None + if max_usd is not None: + try: + cap = float(max_usd) + except (TypeError, ValueError): + cap = 0.0 + if not 0 < cap < float("inf"): + raise Refused(f"--max-usd {max_usd!r}: the run's spending cap in USD, above zero") + unpriced = [m for m in self.models_called() if m not in self.prices] + if unpriced: + raise Refused("a spending cap needs a price for every model the run calls, and there is none for " + + ", ".join(unpriced) + ": nullius prices update, nullius prices set MODEL INPUT OUTPUT, " + "or --price MODEL=INPUT,OUTPUT") + self.max_usd = cap lacking = hostreq.missing(bench, needs["host"], which) if lacking: raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine " @@ -479,6 +498,9 @@ def start(self) -> str: + (["dataset"] if self.ds.get("url") else []) + (["driver"] if install.driver_dir(self.bench) else []) + ([] if control or self.upstream else ["install"]) + ["units"])) + if self.max_usd is not None and self.base: + # one ledger for the job, in a folder of its own: all a unit's relay container is given of the job besides its record + self.ledger = ledgers.Ledger.create(os.path.join(self.job_dir, ".work", "ledger", "ledger.json"), self.max_usd) try: self.ready() except Refused as e: @@ -511,6 +533,7 @@ def start(self) -> str: results = [] u0 = time.time() self._halt, self._refused, self.halted = threading.Event(), [], None + self._cut: List[str] = [] # units the spending cap cut off # a model server on this machine's loopback, which the relays' containers cannot reach as written self.loopback = (container.Loopback(self.base).start() if self.backend == "docker" and self.base and container.on_loopback(self.base) else None) @@ -542,7 +565,8 @@ def work_unit(iur): finally: # a Stop or Ctrl-C: the units not begun are not begun, those running end with their agents pool.shutdown(wait=True, cancel_futures=True) - results = [f.result() for f in futures if not f.cancelled() and f.exception() is None and f.result() is not None] + begun = [not f.cancelled() and f.exception() is None and f.result() is not None for f in futures] + results = [f.result() for f, ran in zip(futures, begun) if ran] for d in self.drivers: d.close() if self.bench_relay is not None: @@ -551,11 +575,20 @@ def work_unit(iur): container.sweep(self.name) # whatever an interrupted unit left if self.loopback is not None: self.loopback.stop() - job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work)) + budget = self.budget([u for n, (_, u, _) in enumerate(work) if n >= len(begun) or not begun[n]]) + job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget) errored = sum(1 for x in results if x.get("exception_info")) self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results)) p.emit("units", "done", seconds=round(time.time() - u0, 1)) self.n_verdicts, self.n_ended = sum(1 for x in results if x.get("verifier_result")), len(results) + if budget and budget["stopped"]: + spent = "unknown" if budget["spent_usd"] is None else ledgers.usd(budget["spent_usd"]) + p.emit("cap", detail=f"a call did not fit the spending cap of {ledgers.usd(budget['max_usd'])} USD", + spent_usd=spent, cut_off=len(budget["cut_off"]), not_run=len(budget["not_run"])) + self.say(f"stopped: a call did not fit the spending cap of {ledgers.usd(budget['max_usd'])} USD " + f"({spent} spent): {len(budget['cut_off'])} cut off, {len(budget['not_run'])} not run") + elif budget: + self.say(f"spent {ledgers.usd(budget['spent_usd'] or 0)} of the {ledgers.usd(budget['max_usd'])} USD cap") if self.halted: self.say(f"stopped: the endpoint refused {REFUSED_IN_A_ROW} units in a row ({self.halted[0]}: " f"{redact(str(self.halted[1]))[:200]}); the {len(work) - len(results)} units not begun were not run") @@ -626,7 +659,7 @@ def ready(self) -> None: # own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's. if self.embed_model and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), - prices=self.prices, model=self.embed_model, + prices=self.prices, model=self.embed_model, ledger=self.ledger_file(), key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -657,6 +690,9 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()): return None # the endpoint refused the run: nothing more is begun + if getattr(self, "ledger", None) is not None and self.ledger.closed(): + self._halt.set() # the spending cap refused a call: every later one is refused too + return None tag = f"u{i:04d}" + (f"r{r}" if self.repeat > 1 else "") self.local.attempt = r trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) @@ -759,6 +795,13 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it score = None + if rl is not None and getattr(self, "ledger", None) is not None: + # a call the spending cap refused, the agent's, the judge's or the simulated user's: the unit was + # cut off by the run's own limit, whatever its agent made of the refusal - never a miss + cut = next((f for f in (relay.failure(rl.calls(t)) for t in (tag, f"{tag}-judge", f"{tag}-simulator")) + if f and f[0] == ledgers.ERROR), None) + if cut: + exc, score = cut, None if self._stopped.is_set() and score is None: # the Stop ended its agent, its relay or its containers under it: what it then failed with is # the Stop's doing, not the agent's nor the environment's (Harbor's name for a stopped trial) @@ -783,8 +826,30 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: **{k: v for k, v in (res.get("agent_result") or {}).items() if k in FEED_TOTALS}, durations={"agent_run": round(produce[1] - produce[0], 3)} if produce[0] is not None else None) self.note_refusal(exc) + if exc and exc[0] == ledgers.ERROR and getattr(self, "_halt", None) is not None: + with self._said: + self._cut.append(unit) + self._halt.set() # the cap is reached: nothing more is begun return res + def budget(self, not_run: List[str]) -> Optional[dict]: + """What ``result.json`` says of the run's spending cap: the cap, + what was spent, what was booked for calls whose cost was never known + (``unsettled_usd``, said only when there is some), whether the cap + stopped the run, the units it cut off and those never begun. None + for a run without a cap.""" + if getattr(self, "ledger", None) is None: + return None + try: + got = self.ledger.state() + except ledgers.Unavailable: + got = {"spent_usd": None, "unsettled_usd": 0.0, "held_usd": 0.0, "closed": True} + unsettled = round(got["unsettled_usd"] + got["held_usd"], 6) + cut = getattr(self, "_cut", []) + return {"max_usd": self.max_usd, "spent_usd": None if got["spent_usd"] is None else round(got["spent_usd"], 6), + **({"unsettled_usd": unsettled} if unsettled else {}), "stopped": bool(got["closed"]), + "cut_off": list(dict.fromkeys(cut)), "not_run": list(dict.fromkeys(not_run)) if got["closed"] else []} + def note_refusal(self, exc: Optional[tuple]) -> None: """A unit's end, counted toward the endpoint refusing the run: a wrong key or an unknown model fails every unit the same way, and running @@ -897,6 +962,11 @@ def prices_record(self) -> Optional[dict]: listed = doc["from"] and any(r["source"] == "list" for r in rows.values()) return {**({"from": doc["from"], "on": doc["on"]} if listed else {}), "models": rows} + def ledger_file(self) -> Optional[str]: + """The file of the run's spending cap, for a relay; None without a cap.""" + led = getattr(self, "ledger", None) + return led.path if led is not None else None + def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the run's, the judge's, the simulator's, an option's embeddings and the @@ -920,7 +990,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header).start() + key_header=key_header, ledger=self.ledger_file()).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -940,7 +1010,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] rl = container.Unit(self.name, tag, record, image=self.image, app=APP, base=door.base if door else self.base, key=self.key, model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows - servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None) + servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, + ledger=self.ledger_file()) rl.mcp_listeners = listeners try: try: diff --git a/app/tests/test_classify.py b/app/tests/test_classify.py index 9008a91..7ab35c9 100644 --- a/app/tests/test_classify.py +++ b/app/tests/test_classify.py @@ -92,6 +92,13 @@ def test_both_boards_charge_the_infrastructure_as_the_classifier_does(self): with self.subTest(name, board=board): self.assertIn(outcome(name), ("agent_error", "timeout")) + def test_a_unit_the_runs_spending_cap_cut_off_is_infrastructure_on_both(self): + """The standard's own: its relay refused the call (``standard.ledger``), so the unit had no attempt.""" + from nullius.standard import job, ledger + self.assertEqual(C.from_verdict(ledger.ERROR, None), C.API_ERROR) + self.assertIn(C.API_ERROR, C.INFRA_OUTCOMES) + self.assertEqual(job.feed_outcome("pass", None, (ledger.ERROR, "the run's spending cap"))["outcome"], "infra_error") + class HarborTaxonomy(unittest.TestCase): def test_provider_failures_are_infra(self): diff --git a/app/tests/test_standard_container.py b/app/tests/test_standard_container.py index 490d186..82fd055 100644 --- a/app/tests/test_standard_container.py +++ b/app/tests/test_standard_container.py @@ -102,6 +102,14 @@ def test_the_relays_container_names_the_key_and_never_holds_it(self): self.assertEqual(a[a.index("--route") + 1], "u0000-judge=the-judge") self.assertEqual(a[a.index("--public") + 1], "http://relay:8080") self.assertEqual(a[a.index("--label") + 1], "nullius.job=j") + self.assertNotIn("--ledger", a) + + def test_the_relays_container_is_given_the_runs_ledger_and_its_folder_alone(self): + a = container.relay_argv("img", name="r", job="j", app="/app", record="/rec", base="b", model="m", routes={}, + ledger="/job/.work/ledger/ledger.json") + self.assertEqual(a[a.index("--ledger") + 1], "/job/.work/ledger/ledger.json") + # where every relay of the job books its calls: that folder, and nothing else of the job but its own record + self.assertEqual([w for w in a if w.endswith(":rw")], ["/rec:/rec:rw", "/job/.work/ledger:/job/.work/ledger:rw"]) def test_an_endpoint_on_this_machines_loopback_is_reached_through_the_bridge(self): self.assertTrue(all(map(container.on_loopback, ["http://localhost:11434/v1", "http://127.0.0.1:8000/v1", "http://[::1]:8000/v1"]))) diff --git a/app/tests/test_standard_ledger.py b/app/tests/test_standard_ledger.py new file mode 100644 index 0000000..8ac64c6 --- /dev/null +++ b/app/tests/test_standard_ledger.py @@ -0,0 +1,138 @@ +"""A run's spending cap: a call is booked before it is sent, sent only while +it fits, and the first that does not closes the ledger for every later one.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import threading +import unittest + +from nullius.standard import ledger + +ROW = {"input": 1.0, "output": 10.0} + + +class LedgerTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.path = os.path.join(self.tmp, "ledger", "ledger.json") + + def test_a_call_is_sent_while_spent_and_in_flight_and_its_booking_fit(self): + led = ledger.Ledger.create(self.path, 1.0) + a, _ = led.reserve(0.4) + b, _ = led.reserve(0.4) + self.assertTrue(a and b and a != b) + self.assertAlmostEqual(led.state()["held_usd"], 0.8) + led.settle(a, 0.05) # the reply: what it cost, in place of what was booked + self.assertAlmostEqual(led.state()["spent_usd"], 0.05) + self.assertAlmostEqual(led.state()["left_usd"], 1.0 - 0.05 - 0.4) + c, _ = led.reserve(0.5) + self.assertTrue(c) + self.assertFalse(led.closed()) + + def test_the_first_call_that_does_not_fit_closes_it_for_every_later_one(self): + led = ledger.Ledger.create(self.path, 1.0) + a, _ = led.reserve(0.7) + refused, seen = led.reserve(0.4) # 0.7 in flight: 0.4 more would pass the cap + self.assertIsNone(refused) + self.assertFalse(seen["closed"]) # the state it was refused in, for its message + self.assertTrue(led.closed()) + led.settle(a, 0.01) + self.assertIsNone(led.reserve(0.0001)[0]) # room again, and still closed: a run stops at one point + said = ledger.refusal(led.state(), 0.0001) + self.assertIn("1.00 USD", said) + self.assertIn("no further call", said) + + def test_a_call_whose_cost_was_never_known_keeps_its_booking_and_an_error_gives_it_back(self): + led = ledger.Ledger.create(self.path, 1.0) + a, _ = led.reserve(0.3) + b, _ = led.reserve(0.3) + led.settle(a, None) # no token count, or cut off: it may have been billed + led.release(b) # the provider's error: not billed + led.release(b) # twice is once + led.settle("unknown", 9.0) # a booking it does not hold changes nothing + got = led.state() + self.assertEqual((got["spent_usd"], got["unsettled_usd"], got["held_usd"]), (0.0, 0.3, 0)) + self.assertAlmostEqual(got["left_usd"], 0.7) + + def test_a_ledger_that_cannot_be_read_lets_no_call_out(self): + led = ledger.Ledger.create(self.path, 1.0) + with open(self.path, "w") as f: + f.write("{not json") + with self.assertRaises(ledger.Unavailable): + led.reserve(0.1) + self.assertTrue(led.closed()) + self.assertTrue(ledger.Ledger(os.path.join(self.tmp, "none.json")).closed()) + + def test_many_calls_at_once_never_book_past_the_cap(self): + led = ledger.Ledger.create(self.path, 1.0) + got = [] + + def ask(): + got.append(led.reserve(0.03)[0]) + + threads = [threading.Thread(target=ask) for _ in range(60)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(sum(1 for b in got if b), 33) # 33 x 0.03 fit in 1.00, the 34th does not + self.assertLessEqual(led.state()["held_usd"], 1.0) + + def test_relays_in_other_processes_book_in_the_same_ledger(self): + """What a docker unit's relay does from its container: each process + books and settles; none of it is lost between them.""" + led = ledger.Ledger.create(self.path, 100.0) + app = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + code = ("import sys\nfrom nullius.standard import ledger\nled = ledger.Ledger(sys.argv[1])\n" + "for _ in range(40):\n b, _ = led.reserve(0.5)\n led.settle(b, 0.01)\n") + procs = [subprocess.Popen([sys.executable, "-c", code, self.path], env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": app}) + for _ in range(4)] + self.assertEqual([p.wait(timeout=60) for p in procs], [0, 0, 0, 0]) + got = led.state() + self.assertAlmostEqual(got["spent_usd"], 4 * 40 * 0.01) + self.assertEqual((got["held_usd"], got["closed"]), (0, False)) + with open(self.path) as f: + self.assertEqual(json.load(f)["held"], {}) + + +class BoundTest(unittest.TestCase): + def bound(self, req, row=ROW, path="/chat/completions"): + return ledger.bound(req, json.dumps(req).encode(), row, path) + + def test_the_limit_the_call_names_is_its_output(self): + req = {"model": "m", "messages": [{"role": "user", "content": "x" * 300}], "max_tokens": 100} + tokens = -(-len(json.dumps(req)) // 3) # its characters over three, rounded up + self.assertAlmostEqual(self.bound(req), (tokens * 1.0 + 100 * 10.0) / 1e6) + for name in ("max_completion_tokens", "max_output_tokens"): + other = {"model": "m", "input": "hi", name: 50} + self.assertAlmostEqual(self.bound(other), (-(-len(json.dumps(other)) // 3) * 1.0 + 50 * 10.0) / 1e6) + + def test_a_call_that_names_none_is_booked_the_models_own_limit_else_the_default(self): + req = {"model": "m", "messages": []} + tokens = -(-len(json.dumps(req)) // 3) + self.assertAlmostEqual(self.bound(req, dict(ROW, max_output=2000)), (tokens + 2000 * 10.0) / 1e6) + self.assertAlmostEqual(self.bound(req), (tokens + ledger.OUTPUT * 10.0) / 1e6) + + def test_input_is_booked_at_the_dearer_of_input_and_cache_write(self): + req = {"model": "m", "messages": [], "max_tokens": 1} + tokens = -(-len(json.dumps(req)) // 3) + self.assertAlmostEqual(self.bound(req, dict(ROW, cache_write=1.25)), (tokens * 1.25 + 10.0) / 1e6) + + def test_an_embeddings_call_has_no_output_and_an_encoded_file_is_not_a_prompt(self): + req = {"model": "e", "input": ["some text"]} + self.assertAlmostEqual(self.bound(req, {"input": 0.02, "output": 5.0}, "/embeddings"), -(-len(json.dumps(req)) // 3) * 0.02 / 1e6) + picture = "A" * 30000 + plain = {"model": "m", "max_tokens": 1, "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": ""}}, {"type": "image", "source": {"type": "base64", "data": ""}}]}]} + with_files = json.loads(json.dumps(plain)) + with_files["messages"][0]["content"][0]["image_url"]["url"] = "data:image/png;base64," + picture + with_files["messages"][0]["content"][1]["source"]["data"] = picture + self.assertAlmostEqual(self.bound(with_files), self.bound(plain)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 3d79e5a..da359c6 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -15,7 +15,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from nullius.runs import classify -from nullius.standard import jsonl, relay +from nullius.standard import jsonl, ledger, relay CHAT = {"choices": [{"message": {"role": "assistant", "content": "hello"}}], "usage": {"prompt_tokens": 10, "completion_tokens": 2, "prompt_tokens_details": {"cached_tokens": 4}}} @@ -44,6 +44,8 @@ def do_POST(self): self.reply(429, b'{"error":{"message":"Rate limit reached"}}', "application/json") elif mode == "401": self.reply(401, b'{"error":{"message":"Incorrect API key"}}', "application/json") + elif mode == "no-usage": + self.reply(200, b'{"choices":[{"message":{"role":"assistant","content":"hello"}}]}', "application/json") elif self.path.endswith("/responses"): self.reply(200, RESP_SSE, "text/event-stream") elif body.get("stream"): @@ -177,6 +179,107 @@ def test_an_advertised_address_is_what_url_hands_out(self): r.stop() +class CapTest(unittest.TestCase): + """A relay of a run with a spending cap: a call is booked before it goes + out and is not sent when it does not fit - the one answer that is the + relay's own, and marked so.""" + PRICES = {"m": {"input": 1.0, "cached": 0.1, "output": 10.0}} + + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + + def capped(self, cap): + tmp = tempfile.mkdtemp() + led = ledger.Ledger.create(os.path.join(tmp, "ledger", "ledger.json"), cap) + r = relay.Relay(self.base, "sk-real", os.path.join(tmp, "relay"), prices=self.PRICES, ledger=led.path).start() + self.addCleanup(r.stop) + return r, led + + def post(self, r, tag, body): + req = urllib.request.Request(r.url(tag) + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req) as got: + return got.status, dict(got.headers.items()), got.read() + except urllib.error.HTTPError as e: + return e.code, dict(e.headers.items()), e.read() + + def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): + r, led = self.capped(1.0) + body = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100} + status, _, _ = self.post(r, "u1", body) + self.assertEqual(status, 200) + rec = r.calls("u1")[-1] + self.assertAlmostEqual(rec["booked_usd"], ledger.bound(body, json.dumps(body).encode(), self.PRICES["m"])) + got = led.state() + self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) + self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + + def test_a_call_that_does_not_fit_is_answered_by_the_relay_and_never_sent(self): + r, led = self.capped(0.0005) # 100 output tokens at 10 USD a million are 0.001 already + sent = len(Upstream.seen) + status, headers, body = self.post(r, "u1", {"model": "m", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}) + self.assertEqual((status, headers.get("X-Nullius-Refused")), (402, "cap")) + error = json.loads(body)["error"] + self.assertEqual(error["type"], "spending_cap") + self.assertIn("nullius relay: the run's spending cap of 0.0005 USD", error["message"]) + rec = r.calls("u1")[-1] + self.assertEqual((rec["status"], rec["refused"]), (402, "cap")) + self.assertNotIn("reply", rec) + self.assertTrue(led.closed()) + # a call small enough to fit is refused too: a run stops at one point + status, _, _ = self.post(r, "u1", {"model": "m", "messages": [], "max_tokens": 1}) + self.assertEqual(status, 402) + self.assertEqual(len(Upstream.seen), sent) # neither reached the provider + self.assertEqual(led.state()["spent_usd"], 0.0) + exc, said = relay.failure(r.calls("u1")) + self.assertEqual(exc, ledger.ERROR) + self.assertIn("spending cap", said) + self.assertEqual(classify.from_verdict(exc, None), classify.API_ERROR) # infrastructure, never a miss + self.assertIn(classify.API_ERROR, classify.INFRA_OUTCOMES) + + def test_a_refused_call_is_the_units_whatever_its_agent_did_next(self): + calls = [{"status": 200}, {"status": 402, "refused": "cap", "error": "the run's spending cap"}, {"status": 200}] + self.assertEqual(relay.failure(calls), (ledger.ERROR, "the run's spending cap")) + + def test_an_error_gives_the_booking_back_and_a_reply_without_tokens_keeps_it(self): + r, led = self.capped(1.0) + status, _, _ = self.post(r, "u1", {"model": "m", "mode": "429", "max_tokens": 100}) + self.assertEqual(status, 429) # the provider's own, passed on as ever + got = led.state() + self.assertEqual((got["spent_usd"], got["unsettled_usd"], got["held_usd"]), (0.0, 0.0, 0)) + status, _, _ = self.post(r, "u1", {"model": "m", "mode": "no-usage", "max_tokens": 100}) + self.assertEqual(status, 200) + rec = r.calls("u1")[-1] + self.assertNotIn("cost_usd", rec) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"]), (0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], rec["booked_usd"]) # it may have been billed + + def test_a_model_without_a_price_is_not_sent_and_closes_nothing(self): + r, led = self.capped(1.0) + sent = len(Upstream.seen) + status, _, body = self.post(r, "u1", {"model": "another", "messages": []}) + self.assertEqual(status, 402) + self.assertIn("no price for the model 'another'", json.loads(body)["error"]["message"]) + self.assertEqual(len(Upstream.seen), sent) + self.assertFalse(led.closed()) + + def test_a_relay_without_a_cap_books_nothing(self): + r = relay.Relay(self.base, "sk-real", tempfile.mkdtemp(), prices=self.PRICES).start() + self.addCleanup(r.stop) + status, _, _ = self.post(r, "u1", {"model": "another", "messages": []}) + self.assertEqual(status, 200) + self.assertNotIn("booked_usd", r.calls("u1")[-1]) + + class EffortTest(unittest.TestCase): def test_the_record_says_the_reasoning_effort_a_request_asked_for(self): # chat's reasoning_effort, the Responses API's reasoning.effort, and none at all @@ -226,6 +329,24 @@ def test_serve_relays_with_the_key_from_its_environment_and_stops_on_sigterm(sel p.kill() p.stdout.close() + def test_serve_books_its_calls_in_the_runs_ledger(self): + """A docker unit's relay: the ledger is a file it is given, shared with every other relay of the job.""" + import signal + led = ledger.Ledger.create(os.path.join(tempfile.mkdtemp(), "ledger", "ledger.json"), 1.0) + p = self.serve("sk-from-env", "--ledger", led.path, "--prices", json.dumps({"the-recipes": {"input": 1.0, "output": 10.0}})) + try: + where = json.loads(p.stdout.readline())["listening"] + req = urllib.request.Request(f"{where}/u1/v1/chat/completions", data=b'{"model":"asked","max_tokens":5}', + headers={"Content-Type": "application/json"}) + urllib.request.urlopen(req).read() + self.assertAlmostEqual(led.state()["spent_usd"], (10 * 1.0 + 2 * 10.0) / 1e6) + p.send_signal(signal.SIGTERM) + self.assertEqual(p.wait(timeout=10), 0) + finally: + if p.poll() is None: + p.kill() + p.stdout.close() + def test_serve_without_the_key_in_its_environment_refuses(self): p = self.serve("") self.assertIn(relay.KEY_VAR, p.stdout.readline()) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index c9856e8..390bf1c 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -13,7 +13,7 @@ from unittest import mock from nullius.runs import classify, parse -from nullius.standard import acp, container, definitions, install, run +from nullius.standard import acp, container, definitions, install, ledger, run ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") # A conversation whose user a model plays: it opens, speaks once more, then is done. @@ -292,6 +292,63 @@ def recorded(doc): # the user's own row over the list's: the list did not say this price, so it is not named self.assertEqual(recorded({"models": {"m-1": listed}, "own": {"m-1": own}}), {"models": {"m-1": {**own, "source": "own"}}}) + def calling(self, **kw): + """Eight units, each one model call (11 tokens in, 3 out), one at a + time. Input is free here and the model's own output limit is 100 + tokens at 10 USD a million, so a call is booked 0.001 and costs 0.00003.""" + here = os.path.join(self.tmp, "calling") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(CALLING_DRIVER) + b = dict(definitions.get("benchmark", "fakebench", ROOT), _here=here, driver={"command": ["python3", "{here}/driver.py"]}) + said = [] + kw = {"prices": {"m-1": {"input": 0.0, "output": 10.0, "max_output": 100}}, **kw} + job_dir, failed = run.run_and_report(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=said.append, unsandboxed=True, proving=True, **kw) + with open(os.path.join(job_dir, "result.json")) as f: + return job_dir, failed, json.load(f), said + + def test_a_spending_cap_stops_the_run_and_its_result_says_where(self): + # 0.001 booked a call, 0.00003 spent by each: the fourth no longer fits 0.00107 + job_dir, failed, result, said = self.calling(max_usd=0.00107) + self.assertEqual(result["budget"], {"max_usd": 0.00107, "spent_usd": 0.00009, "stopped": True, + "cut_off": ["m3"], "not_run": ["m4", "m5", "m6", "m7"]}) + self.assertEqual((result["n_total_trials"], result["stats"]["n_cancelled_trials"], result["stats"]["n_errored_trials"]), (8, 4, 1)) + self.assertFalse(failed) # three units reached a verdict + got, _ = self.outcomes(job_dir) + self.assertEqual(sorted(got), ["m0", "m1", "m2", "m3"]) # the four never begun have no trial + self.assertEqual(got["m3"].exception_type, ledger.ERROR) + self.assertEqual(got["m3"].outcome, classify.API_ERROR) # infrastructure: never a miss + self.assertTrue(all(got[u].outcome == classify.RESOLVED for u in ("m0", "m1", "m2"))) + calls = self.relay_calls(job_dir) + self.assertEqual([c.get("refused") for c in calls].count("cap"), 1) + sent = [c for c in calls if not c.get("refused")] + self.assertEqual(len(sent), 3) # no call went out past the cap + self.assertTrue(all(abs(c["booked_usd"] - 0.001) < 1e-12 for c in sent)) + self.assertLessEqual(sum(c["cost_usd"] for c in sent), 0.00107) + self.assertTrue(any(s.startswith("stopped: a call did not fit the spending cap of 0.0011 USD (0.0001 spent): 1 cut off, 4 not run") + for s in said), said) + self.assert_no_key(job_dir) + + def test_a_spending_cap_the_run_stays_under_changes_nothing_but_the_result(self): + job_dir, failed, result, said = self.calling(max_usd=1.0) + self.assertEqual(result["budget"], {"max_usd": 1.0, "spent_usd": 0.00024, "stopped": False, "cut_off": [], "not_run": []}) + self.assertEqual(len(self.outcomes(job_dir)[0]), 8) + self.assertIn("spent 0.0002 of the 1.00 USD cap", said) + job_dir, _, result, _ = self.calling(name="uncapped") + self.assertNotIn("budget", result) + self.assertFalse(os.path.exists(os.path.join(job_dir, ".work", "ledger"))) + self.assertFalse(any("booked_usd" in c for c in self.relay_calls(job_dir))) + + def test_a_spending_cap_is_refused_without_a_price_for_each_model_or_an_amount(self): + with self.assertRaises(run.Refused) as e: + self.calling(max_usd=1.0, prices={"another": {"input": 1.0, "output": 1.0}}) + self.assertIn("there is none for m-1", str(e.exception)) + for bad in (0, -1, float("inf"), float("nan"), "much"): + with self.assertRaises(run.Refused, msg=bad): + self.calling(max_usd=bad) + self.assertEqual([n for n in os.listdir(self.tmp) if "fakebench" in n], []) # nothing was begun + def test_a_run_with_no_price_has_no_cost_and_says_nothing_of_prices(self): job_dir = self.go("fakebench", "fake") calls = self.relay_calls(job_dir) diff --git a/docs/cli/run.md b/docs/cli/run.md index 7b8c06b..1962596 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -5,7 +5,7 @@ One benchmark and one agent, capability, control or setup, into a job. ```bash title="Run" nullius standard run [@version] [--model M] [--dataset D] [--limit N] [-j N] \ [--unsandboxed | --backend docker] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ - [--price MODEL=INPUT,OUTPUT[,CACHED]]... \ + [--price MODEL=INPUT,OUTPUT[,CACHED]]... [--max-usd USD] \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` @@ -28,6 +28,7 @@ nullius standard run locomo model --model gpt-4o-mini | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | | `--price MODEL=INPUT,OUTPUT[,CACHED]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | +| `--max-usd USD` | The run's spending cap: no call goes out once the spend plus the calls in flight would pass it ([below](#a-spending-cap)) | | `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | | `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--repeat`); by default they stay for the next run, listed and removed in Settings, [Sandbox](../settings/sandbox.md) | | `--judge-model` | Instead of the official judge | @@ -46,6 +47,25 @@ nullius standard run locomo model --model gpt-4o-mini | Progress | stderr, one line per stage (`[nullius] stage=install ...`, `stage=units n=10`, `stage=done`, `stage=refused`), also `progress.jsonl` | | The job | Opens in the app; `events.jsonl` feeds the live board | +## A spending cap + +`--max-usd 5` stops a run at 5 USD, for a prepaid or shared key. Every relay of +the job books each call in one ledger before sending it +(`app/nullius/standard/ledger.py`). + +| Rule | Detail | +| --- | --- | +| Prices first | Every model the run calls (its own, the judge, the simulated user, an option's embeddings, an agent's side models) needs a price in your [price list](../settings/model-prices.md) or a `--price`; else the run is refused before it starts | +| A booking | The most a call is likely to cost: its characters over three as input tokens, and as output the limit the call names, else the model's own (its price's output limit), else 4,096 tokens | +| Sent or refused | A call is sent only while what was spent, what the calls in flight have booked and its own booking stay within the cap. The first that does not fit is refused, and so is every call after it: the run stops at one point | +| Settled | A reply turns the booking into what the call cost. A provider's error gives the booking back. A reply with no token count, or cut off, keeps its whole booking as unsettled: it may have been billed | +| A refused call | Answered by the relay with HTTP 402, naming the relay and the cap, and recorded `refused`: never the provider's error | +| The units | One cut off by the cap ends as `SpendingCapError`, an API error outside the agent's rate, never a miss. Units not begun are not run | +| The result | `result.json` carries `budget`: `max_usd`, `spent_usd`, `unsettled_usd` when there is some, `stopped`, `cut_off` and `not_run`; stderr says `stage=cap` | +| Several at a time | The calls in flight take room: with `-j 4`, four bookings must fit at once. A cap close to one call's booking stops the run early | +| An estimate | Prices are list prices and a booking is not a bill: a call that costs more than was booked takes the spend past the cap by the difference, and the next call is refused | +| A shard | The cap is that shard's alone: split a budget over the shards yourself | +
- [The standard](../standard.md#a-run) diff --git a/docs/field-guide.md b/docs/field-guide.md index c0fede5..893780d 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -231,7 +231,7 @@ Eleven classes. The three marked **yes** under Infrastructure leave the adjusted | `no_patch` | No patch produced | no | Finished without changing anything. | | `agent_error` | Agent error | no | Timed out, filled its context, truncated its output, refused, or exited non-zero. A result, counted against the agent. | | `rate_limited` | Rate limited | yes | An HTTP 429 ended the trial. | -| `api_error` | API error | yes | Any other provider failure: overloaded, unreachable, out of quota, key rejected. | +| `api_error` | API error | yes | Any other provider failure: overloaded, unreachable, out of quota, key rejected. Also a standard run's unit cut off by its own spending cap (`SpendingCapError`). | | `setup_error` | Setup error | yes | The environment or image never came up, or the unit was not run for want of a secret or docker it needs, so the agent never ran. | | `unknown` | Unknown | no | No verdict, or an exception on neither list. Counted, charged to nobody; the trial page shows the recorded type. | diff --git a/docs/settings/model-prices.md b/docs/settings/model-prices.md index 5dec9ed..f9d6158 100644 --- a/docs/settings/model-prices.md +++ b/docs/settings/model-prices.md @@ -20,6 +20,7 @@ its list is filled once. | Recorded | A job's `config.json` keeps the prices of the models it calls, each with its source (`list`, `own`, or `run` for `--price`), and where and when the list was fetched when a price is the list's | | Docker and hosts | A unit's relay in a container gets its models' prices from the run; a run on a host takes this machine's list with it | | An estimate | List prices, not a bill: [Metrics](../field-guide.md) | +| A spending cap | `--max-usd` books each call at these prices before sending it, so it needs one for every model the run calls: [Run a benchmark](../cli/run.md#a-spending-cap) | | Code | `app/nullius/prices.py` | ## From a shell diff --git a/docs/standard.md b/docs/standard.md index 2a46f71..9cad1ae 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -138,6 +138,7 @@ record); verifier/reward.txt and score.json | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | +| A spending cap (`--max-usd`) | Every relay of the job books a call in one ledger (`.work/ledger`, `standard/ledger.py`) before sending it; the first that does not fit is refused by the relay itself (HTTP 402, recorded `refused`), and every call after it. A unit cut off is a `SpendingCapError`, an API error outside the agent's rate; units not begun are not run; `result.json` carries `budget`. [Run a benchmark](cli/run.md#a-spending-cap) | ## Safety @@ -296,7 +297,7 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | Rule | Detail | | --- | --- | -| No central relay | Each unit's relay runs on its own machine with the key it was handed; split a launch over keys of their own if one cannot carry it | +| No central relay | Each unit's relay runs on its own machine with the key it was handed; split a launch over keys of their own if one cannot carry it. A spending cap (`--max-usd`) is each shard's own | | Shards by the unit's id | `sha256(unit id) mod N` (`standard/shard.py`); changing N moves most units. Order: the driver's `units`, the shard, then `--limit` (`--shard 1/2 --limit 6` is the first six units of shard 1) | | A unit keeps its place | `u0042` is the 43rd unit of the whole dataset in every shard | | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model) | From b4393c5dba7b8a733a3c18f96cafac350077769d Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 22:31:44 +0100 Subject: [PATCH 6/8] Give a unit the spending cap cut off its own outcome A unit the run's cap cut off read as an API error, as if the provider had failed. Nothing failed: the relay refused the call at the user's own cap. It is now the outcome spending_cap (Spending cap), outside the adjusted rate like the other infrastructure outcomes, in a bronze of its own in both themes. On the live feed it is budget, an outcome the board already knew and nothing produced; the board's adjusted rate leaves it out as it does infra_error. The judge is not offered it as a likely outcome and no cause suggests it: only the relay's refusal makes it, recorded, never read from a transcript. --- app/nullius/judge/reconcile.py | 3 +++ app/nullius/runs/classify.py | 22 ++++++++++++++++++---- app/nullius/runs/live.py | 2 ++ app/nullius/standard/job.py | 7 ++++--- app/nullius/standard/ledger.py | 2 +- app/static/runs/board.css | 1 + app/static/runs/launches.css | 1 + app/static/runs/live-board.css | 2 +- app/static/runs/live-board.js | 4 ++-- app/static/style.css | 8 ++++++-- app/tests/test_classify.py | 19 +++++++++++++------ app/tests/test_judge.py | 2 +- app/tests/test_standard_relay.py | 4 ++-- app/tests/test_standard_run.py | 2 +- docs/cli/run.md | 2 +- docs/field-guide.md | 7 ++++--- docs/standard.md | 2 +- 17 files changed, 62 insertions(+), 28 deletions(-) diff --git a/app/nullius/judge/reconcile.py b/app/nullius/judge/reconcile.py index f77d610..8e24d6a 100644 --- a/app/nullius/judge/reconcile.py +++ b/app/nullius/judge/reconcile.py @@ -65,6 +65,9 @@ # The judge's option names API errors alongside rate limits and timeouts. classify.API_ERROR: ("rate_limit_or_timeout",), classify.SETUP_ERROR: ("environment_error",), + # The relay's own refusal at the run's spending cap: recorded, never read + # from a transcript, so nothing agrees or disagrees with it and no cause suggests it. + classify.SPENDING_CAP: (), # No claim made: not comparable. The judge's pick becomes a suggestion. classify.UNKNOWN: (), } diff --git a/app/nullius/runs/classify.py b/app/nullius/runs/classify.py index 2b37bf2..d8816fa 100644 --- a/app/nullius/runs/classify.py +++ b/app/nullius/runs/classify.py @@ -24,11 +24,15 @@ RATE_LIMITED = "rate_limited" API_ERROR = "api_error" SETUP_ERROR = "setup_error" +# The run's own spending cap refused a model call the unit needed (the +# standard's relay, ``standard.ledger``): the unit was cut off. Nothing +# failed, and no attempt was had: the money ran out, by the user's own word. +SPENDING_CAP = "spending_cap" AGENT_ERROR = "agent_error" UNKNOWN = "unknown" # Outcomes that reflect harness/infra problems, not the agent's ability. -INFRA_OUTCOMES = {RATE_LIMITED, API_ERROR, SETUP_ERROR} +INFRA_OUTCOMES = {RATE_LIMITED, API_ERROR, SETUP_ERROR, SPENDING_CAP} ORDER = [ RESOLVED, @@ -41,6 +45,7 @@ RATE_LIMITED, API_ERROR, SETUP_ERROR, + SPENDING_CAP, UNKNOWN, ] @@ -57,8 +62,10 @@ "modelnotfounderror", # the model name is wrong; the agent never ran "networkconnectionerror", # DNS, refused connection, TLS — transport, not skill "unknownapierror", # Harbor's own "some provider error"; an ApiError without the prefix - "spendingcaperror", # the run's own spending cap refused the call (the standard's relay): no attempt was had } +# What a unit the run's spending cap cut off carries (``standard.ledger.ERROR``): recorded by the relay that +# refused the call, never inferred. +_CAP_EXCEPTION = "spendingcaperror" # What Harbor's agents raise when the agent itself fails — its budget spent, # its own context filled, its own output truncated, a refusal, its process @@ -93,6 +100,7 @@ RATE_LIMITED: "Rate limited", API_ERROR: "API error", SETUP_ERROR: "Setup error", + SPENDING_CAP: "Spending cap", UNKNOWN: "Unknown", } @@ -132,14 +140,18 @@ ), API_ERROR: ( "The model API failed some other way — overloaded, unreachable, out of " - "quota, or the key was rejected — or the run's own spending cap refused " - "the call. The agent never got a fair attempt, so " + "quota, or the key was rejected. The agent never got a fair attempt, so " "this is infrastructure too and is excluded from the adjusted rate." ), SETUP_ERROR: ( "The environment or image failed to build or start, so the agent never ran. " "Infrastructure, not capability — excluded from the adjusted rate." ), + SPENDING_CAP: ( + "The run reached its spending cap: its relay refused a model call this " + "trial needed, so the trial was cut off. Nothing failed and no attempt " + "was had, so it is excluded from the adjusted rate." + ), UNKNOWN: ( "No verifier report and no exception, or an exception this reader does not " "recognise — a crash in the harness, the verifier or the reader itself. The " @@ -164,6 +176,8 @@ def from_verdict(exception_type, verdict): """ if exception_type: et = exception_type.lower().rpartition(".")[2] # tolerate dotted paths + if et == _CAP_EXCEPTION: + return SPENDING_CAP # Rate limiting first: it is an Api* class too, but common enough to be # worth seeing on its own rather than folded into the generic bucket. if "ratelimit" in et or "rate_limit" in et or "429" in et: diff --git a/app/nullius/runs/live.py b/app/nullius/runs/live.py index 41e91ae..6f2affc 100644 --- a/app/nullius/runs/live.py +++ b/app/nullius/runs/live.py @@ -70,6 +70,8 @@ #: mean over the trial's parts, how many (``parts``). The snapshot says so only #: where there is one: a ``scored`` count, and ``score`` / ``parts`` on that #: trial. A feed without one folds to a snapshot with none of those keys. +#: ``budget`` is a run's spending cap: a unit its relay cut off there (the +#: standard's ``--max-usd``), outside the rates as ``infra_error`` is. OUTCOMES = ("resolved", "unresolved", "scored", "agent_error", "infra_error", "cancelled", "timeout", "budget") ERROR_OUTCOMES = {"agent_error", "infra_error", "timeout", "budget"} diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 58cc1c3..ca7e3ae 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -486,17 +486,18 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str #: an error kind naming the machine or the benchmark's side rather than the agent: the live board's infra_error _INFRA = ("Setup", "Verifier", "Api", "RateLimit", "Environment", "Docker", "Build", "Connection", "Network", "Authentication", - "ModelNotFound", "SpendingCap") + "ModelNotFound") def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict: """A unit's end in the live feed's words (``nullius.runs.live``): its outcome, with the reward (a pass benchmark) or the score, and the error's - kind as evidence.""" + kind as evidence. A unit the run's spending cap cut off (``ledger.ERROR``) + is the feed's ``budget``.""" score = score or {} if exception: k = exception[0] - return {"outcome": "cancelled" if k == "CancelledError" else "timeout" if "Timeout" in k else "infra_error" if any(w in k for w in _INFRA) else "agent_error", + return {"outcome": "cancelled" if k == "CancelledError" else "budget" if k == "SpendingCapError" else "timeout" if "Timeout" in k else "infra_error" if any(w in k for w in _INFRA) else "agent_error", "exc": k, "evidence": (exception[1] or k)[:200]} if kind == "pass": reward = score.get("reward") diff --git a/app/nullius/standard/ledger.py b/app/nullius/standard/ledger.py index 72f07a6..7347007 100644 --- a/app/nullius/standard/ledger.py +++ b/app/nullius/standard/ledger.py @@ -35,7 +35,7 @@ from typing import Optional, Tuple OUTPUT = 4096 # output tokens booked when neither the call nor the model's price row names a limit -ERROR = "SpendingCapError" # what a unit the cap ended carries: infrastructure, never a miss (runs.classify) +ERROR = "SpendingCapError" # what a unit the cap ended carries: its own outcome outside the rate, never a miss (runs.classify) _LIMITS = ("max_completion_tokens", "max_output_tokens", "max_tokens") diff --git a/app/static/runs/board.css b/app/static/runs/board.css index 85fd86e..69a70ce 100644 --- a/app/static/runs/board.css +++ b/app/static/runs/board.css @@ -40,6 +40,7 @@ .rp .oc-no_patch { --oc: var(--no_patch); } .rp .oc-agent_error { --oc: var(--agent_error); } .rp .oc-rate_limited { --oc: var(--rate_limited); } .rp .oc-api_error { --oc: var(--api_error); } .rp .oc-setup_error { --oc: var(--setup_error); } .rp .oc-unknown { --oc: var(--unknown-mark); } +.rp .oc-spending_cap { --oc: var(--spending_cap); } .rp .sw { display: inline-block; width: 10px; height: 10px; border-radius: 2px; background: var(--oc); flex: none; } .rp .odot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: var(--oc); flex: none; } .rp .adv { display: inline-block; font: 500 10.5px/16px var(--sans); letter-spacing: .06em; text-transform: uppercase; color: var(--text-faint); diff --git a/app/static/runs/launches.css b/app/static/runs/launches.css index ee5f7c2..1d02ee6 100644 --- a/app/static/runs/launches.css +++ b/app/static/runs/launches.css @@ -64,6 +64,7 @@ .la-mix .oc-no_patch { background: var(--no_patch); } .la-mix .oc-agent_error { background: var(--agent_error); } .la-mix .oc-rate_limited { background: var(--rate_limited); } .la-mix .oc-api_error { background: var(--api_error); } .la-mix .oc-setup_error { background: var(--setup_error); } .la-mix .oc-unknown { background: var(--unknown); } +.la-mix .oc-spending_cap { background: var(--spending_cap); } .la-mix span + span { box-shadow: inset 1.5px 0 0 var(--surface); } .la-go { text-align: right; white-space: nowrap; font-size: 12.5px; } .launch-foot { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; padding: 10px 18px; border-top: 1px solid var(--border-soft); font-size: 12.5px; color: var(--text-dim); } diff --git a/app/static/runs/live-board.css b/app/static/runs/live-board.css index db01ab6..4ef11b1 100644 --- a/app/static/runs/live-board.css +++ b/app/static/runs/live-board.css @@ -18,7 +18,7 @@ .lb { --lb-track: color-mix(in srgb, var(--text-faint) 16%, var(--surface)); --lb-hair: var(--border-soft); } .lb .lbo-resolved { --c: var(--resolved); } .lb .lbo-unresolved { --c: var(--failed_verification); } .lb .lbo-scored { --c: var(--scored); } .lb .lbo-agent_error { --c: var(--agent_error); } .lb .lbo-infra_error { --c: var(--rate_limited); } -.lb .lbo-timeout { --c: var(--patch_failed_tests); } .lb .lbo-budget { --c: var(--api_error); } .lb .lbo-cancelled { --c: var(--unknown); } +.lb .lbo-timeout { --c: var(--patch_failed_tests); } .lb .lbo-budget { --c: var(--spending_cap); } .lb .lbo-cancelled { --c: var(--unknown); } .lb .lbp-env_setup { --c: var(--rate_limited); } .lb .lbp-agent_setup { --c: var(--patch_failed_tests); } .lb .lbp-agent_run { --c: var(--accent); } .lb .lbp-verify { --c: var(--ph-verify); } diff --git a/app/static/runs/live-board.js b/app/static/runs/live-board.js index e19c404..b969a07 100644 --- a/app/static/runs/live-board.js +++ b/app/static/runs/live-board.js @@ -28,7 +28,7 @@ const PHASES = ['env_setup', 'agent_setup', 'agent_run', 'verify']; const PHASE_LABEL = {env_setup: 'env setup', agent_setup: 'agent setup', agent_run: 'agent run', verify: 'verify'}; const OUTCOMES = ['resolved', 'unresolved', 'scored', 'agent_error', 'infra_error', 'timeout', 'budget', 'cancelled']; - const OUTCOME_LABEL = {resolved: 'Resolved', unresolved: 'Unresolved', scored: 'Scored', agent_error: 'Agent error', infra_error: 'Infra error', cancelled: 'Cancelled', timeout: 'Timeout', budget: 'Budget'}; + const OUTCOME_LABEL = {resolved: 'Resolved', unresolved: 'Unresolved', scored: 'Scored', agent_error: 'Agent error', infra_error: 'Infra error', cancelled: 'Cancelled', timeout: 'Timeout', budget: 'Spending cap'}; const STATE_LABEL = {preparing: 'Preparing', starting: 'Starting', running: 'Running', finished: 'Finished', unknown: 'Waiting for the feed'}; const FLIGHT_SHOWN = 12; // past this many in flight, the nearest their limit come first and the rest wait behind a button const MANY_TILES = 60; // past this many trials, the tiles are drawn smaller @@ -99,7 +99,7 @@ const C = s.counts || {}; const queued = C.total != null ? Math.max(0, C.total - C.completed - C.running) : null; const rate = s.elapsed_sec > 60 && C.completed ? C.completed / (s.elapsed_sec / 60) : null; - const ran = C.completed - (oc.infra_error || 0); + const ran = C.completed - (oc.infra_error || 0) - (oc.budget || 0); const when = state === 'finished' ? 'on the feed' : 'so far'; // the feed's outcomes, not the classifier's: Results has those // A benchmark that gives a score and no pass or fail has no solve rate: the mean score, over every part scored, stands there. const scored = (s.trials || []).filter(t => t.state === 'done' && t.outcome === 'scored' && t.score != null); diff --git a/app/static/style.css b/app/static/style.css index 9631c7b..e228d89 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -35,6 +35,7 @@ --rate_limited: #7d8595; --api_error: #4aa8ee; /* 35.8 from its nearest, rate_limited */ --setup_error: #e58ac0; + --spending_cap: #b68a6c; /* bronze, a coin: 34.8 from its nearest, rate_limited */ --unknown: #454c58; /* Tool categories and run phases own their colours rather than borrowing @@ -109,6 +110,7 @@ --rate_limited: #626b7a; --api_error: #125db2; /* 43.5 from its nearest, rate_limited */ --setup_error: #a03089; + --spending_cap: #895741; /* 34.5 from its nearest, patch_failed_tests */ --unknown: #aab1bb; /* Same treatment, inverted: worst pair 16.7. */ @@ -998,6 +1000,7 @@ tr.control-row td { color: var(--text-dim); } .oc-rate_limited::before, .seg.oc-rate_limited, .tag.oc-rate_limited::before { background: var(--rate_limited); } .oc-api_error::before, .seg.oc-api_error, .tag.oc-api_error::before { background: var(--api_error); } .oc-setup_error::before, .seg.oc-setup_error, .tag.oc-setup_error::before { background: var(--setup_error); } +.oc-spending_cap::before, .seg.oc-spending_cap, .tag.oc-spending_cap::before { background: var(--spending_cap); } .seg.oc-unknown { background: var(--unknown); } /* outcome tag inside a table row */ @@ -1019,15 +1022,16 @@ tr.control-row td { color: var(--text-dim); } .tag.oc-rate_limited { --outcome-color: var(--rate_limited); } .tag.oc-api_error { --outcome-color: var(--api_error); } .tag.oc-setup_error { --outcome-color: var(--setup_error); } +.tag.oc-spending_cap { --outcome-color: var(--spending_cap); } .tag.oc-unknown { --outcome-color: var(--unknown); } -.tag:is(.oc-resolved, .oc-scored, .oc-patch_failed_tests, .oc-failed_verification, .oc-patch_not_applied, .oc-no_patch, .oc-agent_error, .oc-rate_limited, .oc-api_error, .oc-setup_error, .oc-unknown) { +.tag:is(.oc-resolved, .oc-scored, .oc-patch_failed_tests, .oc-failed_verification, .oc-patch_not_applied, .oc-no_patch, .oc-agent_error, .oc-rate_limited, .oc-api_error, .oc-setup_error, .oc-spending_cap, .oc-unknown) { color: color-mix(in srgb, var(--outcome-color) 75%, var(--text)); background: color-mix(in srgb, var(--outcome-color) 16%, var(--surface)); border-color: color-mix(in srgb, var(--outcome-color) 42%, var(--border)); font-family: var(--sans); font-size: 12px; font-weight: 500; line-height: 1.45; padding: .22rem .55rem; border-radius: 5px; } -.tag:is(.oc-resolved, .oc-scored, .oc-patch_failed_tests, .oc-failed_verification, .oc-patch_not_applied, .oc-no_patch, .oc-agent_error, .oc-rate_limited, .oc-api_error, .oc-setup_error, .oc-unknown)::before { width: 7px; height: 7px; flex: none; } +.tag:is(.oc-resolved, .oc-scored, .oc-patch_failed_tests, .oc-failed_verification, .oc-patch_not_applied, .oc-no_patch, .oc-agent_error, .oc-rate_limited, .oc-api_error, .oc-setup_error, .oc-spending_cap, .oc-unknown)::before { width: 7px; height: 7px; flex: none; } .tag.oc-unknown { color: var(--text-dim); } .tag.oc-unknown::before { background: var(--text-dim); } diff --git a/app/tests/test_classify.py b/app/tests/test_classify.py index 7ab35c9..9bd7ca1 100644 --- a/app/tests/test_classify.py +++ b/app/tests/test_classify.py @@ -92,12 +92,19 @@ def test_both_boards_charge_the_infrastructure_as_the_classifier_does(self): with self.subTest(name, board=board): self.assertIn(outcome(name), ("agent_error", "timeout")) - def test_a_unit_the_runs_spending_cap_cut_off_is_infrastructure_on_both(self): - """The standard's own: its relay refused the call (``standard.ledger``), so the unit had no attempt.""" + def test_a_unit_the_runs_spending_cap_cut_off_has_its_own_outcome_outside_the_rate_on_both(self): + """The standard's own: its relay refused the call (``standard.ledger``), so the unit had no attempt. + Not an API error: nothing failed. The live feed's word for it is ``budget``.""" + from nullius.runs import live from nullius.standard import job, ledger - self.assertEqual(C.from_verdict(ledger.ERROR, None), C.API_ERROR) - self.assertIn(C.API_ERROR, C.INFRA_OUTCOMES) - self.assertEqual(job.feed_outcome("pass", None, (ledger.ERROR, "the run's spending cap"))["outcome"], "infra_error") + self.assertEqual(C.from_verdict(ledger.ERROR, None), C.SPENDING_CAP) + self.assertEqual(C.from_verdict("nullius.standard." + ledger.ERROR, None), C.SPENDING_CAP) + self.assertIn(C.SPENDING_CAP, C.INFRA_OUTCOMES) + self.assertEqual(C.ORDER.index(C.SPENDING_CAP), C.ORDER.index(C.UNKNOWN) - 1) + self.assertNotIn("—", C.DESCRIPTIONS[C.SPENDING_CAP]) + end = job.feed_outcome("pass", None, (ledger.ERROR, "the run's spending cap")) + self.assertEqual((end["outcome"], end["exc"]), ("budget", ledger.ERROR)) + self.assertIn("budget", live.OUTCOMES) class HarborTaxonomy(unittest.TestCase): @@ -242,7 +249,7 @@ def test_every_outcome_is_ordered(self): """ORDER drives sorting; an outcome missing from it raises on compare.""" produced = {C.RESOLVED, C.SCORED, C.PATCH_FAILED_TESTS, C.FAILED_VERIFICATION, C.PATCH_NOT_APPLIED, C.NO_PATCH, C.AGENT_ERROR, - C.RATE_LIMITED, C.API_ERROR, C.SETUP_ERROR, C.UNKNOWN} + C.RATE_LIMITED, C.API_ERROR, C.SETUP_ERROR, C.SPENDING_CAP, C.UNKNOWN} self.assertEqual(produced, set(C.ORDER)) diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index 65e1f95..b90fad5 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -61,7 +61,7 @@ def test_likely_outcome_is_asked_of_unknown_trials_only_over_the_classifiers_own from nullius.runs import classify q = questions.QUESTIONS["likely_outcome"] self.assertEqual((q["type"], q["state"], q["when"]), ("choice", "failure", "unknown")) - self.assertEqual(sorted(q["criteria"]), sorted([l for l in classify.ORDER if l not in (classify.UNKNOWN, classify.SCORED)] + ["other"])) # the classifier's labels, unknown itself and scored (only a benchmark's score makes it) excluded, an escape + self.assertEqual(sorted(q["criteria"]), sorted([l for l in classify.ORDER if l not in (classify.UNKNOWN, classify.SCORED, classify.SPENDING_CAP)] + ["other"])) # the classifier's labels, unknown itself, scored (only a benchmark's score makes it) and the spending cap (only the relay's refusal makes it) excluded, an escape self.assertIn("likely_outcome", questions.for_trial(False, "unknown")); self.assertNotIn("failure_cause", questions.for_trial(True, "resolved")) for outcome in ("resolved", "patch_failed_tests", "agent_error", None): self.assertNotIn("likely_outcome", questions.for_trial(outcome == "resolved", outcome), outcome) diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 88928d9..d5bd83e 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -243,8 +243,8 @@ def test_a_call_that_does_not_fit_is_answered_by_the_relay_and_never_sent(self): exc, said = relay.failure(r.calls("u1")) self.assertEqual(exc, ledger.ERROR) self.assertIn("spending cap", said) - self.assertEqual(classify.from_verdict(exc, None), classify.API_ERROR) # infrastructure, never a miss - self.assertIn(classify.API_ERROR, classify.INFRA_OUTCOMES) + self.assertEqual(classify.from_verdict(exc, None), classify.SPENDING_CAP) # its own outcome, never a miss + self.assertIn(classify.SPENDING_CAP, classify.INFRA_OUTCOMES) def test_a_refused_call_is_the_units_whatever_its_agent_did_next(self): calls = [{"status": 200}, {"status": 402, "refused": "cap", "error": "the run's spending cap"}, {"status": 200}] diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 390bf1c..3a21656 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -318,7 +318,7 @@ def test_a_spending_cap_stops_the_run_and_its_result_says_where(self): got, _ = self.outcomes(job_dir) self.assertEqual(sorted(got), ["m0", "m1", "m2", "m3"]) # the four never begun have no trial self.assertEqual(got["m3"].exception_type, ledger.ERROR) - self.assertEqual(got["m3"].outcome, classify.API_ERROR) # infrastructure: never a miss + self.assertEqual(got["m3"].outcome, classify.SPENDING_CAP) # its own outcome, outside the rate: never a miss self.assertTrue(all(got[u].outcome == classify.RESOLVED for u in ("m0", "m1", "m2"))) calls = self.relay_calls(job_dir) self.assertEqual([c.get("refused") for c in calls].count("cap"), 1) diff --git a/docs/cli/run.md b/docs/cli/run.md index 1962596..25e4d3a 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -60,7 +60,7 @@ the job books each call in one ledger before sending it | Sent or refused | A call is sent only while what was spent, what the calls in flight have booked and its own booking stay within the cap. The first that does not fit is refused, and so is every call after it: the run stops at one point | | Settled | A reply turns the booking into what the call cost. A provider's error gives the booking back. A reply with no token count, or cut off, keeps its whole booking as unsettled: it may have been billed | | A refused call | Answered by the relay with HTTP 402, naming the relay and the cap, and recorded `refused`: never the provider's error | -| The units | One cut off by the cap ends as `SpendingCapError`, an API error outside the agent's rate, never a miss. Units not begun are not run | +| The units | One cut off by the cap ends as `SpendingCapError`: the outcome Spending cap, outside the agent's rate, never a miss. Units not begun are not run | | The result | `result.json` carries `budget`: `max_usd`, `spent_usd`, `unsettled_usd` when there is some, `stopped`, `cut_off` and `not_run`; stderr says `stage=cap` | | Several at a time | The calls in flight take room: with `-j 4`, four bookings must fit at once. A cap close to one call's booking stops the run early | | An estimate | Prices are list prices and a booking is not a bill: a call that costs more than was booked takes the spend past the cap by the difference, and the next call is refused | diff --git a/docs/field-guide.md b/docs/field-guide.md index 893780d..93549c6 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -150,7 +150,7 @@ How the agent worked, not whether it won. Computed over layer 3. | Field | What it is | Source | Works on | | --- | --- | --- | --- | -| `outcome` | One of eleven classes: from the exception if there was one, else from the verifier's verdict. Full vocabulary below. | inferred | mixed | +| `outcome` | One of twelve classes: from the exception if there was one, else from the verifier's verdict. Full vocabulary below. | inferred | mixed | | `score · n_parts` | A graded score the benchmark gave, and how many parts it averages. With no reward and no exception, the outcome is *Scored*. | measured | mixed | | `reward` | The verifier's raw reward. Where no report says more, 1.0 or above is a solve. | measured | any | | `f2p_pass / f2p_fail` | Checks that had to start passing: SWE-bench's FAIL_TO_PASS, or every test of a Terminal-Bench task. | measured | mixed | @@ -219,7 +219,7 @@ Computed here over the trajectory's tool calls, when present. They describe cond ## Outcome vocabulary {#outcomes} -Eleven classes. The three marked **yes** under Infrastructure leave the adjusted solve rate's denominator. +Twelve classes. The four marked **yes** under Infrastructure leave the adjusted solve rate's denominator. | Outcome | Label | Infrastructure | Meaning | | --- | --- | --- | --- | @@ -231,8 +231,9 @@ Eleven classes. The three marked **yes** under Infrastructure leave the adjusted | `no_patch` | No patch produced | no | Finished without changing anything. | | `agent_error` | Agent error | no | Timed out, filled its context, truncated its output, refused, or exited non-zero. A result, counted against the agent. | | `rate_limited` | Rate limited | yes | An HTTP 429 ended the trial. | -| `api_error` | API error | yes | Any other provider failure: overloaded, unreachable, out of quota, key rejected. Also a standard run's unit cut off by its own spending cap (`SpendingCapError`). | +| `api_error` | API error | yes | Any other provider failure: overloaded, unreachable, out of quota, key rejected. | | `setup_error` | Setup error | yes | The environment or image never came up, or the unit was not run for want of a secret or docker it needs, so the agent never ran. | +| `spending_cap` | Spending cap | yes | A standard run reached its spending cap (`--max-usd`): its relay refused a model call the unit needed (`SpendingCapError`). Nothing failed; no attempt was had. | | `unknown` | Unknown | no | No verdict, or an exception on neither list. Counted, charged to nobody; the trial page shows the recorded type. | ## Caveats {#caveats} diff --git a/docs/standard.md b/docs/standard.md index 9f90f84..04910e0 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -138,7 +138,7 @@ record); verifier/reward.txt and score.json | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | -| A spending cap (`--max-usd`) | Every relay of the job books a call in one ledger (`.work/ledger`, `standard/ledger.py`) before sending it; the first that does not fit is refused by the relay itself (HTTP 402, recorded `refused`), and every call after it. A unit cut off is a `SpendingCapError`, an API error outside the agent's rate; units not begun are not run; `result.json` carries `budget`. [Run a benchmark](cli/run.md#a-spending-cap) | +| A spending cap (`--max-usd`) | Every relay of the job books a call in one ledger (`.work/ledger`, `standard/ledger.py`) before sending it; the first that does not fit is refused by the relay itself (HTTP 402, recorded `refused`), and every call after it. A unit cut off is a `SpendingCapError`, the outcome `spending_cap` outside the agent's rate; units not begun are not run; `result.json` carries `budget`. [Run a benchmark](cli/run.md#a-spending-cap) | ## Safety From 8c9f655f07257678a5d681c4e98157b13082dcac Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 23:06:53 +0100 Subject: [PATCH 7/8] Re-read cached jobs for the Spending cap outcome A workspace caches parsed jobs by the parse's schema. A unit the cap cut off now classifies as spending_cap where it read api_error, so a cache written by the earlier parse would be read back as current: the schema moves to 13. --- app/nullius/runs/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/nullius/runs/parse.py b/app/nullius/runs/parse.py index 5abc965..aaadc1c 100644 --- a/app/nullius/runs/parse.py +++ b/app/nullius/runs/parse.py @@ -30,7 +30,7 @@ # The shape this module produces. A workspace caches parsed Jobs keyed on it, # so bump it when a Trial or Job field is added, renamed or means something # new — otherwise a cache written by the old parse is read back as current. -SCHEMA = "12" +SCHEMA = "13" # What a rate-limit record in `agent/gateway.log` looks like. The agent's SDK # retries a 429 below the harness (an agent's `max_retries` reaching its From db813ca31a505f16f1108d23e6dc59226641b01b Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 10 Nov 2026 23:51:57 +0100 Subject: [PATCH 8/8] Book a call in flight at what the run's replies have been A call was booked at the model's whole output limit when it named none. On a real provider that was 0.026 USD for calls that cost 0.0002: fifteen problems at a time under a 0.30 USD cap sent eleven calls and stopped with 0.002 spent. What was spent was already each reply's real cost. Only a call in flight has to be estimated, since its cost is known when its reply comes back. It is now counted as the largest reply the run has had from that model so far, which the ledger keeps, and as 4,096 output tokens before any reply; never more than the limit the call names or the model's own. --- app/nullius/standard/ledger.py | 55 +++++++++++++++++++++---------- app/nullius/standard/relay.py | 4 +-- app/tests/test_standard_ledger.py | 34 +++++++++++++++++++ app/tests/test_standard_relay.py | 16 +++++++++ app/tests/test_standard_run.py | 19 ++++++----- docs/cli/run.md | 6 ++-- 6 files changed, 102 insertions(+), 32 deletions(-) diff --git a/app/nullius/standard/ledger.py b/app/nullius/standard/ledger.py index 7347007..ec87b42 100644 --- a/app/nullius/standard/ledger.py +++ b/app/nullius/standard/ledger.py @@ -2,8 +2,8 @@ A capped run (``run --max-usd``) gives each of its relays - the units' own, in this process or in their containers, and the benchmark's - the same -ledger. Before a call goes out, its relay books the most the call is likely -to cost (:func:`bound`). The call is sent only while what was spent, what the +ledger. Before a call goes out, its relay books what the call is likely to +cost (:func:`bound`). The call is sent only while what was spent, what the calls in flight have booked and this booking stay within the cap. The first call that does not fit is refused and closes the ledger: every later call is refused too, so a run stops at one point instead of letting its small calls @@ -14,11 +14,16 @@ whose cost was never known - a reply with no token count, a reply cut off - keeps its whole booking, as ``unsettled``: it may have been billed. -A booking is an estimate, not a bill: the prompt's characters over three as -input tokens, and as output the limit the call names, else the model's own -(its price row's ``max_output``), else ``OUTPUT``. A call that costs more -than was booked for it takes the spend past the cap by the difference, and -the next call is refused. +What was spent is each reply's real cost. Only a call in flight is an +estimate, since its cost is known when its reply comes back: the prompt's +characters over three as input tokens, and as output the largest reply +the run has had from that model so far (the ledger keeps it, ``seen``) - +``OUTPUT`` before any - and never more than the limit the call names or +the model's own (its price row's ``max_output``). The model's limit alone +would book a short answer at a hundred times its cost, and a few calls +at a time would fill a cap nothing had been spent of. A call that costs +more than was booked for it takes the spend past the cap by the +difference, and the next call is refused. The file is rewritten whole under a lock held for one read and one write (``flock`` on a file beside it, as an install is locked): a relay that dies @@ -34,7 +39,7 @@ import uuid from typing import Optional, Tuple -OUTPUT = 4096 # output tokens booked when neither the call nor the model's price row names a limit +OUTPUT = 4096 # output tokens booked for a call before the run has had a reply from its model ERROR = "SpendingCapError" # what a unit the cap ended carries: its own outcome outside the rate, never a miss (runs.classify) _LIMITS = ("max_completion_tokens", "max_output_tokens", "max_tokens") @@ -43,18 +48,19 @@ class Unavailable(Exception): """The ledger could not be read or written: a capped call is then not sent.""" -def bound(req: dict, raw: bytes, row: dict, path: str = "") -> float: - """The most a call is likely to cost, in USD, at ``row`` (a price row, - USD per million tokens). Input: the request's characters over three - - its JSON whole, less what it carries as base64 - at the dearer of the - input and the cache-write price. Output: the limit the call names, else - the row's, else ``OUTPUT``; an embeddings call has none.""" +def bound(req: dict, raw: bytes, row: dict, path: str = "", seen: Optional[int] = None) -> float: + """What a call is likely to cost, in USD, at ``row`` (a price row, USD + per million tokens). Input: the request's characters over three - its + JSON whole, less what it carries as base64 - at the dearer of the input + and the cache-write price. Output: ``seen``, the largest reply the run + has had from this model, or ``OUTPUT`` before any; never more than the + limit the call names or the row's own. An embeddings call has none.""" tokens = math.ceil(max(0, len(raw) - _blobs(req)) / 3) rate = max(row["input"], row.get("cache_write") or 0) out = 0 if not path.split("?")[0].endswith("/embeddings"): named = [req.get(k) for k in _LIMITS if isinstance(req.get(k), int) and not isinstance(req.get(k), bool) and req.get(k) > 0] - out = named[0] if named else row.get("max_output") or OUTPUT + out = min(x for x in (named[0] if named else None, row.get("max_output"), seen or OUTPUT) if x) return (tokens * rate + out * row["output"]) / 1e6 @@ -80,7 +86,7 @@ def __init__(self, path: str): def create(cls, path: str, max_usd: float) -> "Ledger": os.makedirs(os.path.dirname(path), exist_ok=True) led = cls(path) - led._write({"max_usd": float(max_usd), "spent_usd": 0.0, "unsettled_usd": 0.0, "held": {}, "closed": False}) + led._write({"max_usd": float(max_usd), "spent_usd": 0.0, "unsettled_usd": 0.0, "held": {}, "closed": False, "seen": {}}) return led @contextlib.contextmanager @@ -99,6 +105,8 @@ def _read(self) -> dict: with open(self.path, encoding="utf-8") as f: doc = json.load(f) float(doc["max_usd"]), float(doc["spent_usd"]), float(doc["unsettled_usd"]), sum(doc["held"].values()) + if not isinstance(doc.get("seen"), dict): + doc["seen"] = {} return doc def _write(self, doc: dict) -> None: @@ -124,13 +132,17 @@ def reserve(self, amount: float) -> Tuple[Optional[str], dict]: self._write(doc) return booking, seen - def settle(self, booking: str, cost: Optional[float]) -> None: + def settle(self, booking: str, cost: Optional[float], model: Optional[str] = None, output: Optional[int] = None) -> None: """The call's cost in place of its booking; None, its cost was never - known: the whole booking is kept, as unsettled.""" + known: the whole booking is kept, as unsettled. ``output``: the + reply's output tokens, kept when they are the most ``model`` has + answered in this run - what its next calls are booked at.""" with self._locked() as doc: amount = doc["held"].pop(booking, None) if amount is None: return + if model and isinstance(output, int) and not isinstance(output, bool) and output > doc["seen"].get(model, 0): + doc["seen"][model] = output if cost is None: doc["unsettled_usd"] += amount else: @@ -143,6 +155,13 @@ def release(self, booking: str) -> None: if doc["held"].pop(booking, None) is not None: self._write(doc) + def largest(self, model) -> Optional[int]: + """The most output tokens ``model`` has answered a call of this run with; None before any. Read without the lock.""" + try: + return self._read()["seen"].get(model) or None + except (OSError, ValueError, KeyError, TypeError) as e: + raise Unavailable(f"the spending cap's ledger could not be read: {type(e).__name__}: {e}") + def state(self) -> dict: """``max_usd``, ``spent_usd``, ``unsettled_usd``, ``held_usd`` (the calls in flight), ``left_usd`` and ``closed``, read without the lock.""" diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index bd12da3..9804324 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -489,7 +489,7 @@ def record(body): rec["booked_usd"] = booked try: if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole - self.ledger.settle(booking, cost) + self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output")) else: # an error of the provider's, or never reached: not billed self.ledger.release(booking) except ledgers.Unavailable: @@ -551,7 +551,7 @@ def _book(self, req: dict, raw: bytes, rest: str) -> tuple: if not row: return None, None, f"no price for the model {req.get('model')!r}: a run with a spending cap sends no call it cannot price" try: - amount = ledgers.bound(req, raw, row, rest) + amount = ledgers.bound(req, raw, row, rest, self.ledger.largest(req.get("model"))) booking, seen = self.ledger.reserve(amount) except ledgers.Unavailable as e: return None, None, str(e) diff --git a/app/tests/test_standard_ledger.py b/app/tests/test_standard_ledger.py index 8ac64c6..cf293d0 100644 --- a/app/tests/test_standard_ledger.py +++ b/app/tests/test_standard_ledger.py @@ -20,6 +20,23 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self.path = os.path.join(self.tmp, "ledger", "ledger.json") + def test_the_ledger_keeps_the_largest_reply_each_model_has_answered_with(self): + led = ledger.Ledger.create(self.path, 1.0) + self.assertIsNone(led.largest("m")) # before any reply + for cost, model, output in ((0.01, "m", 300), (0.01, "m", 120), (0.01, "other", 9000), (None, "m", None), (0.01, "m", True)): + booking, _ = led.reserve(0.02) + led.settle(booking, cost, model, output) + self.assertEqual((led.largest("m"), led.largest("other"), led.largest("never")), (300, 9000, None)) + with open(self.path) as f: # a ledger written before the field is read as one with none + doc = json.load(f) + del doc["seen"] + with open(self.path, "w") as f: + json.dump(doc, f) + self.assertIsNone(led.largest("m")) + booking, _ = led.reserve(0.02) + led.settle(booking, 0.01, "m", 50) + self.assertEqual(led.largest("m"), 50) + def test_a_call_is_sent_while_spent_and_in_flight_and_its_booking_fit(self): led = ledger.Ledger.create(self.path, 1.0) a, _ = led.reserve(0.4) @@ -117,6 +134,23 @@ def test_a_call_that_names_none_is_booked_the_models_own_limit_else_the_default( self.assertAlmostEqual(self.bound(req, dict(ROW, max_output=2000)), (tokens + 2000 * 10.0) / 1e6) self.assertAlmostEqual(self.bound(req), (tokens + ledger.OUTPUT * 10.0) / 1e6) + def test_a_call_in_flight_is_booked_the_largest_reply_the_run_has_seen_never_past_a_limit(self): + """Its cost is known only when its reply comes back: until then it counts as the largest reply the run + has had from that model, not as the model's limit, which a short answer is a hundredth of.""" + req = {"model": "m", "messages": []} + tokens = -(-len(json.dumps(req)) // 3) + out = lambda row=ROW, seen=None, **named: round((ledger.bound({**req, **named}, json.dumps({**req, **named}).encode(), row, "/chat/completions", seen) * 1e6 + - -(-len(json.dumps({**req, **named})) // 3)) / 10.0) + self.assertEqual(out(seen=300), 300) # what its replies have been + self.assertEqual(out(), ledger.OUTPUT) # before any reply + self.assertEqual(out(dict(ROW, max_output=131072)), ledger.OUTPUT) # not the model's whole limit + self.assertEqual(out(dict(ROW, max_output=131072), seen=900), 900) + self.assertEqual(out(seen=300, max_tokens=100), 100) # never more than the call asks for + self.assertEqual(out(dict(ROW, max_output=2000), seen=5000), 2000) # nor than the model can answer + self.assertEqual(out(max_tokens=64000), ledger.OUTPUT) # a large limit named is no reply seen + self.assertEqual(out(seen=7000, max_tokens=64000), 7000) + self.assertAlmostEqual(ledger.bound(req, json.dumps(req).encode(), ROW, "/chat/completions", 300), (tokens + 300 * 10.0) / 1e6) + def test_input_is_booked_at_the_dearer_of_input_and_cache_write(self): req = {"model": "m", "messages": [], "max_tokens": 1} tokens = -(-len(json.dumps(req)) // 3) diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index d5bd83e..a6f5e7e 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -223,6 +223,22 @@ def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + def test_a_call_is_booked_at_what_the_runs_replies_have_been_once_one_came_back(self): + """Fifteen calls at the model's limit would fill a cap nothing was spent of: after the first reply + (2 output tokens here) a call that names no limit is booked those, at the same price.""" + r, led = self.capped(1.0) + body = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} + raw = json.dumps(body).encode() + self.assertEqual(self.post(r, "u1", body)[0], 200) + first = r.calls("u1")[-1] + self.assertAlmostEqual(first["booked_usd"], ledger.bound(body, raw, self.PRICES["m"])) # no reply yet: the start + self.assertEqual(led.largest("m"), first["reply"]["usage"]["output"]) + self.assertEqual(self.post(r, "u2", body)[0], 200) + second = r.calls("u2")[-1] + self.assertAlmostEqual(second["booked_usd"], ledger.bound(body, raw, self.PRICES["m"], seen=first["reply"]["usage"]["output"])) + self.assertLess(second["booked_usd"], first["booked_usd"]) + self.assertAlmostEqual(led.state()["spent_usd"], first["cost_usd"] + second["cost_usd"]) # what was spent is what the calls cost + def test_a_call_that_does_not_fit_is_answered_by_the_relay_and_never_sent(self): r, led = self.capped(0.0005) # 100 output tokens at 10 USD a million are 0.001 already sent = len(Upstream.seen) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 3a21656..52d418d 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -294,24 +294,25 @@ def recorded(doc): def calling(self, **kw): """Eight units, each one model call (11 tokens in, 3 out), one at a - time. Input is free here and the model's own output limit is 100 - tokens at 10 USD a million, so a call is booked 0.001 and costs 0.00003.""" + time. Input is free here and the model's own output limit is 10 + tokens at 10 USD a million: the first call is booked 0.0001, and once + its reply is seen each call is booked what a reply is, 0.00003, its cost.""" here = os.path.join(self.tmp, "calling") os.makedirs(here, exist_ok=True) with open(os.path.join(here, "driver.py"), "w") as f: f.write(CALLING_DRIVER) b = dict(definitions.get("benchmark", "fakebench", ROOT), _here=here, driver={"command": ["python3", "{here}/driver.py"]}) said = [] - kw = {"prices": {"m-1": {"input": 0.0, "output": 10.0, "max_output": 100}}, **kw} + kw = {"prices": {"m-1": {"input": 0.0, "output": 10.0, "max_output": 10}}, **kw} job_dir, failed = run.run_and_report(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, concurrency=1, env=self.env, say=said.append, unsandboxed=True, proving=True, **kw) with open(os.path.join(job_dir, "result.json")) as f: return job_dir, failed, json.load(f), said def test_a_spending_cap_stops_the_run_and_its_result_says_where(self): - # 0.001 booked a call, 0.00003 spent by each: the fourth no longer fits 0.00107 - job_dir, failed, result, said = self.calling(max_usd=0.00107) - self.assertEqual(result["budget"], {"max_usd": 0.00107, "spent_usd": 0.00009, "stopped": True, + # 0.00003 spent by each call and as much booked for the next: with 0.00009 spent the fourth no longer fits 0.00011 + job_dir, failed, result, said = self.calling(max_usd=0.00011) + self.assertEqual(result["budget"], {"max_usd": 0.00011, "spent_usd": 0.00009, "stopped": True, "cut_off": ["m3"], "not_run": ["m4", "m5", "m6", "m7"]}) self.assertEqual((result["n_total_trials"], result["stats"]["n_cancelled_trials"], result["stats"]["n_errored_trials"]), (8, 4, 1)) self.assertFalse(failed) # three units reached a verdict @@ -324,9 +325,9 @@ def test_a_spending_cap_stops_the_run_and_its_result_says_where(self): self.assertEqual([c.get("refused") for c in calls].count("cap"), 1) sent = [c for c in calls if not c.get("refused")] self.assertEqual(len(sent), 3) # no call went out past the cap - self.assertTrue(all(abs(c["booked_usd"] - 0.001) < 1e-12 for c in sent)) - self.assertLessEqual(sum(c["cost_usd"] for c in sent), 0.00107) - self.assertTrue(any(s.startswith("stopped: a call did not fit the spending cap of 0.0011 USD (0.0001 spent): 1 cut off, 4 not run") + self.assertEqual(sorted(round(c["booked_usd"], 8) for c in sent), [0.00003, 0.00003, 0.0001]) # the start once, then what a reply has been + self.assertLessEqual(sum(c["cost_usd"] for c in sent), 0.00011) + self.assertTrue(any(s.startswith("stopped: a call did not fit the spending cap of 0.0001 USD (0.0001 spent): 1 cut off, 4 not run") for s in said), said) self.assert_no_key(job_dir) diff --git a/docs/cli/run.md b/docs/cli/run.md index 25e4d3a..aa42329 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -56,14 +56,14 @@ the job books each call in one ledger before sending it | Rule | Detail | | --- | --- | | Prices first | Every model the run calls (its own, the judge, the simulated user, an option's embeddings, an agent's side models) needs a price in your [price list](../settings/model-prices.md) or a `--price`; else the run is refused before it starts | -| A booking | The most a call is likely to cost: its characters over three as input tokens, and as output the limit the call names, else the model's own (its price's output limit), else 4,096 tokens | +| A booking | What a call in flight is counted as, since its cost is known only when its reply comes back: its characters over three as input tokens, and as output the largest reply the run has had from that model so far, 4,096 tokens before any; never more than the limit the call names or the model's own | | Sent or refused | A call is sent only while what was spent, what the calls in flight have booked and its own booking stay within the cap. The first that does not fit is refused, and so is every call after it: the run stops at one point | | Settled | A reply turns the booking into what the call cost. A provider's error gives the booking back. A reply with no token count, or cut off, keeps its whole booking as unsettled: it may have been billed | | A refused call | Answered by the relay with HTTP 402, naming the relay and the cap, and recorded `refused`: never the provider's error | | The units | One cut off by the cap ends as `SpendingCapError`: the outcome Spending cap, outside the agent's rate, never a miss. Units not begun are not run | | The result | `result.json` carries `budget`: `max_usd`, `spent_usd`, `unsettled_usd` when there is some, `stopped`, `cut_off` and `not_run`; stderr says `stage=cap` | -| Several at a time | The calls in flight take room: with `-j 4`, four bookings must fit at once. A cap close to one call's booking stops the run early | -| An estimate | Prices are list prices and a booking is not a bill: a call that costs more than was booked takes the spend past the cap by the difference, and the next call is refused | +| Several at a time | The calls in flight take room: with `-j 4`, four bookings must fit at once. The first calls, before any reply, are booked 4,096 output tokens each | +| An estimate | What was spent is each reply's cost, the provider's own figure when its reply carries one. A booking is not a bill: a reply longer than any before it costs more than was booked and takes the spend past the cap by the difference; the next call is then refused | | A shard | The cap is that shard's alone: split a budget over the shards yourself |