diff --git a/app/nullius/runs/parse.py b/app/nullius/runs/parse.py index aaadc1c..2633468 100644 --- a/app/nullius/runs/parse.py +++ b/app/nullius/runs/parse.py @@ -30,7 +30,7 @@ # The shape this module produces. A workspace caches parsed Jobs keyed on it, # so bump it when a Trial or Job field is added, renamed or means something # new — otherwise a cache written by the old parse is read back as current. -SCHEMA = "13" +SCHEMA = "14" # What a rate-limit record in `agent/gateway.log` looks like. The agent's SDK # retries a 429 below the harness (an agent's `max_retries` reaching its @@ -283,6 +283,9 @@ class Job: # defaults filled in (launch_of): what "New run like this" rebuilds a # recipe from when the run kept none. Empty when config.json is absent. launch: dict = field(default_factory=dict) + # A run with a spending cap (a Nullius standard job's ``result.json``, ``budget``): the cap and what was + # spent in USD, whether the cap stopped the run, the units it cut off and those never begun. {} without one. + budget: dict = field(default_factory=dict) # A Harbor job run with several agents or models (`-m a -m b`, or a config # listing several): each one's {name, model, effort}, which split_agents # turns into one view per agent. [] for a job of one agent. @@ -830,6 +833,17 @@ def launch_of(cfg: dict) -> dict: } +def _budget(b) -> dict: + """A job's recorded spending cap, read leniently: {} unless it names a cap; an amount that is no number is + None (not reported), never zero.""" + num = lambda v: v if isinstance(v, (int, float)) and not isinstance(v, bool) else None + if not isinstance(b, dict) or not num(b.get("max_usd")): + return {} + names = lambda v: [str(x) for x in v] if isinstance(v, list) else [] + return {"max_usd": b["max_usd"], "spent_usd": num(b.get("spent_usd")), "unsettled_usd": num(b.get("unsettled_usd")), + "stopped": b.get("stopped") is True, "cut_off": names(b.get("cut_off")), "not_run": names(b.get("not_run"))} + + def parse_job(job_dir: str, reuse=None) -> Job: """One job directory. ``reuse(trial_dir)``: a Trial already parsed from files that have not changed since (a workspace's cache, for a running @@ -906,6 +920,7 @@ def parse_job(job_dir: str, reuse=None) -> Job: ctl = std.get("control") if isinstance(std.get("control"), dict) else {} job.control_source = str(ctl.get("source") or "control") if ctl.get("name") == job.agent_label.lower() else "" job.launch_id = str(std.get("launch") or "") + job.budget = _budget(res.get("budget")) job.standard = bool(std) # a runner's job was never a `harbor run`: there is no command to rebuild job.harbor_command = "" if job.runner else redact(harbor_command( diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index d0642c5..00848cc 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -26,7 +26,7 @@ import json import os import shutil -from typing import Dict, List +from typing import Dict, List, Optional from nullius.standard import job @@ -154,6 +154,21 @@ def check(job_dirs: List[str], force: bool = False) -> dict: return {"shards": shards, "by_unit": by_unit, "notes": notes, "missing": missing, "units_total": total, "differ": differ} +def _budget(budgets: list) -> Optional[dict]: + """The shards' spending caps as one (``run.Run.budget``): the caps and what was spent added up, stopped + when any shard was, their units cut off and not run together. None unless every shard had a cap: a cap + over part of a launch is not the launch's.""" + if not budgets or not all(isinstance(x, dict) and isinstance(x.get("max_usd"), (int, float)) for x in budgets): + return None + spent = [x.get("spent_usd") for x in budgets] + unsettled = round(sum(x.get("unsettled_usd") or 0 for x in budgets), 6) + return {"max_usd": round(sum(x["max_usd"] for x in budgets), 6), + "spent_usd": None if any(not isinstance(v, (int, float)) for v in spent) else round(sum(spent), 6), + **({"unsettled_usd": unsettled} if unsettled else {}), "stopped": any(x.get("stopped") for x in budgets), + "cut_off": [u for x in budgets for u in x.get("cut_off") or []], + "not_run": [u for x in budgets for u in x.get("not_run") or []]} + + def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: """Merge the shard jobs ``job_dirs`` into the new job ``out``. {out, trials, merged_from, notes, missing}.""" @@ -183,6 +198,7 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: errored = sum(1 for r in results if r.get("exception_info")) rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results] scores = [w["score"] for w in rewards if "score" in w] + budget = _budget([s["result"].get("budget") for s in shards]) with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f: # ISO stamps in UTC sort as their instants json.dump({"id": name, "started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None, @@ -190,7 +206,8 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), - "scored": len(scores), "mean_score": job.mean_score(results)}}, f, indent=1) + "scored": len(scores), "mean_score": job.mean_score(results)}, + **({"budget": budget} if budget else {})}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 682556a..dd8167f 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -31,13 +31,14 @@ """ from __future__ import annotations +import math import re import shlex from typing import Callable, Dict, List, Optional from nullius import hosts, providers from nullius.providers import PROVIDER_LABEL, PROVIDERS, STEP -from nullius.standard import hostreq, definitions, match, ports, prove +from nullius.standard import hostreq, definitions, ledger, match, ports, prove HARNESS = "standard" # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's. @@ -253,7 +254,8 @@ def _int(v, low: int, high: Optional[int] = None) -> Optional[int]: def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, - docker_unusable: Optional[Callable[[], Optional[str]]] = None) -> dict: + docker_unusable: Optional[Callable[[], Optional[str]]] = None, + unpriced: Optional[Callable[[List[str]], List[str]]] = None) -> dict: """{errors, notes, review, jobs, confirm, script, parts, model} for one recipe; errors name the step that edits what is wrong. And ``shards``: one per machine of *Where*, each @@ -265,7 +267,15 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, machine, why docker cannot run here (``container.unusable``) - a reason is the Where step's error. Not given, it is not asked: compiling stays a function of the definitions alone, and the routes that preview or start - a run pass it.""" + a run pass it. + + ``max_usd``: the run's spending cap (``standard.ledger``), one amount for + the whole run: its jobs that call a model - each dataset with each + participant, on each machine - take equal shares (``--max-usd``), so + none spends another's and the amount typed is the most the run spends. + ``unpriced``: asked, for a recipe with a cap, which of the models it + names have no price in the user's list (``nullius.prices``): a run + with a cap is refused without one, and the Limits step says so first.""" d = defs or _all() errors, notes = [], [] add = lambda step, message: errors.append({"step": STEP[step], "message": message}) @@ -413,6 +423,13 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model + # the run's spending cap: an amount in USD, or none + cap = recipe.get("max_usd") + if cap in ("", None): + cap = None + elif isinstance(cap, bool) or not isinstance(cap, (int, float)) or not 0 < cap < float("inf"): + add("limits", "The spending cap is an amount in USD above zero, or empty for none.") + cap = None limit = recipe.get("limit") if limit not in ("", None) and _int(limit, 1) is None: add("limits", "The number of units is a whole number of at least 1, or empty for all of them.") @@ -426,6 +443,25 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, why = docker_unusable() if why: add("where", f"The docker backend cannot run on this machine: {why}. Choose local processes, or a host with docker.") + # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share + spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"]) + paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) + share = None + if cap is not None and paying: + share = math.floor(cap / paying * 1e6) / 1e6 # never more than the cap in all + if not share: + add("limits", "The spending cap is too small" + (f" to share over {paying} jobs." if paying > 1 else ".")) + elif unpriced is not None: + # the models a run names as it sends them (run.Run.models_called): its own or a setup's, the judge's, + # the simulated user's, an option's embeddings + judge, sim = (b.get("score") or {}).get("judge"), b.get("simulator") + called = [*[(p["o"].get("sent") or p.get("model")) if p.get("setup") else model for p in picks if not p["control"]], + (judge_model or providers.served_as(judge["model"], provider)) if judge else "", + (sim_model or providers.served_as(sim["model"], provider)) if sim and any(not p["control"] for p in picks) else "", + *needs["embeddings"][:1]] + missing = unpriced([m for m in dict.fromkeys(called) if isinstance(m, str) and m]) + if missing: + add("limits", f"A spending cap needs a price for {', '.join(missing)}: update the price list in Settings, or set a price there.") if errors: return out nouns = (b.get("units") or {}).get("nouns", "units") @@ -458,6 +494,7 @@ def command_for(d: dict, p: dict, index: int) -> str: "--dataset", d["id"], *(["--limit", str(limit)] if isinstance(limit, int) else []), "-j", str(conc), *[w for k, v in options_for_pick(d, p).items() for w in ("--bench-option", f"{k}={v}")], *(["--repeat", str(repeat)] if repeat > 1 else []), + *(["--max-usd", f"{share:.6f}".rstrip("0").rstrip(".")] if share and spends(p) else []), *(["--simulator-model", sim_model] if sim_model and not p["control"] else []), *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], @@ -537,6 +574,8 @@ def row(p: dict) -> list: + (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "") + (" · a setup counts " + next(p["o"]["units"]["nouns"] for p in picks if p.get("setup")) if any(p.get("setup") for p in picks) and not all(p.get("setup") for p in picks) else "")], + *([[STEP["limits"], "Spending cap", f"${ledger.usd(cap)}" + + (f" in all · ${ledger.usd(share)} for each of {paying} jobs" if paying > 1 else "")]] if share else []), [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") + ", ".join(where_label(t) for t in targets) + f" · {backend} backend"]] if agents_local: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index e99e51b..34340e6 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -527,7 +527,8 @@ def start(self) -> str: work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] self.n_units = len(work) # what the progress counts: every run of every unit self.feed = live_feed.Writer(os.path.join(self.job_dir, live_feed.FEED), self.name) - self.feed.emit("job.start", n_total=len(work), n_concurrent=self.concurrency, agents=[self.other["key"]], tasks=units) + self.feed.emit("job.start", n_total=len(work), n_concurrent=self.concurrency, agents=[self.other["key"]], tasks=units, + **({"budget": {"max_usd": self.max_usd}} if self.ledger is not None else {})) # the live board says the cap for pos, (i, u, r) in enumerate(work): self.feed.emit("trial.queued", self.trial_name(i, u, r), agent=self.other["key"], position=pos) results = [] diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 5d2ca8f..389dc20 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -135,6 +135,8 @@ if ((Number.isInteger(raw.concurrency) && raw.concurrency >= 1) || raw.concurrency === '') r.concurrency = raw.concurrency; // attempts per unit and the judge's and user simulator's models: a reopened recipe runs as it was saved if (Number.isInteger(raw.repeat) && raw.repeat >= 1 && raw.repeat <= 32) r.repeat = raw.repeat; + // the run's spending cap in USD, shared by its jobs (recipe.py): an amount above zero, or none + if (typeof raw.max_usd === 'number' && Number.isFinite(raw.max_usd) && raw.max_usd > 0) r.max_usd = raw.max_usd; for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); r.unsandboxed = raw.unsandboxed === true; if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id'; diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index a298f2a..ddb03c8 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -623,7 +623,9 @@ return '

Scope

' + field('Number of ' + u.nouns, 'limit', rec.limit, {type: 'number', min: 1, placeholder: all, hint: 'The first ones' + per + ', in its order.'}) + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: '1'}) + '
' - + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}); + + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + // the whole run's, shared by its jobs (recipe.py); a run that calls no model spends nothing: no cap to offer + + (C.standardKeyless(rec) ? '' : field('Spending cap (USD)', 'max_usd', rec.max_usd ?? '', {type: 'number', min: 0, placeholder: 'none'})); } return ''; } diff --git a/app/static/runs/board.css b/app/static/runs/board.css index 69a70ce..d41c48f 100644 --- a/app/static/runs/board.css +++ b/app/static/runs/board.css @@ -212,6 +212,8 @@ .rp .popm { position: absolute; z-index: 6; left: 0; top: calc(100% + 4px); display: grid; grid-template-columns: auto auto; gap: 4px 14px; padding: 9px 12px; border-radius: 8px; background: var(--surface-2); border: 1px solid var(--border); box-shadow: var(--shadow); font-size: 12px; color: var(--text-faint); white-space: nowrap; } .rp .popm b { font: 500 12px var(--mono); color: var(--text); text-align: right; } +/* the units a spending cap left out: ids, as many as there are */ +.rp .popm.cap-units { grid-template-columns: repeat(4, auto); max-height: 220px; overflow-y: auto; font-family: var(--mono); color: var(--text-dim); } /* linked preview: a hovered row lights its board marks and dims the rest; board counts never change */ .rp .board.pv .hl:not(.lr) { opacity: .28; } diff --git a/app/static/runs/live-board.js b/app/static/runs/live-board.js index b969a07..384ee65 100644 --- a/app/static/runs/live-board.js +++ b/app/static/runs/live-board.js @@ -87,7 +87,13 @@ } // measured, or left out: a rate the window could not measure, and a cost nobody reported, are absent, never 0 if (s.tokens_per_sec != null) bits.push(`${count(Math.round(s.tokens_per_sec))} tokens / s, last 30 s`); - if (T.cost_usd != null) bits.push(`$${Number(T.cost_usd).toFixed(2)} spent`); + // a run with a spending cap (job.start's budget) says it beside what was spent, or alone while nothing was + // the cap as it was given: cents, and up to two digits more when it has them (0.395), never rounded to a cent it is not + const capped = v => { const [whole, part = ''] = Number(v).toFixed(4).replace(/0+$/, '').split('.'); return whole + '.' + part.padEnd(2, '0'); }; + const cap = s.budget && Number(s.budget.max_usd) > 0 ? ` of the $${capped(s.budget.max_usd)} cap` : ''; + // under a cent, four decimals as Results shows it: beside a cap, $0.00 spent would read as nothing spent + if (T.cost_usd != null) bits.push(`$${Number(T.cost_usd).toFixed(cap && T.cost_usd < 0.01 ? 4 : 2)} spent${cap}`); + else if (cap) bits.push(`spending cap $${capped(s.budget.max_usd)}`); if (s.wall_sec) bits.push(`wall ${clock(s.wall_sec)} per trial`); return bits.join(''); } diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index 962842a..ec90fb2 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -56,6 +56,14 @@ {% endif %} +{% if job.budget and job.budget.stopped %} +{# The run's own spending cap ended it (a Nullius standard run's --max-usd): nothing failed, so the notice is neutral. #} +
+ Stopped at its spending cap of {{ job.budget.max_usd|cap }}{% if job.budget.spent_usd is not none %}, with {{ job.budget.spent_usd|usd }} spent{% endif %}: + {{ job.budget.cut_off|length }} cut off, {{ job.budget.not_run|length }} not run. +
+{% endif %} + {% if live_board and live_board.error %}
The live feed could not be read — {{ live_board.error }}. The sections below come from the trial results and are unaffected.
{% endif %} @@ -305,6 +313,9 @@
in · cached · out
input{{ '{:,}'.format(job.total_input_tokens) }}cached (part of input){{ '{:,}'.format(job.total_cache_tokens) }}output{{ '{:,}'.format(job.total_output_tokens) }}
Cost{{ job.total_cost_usd|usd(job.kind) }}
+ {% if job.budget %}
Spending cap{{ job.budget.max_usd|cap }}
+ {% if job.budget.not_run %}
{{ job.budget.not_run|length }} not run +
{% for u in job.budget.not_run %}{{ u }}{% endfor %}
{% endif %}{% endif %} {% if job.mean_score is not none %}
Mean score{{ '%.2f'|format(job.mean_score) }}
{% endif %} {% for g in job.score_by_group %}
{{ job.parts_by|capitalize }} {{ g.group }}{{ '%.2f'|format(g.mean) }} {{ g.n }}
{% endfor %} {% if job.duration_min %}
Duration{{ job.duration_min|runtime }}
{% endif %} diff --git a/app/tests/field_contract.json b/app/tests/field_contract.json index 6b11649..62125f9 100644 --- a/app/tests/field_contract.json +++ b/app/tests/field_contract.json @@ -258,6 +258,7 @@ "agent_name", "agent_version", "agents", + "budget", "control_source", "counts", "dataset", diff --git a/app/tests/test_standard_recipe.py b/app/tests/test_standard_recipe.py index 3a5a965..d262c92 100644 --- a/app/tests/test_standard_recipe.py +++ b/app/tests/test_standard_recipe.py @@ -135,6 +135,65 @@ def test_a_value_cannot_become_a_shell_word(self): self.assertEqual(out["errors"][0]["step"], recipe.STEP["model"]) +class SpendingCap(unittest.TestCase): + """*Limits*: one cap for the whole run (``max_usd``), shared in equal parts by its jobs that call a model.""" + + def twin(self): + return dict(definitions.load("agent", ROOT)[0][0], key="twin", label="Twin agent") + + def caps(self, out): + return [c.split("--max-usd ")[1].split()[0] if "--max-usd" in c else None for sh in out["shards"] for c in sh["command"].split("\n")] + + def test_one_job_takes_the_whole_cap_and_no_cap_adds_nothing(self): + out = recipe.compile(a_recipe(max_usd=2), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("-j 2 --max-usd 2 --name fake-0101-120000", out["script"]) + self.assertIn([recipe.STEP["limits"], "Spending cap", "$2.00"], out["review"]) + for none in ({}, {"max_usd": ""}, {"max_usd": None}): + plain = recipe.compile(a_recipe(**none), defs()) + self.assertEqual((plain["errors"], self.caps(plain)), ([], [None])) + self.assertNotIn("Spending cap", [row[1] for row in plain["review"]]) + + def test_the_jobs_of_a_run_share_the_cap_in_equal_parts_and_never_pass_it(self): + out = recipe.compile(a_recipe(others=["fake", "twin"], where=["local", "me@bench"], max_usd=1), defs(agent=[self.twin()])) + self.assertEqual(out["errors"], []) + self.assertEqual(self.caps(out), ["0.25"] * 4) # two agents on two machines: four jobs + self.assertIn([recipe.STEP["limits"], "Spending cap", "$1.00 in all · $0.25 for each of 4 jobs"], out["review"]) + three = recipe.compile(a_recipe(where=["local", "me@bench", "gpu-box"], max_usd=1), defs()) + self.assertEqual(self.caps(three), ["0.333333"] * 3) # rounded down: the shares never add up to more + self.assertLessEqual(sum(float(c) for c in self.caps(three)), 1) + + def test_a_job_that_calls_no_model_takes_no_share(self): + bench = dict(definitions.find(definitions.load("benchmark", ROOT)[0], "fakebench"), key="checked", + controls={"nop": {"source": "nullius", "from": "an empty answer"}}) + out = recipe.compile(a_recipe(benchmark="checked", others=["fake", "nop"], max_usd=2), defs(benchmark=[bench])) + self.assertEqual(out["errors"], []) + self.assertEqual(self.caps(out), ["2", None]) # the control spends nothing + self.assertIn([recipe.STEP["limits"], "Spending cap", "$2.00"], out["review"]) + alone = recipe.compile(a_recipe(benchmark="checked", others=["nop"], max_usd=2), defs(benchmark=[bench])) + self.assertEqual((alone["errors"], self.caps(alone)), ([], [None])) + self.assertNotIn("Spending cap", [row[1] for row in alone["review"]]) + + def test_a_cap_that_is_no_amount_names_limits(self): + for bad in (0, -1, "2", True, float("inf"), float("nan"), [2]): + with self.subTest(bad): + errors = recipe.compile(a_recipe(max_usd=bad), defs())["errors"] + self.assertEqual([(e["step"], e["message"]) for e in errors], + [(recipe.STEP["limits"], "The spending cap is an amount in USD above zero, or empty for none.")]) + small = recipe.compile(a_recipe(where=["local", "me@bench"], max_usd=1e-6), defs())["errors"] + self.assertEqual([(e["step"], e["message"]) for e in small], [(recipe.STEP["limits"], "The spending cap is too small to share over 2 jobs.")]) + + def test_a_cap_without_a_price_is_said_at_limits_and_only_a_cap_asks(self): + asked = [] + out = recipe.compile(a_recipe(max_usd=2), defs(), unpriced=lambda models: asked.append(list(models)) or list(models)) + self.assertEqual(asked, [["gpt-5-mini"]]) # the model as the run sends it + self.assertEqual([(e["step"], e["message"]) for e in out["errors"]], + [(recipe.STEP["limits"], "A spending cap needs a price for gpt-5-mini: update the price list in Settings, or set a price there.")]) + self.assertEqual(recipe.compile(a_recipe(max_usd=2), defs(), unpriced=lambda models: [])["errors"], []) + never = lambda models: self.fail("a recipe without a cap asks for no price") + self.assertEqual(recipe.compile(a_recipe(), defs(), unpriced=never)["errors"], []) + + class ExecutionScripts(unittest.TestCase): """Execution scripts dispatch only to stubs, with no installed package visible to the shell's python3 and no developer settings or credentials.""" diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 52d418d..93b552d 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -12,7 +12,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest import mock -from nullius.runs import classify, parse +from nullius.runs import classify, live, parse from nullius.standard import acp, container, definitions, install, ledger, run ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") @@ -329,6 +329,11 @@ def test_a_spending_cap_stops_the_run_and_its_result_says_where(self): self.assertLessEqual(sum(c["cost_usd"] for c in sent), 0.00011) self.assertTrue(any(s.startswith("stopped: a call did not fit the spending cap of 0.0001 USD (0.0001 spent): 1 cut off, 4 not run") for s in said), said) + # the live board is told the cap as the units start, and the unit it cut off ends there as budget + with open(os.path.join(job_dir, "events.jsonl")) as f: + board = live.fold([live.parse_line(line) for line in f if line.strip()]) + self.assertEqual(board.budget, {"max_usd": 0.00011}) + self.assertEqual({t.trial.split("__")[0]: t.outcome for t in board.trials if t.outcome}, {"m0": "resolved", "m1": "resolved", "m2": "resolved", "m3": "budget"}) self.assert_no_key(job_dir) def test_a_spending_cap_the_run_stays_under_changes_nothing_but_the_result(self): diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index c1bd8bf..526e4a1 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -61,7 +61,7 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=()): + planned=None, t0=1_760_000_000.0, failed=(), budget=None): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) @@ -80,7 +80,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": 1.0 if place % 2 else 0.0}, exception=exc, calls=[], trajectory=None)) - job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned)) + job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) return d @@ -117,6 +117,31 @@ def test_the_merged_job_reads_as_one_run_of_every_trial(self): self.assertTrue(os.path.samefile(os.path.join(src, "workspace", "solution.py"), os.path.join(got["out"], "t-1__u0001", "workspace", "solution.py"))) + def test_the_merged_job_carries_the_shards_spending_caps_as_one(self): + """Each shard ran under its share of the run's cap: the merged job says the whole cap, what all of them + spent, and every unit a cap cut off or left out.""" + caps = {1: {"max_usd": 0.5, "spent_usd": 0.2, "stopped": False, "cut_off": [], "not_run": []}, + 2: {"max_usd": 0.5, "spent_usd": 0.41, "unsettled_usd": 0.03, "stopped": True, "cut_off": ["t-9"], "not_run": ["t-11", "t-12"]}, + 3: {"max_usd": 0.5, "spent_usd": 0.1, "stopped": False, "cut_off": [], "not_run": []}} + dirs = [write_shard(self.tmp, f"run-shard-{i}of3", self.split[i], i, 3, budget=caps[i]) for i in (1, 2, 3)] + got = merge.merge(dirs, os.path.join(self.tmp, "capped", "run")) + want = {"max_usd": 1.5, "spent_usd": 0.71, "unsettled_usd": 0.03, "stopped": True, "cut_off": ["t-9"], "not_run": ["t-11", "t-12"]} + self.assertEqual(read(os.path.join(got["out"], "result.json"))["budget"], want) + self.assertEqual(parse.parse_job(got["out"]).budget, want) + self.assertGreaterEqual(int(parse.SCHEMA), 14) # a job cached before the field is read again + # a cap over part of a launch is not the launch's: nothing is claimed + shutil.rmtree(dirs[2]) + dirs[2] = write_shard(self.tmp, "run-shard-3of3", self.split[3], 3, 3) + part = merge.merge(dirs, os.path.join(self.tmp, "partly", "run")) + self.assertNotIn("budget", read(os.path.join(part["out"], "result.json"))) + self.assertEqual(parse.parse_job(part["out"]).budget, {}) + + def test_a_recorded_cap_is_read_leniently_and_an_amount_missing_is_not_zero(self): + self.assertEqual(parse._budget(None), {}) + self.assertEqual(parse._budget({"max_usd": "2", "spent_usd": 1}), {}) # no cap named: no cap + self.assertEqual(parse._budget({"max_usd": 2, "stopped": "yes", "cut_off": "t-1"}), + {"max_usd": 2, "spent_usd": None, "unsettled_usd": None, "stopped": False, "cut_off": [], "not_run": []}) + def test_shards_of_different_launches_models_or_versions_are_refused_unless_forced(self): a = write_shard(self.tmp, "a", self.split[1], 1, 3) for kw, word in (({"launch": "L2"}, "launch"), ({"model": "gpt-4o"}, "model"), diff --git a/app/ui_tests/test_capped_job.py b/app/ui_tests/test_capped_job.py new file mode 100644 index 0000000..ea8b407 --- /dev/null +++ b/app/ui_tests/test_capped_job.py @@ -0,0 +1,118 @@ +"""A run with a spending cap, as its pages show it: the job page says the cap, +that it stopped the run, and the units it cut off or never began, read from +the job as a standard run writes it; the builder keeps a recipe's cap.""" +from __future__ import annotations + +import re +import tempfile +import unittest +from contextlib import ExitStack +from html import unescape +from pathlib import Path +from unittest.mock import patch + +from nullius import workspace +from nullius.runs import classify +from nullius.standard import job, ledger +from ui_tests.builder_support import AGENT as CUSTOM_AGENT, APP, PIN, CompilerCase +from .test_routes import web + +BENCH = {"key": "fakebench", "_ref": "fakebench", "family": "agent", "needs": ["agent.workspace/v1"], + "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "pass", "says": "the tests pass"}} +AGENT = {"key": "fake", "_kind": "agent", "_ref": "fake"} + + +def capped_job(root: Path, name: str, ran: int, planned: int, budget=None, cut=()): + """A standard job of ``planned`` units of which the first ``ran`` have a trial; those in ``cut`` ended at the cap.""" + d = root / name + d.mkdir(parents=True) + units = [f"t-{i}" for i in range(planned)] + job.write_config(str(d), job=name, bench=BENCH, other=AGENT, model="m-1", provider="openai", dataset="all", units=units, + concurrency=1, granted=True) + results = [] + for k, u in enumerate(units[:ran]): + trial = d / f"{u}__u{k:04d}" + trial.mkdir() + exc = (ledger.ERROR, "the run's spending cap") if u in cut else None + results.append(job.write_trial(str(trial), job=name, bench=BENCH, other=AGENT, model="m-1", unit=u, t0=1_760_000_000.0 + k, + t1=1_760_000_005.0 + k, produce=(1_760_000_000.0 + k, 1_760_000_004.0 + k), + score=None if exc else {"reward": 1.0}, exception=exc, calls=[], trajectory=None)) + job.write_result(str(d), name, 1_760_000_000.0, 1_760_000_060.0, results, planned, budget=budget) + + +class CappedJobPage(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp = tempfile.TemporaryDirectory(prefix="nullius-capped-job-") + tmp = Path(cls.temp.name) + capped_job(tmp / "jobs", "stopped", 6, 10, cut=("t-5",), + budget={"max_usd": 0.5, "spent_usd": 0.41, "unsettled_usd": 0.03, "stopped": True, "cut_off": ["t-5"], + "not_run": ["t-6", "t-7", "t-8", "t-9"]}) + capped_job(tmp / "jobs", "within", 4, 4, budget={"max_usd": 0.395, "spent_usd": 0.004, "stopped": False, "cut_off": [], "not_run": []}) + capped_job(tmp / "jobs", "uncapped", 4, 4) + cls.stack = ExitStack() + cls.stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "ws"))) + cls.stack.enter_context(patch.dict(web.app.config, TESTING=True)) + cls.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []})) + cls.ws = workspace.create(str(tmp / "ws"), [str(tmp / "jobs")], label="capped") + web.shared.open_workspace(cls.ws) + cls.client = web.app.test_client() + + @classmethod + def tearDownClass(cls): + cls.stack.close() + cls.temp.cleanup() + + def page(self, name): + r = self.client.get(f"/job/{name}?view=results"); self.addCleanup(r.close) + self.assertEqual(r.status_code, 200, name) + return r.get_data(as_text=True) + + def text(self, html, pattern): + got = re.search(pattern, html, re.S) + self.assertIsNotNone(got, pattern) + return re.sub(r"\s+", " ", unescape(re.sub(r"<[^>]+>", " ", got[0]))).strip() + + def test_a_run_the_cap_stopped_says_so_with_what_was_spent_and_the_units_left_out(self): + html = self.page("stopped") + self.assertEqual(self.text(html, r'
.*?
'), + "Stopped at its spending cap of $0.50, with $0.41 spent: 1 cut off, 4 not run.") + self.assertNotIn('class="alert error" id="cap-stopped"', html) # nothing failed: a neutral notice + self.assertEqual(self.text(html, r'
'), "Spending cap $0.50") + self.assertIn("$0.03 stays booked for calls whose cost was never reported", unescape(html)) + self.assertEqual(self.text(html, r'
'), "t-6 t-7 t-8 t-9") + self.assertIn("4 not run", html) + # the unit it cut off is a trial with its own outcome, outside the rate + self.assertRegex(html, r'class="[^"]*oc-spending_cap[^"]*"[^>]*data-o="spending_cap"') + self.assertIn(classify.LABELS[classify.SPENDING_CAP] + 'infra', html) + self.assertIn("6 of 10 planned", self.text(html, r'
')) + + def test_a_run_within_its_cap_says_the_cap_and_no_more_and_one_without_says_nothing(self): + html = self.page("within") + self.assertEqual(self.text(html, r'
'), "Spending cap $0.395") # as it was given, not rounded to $0.40 + for absent in ('id="cap-stopped"', "cap-units", "not run", "oc-spending_cap"): + self.assertNotIn(absent, html) + plain = self.page("uncapped") + for absent in ('id="run-cap"', 'id="cap-stopped"', "Spending cap"): + self.assertNotIn(absent, plain) + + +class BuilderKeepsTheCap(CompilerCase): + def test_a_saved_recipe_keeps_an_amount_above_zero_and_nothing_else(self): + out = self.js('const back=v=>C.fromJSON(JSON.stringify({...C.createStandard("gsm8k"),max_usd:v})).max_usd;' + 'process.stdout.write(JSON.stringify([back(2),back(0.25),back(0),back(-1),back("2"),back(true),back(null),' + '"max_usd" in C.createStandard("gsm8k")].map(v=>v===undefined?"none":v)));') + self.assertEqual(out, [2, 0.25, "none", "none", "none", "none", "none", False]) + + def test_a_harbor_recipe_takes_no_cap(self): + """Harbor's calls do not pass the relay, so nothing could hold them to one: the field is the standard's alone.""" + out = self.js('const r=C.fromJSON(JSON.stringify({...C.create("harbor"),max_usd:2}));' + CUSTOM_AGENT + PIN + + 'process.stdout.write(JSON.stringify(["max_usd" in r,/max.usd/i.test(JSON.stringify(C.compile(r)))]));') + self.assertEqual(out, [False, False]) + page = (APP / "static" / "builder" / "command-builder.js").read_text(encoding="utf-8") + self.assertEqual(page.count("'max_usd'"), 1) # offered once, on the standard's Limits step + self.assertIn("C.standardKeyless(rec) ? '' : field('Spending cap (USD)', 'max_usd'", page) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/builder.py b/app/web/builder.py index 2cec06c..ef77f54 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -87,7 +87,7 @@ def new_run_runner(): if len(json.dumps(recipe)) > 200_000: abort(413) if recipe.get("harness") == standard_recipe.HARNESS: - out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem) + out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem, unpriced=execute.unpriced) return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm")} return {"errors": [{"step": providers.STEP["runner"], "message": providers.GONE}], "notes": [], "review": []}, 404 diff --git a/app/web/execute.py b/app/web/execute.py index e9fa777..faadcbd 100644 --- a/app/web/execute.py +++ b/app/web/execute.py @@ -5,7 +5,7 @@ import os import re from flask import abort, jsonify, request, url_for -from nullius import execution, hosts, prerequisites, providers +from nullius import execution, hosts, prerequisites, prices, providers from nullius.harbor import source as harbor_source from nullius.standard import recipe as standard_recipe @@ -115,7 +115,7 @@ def _start_standard(recipe: dict): one checked before any starts.""" if len(json.dumps(recipe)) > 200_000: return jsonify(ok=False, error="the recipe is too large to keep with the run"), 400 - out = standard_recipe.compile(recipe, docker_unusable=docker_problem) + out = standard_recipe.compile(recipe, docker_unusable=docker_problem, unpriced=unpriced) if out["errors"]: return jsonify(ok=False, error=" ".join(e["message"] for e in out["errors"])), 400 # a run of controls calls no model: it is handed no key, so none need be set @@ -141,6 +141,13 @@ def _start_standard(recipe: dict): **started[0].summary()) +def unpriced(models: list) -> list: + """Which of ``models`` have no price in the user's list (``nullius.prices``): what a recipe with a spending + cap is told at its Limits step, since a run with a cap is refused without a price for each model it calls.""" + rows = prices.table() + return [m for m in models if m not in rows] + + def docker_problem(): """Why docker cannot run containers on this machine, or None; kept for a minute (``prerequisites.docker_here``).""" diff --git a/app/web/filters.py b/app/web/filters.py index 29d6422..4b2e046 100644 --- a/app/web/filters.py +++ b/app/web/filters.py @@ -8,7 +8,7 @@ from nullius import importers as harnesses from nullius.runs import classify from nullius import workspace -from nullius.standard import ports, definitions +from nullius.standard import ledger, ports, definitions from .core import app from . import shared @@ -97,6 +97,13 @@ def _usd(value, kind="agent"): return f"${value:.4f}" if value < 0.01 else f"${value:.2f}" +@app.template_filter("cap") +def _cap(value): + """A spending cap as it was given: cents, and up to two digits more when it has them ($0.395). The cost + filter rounds to the cent, which would show a cap the run never had.""" + return "$" + ledger.usd(value) + + @app.template_filter("dsname") def _dsname(value): return shared.dataset_title(value) diff --git a/docs/builder.md b/docs/builder.md index c227b43..dd49d73 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -27,7 +27,7 @@ Then [read its results](results.md). | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs and the licences its units may need | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled | -| Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes | +| Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a spending cap in USD for the whole run: its jobs that call a model share it in equal parts ([A spending cap](cli/run.md#a-spending-cap)) | | Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Docker*. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | diff --git a/docs/cli/run.md b/docs/cli/run.md index aa42329..84c8eaa 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -64,7 +64,9 @@ the job books each call in one ledger before sending it | The result | `result.json` carries `budget`: `max_usd`, `spent_usd`, `unsettled_usd` when there is some, `stopped`, `cut_off` and `not_run`; stderr says `stage=cap` | | Several at a time | The calls in flight take room: with `-j 4`, four bookings must fit at once. The first calls, before any reply, are booked 4,096 output tokens each | | An estimate | What was spent is each reply's cost, the provider's own figure when its reply carries one. A booking is not a bill: a reply longer than any before it costs more than was booked and takes the spend past the cap by the difference; the next call is then refused | -| A shard | The cap is that shard's alone: split a budget over the shards yourself | +| A shard | On the command line the cap is that shard's alone. `merge` adds the shards' caps and spending into one `budget`, when every shard had a cap | +| From the builder | *Limits* takes one cap for the whole run. Its jobs that call a model (each dataset with each participant, on each machine) take equal shares, rounded down, so the amount typed is the most the run spends. A model without a price is named there, before Run. Not offered for a Harbor recipe, whose calls do not pass the relay | +| The job page | Run totals shows the cap. A run the cap stopped says so, with what was spent, and lists the units not run; the units cut off are its Spending cap trials. The live board says what was spent of the cap |
diff --git a/docs/cli/shard-and-merge.md b/docs/cli/shard-and-merge.md index 58a3e8c..1720df2 100644 --- a/docs/cli/shard-and-merge.md +++ b/docs/cli/shard-and-merge.md @@ -15,6 +15,7 @@ nullius standard merge jobs_standard/*__shard-* -o jobs_standard/humaneval-oracl | A unit keeps its place | `u0042` is the 43rd unit of the dataset in every shard; shard jobs never share a trial name | | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch`: `--launch`, or the day (UTC), benchmark, agent or capability and model. The runs page groups them until merged | | `merge JOB_DIR... -o OUT` | One job (`app/nullius/standard/merge.py`): trials hard-linked, each `result.json` naming the merged job and shard; the job's `result.json` recomputed; `config.json` with `merged_from` for `shard`. Different launches, versions, datasets, agents, models, backends or N refused unless `--force`; two trials of one unit always. Units no shard ran are named | +| A spending cap | `--max-usd` is each shard's own; the builder shares a recipe's cap over its shards in equal parts. `merge` adds the caps and what was spent into the merged job's `budget`, with every unit cut off or not run, when every shard had a cap | | `prove --shard I/N` | Controls and leak on that shard, units on all; recorded as `shard` | The builder's *Where* does the same: one shard per machine. diff --git a/docs/field-guide.md b/docs/field-guide.md index 93549c6..698a6ab 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -139,6 +139,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `total_output_tokens` | Completion tokens. | derived | any | | `total_cost_usd` | The trials' recorded costs, summed: an estimate, not a bill. Shows — when no trial recorded one. | derived | any | | `duration_min` | The whole run, start to finish. | derived | any | +| `budget` | A standard run's spending cap, from its `result.json`: the cap and what its ledger counted as spent (`max_usd`, `spent_usd`), whether the cap stopped the run, and the units cut off or never begun. A merged job's is its shards' added up. Empty without a cap. | measured | any | | `agent_label` | The agent's name: the class of a `module:Class` path, or the trials' own name when the config gives none. | measured | any | | `agent_version` | The version the trial's `agent_info` reports. | measured | any | | `is_control` | An `oracle` or `nop` run: a check on the tasks, kept out of rankings and totals. Its row reads as a verdict, not a rate. A standard run is one only when its benchmark declares that control. An imported runner's arm without that declaration is not a control. On a benchmark that scores, a control's score of 1 reads as accepted and 0 as rejected. | measured | any | diff --git a/docs/settings/model-prices.md b/docs/settings/model-prices.md index f9d6158..50d7824 100644 --- a/docs/settings/model-prices.md +++ b/docs/settings/model-prices.md @@ -20,7 +20,7 @@ its list is filled once. | Recorded | A job's `config.json` keeps the prices of the models it calls, each with its source (`list`, `own`, or `run` for `--price`), and where and when the list was fetched when a price is the list's | | Docker and hosts | A unit's relay in a container gets its models' prices from the run; a run on a host takes this machine's list with it | | An estimate | List prices, not a bill: [Metrics](../field-guide.md) | -| A spending cap | `--max-usd` books each call at these prices before sending it, so it needs one for every model the run calls: [Run a benchmark](../cli/run.md#a-spending-cap) | +| A spending cap | `--max-usd`, or the builder's spending cap, books each call at these prices before sending it, so it needs one for every model the run calls: [Run a benchmark](../cli/run.md#a-spending-cap) | | Code | `app/nullius/prices.py` | ## From a shell diff --git a/docs/standard.md b/docs/standard.md index 04910e0..2aea931 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -138,7 +138,7 @@ record); verifier/reward.txt and score.json | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | -| A spending cap (`--max-usd`) | Every relay of the job books a call in one ledger (`.work/ledger`, `standard/ledger.py`) before sending it; the first that does not fit is refused by the relay itself (HTTP 402, recorded `refused`), and every call after it. A unit cut off is a `SpendingCapError`, the outcome `spending_cap` outside the agent's rate; units not begun are not run; `result.json` carries `budget`. [Run a benchmark](cli/run.md#a-spending-cap) | +| A spending cap (`--max-usd`) | Every relay of the job books a call in one ledger (`.work/ledger`, `standard/ledger.py`) before sending it; the first that does not fit is refused by the relay itself (HTTP 402, recorded `refused`), and every call after it. A unit cut off is a `SpendingCapError`, the outcome `spending_cap` outside the agent's rate; units not begun are not run; `result.json` carries `budget`, which the job page shows. The builder's *Limits* takes one cap for the whole run, shared in equal parts by its jobs. [Run a benchmark](cli/run.md#a-spending-cap) | ## Safety @@ -297,7 +297,7 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | Rule | Detail | | --- | --- | -| No central relay | Each unit's relay runs on its own machine with the key it was handed; split a launch over keys of their own if one cannot carry it. A spending cap (`--max-usd`) is each shard's own | +| No central relay | Each unit's relay runs on its own machine with the key it was handed; split a launch over keys of their own if one cannot carry it. A spending cap (`--max-usd`) is each shard's own: the builder shares a recipe's cap over its shards in equal parts, and `merge` adds them up | | Shards by the unit's id | `sha256(unit id) mod N` (`standard/shard.py`); changing N moves most units. Order: the driver's `units`, the shard, then `--limit` (`--shard 1/2 --limit 6` is the first six units of shard 1) | | A unit keeps its place | `u0042` is the 43rd unit of the whole dataset in every shard | | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model) |